From 741a9c517c0252e80e8a12138b3cc1ab7c250247 Mon Sep 17 00:00:00 2001 From: Lucas Esclapez <13371051+esclapez@users.noreply.github.com> Date: Mon, 23 May 2022 11:35:00 -0700 Subject: [PATCH] Completely remove the Fortran and switch off BL_USE_FORT (#232) * Remove remnants of Fortran from the code. * A few more clean in Make.PeleLM * Update nvcc dependency. * Update CMake * Update Sundials submodules to use main branch * Explicitly set main in .gitmodules * Missing a couple of CUDA pieces. * Disable FPE trap by default in input.2d-regt, turn it on as command line keys in CI. * Add missing source for EB in CMake * Update IAMR * Move to CEPTR generated chmeistry files. * Update CMake for CEPTR --- .../dependencies/dependencies_nvcc.sh | 20 +- .gitmodules | 1 + Exec/RegTests/FlameSheet/inputs.2d-regt | 6 +- Source/PeleLM.cpp | 258 +++++------------- Submodules/IAMR | 2 +- Submodules/PelePhysics | 2 +- Submodules/sundials | 2 +- Tools/CMake/BuildPeleLMExe.cmake | 10 +- Tools/CMake/SetAmrexOptions.cmake | 5 +- Tools/Make/Make.PeleLM | 100 ++----- 10 files changed, 120 insertions(+), 286 deletions(-) diff --git a/.github/workflows/dependencies/dependencies_nvcc.sh b/.github/workflows/dependencies/dependencies_nvcc.sh index 2e5b7c2a..0d52e756 100755 --- a/.github/workflows/dependencies/dependencies_nvcc.sh +++ b/.github/workflows/dependencies/dependencies_nvcc.sh @@ -22,9 +22,19 @@ sudo apt-get install -y --no-install-recommends\ pkg-config \ wget -sudo wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub -sudo apt-key add 7fa2af80.pub -echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" | sudo tee /etc/apt/sources.list.d/cuda.list +sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub +echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" \ + | sudo tee /etc/apt/sources.list.d/cuda.list sudo apt-get update -sudo apt-get install -y cuda-command-line-tools-11-0 cuda-compiler-11-0 cuda-cupti-dev-11-0 cuda-minimal-build-11-0 cuda-nvml-dev-11-0 cuda-nvtx-11-0 libcurand-dev-11-0 libcusparse-dev-11-0 libcusolver-dev-11-0 libcublas-dev-11-0 -sudo ln -s cuda-11.0 /usr/local/cuda +sudo apt-get install -y \ + cuda-command-line-tools-11-2 \ + cuda-compiler-11-2 \ + cuda-cupti-dev-11-2 \ + cuda-minimal-build-11-2 \ + cuda-nvml-dev-11-2 \ + cuda-nvtx-11-2 \ + libcurand-dev-11-2 \ + libcusparse-dev-11-2 \ + libcusolver-dev-11-2 \ + libcublas-dev-11-2 +sudo ln -s cuda-11.2 /usr/local/cuda diff --git a/.gitmodules b/.gitmodules index b12cd596..9b0fc263 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,6 +7,7 @@ [submodule "Submodules/sundials"] path = Submodules/sundials url = https://github.com/LLNL/sundials.git + branch = main [submodule "Submodules/IAMR"] path = Submodules/IAMR url = https://github.com/AMReX-Codes/IAMR.git diff --git a/Exec/RegTests/FlameSheet/inputs.2d-regt b/Exec/RegTests/FlameSheet/inputs.2d-regt index a0ef60e9..915c2bcb 100644 --- a/Exec/RegTests/FlameSheet/inputs.2d-regt +++ b/Exec/RegTests/FlameSheet/inputs.2d-regt @@ -149,6 +149,6 @@ mac_proj.verbose = 0 #--------------------------DEBUG/REGTESTS INPUTS----------------------------- amrex.regtest_reduction=1 -amrex.fpe_trap_invalid = 1 -amrex.fpe_trap_zero = 1 -amrex.fpe_trap_overflow = 1 +#amrex.fpe_trap_invalid = 1 +#amrex.fpe_trap_zero = 1 +#amrex.fpe_trap_overflow = 1 diff --git a/Source/PeleLM.cpp b/Source/PeleLM.cpp index e520cfe4..3d126968 100644 --- a/Source/PeleLM.cpp +++ b/Source/PeleLM.cpp @@ -31,18 +31,11 @@ #include #include -#include #include #include #include #include - -#include -#include -#if defined(AMREX_USE_NEWMECH) || defined(AMREX_USE_VELOCITY) -#include -#include -#endif +#include #ifdef AMREX_USE_GPU #include @@ -1775,70 +1768,9 @@ PeleLM::initData () MultiFab& S_new = get_new_data(State_Type); MultiFab& P_new = get_new_data(Press_Type); -#ifdef AMREX_USE_NEWMECH - // - // This code has a few drawbacks. It assumes that the physical - // domain size of the current problem is the same as that of the - // one that generated the pltfile. It also assumes that the pltfile - // has at least as many levels as does the current problem. If - // either of these are false this code is likely to core dump. - // - ParmParse pp("ht"); - - std::string pltfile; - pp.query("pltfile", pltfile); - if (pltfile.empty()) - amrex::Abort("You must specify `pltfile'"); - if (verbose) - amrex::Print() << "initData: reading data from: " << pltfile << '\n'; - - DataServices::SetBatchMode(); - FileType fileType(NEWPLT); - DataServices dataServices(pltfile, fileType); - - if (!dataServices.AmrDataOk()) - // - // This calls ParallelDescriptor::EndParallel() and exit() - // - DataServices::Dispatch(DataServices::ExitRequest, NULL); - - AmrData& amrData = dataServices.AmrDataRef(); - Vector names; - pele::physics::eos::speciesNames(names); - Vector plotnames = amrData.PlotVarNames(); - - int idT = -1, idX = -1; - for (int i = 0; i < plotnames.size(); ++i) - { - if (plotnames[i] == "temp") idT = i; - if (plotnames[i] == "x_velocity") idX = i; - } - // - // In the plotfile the mass fractions directly follow the velocities. - // - int idSpec = idX + AMREX_SPACEDIM; - - for (int i = 0; i < AMREX_SPACEDIM; i++) - { - amrData.FillVar(S_new, level, plotnames[idX+i], Xvel+i); - amrData.FlushGrids(idX+i); - } - amrData.FillVar(S_new, level, plotnames[idT], Temp); - amrData.FlushGrids(idT); - - for (int i = 0; i < NUM_SPECIES; i++) - { - amrData.FillVar(S_new, level, plotnames[idSpec+i], first_spec+i); - amrData.FlushGrids(idSpec+i); - } - - if (verbose) amrex::Print() << "initData: finished init from pltfile" << '\n'; -#endif - - ParmParse pp("pelelm"); - if (pp.countval("pltfile_for_init") > 0) { - S_new.setVal(0.0); - P_new.setVal(0.0); + // If the user provide an initDataPlt, fill the data from there. + ParmParse pp("peleLM"); + if (pp.countval("initDataPlt") > 0) { // // This code has a few drawbacks. It assumes that the physical @@ -1847,80 +1779,66 @@ PeleLM::initData () // has at least as many levels as does the current problem. If // either of these are false this code is likely to core dump. // + S_new.setVal(0.0); + P_new.setVal(0.0); std::string pltfile; - pp.get("pltfile_for_init", pltfile); + pp.query("initDataPlt", pltfile); if (verbose) amrex::Print() << "initData: reading data from: " << pltfile << '\n'; - DataServices::SetBatchMode(); - Amrvis::FileType fileType(Amrvis::NEWPLT); - DataServices dataServices(pltfile, fileType); - if (!dataServices.AmrDataOk()) { - DataServices::Dispatch(DataServices::ExitRequest, NULL); - } - AmrData& amrData = dataServices.AmrDataRef(); + // Use PelePhysics PltFileManager + pele::physics::pltfilemanager::PltFileManager pltData(pltfile); + Vector plotnames = pltData.getVariableList(); + + // Find required data in pltfile Vector names; pele::physics::eos::speciesNames(names); - Vector plotnames = amrData.PlotVarNames(); - - int idT = -1, idV = -1, idX = -1, idY = -1; - for (int i = 0; i < plotnames.size(); ++i) { - if (plotnames[i] == "temp") idT = i; - if (plotnames[i] == "x_velocity") idV = i; - if (plotnames[i] == "X("+names[0]+")") idX = i; - if (plotnames[i] == "Y("+names[0]+")") idY = i; - } - if (verbose) { - Print() << "Initializing data from pltfile: \"" << pltfile << "\" for level " << level << std::endl; - } - for (int i = 0; i < AMREX_SPACEDIM; i++) { - amrData.FillVar(S_new, level, plotnames[idV+i], Xvel+i); - amrData.FlushGrids(idV+i); - } - amrData.FillVar(S_new, level, plotnames[idT], Temp); - amrData.FlushGrids(idT); - if (idY>=0) { - for (int i = 0; i < NUM_SPECIES; i++) { - amrData.FillVar(S_new, level, plotnames[idY+i], first_spec+i); - amrData.FlushGrids(idY+i); + int idT = -1, idX = -1, idY = -1, nSpecPlt = 0; + for (int i = 0; i < plotnames.size(); ++i) + { + std::string firstChars = plotnames[i].substr(0, 2); + if (plotnames[i] == "temp") idT = i; + if (plotnames[i] == "x_velocity") idX = i; + if (firstChars == "Y(") nSpecPlt += 1; + if (firstChars == "Y(" && idY < 0 ) { // species might not be ordered in the order of the current mech. + idY = i; } } - else if (idX>=0) { - for (int i = 0; i < NUM_SPECIES; i++) { - amrData.FillVar(S_new, level, plotnames[idX+i], first_spec+i); - amrData.FlushGrids(idX+i); - } - -#ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - const Box& box = mfi.tilebox(); - auto sfab = S_new.array(mfi); - - amrex::ParallelFor(box, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - amrex::Real Yt[NUM_SPECIES] = {0.0}; - amrex::Real Xt[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { - Yt[n] = sfab(i,j,k,n+1); - } - auto eos = pele::physics::PhysicsType::eos(); - eos.X2Y(Xt,Yt); - for (int n = 0; n < NUM_SPECIES; n++) { - sfab(i,j,k,n) = Xt[n]; + // + // In the plotfile the mass fractions directly follow the velocities. + // + int idSpec = idX + AMREX_SPACEDIM; + + // Velocity + pltData.fillPatchFromPlt(level, geom, idX, Xvel, AMREX_SPACEDIM, + S_new); + + // Temperature + pltData.fillPatchFromPlt(level, geom, idT, Temp, 1, + S_new); + + // Species + // Hold the species in temporary MF before copying to level data + // in case the number of species differs. + MultiFab speciesPlt(grids, dmap, nSpecPlt, 0); + pltData.fillPatchFromPlt(level, geom, idSpec, 0, nSpecPlt, + speciesPlt); + for (int i = 0; i < NUM_SPECIES; i++) { + std::string specString = "Y("+spec_names[i]+")"; + int foundSpec = 0; + for (int iplt = 0; iplt < nSpecPlt; iplt++) { + if ( specString == plotnames[idY+iplt] ) { + MultiFab::Copy(S_new, speciesPlt, iplt, DEF_first_spec+i, 1, 0); + foundSpec = 1; } - }); - } - } - else { - Abort("pltfile_for_init: plotfile not compatible, cannot initialize"); + } + if (!foundSpec) S_new.setVal(0.0,DEF_first_spec+i,1); } + if (verbose) amrex::Print() << "initData: finished init from pltfile" << '\n'; + #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif @@ -1966,11 +1884,7 @@ PeleLM::initData () amrex::ParallelFor(S_new, [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept { -#ifdef AMREX_USE_NEWMECH - amrex::Abort("USE_NEWMECH feature no longer working and has to be fixed/redone"); -#else pelelm_initdata(i, j, k, sma[box_no], geomdata, *lprobparm, lpmfdata); -#endif }); } @@ -1981,71 +1895,43 @@ PeleLM::initData () // computeGradP(state[Press_Type].curTime()); -#ifdef AMREX_USE_VELOCITY // - // We want to add the velocity from the supplied plotfile - // to what we already put into the velocity field via FORT_INITDATA. + // We might want to add the velocity from the supplied plotfile + // to what we already put into the velocity field via init_data // - // This code has a few drawbacks. It assumes that the physical - // domain size of the current problem is the same as that of the - // one that generated the pltfile. It also assumes that the pltfile - // has at least as many levels (with the same refinement ratios) as does - // the current problem. If either of these are false this code is - // likely to core dump. - // - ParmParse pp("ht"); - std::string velocity_plotfile; - pp.query("velocity_plotfile", velocity_plotfile); + pp.query("initVelocityPlt", velocity_plotfile); - if (!velocity_plotfile.empty()) - { + if (!velocity_plotfile.empty()) { if (verbose) - amrex::Print() << "initData: reading data from: " << velocity_plotfile << '\n'; - - DataServices::SetBatchMode(); - Amrvis::FileType fileType(Amrvis::NEWPLT); - DataServices dataServices(velocity_plotfile, fileType); - - if (!dataServices.AmrDataOk()) - // - // This calls ParallelDescriptor::EndParallel() and exit() - // - DataServices::Dispatch(DataServices::ExitRequest, NULL); - - AmrData& amrData = dataServices.AmrDataRef(); - Vector plotnames = amrData.PlotVarNames(); + amrex::Print() << "initData: reading velocity from: " << velocity_plotfile << '\n'; - if (amrData.FinestLevel() < level) - amrex::Abort("initData: not enough levels in plotfile"); + // Use PelePhysics PltFileManager + pele::physics::pltfilemanager::PltFileManager pltData(velocity_plotfile); + Vector plotnames = pltData.getVariableList(); - if (amrData.ProbDomain()[level] != Domain()) - amrex::Abort("initData: problem domains do not match"); + // Find required data in pltfile + Vector names; + pele::physics::eos::speciesNames(names); int idX = -1; for (int i = 0; i < plotnames.size(); ++i) + { if (plotnames[i] == "x_velocity") idX = i; - + } if (idX == -1) - amrex::Abort("Could not find velocity fields in supplied velocity_plotfile"); - + amrex::Abort("Could not find velocity fields in supplied peleLM.initVelocityPlt"); + MultiFab tmp(S_new.boxArray(), S_new.DistributionMap(), 1, 0); - for (int i = 0; i < AMREX_SPACEDIM; i++) - { - amrData.FillVar(tmp, level, plotnames[idX+i], 0); -#ifdef AMREX_USE_OMP -#pragma omp parallel -#endif - for (MFIter mfi(tmp,true); mfi.isValid(); ++mfi) { - S_new[mfi].plus(tmp[mfi], mfi.tilebox(), 0, Xvel+i, 1); // This will not work on GPU - } - amrData.FlushGrids(idX+i); - } + + // Velocity + pltData.fillPatchFromPlt(level, geom, idX, 0, AMREX_SPACEDIM, + tmp); + MultiFab::Add(S_new,tmp,0,Xvel,AMREX_SPACEDIM,0); if (verbose) - amrex::Print() << "initData: finished init from velocity_plotfile" << '\n'; + amrex::Print() << "initData: finished init from peleLM.initVelocityPlt" << '\n'; } -#endif /*AMREX_USE_VELOCITY*/ make_rho_prev_time(); make_rho_curr_time(); diff --git a/Submodules/IAMR b/Submodules/IAMR index 8073b03a..af116aa2 160000 --- a/Submodules/IAMR +++ b/Submodules/IAMR @@ -1 +1 @@ -Subproject commit 8073b03a3e0d55cb3f47c74b34186790dfebf795 +Subproject commit af116aa290f6849e15a5e1bd4095c64c797831c7 diff --git a/Submodules/PelePhysics b/Submodules/PelePhysics index 3598a2a8..b0d3f35b 160000 --- a/Submodules/PelePhysics +++ b/Submodules/PelePhysics @@ -1 +1 @@ -Subproject commit 3598a2a8b9c480fec8c00330c6e6c160c630a0ab +Subproject commit b0d3f35b0cd1e646fccb9d61daeeef66457e5941 diff --git a/Submodules/sundials b/Submodules/sundials index c0097c0b..e2f29c34 160000 --- a/Submodules/sundials +++ b/Submodules/sundials @@ -1 +1 @@ -Subproject commit c0097c0bfd56e8f94dbb588e0894fdceacf6f737 +Subproject commit e2f29c34f324829302037a1492db480be8828084 diff --git a/Tools/CMake/BuildPeleLMExe.cmake b/Tools/CMake/BuildPeleLMExe.cmake index fb063454..1c906cd2 100644 --- a/Tools/CMake/BuildPeleLMExe.cmake +++ b/Tools/CMake/BuildPeleLMExe.cmake @@ -67,7 +67,7 @@ function(build_pelelm_exe pelelm_exe_name) target_compile_definitions(${pelelm_exe_name} PRIVATE USE_SRK_EOS) endif() - set(PELELM_MECHANISM_DIR "${PELE_PHYSICS_SRC_DIR}/Support/Fuego/Mechanism/Models/${PELELM_CHEMISTRY_MODEL}") + set(PELELM_MECHANISM_DIR "${PELE_PHYSICS_SRC_DIR}/Support/Mechanism/Models/${PELELM_CHEMISTRY_MODEL}") target_sources(${pelelm_exe_name} PRIVATE ${PELELM_MECHANISM_DIR}/mechanism.cpp ${PELELM_MECHANISM_DIR}/mechanism.H) @@ -98,10 +98,10 @@ function(build_pelelm_exe pelelm_exe_name) set_source_files_properties(${PELELM_MECHANISM_DIR}/mechanism.cpp PROPERTIES COMPILE_OPTIONS "${MY_CXX_FLAGS}") set_source_files_properties(${PELELM_MECHANISM_DIR}/mechanism.H PROPERTIES COMPILE_OPTIONS "${MY_CXX_FLAGS}") target_include_directories(${pelelm_exe_name} SYSTEM PRIVATE ${PELELM_MECHANISM_DIR}) - target_include_directories(${pelelm_exe_name} SYSTEM PRIVATE ${PELE_PHYSICS_SRC_DIR}/Support/Fuego/Evaluation) target_include_directories(${pelelm_exe_name} SYSTEM PRIVATE ${PELE_PHYSICS_SRC_DIR}/Utility/PMF) target_include_directories(${pelelm_exe_name} SYSTEM PRIVATE ${PELE_PHYSICS_SRC_DIR}/Utility/TurbInflow) - target_include_directories(${pelelm_exe_name} SYSTEM PRIVATE ${PELE_PHYSICS_SRC_DIR}/Support/Fuego/Mechanism/Models) + target_include_directories(${pelelm_exe_name} SYSTEM PRIVATE ${PELE_PHYSICS_SRC_DIR}/Utility/PltFileManager) + target_include_directories(${pelelm_exe_name} SYSTEM PRIVATE ${PELE_PHYSICS_SRC_DIR}/Support/Mechanism/Models) target_sources(${pelelm_exe_name} PRIVATE @@ -172,6 +172,7 @@ function(build_pelelm_exe pelelm_exe_name) PRIVATE ${IAMR_SRC_DIR}/SyncRegister.cpp ${IAMR_SRC_DIR}/NS_init_eb2.cpp + ${IAMR_SRC_DIR}/EBUserDefined.H ${IAMR_SRC_DIR}/NS_getForce.cpp ${IAMR_SRC_DIR}/OutFlowBC.cpp ${IAMR_SRC_DIR}/FluxBoxes.cpp @@ -271,6 +272,9 @@ function(build_pelelm_exe pelelm_exe_name) ${PELE_PHYSICS_SRC_DIR}/Utility/PMF/PMFData.H ${PELE_PHYSICS_SRC_DIR}/Utility/TurbInflow/turbinflow.cpp ${PELE_PHYSICS_SRC_DIR}/Utility/TurbInflow/turbinflow.H + ${PELE_PHYSICS_SRC_DIR}/Utility/PltFileManager/PltFileManager.cpp + ${PELE_PHYSICS_SRC_DIR}/Utility/PltFileManager/PltFileManager.H + ${PELE_PHYSICS_SRC_DIR}/Utility/PltFileManager/PltFileManagerBCFill.H ) if(NOT "${pelelm_exe_name}" STREQUAL "PeleLM-UnitTests") diff --git a/Tools/CMake/SetAmrexOptions.cmake b/Tools/CMake/SetAmrexOptions.cmake index ada7b242..47257567 100644 --- a/Tools/CMake/SetAmrexOptions.cmake +++ b/Tools/CMake/SetAmrexOptions.cmake @@ -9,12 +9,11 @@ set(AMReX_DPCPP ${PELELM_ENABLE_DPCPP}) set(AMReX_HIP ${PELELM_ENABLE_HIP}) set(AMReX_PLOTFILE_TOOLS ${PELELM_ENABLE_FCOMPARE}) set(AMReX_SUNDIALS OFF) -set(AMReX_FORTRAN ON) -set(AMReX_FORTRAN_INTERFACES ON) +set(AMReX_FORTRAN OFF) +set(AMReX_FORTRAN_INTERFACES OFF) set(AMReX_PIC OFF) set(AMReX_PRECISION "${PELELM_PRECISION}" CACHE STRING "Floating point precision" FORCE) set(AMReX_LINEAR_SOLVERS ON) -set(AMReX_AMRDATA ON) set(AMReX_ASCENT OFF) set(AMReX_SENSEI OFF) set(AMReX_CONDUIT OFF) diff --git a/Tools/Make/Make.PeleLM b/Tools/Make/Make.PeleLM index 4be4c5be..58dc097e 100644 --- a/Tools/Make/Make.PeleLM +++ b/Tools/Make/Make.PeleLM @@ -2,6 +2,7 @@ EBASE = PeleLM PELE_PHYSICS_HOME ?= $(SUBMODS)/PelePhysics USE_SUNDIALS = TRUE +BL_NO_FORT = TRUE include $(AMREX_HOME)/Tools/GNUMake/Make.defs #$(info LIBRARIES_BeginPeleMake="$(LIBRARIES)") @@ -29,21 +30,7 @@ INCLUDE_LOCATIONS += $(AMREX_HOME)/Tools/C_scripts ############################################################## # PelePhysics ############################################################## -# Core -PP_SRC_HOME = $(PELE_PHYSICS_HOME)/Source -INCLUDE_LOCATIONS += $(PP_SRC_HOME) -VPATH_LOCATIONS += $(PP_SRC_HOME) -Bpack += $(PP_SRC_HOME)/Make.package -Blocs += $(PP_SRC_HOME) - -# Utilities -Bpack += $(PELE_PHYSICS_HOME)/Utility/PMF/Make.package -Blocs += $(PELE_PHYSICS_HOME)/Utility/PMF -Bpack += $(PELE_PHYSICS_HOME)/Utility/TurbInflow/Make.package -Blocs += $(PELE_PHYSICS_HOME)/Utility/TurbInflow - -# EOS -EOS_HOME = $(PELE_PHYSICS_HOME)/Eos +# EOS models switches ifeq ($(Eos_Model),$(filter $(Eos_Model),GammaLaw)) DEFINES += -DUSE_GAMMALAW_EOS endif @@ -51,37 +38,10 @@ ifeq ($(Eos_Model),$(filter $(Eos_Model),Fuego)) DEFINES += -DUSE_FUEGO_EOS endif ifeq ($(Eos_Model),$(filter $(Eos_Model),Soave-Redlich-Kwong)) - $(error EOS Soave-Redlich-Kwong not available in PeleLM) -endif -INCLUDE_LOCATIONS += $(EOS_HOME) -VPATH_LOCATIONS += $(EOS_HOME) -Bpack += $(EOS_HOME)/Make.package -Blocs += $(EOS_HOME) - -# Reactions -REACTIONS_HOME = $(PELE_PHYSICS_HOME)/Reactions -EXTERN_CORE += $(REACTIONS_HOME) -INCLUDE_LOCATIONS += $(REACTIONS_HOME) -VPATH_LOCATIONS += $(REACTIONS_HOME) -Bpack += $(REACTIONS_HOME)/Make.package -Blocs += $(REACTIONS_HOME) -include $(PELE_PHYSICS_HOME)/ThirdParty/Make.ThirdParty + DEFINES += -DUSE_SRK_EOS +endif -ifdef Chemistry_Model - USE_FUEGO = TRUE - CHEM_HOME = $(PELE_PHYSICS_HOME)/Support/Fuego/Mechanism/Models/$(Chemistry_Model) - CHEM_ALL = $(PELE_PHYSICS_HOME)/Support/Fuego/Mechanism/Models - Blocs += $(PELE_PHYSICS_HOME)/Support/Fuego/Evaluation -endif -VPATH_LOCATIONS += $(CHEM_HOME) -Bpack += $(CHEM_HOME)/Make.package $(CHEM_ALL)/Make.package -Blocs += $(CHEM_HOME) $(CHEM_ALL) - -# Transport -TRANSPORT_HOME = $(PELE_PHYSICS_HOME)/Transport/ -include $(TRANSPORT_HOME)/Make.package -INCLUDE_LOCATIONS += $(TRANSPORT_HOME) -VPATH_LOCATIONS += $(TRANSPORT_HOME) +# Transport model switches ifeq ($(Transport_Model), Simple) DEFINES += -DUSE_SIMPLE_TRANSPORT endif @@ -96,6 +56,18 @@ ifeq ($(Transport_Model), Sutherland) DEFINES += -DUSE_SUTHERLAND_TRANSPORT endif +ifeq ($(PELE_USE_KLU), TRUE) + DEFINES += -DPELE_USE_KLU +endif + +ChemDir = Support/Mechanism/Models/$(Chemistry_Model) + +PPdirs := Utility/PMF Utility/TurbInflow Utility/PltFileManager Source $(ChemDir) Reactions Eos Transport + +Bpack += $(foreach dir, $(PPdirs), $(PELE_PHYSICS_HOME)/$(dir)/Make.package) +Blocs += $(foreach dir, $(PPdirs), $(PELE_PHYSICS_HOME)/$(dir)) +include $(PELE_PHYSICS_HOME)/ThirdParty/Make.ThirdParty + # # Tells IAMR not to include the following: NS_error.cpp NS_setup.cpp NSBld.cpp # NavierStokes.cpp NavierStokes.H @@ -145,44 +117,8 @@ VPATH_LOCATIONS += $(Blocs) #INCLUDE_LOCATIONS += $(AMREX_HOME)/Src/F_Interfaces/Base #VPATH_LOCATIONS += $(AMREX_HOME)/Src/F_Interfaces/Base -ifeq ($(USE_FLCTS), TRUE) - DEFINES += -DBL_DO_FLCT - include $(IAMR_HOME)/Source/TurbInflow/Make.package - VPATH_LOCATIONS += $(IAMR_HOME)/Source/TurbInflow - EXTERN_CORE += $(IAMR_HOME)/Source/TurbInflow -endif - -# runtime parameter support for extern/ routines -# Right now, only USE_FLCTS requires those -#ifeq ($(USE_FLCTS), TRUE) -# f90EXE_sources += extern.f90 -# -# EXTERN_TEMPLATE := $(PELE_PHYSICS_HOME)/Testing/Source/extern_probin.template -# -# EXTERN_PARAMETERS := $(shell $(AMREX_HOME)/Tools/F_scripts/findparams.py $(EXTERN_CORE)) -# -# extern.f90: $(EXTERN_PARAMETERS) $(EXTERN_TEMPLATE) -# $(AMREX_HOME)/Tools/F_scripts/write_probin.py \ -# -t $(EXTERN_TEMPLATE) -o extern.f90 -n extern \ -# --pa "$(EXTERN_PARAMETERS)" -#endif - # Hack in some LM-specific stuff -AMRVIS_HOME = $(AMREX_HOME)/Src/Extern/amrdata -include $(PELELM_HOME)/Tools/Make/AmrData.mk -INCLUDE_LOCATIONS += $(AMRVIS_HOME) -VPATH_LOCATIONS += $(AMRVIS_HOME) -DEFINES += -DAMREX_NOLINEVALUES -DAMREX_PARALLEL_IO - -ifeq ($(USE_VELOCITY), TRUE) - DEFINES += -DAMREX_USE_VELOCITY -endif - -ifeq ($(USE_NEWMECH), TRUE) - DEFINES += -DAMREX_USE_NEWMECH -endif - # Enable time-dependent turbulence forcing term ifeq ($(USE_TDEPFORCING), TRUE) ifeq ($(DIM), 3) @@ -218,7 +154,6 @@ AMReX_buildInfo.cpp: $(AMREX_HOME)/Tools/C_scripts/makebuildinfo_C.py \ --amrex_home "$(AMREX_HOME)" \ --COMP "$(COMP)" --COMP_VERSION "$(COMP_VERSION)" \ - --FCOMP "$(FCOMP)" --FCOMP_VERSION "$(FCOMP_VERSION)" \ --MODULES "Chemistry=$(Chemistry_Model)" \ --GIT "$(PELELM_HOME) $(AMREX_HOME) $(IAMR_HOME) $(AMREX_HYDRO_HOME) $(PELE_PHYSICS_HOME)" @@ -233,7 +168,6 @@ endif #$(info LIBRARIES_EndPeleMake="$(LIBRARIES)") clean:: - $(SILENT) $(RM) extern_parameters.cpp extern_parameters_F.H extern_parameters.H extern.f90 $(SILENT) $(RM) AMReX_buildInfo.cpp TPL: