From f7005d2992faac8ed33f883c7c12fd63b0ed842d Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 17 May 2024 09:25:58 -0600 Subject: [PATCH 01/43] compiles; a couple things still need to be added --- .../equation_systems/vof/volume_fractions.H | 60 +++ amr-wind/incflo.cpp | 3 + amr-wind/overset/OversetOps.H | 42 ++ amr-wind/overset/OversetOps.cpp | 331 +++++++++++- amr-wind/overset/overset_ops_K.H | 119 +++++ amr-wind/overset/overset_ops_routines.H | 499 ++++++++++++++++++ amr-wind/physics/multiphase/MultiPhase.H | 4 + amr-wind/physics/multiphase/MultiPhase.cpp | 79 +++ 8 files changed, 1134 insertions(+), 3 deletions(-) create mode 100644 amr-wind/overset/overset_ops_K.H create mode 100644 amr-wind/overset/overset_ops_routines.H diff --git a/amr-wind/equation_systems/vof/volume_fractions.H b/amr-wind/equation_systems/vof/volume_fractions.H index b67988a7aa..5765840063 100644 --- a/amr-wind/equation_systems/vof/volume_fractions.H +++ b/amr-wind/equation_systems/vof/volume_fractions.H @@ -64,6 +64,66 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal( mz = mm1 - mm2; } + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal_neumann( + int i, + int j, + int k, + int ibdy, + int jbdy, + int kbdy, + amrex::Array4 const& volfrac, + amrex::Real& mx, + amrex::Real& my, + amrex::Real& mz) noexcept +{ + amrex::Real mm1, mm2; + + // Do neumann condition via indices + int im1 = ibdy == -1 ? i : i - 1; + int jm1 = jbdy == -1 ? j : j - 1; + int km1 = kbdy == -1 ? k : k - 1; + int ip1 = ibdy == +1 ? i : i + 1; + int jp1 = jbdy == +1 ? j : j + 1; + int kp1 = kbdy == +1 ? k : k + 1; + + mm1 = volfrac(im1, jm1, km1) + volfrac(im1, jm1, kp1) + + volfrac(im1, jp1, km1) + volfrac(im1, jp1, kp1) + + 2.0 * (volfrac(im1, jm1, k) + volfrac(im1, jp1, k) + + volfrac(im1, j, km1) + volfrac(im1, j, kp1)) + + 4.0 * volfrac(im1, j, k); + mm2 = volfrac(ip1, jm1, km1) + volfrac(ip1, jm1, kp1) + + volfrac(ip1, jp1, km1) + volfrac(ip1, jp1, kp1) + + 2.0 * (volfrac(ip1, jm1, k) + volfrac(ip1, jp1, k) + + volfrac(ip1, j, km1) + volfrac(ip1, j, kp1)) + + 4.0 * volfrac(ip1, j, k); + mx = mm1 - mm2; + + mm1 = volfrac(im1, jm1, km1) + volfrac(im1, jm1, kp1) + + volfrac(ip1, jm1, km1) + volfrac(ip1, jm1, kp1) + + 2.0 * (volfrac(im1, jm1, k) + volfrac(ip1, jm1, k) + + volfrac(i, jm1, km1) + volfrac(i, jm1, kp1)) + + 4.0 * volfrac(i, jm1, k); + mm2 = volfrac(im1, jp1, km1) + volfrac(im1, jp1, kp1) + + volfrac(ip1, jp1, km1) + volfrac(ip1, jp1, kp1) + + 2.0 * (volfrac(im1, jp1, k) + volfrac(ip1, jp1, k) + + volfrac(i, jp1, km1) + volfrac(i, jp1, kp1)) + + 4.0 * volfrac(i, jp1, k); + my = mm1 - mm2; + + mm1 = volfrac(im1, jm1, km1) + volfrac(im1, jp1, km1) + + volfrac(ip1, jm1, km1) + volfrac(ip1, jp1, km1) + + 2.0 * (volfrac(im1, j, km1) + volfrac(ip1, j, km1) + + volfrac(i, jm1, km1) + volfrac(i, jp1, km1)) + + 4.0 * volfrac(i, j, km1); + mm2 = volfrac(im1, jm1, kp1) + volfrac(im1, jp1, kp1) + + volfrac(ip1, jm1, kp1) + volfrac(ip1, jp1, kp1) + + 2.0 * (volfrac(im1, j, kp1) + volfrac(ip1, j, kp1) + + volfrac(i, jm1, kp1) + volfrac(i, jp1, kp1)) + + 4.0 * volfrac(i, j, kp1); + mz = mm1 - mm2; +} + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mixed_youngs_central_normal( int i, int j, diff --git a/amr-wind/incflo.cpp b/amr-wind/incflo.cpp index 750cfe2e0e..4cc56b030a 100644 --- a/amr-wind/incflo.cpp +++ b/amr-wind/incflo.cpp @@ -319,6 +319,9 @@ void incflo::do_advance() } else { advance(); } + if (m_sim.has_overset()) { + m_ovst_ops.post_advance_work(); + } } // Make a new level from scratch using provided BoxArray and diff --git a/amr-wind/overset/OversetOps.H b/amr-wind/overset/OversetOps.H index 1220e99570..c7893c34d8 100644 --- a/amr-wind/overset/OversetOps.H +++ b/amr-wind/overset/OversetOps.H @@ -13,10 +13,52 @@ public: void initialize(CFDSim& sim); void pre_advance_work(); + void post_advance_work(); void update_gradp(); private: + // Functions called within public functions + void parameter_output(); + void sharpen_nalu_data(); + void set_hydrostatic_gradp(); + void replace_masked_gradp(); + + // Check for multiphase sim + bool m_vof_exists{false}; + // Check for perturbational pressure + // (will be removed soon) + bool m_perturb_p{false}; + + // This is the only option for now + const bool m_use_hs_pgrad{true}; + + // Coupling options + bool m_disable_nodal_proj{false}; + bool m_disable_mac_proj{false}; + bool m_replace_gp{false}; + + // Verbosity + int m_verbose{0}; + + // Reinitialization parameters + int m_niterations{10}; + int m_calcconvint{1}; // calctolniter, cconv + amrex::Real m_tol = 1e-12; + amrex::Real m_rlscale = 1.5; + amrex::Real m_margin = 0.1; + amrex::Real m_target_cutoff = 0.0; // proc_tgvof_tol + + // Tolerance for VOF-related bound checks + const amrex::Real m_vof_tol = 1e-12; + // Small number for approximate signed distance function + const amrex::Real m_asdf_tiny = 1e-12; + + // Pointer for pressure gradient copy field + amr_wind::Field* m_gp_copy{nullptr}; + // Pointer for MultiPhase physics + amr_wind::MultiPhase* m_mphase{nullptr}; + CFDSim* m_sim_ptr; }; diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 31f2f1e94e..f869fe613a 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -1,4 +1,5 @@ #include "amr-wind/overset/OversetOps.H" +#include "amr-wind/overset/overset_ops_routines.H" #include "amr-wind/core/field_ops.H" #include "AMReX_ParmParse.H" #include "AMReX_MultiFabUtil.H" @@ -8,12 +9,79 @@ namespace amr_wind { -void OversetOps::initialize(CFDSim& sim) { m_sim_ptr = ∼ } +void OversetOps::initialize(CFDSim& sim) +{ + m_sim_ptr = ∼ + // Queries for reinitialization options + amrex::ParmParse pp("Overset"); + pp.query("reinit_iterations", m_niterations); + pp.query("reinit_convg_interval", m_calcconvint); + pp.query("reinit_convg_tolerance", m_tol); + pp.query("reinit_rlscale", m_niterations); + pp.query("reinit_upw_margin", m_margin); + pp.query("reinit_target_cutoff", m_target_cutoff); + + // Queries for coupling options + pp.query("disable_coupled_nodal_proj", m_disable_nodal_proj); + pp.query("disable_coupled_mac_proj", m_disable_mac_proj); + pp.query("replace_gradp_postsolve", m_replace_gp); + + // Verbosity + pp.query("verbose", m_verbose); + + // Check for perturbational pressure + // (will be removed soon) + amrex::ParmParse pp_icns("ICNS"); + pp_icns.query("use_perturb_pressure", m_perturb_p); + + // Check for vof to determine if multiphase sim + m_vof_exists = (*m_sim_ptr).repo().field_exists("vof"); + + // Set up pointer to MultiPhase physics + if (m_vof_exists) { + m_mphase = &(*m_sim_ptr).physics_manager().get(); + } + + // Set up field to store pressure gradient + if (m_replace_gp) { + m_gp_copy = &(*m_sim_ptr).repo().declare_field("gp_copy", 3); + } + + // Output parameters if verbose + parameter_output(); + + // TODO: + // * set up coupling options + // * different behavior depending on prescribe flag +} void OversetOps::pre_advance_work() { - // Update pressure gradient using updated overset pressure field - update_gradp(); + // Pressure gradient not updated for current multiphase approach + if (!(m_vof_exists && m_use_hs_pgrad)) { + // Update pressure gradient using updated overset pressure field + update_gradp(); + } + + if (m_vof_exists) { + // Reinitialize fields + sharpen_nalu_data(); + if (m_use_hs_pgrad) { + // Use hydrostatic pressure gradient + set_hydrostatic_gradp(); + } + } + + // If pressure gradient will be replaced, store current pressure gradient + if (m_replace_gp) { + auto& gp = (*m_sim_ptr).repo().get_field("gp"); + for (int lev = 0; lev < (*m_sim_ptr).repo().num_active_levels(); + ++lev) { + amrex::MultiFab::Copy( + (*m_gp_copy)(lev), gp(lev), 0, 0, gp(lev).nComp(), + (m_gp_copy)->num_grow()); + } + } } void OversetOps::update_gradp() @@ -73,4 +141,261 @@ void OversetOps::update_gradp() // Averaging down here would be unnecessary; it is built into calcGradPhi } +void OversetOps::post_advance_work() +{ + // Replace and reapply pressure gradient if requested + if (m_replace_gp) { + replace_masked_gradp(); + } +} + +/* ----------------------------------------------- */ +/* PUBLIC FUNCTIONS ABOVE, PRIVATE FUNCTIONS BELOW */ +/* ----------------------------------------------- */ + +void OversetOps::parameter_output() +{ + // Print the details + if (m_verbose > 0) { + // Important parameters + amrex::Print() << "Overset Coupling Parameters: \n" + << "---- Coupled nodal projection : " + << !m_disable_nodal_proj << std::endl + << "---- Coupled MAC projection : " + << !m_disable_mac_proj << std::endl + << "---- Replace overset pres grad: " << m_replace_gp + << std::endl; + if (m_vof_exists) { + amrex::Print() << "Overset Reinitialization Parameters:\n" + << "---- Maximum iterations : " << m_niterations + << std::endl + << "---- Convergence tolerance: " << m_tol + << std::endl + << "---- Relative length scale: " << m_rlscale + << std::endl + << "---- Upwinding VOF margin : " << m_margin + << std::endl; + } + } + if (m_verbose > 1 && m_vof_exists) { + // Less important or less used parameters + amrex::Print() << "---- Calc. conv. interval : " << m_calcconvint + << std::endl + << "---- Target field cutoff : " << m_target_cutoff + << std::endl; + } +} + +void OversetOps::sharpen_nalu_data() +{ + auto& repo = (*m_sim_ptr).repo(); + auto nlevels = repo.num_active_levels(); + auto geom = (*m_sim_ptr).mesh().Geom(); + + // Get blanking for cells + auto& iblank_cell = repo.get_int_field("iblank_cell"); + + // Get fields that will be modified + auto& vof = repo.get_field("vof"); + auto& levelset = repo.get_field("levelset"); + auto& rho = repo.get_field("density"); + auto& velocity = repo.get_field("velocity"); + + // Create scratch fields for fluxes + // 5 components are vof, density, and 3 of velocity + auto flux_x = repo.create_scratch_field(5, 0, amr_wind::FieldLoc::XFACE); + auto flux_y = repo.create_scratch_field(5, 0, amr_wind::FieldLoc::YFACE); + auto flux_z = repo.create_scratch_field(5, 0, amr_wind::FieldLoc::ZFACE); + // Create scratch field for approximate signed distance function and grad + // (components 0-2 are gradient, 3 is asdf) + auto normal_vec = repo.create_scratch_field(3, vof.num_grow()[0] - 1); + + auto target_vof = repo.create_scratch_field(1, vof.num_grow()[0]); + + // Create levelset field + for (int lev = 0; lev < nlevels; ++lev) { + // Thickness used here is user parameter, whatever works best + auto dx = (geom[lev]).CellSizeArray(); + const amrex::Real i_th = m_rlscale * std::cbrt(dx[0] * dx[1] * dx[2]); + + // Populate approximate signed distance function + overset_ops::populate_psi(levelset(lev), vof(lev), i_th, m_asdf_tiny); + } + + // Convert levelset to vof to get target_vof + m_mphase->levelset2vof(iblank_cell, *target_vof); + + // Process target vof for tiny margins from single-phase + for (int lev = 0; lev < nlevels; ++lev) { + // A tolerance of 0 should do nothing + overset_ops::process_vof((*target_vof)(lev), m_target_cutoff); + } + + // Replace vof with original values in amr domain + for (int lev = 0; lev < nlevels; ++lev) { + overset_ops::harmonize_vof( + (*target_vof)(lev), vof(lev), iblank_cell(lev)); + } + + // Put fluxes in vector for averaging down during iterations + amrex::Vector> fluxes( + repo.num_active_levels()); + for (int lev = 0; lev < nlevels; ++lev) { + fluxes[lev][0] = &(*flux_x)(lev); + fluxes[lev][1] = &(*flux_y)(lev); + fluxes[lev][2] = &(*flux_z)(lev); + } + + // Pseudo-time loop + amrex::Real err = 100.0 * m_tol; + int n = 0; + while (n < m_niterations && err > m_tol) { + // Increment step counter + ++n; + + // Determine if convergence error is calculated this step + bool cconv = n % m_calcconvint == 0; + // Zero error if being calculated this step + err = cconv ? 0.0 : err; + + // Maximum possible value of pseudo time factor (dtau) + amrex::Real ptfac = 1.0; + // Maximum pseudoCFL, 0.5 seems to work well + const amrex::Real pCFL = 0.5; + + for (int lev = 0; lev < nlevels; ++lev) { + // Populate normal vector + overset_ops::populate_normal_vector( + (*normal_vec)(lev), vof(lev), iblank_cell(lev)); + + // Sharpening fluxes for vof, density, and momentum + overset_ops::populate_sharpen_fluxes( + (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), vof(lev), + (*target_vof)(lev), (*normal_vec)(lev), velocity(lev), m_margin, + m_mphase->rho1(), m_mphase->rho2()); + + // Process fluxes + overset_ops::process_fluxes( + (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), + iblank_cell(lev)); + + // Measure convergence to determine if loop can stop + if (cconv) { + // Update error at specified interval of steps + const amrex::Real err_lev = overset_ops::measure_convergence( + (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev)); + err = amrex::max(err, err_lev); + } + } + + // Average down fluxes across levels for consistency + for (int lev = nlevels - 1; lev > 0; --lev) { + amrex::IntVect rr = + geom[lev].Domain().size() / geom[lev - 1].Domain().size(); + amrex::average_down_faces( + GetArrOfConstPtrs(fluxes[lev]), fluxes[lev - 1], rr, + geom[lev - 1]); + } + + // Get pseudo dt (dtau) + for (int lev = 0; lev < nlevels; ++lev) { + // Compare vof fluxes to vof in source cells + // Convergence tolerance determines what size of fluxes matter + const amrex::Real ptfac_lev = overset_ops::calculate_pseudo_dt_flux( + (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), vof(lev), + m_tol); + ptfac = amrex::min(ptfac, ptfac_lev); + } + amrex::ParallelDescriptor::ReduceRealMin(ptfac); + + // Conform pseudo dt (dtau) to pseudo CFL + ptfac = pCFL * ptfac; + + // Apply fluxes + for (int lev = 0; lev < nlevels; ++lev) { + overset_ops::apply_fluxes( + (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), vof(lev), + rho(lev), velocity(lev), ptfac, m_vof_tol); + } + + // Fillpatch for ghost cells + vof.fillpatch((*m_sim_ptr).time().current_time()); + velocity.fillpatch((*m_sim_ptr).time().current_time()); + + // Update density (fillpatch built in) + m_mphase->set_density_via_vof(); + + // Ensure that err is same across processors + if (cconv) { + amrex::ParallelDescriptor::ReduceRealMax(err); + } + + amrex::Print() << "sharpen step " << n << " " << err << " " << m_tol + << std::endl; + } + + // Purely for debugging via visualization, should be removed later + // Currently set up to overwrite the levelset field (not used as time + // evolves) with the post-sharpening velocity magnitude + for (int lev = 0; lev < nlevels; ++lev) { + overset_ops::equate_field(levelset(lev), velocity(lev)); + } +} + +void OversetOps::set_hydrostatic_gradp() +{ + auto& repo = (*m_sim_ptr).repo(); + auto nlevels = repo.num_active_levels(); + auto geom = (*m_sim_ptr).mesh().Geom(); + + // Get blanking for cells + auto& iblank_cell = repo.get_int_field("iblank_cell"); + + // Get fields that will be modified or used + Field* rho0{nullptr}; + auto& rho = repo.get_field("density"); + auto& gp = repo.get_field("gp"); + if (m_perturb_p) { + rho0 = &((*m_sim_ptr).repo().get_field("reference_density")); + } else { + // Point to existing field, won't be used + rho0 = ρ + } + + // Replace initial gp with best guess (hydrostatic) + for (int lev = 0; lev < nlevels; ++lev) { + overset_ops::replace_gradp_hs( + gp(lev), rho(lev), (*rho0)(lev), iblank_cell(lev), + m_mphase->gravity()[2], m_perturb_p); + } +} + +void OversetOps::replace_masked_gradp() +{ + auto& repo = (*m_sim_ptr).repo(); + auto nlevels = repo.num_active_levels(); + + // Get timestep + const amrex::Real dt = (*m_sim_ptr).time().deltaT(); + + // Get blanking for cells + auto& iblank_cell = repo.get_int_field("iblank_cell"); + + // Get fields that will be modified or used + auto& vel = repo.get_field("velocity"); + auto& rho = repo.get_field("density"); + auto& gp = repo.get_field("gp"); + + // For iblanked cells, replace gp with original gp, to get original vel + for (int lev = 0; lev < nlevels; ++lev) { + // Remove pressure gradient term + overset_ops::apply_pressure_gradient(vel(lev), rho(lev), gp(lev), -dt); + // Modify pressure gradient + overset_ops::replace_gradp( + gp(lev), (*m_gp_copy)(lev), iblank_cell(lev)); + // Reapply pressure gradient term + overset_ops::apply_pressure_gradient(vel(lev), rho(lev), gp(lev), dt); + } +} + } // namespace amr_wind \ No newline at end of file diff --git a/amr-wind/overset/overset_ops_K.H b/amr-wind/overset/overset_ops_K.H new file mode 100644 index 0000000000..89b0780c7b --- /dev/null +++ b/amr-wind/overset/overset_ops_K.H @@ -0,0 +1,119 @@ +#ifndef OVERSET_OPS_K_H_ +#define OVERSET_OPS_K_H_ + +#include + +namespace amr_wind::overset_ops { + +// Approximate signed distance function +amrex::Real AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +asdf(const amrex::Real a_vof, const amrex::Real i_th, const amrex::Real tiny) +{ + // function of local vof value and interface thickness + return (i_th * log((a_vof + tiny) / (1. - a_vof + tiny))); +} + +amrex::Real AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE alpha_flux( + int i, + int j, + int k, + int dir, + const amrex::Real margin, + amrex::Array4 const& vof, + amrex::Array4 const& tg_vof, + amrex::Array4 const& normal) +{ + // Set up neighbor indices + int ii = i; + int jj = j; + int kk = k; + ii += (dir == 0) ? -1 : 0; + jj += (dir == 1) ? -1 : 0; + kk += (dir == 2) ? -1 : 0; + + // Gradient of phi normal to interface + const amrex::Real gphi = (vof(i, j, k) - vof(ii, jj, kk)); + // Normal vector in each cell (already normalized) + const amrex::Real norm_ = normal(i, j, k, dir); + const amrex::Real norm_nb = normal(ii, jj, kk, dir); + + // Determine which delta_phi (and multiply by normal) + // The sign depends on side of flux face (like upwinding) + const amrex::Real dphi_ = (tg_vof(i, j, k) - vof(i, j, k)) * (-norm_); + const amrex::Real dphi_nb = + (tg_vof(ii, jj, kk) - vof(ii, jj, kk)) * norm_nb; + // Average value used across the interface + amrex::Real dphi_eval = 0.5 * (dphi_ + dphi_nb); + // Upwinding when on the gas side, downwinding on the liquid + // Across the interface defined as crossing 0.5 or within margin of 0.5 + if ((std::abs(vof(i, j, k) - 0.5) > margin || + std::abs(vof(ii, jj, kk) - 0.5) > margin)) { + if (gphi > 0.0) { + dphi_eval = (vof(ii, jj, kk) < 0.5 && vof(i, j, k) <= 0.5 + margin) + ? dphi_nb + : dphi_eval; + dphi_eval = (vof(ii, jj, kk) >= 0.5 - margin && vof(i, j, k) > 0.5) + ? dphi_ + : dphi_eval; + } + if (gphi < 0.0) { + dphi_eval = (vof(i, j, k) < 0.5 && vof(ii, jj, kk) <= 0.5 + margin) + ? dphi_ + : dphi_eval; + dphi_eval = (vof(i, j, k) >= 0.5 - margin && vof(ii, jj, kk) > 0.5) + ? dphi_nb + : dphi_eval; + } + } + return dphi_eval; +} + +void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE velocity_face( + int i, + int j, + int k, + int dir, + amrex::Array4 const& vof, + amrex::Array4 const& velocity, + amrex::Real& uface, + amrex::Real& vface, + amrex::Real& wface) +{ + // Set up neighbor indices + int ii = i; + int jj = j; + int kk = k; + ii += (dir == 0) ? -1 : 0; + jj += (dir == 1) ? -1 : 0; + kk += (dir == 2) ? -1 : 0; + + // Gradient of phi normal to interface + const amrex::Real gphi = (vof(i, j, k) - vof(ii, jj, kk)); + + // Get velocities on both sides + const amrex::Real u_ = velocity(i, j, k, 0); + const amrex::Real v_ = velocity(i, j, k, 1); + const amrex::Real w_ = velocity(i, j, k, 2); + const amrex::Real u_nb = velocity(ii, jj, kk, 0); + const amrex::Real v_nb = velocity(ii, jj, kk, 1); + const amrex::Real w_nb = velocity(ii, jj, kk, 2); + // Average value when gphi = 0 + uface = 0.5 * (u_ + u_nb); + vface = 0.5 * (v_ + v_nb); + wface = 0.5 * (w_ + w_nb); + // Use simple upwinding + if (gphi > 0.0) { + uface = u_nb; + vface = v_nb; + wface = w_nb; + } + if (gphi < 0.0) { + uface = u_; + vface = v_; + wface = w_; + } +} + +} // namespace amr_wind::overset_ops + +#endif \ No newline at end of file diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H new file mode 100644 index 0000000000..bb47c32e42 --- /dev/null +++ b/amr-wind/overset/overset_ops_routines.H @@ -0,0 +1,499 @@ +#ifndef OVERSET_OPS_ROUTINES_H_ +#define OVERSET_OPS_ROUTINES_H_ + +#include "amr-wind/equation_systems/vof/volume_fractions.H" +#include "amr-wind/overset/overset_ops_K.H" + +namespace amr_wind::overset_ops { + +// Populate approximate signed distance function using vof field +void populate_psi( + amrex::MultiFab& mf_psi, + amrex::MultiFab& mf_vof, + const amrex::Real i_th, + const amrex::Real asdf_tiny) +{ + for (amrex::MFIter mfi(mf_psi); mfi.isValid(); ++mfi) { + const auto& gbx = mfi.growntilebox(); + const amrex::Array4& psi = mf_psi.array(mfi); + const amrex::Array4& vof = mf_vof.const_array(mfi); + amrex::ParallelFor( + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + psi(i, j, k) = asdf(vof(i, j, k), i_th, asdf_tiny); + }); + } +} + +// Modify a vof field to not have values that barely differ from 0 or 1 +void process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) +{ + for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { + const auto& gbx = mfi.growntilebox(); + const amrex::Array4& vof = mf_vof.array(mfi); + amrex::ParallelFor( + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + vof(i, j, k) = + vof(i, j, k) < vof_tol + ? 0.0 + : (vof(i, j, k) > 1. - vof_tol ? 1. : vof(i, j, k)); + }); + } +} + +// Combine overset target vof field with current non-overset vof field +void harmonize_vof( + amrex::MultiFab& mf_vof_target, + amrex::MultiFab& mf_vof_original, + amrex::iMultiFab& mf_iblank) +{ + for (amrex::MFIter mfi(mf_vof_target); mfi.isValid(); ++mfi) { + const auto& vbx = mfi.validbox(); + const amrex::Array4& tg_vof = mf_vof_target.array(mfi); + const amrex::Array4& og_vof = + mf_vof_original.const_array(mfi); + const amrex::Array4& iblank = mf_iblank.const_array(mfi); + amrex::ParallelFor( + vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Replace amr-wind vof values with originals + if (iblank(i, j, k) != -1) { + tg_vof(i, j, k) = og_vof(i, j, k); + } + }); + } +} + +// Populate normal vector with special treatment of overset boundary +void populate_normal_vector( + amrex::MultiFab& mf_normvec, + amrex::MultiFab& mf_vof, + amrex::iMultiFab& mf_iblank) +{ + for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { + const auto& gbxm1 = grow(mfi.growntilebox(), -1); + const amrex::Array4& normvec = mf_normvec.array(mfi); + const amrex::Array4& vof = mf_vof.const_array(mfi); + const amrex::Array4& iblank = mf_iblank.const_array(mfi); + // Calculate gradients in each direction with centered diff + amrex::ParallelFor( + gbxm1, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Neumann condition across nalu bdy + int ibdy = 0; + int jbdy = 0; + int kbdy = 0; + if (iblank(i, j, k) != iblank(i - 1, j, k)) { + ibdy = -1; + } + if (iblank(i, j, k) != iblank(i, j - 1, k)) { + jbdy = -1; + } + if (iblank(i, j, k) != iblank(i, j, k - 1)) { + kbdy = -1; + } + // no cell should be isolated such that -1 and 1 are needed + if (iblank(i, j, k) != iblank(i + 1, j, k)) { + ibdy = +1; + } + if (iblank(i, j, k) != iblank(i, j + 1, k)) { + jbdy = +1; + } + if (iblank(i, j, k) != iblank(i, j, k + 1)) { + kbdy = +1; + } + // Calculate normal + amrex::Real mx, my, mz, mmag; + multiphase::youngs_fd_normal_neumann( + i, j, k, ibdy, jbdy, kbdy, vof, mx, my, mz); + // Normalize normal + mmag = std::sqrt(mx * mx + my * my + mz * mz + 1e-20); + // Save normal + normvec(i, j, k, 0) = mx / mmag; + normvec(i, j, k, 1) = my / mmag; + normvec(i, j, k, 2) = mz / mmag; + }); + } +} + +// Calculate fluxes for reinitialization over entire domain without concern for +// overset bdy +void populate_sharpen_fluxes( + amrex::MultiFab& mf_fx, + amrex::MultiFab& mf_fy, + amrex::MultiFab& mf_fz, + amrex::MultiFab& mf_vof, + amrex::MultiFab& mf_target_vof, + amrex::MultiFab& mf_norm, + amrex::MultiFab& mf_velocity, + const amrex::Real margin, + const amrex::Real rho1, + const amrex::Real rho2) +{ + for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { + const auto& vbx = mfi.validbox(); + const auto& xbx = amrex::surroundingNodes(vbx, 0); + const auto& ybx = amrex::surroundingNodes(vbx, 1); + const auto& zbx = amrex::surroundingNodes(vbx, 2); + const amrex::Array4& fx = mf_fx.array(mfi); + const amrex::Array4& fy = mf_fy.array(mfi); + const amrex::Array4& fz = mf_fz.array(mfi); + const amrex::Array4& vof = mf_vof.const_array(mfi); + const amrex::Array4& tg_vof = + mf_target_vof.const_array(mfi); + const amrex::Array4& norm = mf_norm.const_array(mfi); + const amrex::Array4& vel = + mf_velocity.const_array(mfi); + // Populate vof and density fluxes for each direction + amrex::ParallelFor( + xbx, ybx, zbx, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // vof flux + amrex::Real flux = + alpha_flux(i, j, k, 0, margin, vof, tg_vof, norm); + fx(i, j, k, 0) = flux; + // density flux + flux *= (rho1 - rho2); + fx(i, j, k, 1) = flux; + // momentum fluxes (dens flux * face vel) + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 0, vof, vel, uf, vf, wf); + fx(i, j, k, 2) = flux * uf; + fx(i, j, k, 3) = flux * vf; + fx(i, j, k, 4) = flux * wf; + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real flux = + alpha_flux(i, j, k, 1, margin, vof, tg_vof, norm); + fy(i, j, k, 0) = flux; + flux *= (rho1 - rho2); + fy(i, j, k, 1) = flux; + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 1, vof, vel, uf, vf, wf); + fy(i, j, k, 2) = flux * uf; + fy(i, j, k, 3) = flux * vf; + fy(i, j, k, 4) = flux * wf; + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real flux = + alpha_flux(i, j, k, 2, margin, vof, tg_vof, norm); + fz(i, j, k, 0) = flux; + flux *= (rho1 - rho2); + fz(i, j, k, 1) = flux; + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 2, vof, vel, uf, vf, wf); + fz(i, j, k, 2) = flux * uf; + fz(i, j, k, 3) = flux * vf; + fz(i, j, k, 4) = flux * wf; + }); + } +} + +// Process reinitialization fluxes - zero non-internal to overset region +void process_fluxes( + amrex::MultiFab& mf_fx, + amrex::MultiFab& mf_fy, + amrex::MultiFab& mf_fz, + amrex::iMultiFab& mf_iblank) +{ + int ncompx = mf_fx.nComp(); + int ncompy = mf_fy.nComp(); + int ncompz = mf_fz.nComp(); + for (amrex::MFIter mfi(mf_iblank); mfi.isValid(); ++mfi) { + const auto& vbx = mfi.validbox(); + const auto& xbx = amrex::surroundingNodes(vbx, 0); + const auto& ybx = amrex::surroundingNodes(vbx, 1); + const auto& zbx = amrex::surroundingNodes(vbx, 2); + const amrex::Array4& fx = mf_fx.array(mfi); + const amrex::Array4& fy = mf_fy.array(mfi); + const amrex::Array4& fz = mf_fz.array(mfi); + const amrex::Array4& iblank = mf_iblank.const_array(mfi); + // Only faces with iblank = -1 on both sides can have nonzero flux + amrex::ParallelFor( + xbx, ybx, zbx, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bool zero_all = (iblank(i - 1, j, k) + iblank(i, j, k) > -2); + for (int n = 0; n < ncompx; ++n) { + fx(i, j, k, n) *= zero_all ? 0. : 1.; + } + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bool zero_all = (iblank(i, j - 1, k) + iblank(i, j, k) > -2); + for (int n = 0; n < ncompy; ++n) { + fy(i, j, k, n) *= zero_all ? 0. : 1.; + } + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bool zero_all = (iblank(i, j, k - 1) + iblank(i, j, k) > -2); + for (int n = 0; n < ncompz; ++n) { + fz(i, j, k, n) *= zero_all ? 0. : 1.; + } + }); + } +} + +// Calculate a type of CFL by measuring how much % VOF is being removed per cell +amrex::Real calculate_pseudo_dt_flux( + amrex::MultiFab& mf_fx, + amrex::MultiFab& mf_fy, + amrex::MultiFab& mf_fz, + amrex::MultiFab& mf_vof, + amrex::Real tol) +{ + // Get the maximum flux magnitude, but just for vof fluxes + const amrex::Real pdt_fx = amrex::ReduceMin( + mf_fx, mf_vof, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, amrex::Array4 const& fx, + amrex::Array4 const& vof) -> amrex::Real { + amrex::Real pdt_fab = 1.0; + amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { + amrex::Real pdt_lim = 1.0; + if (fx(i, j, k, 0) > tol && vof(i, j, k) > tol) { + // VOF is removed from cell i + pdt_lim = vof(i, j, k) / fx(i, j, k, 0); + } else if (fx(i, j, k, 0) < -tol && vof(i - 1, j, k) > tol) { + // VOF is removed from cell i-1 + pdt_lim = vof(i - 1, j, k) / -fx(i, j, k, 0); + } + pdt_fab = amrex::min(pdt_fab, pdt_lim); + }); + return pdt_fab; + }); + const amrex::Real pdt_fy = amrex::ReduceMin( + mf_fy, mf_vof, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, amrex::Array4 const& fy, + amrex::Array4 const& vof) -> amrex::Real { + amrex::Real pdt_fab = 1.0; + amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { + amrex::Real pdt_lim = 1.0; + if (fy(i, j, k, 0) > tol && vof(i, j, k) > tol) { + // VOF is removed from cell j + pdt_lim = vof(i, j, k) / fy(i, j, k, 0); + } else if (fy(i, j, k, 0) < -tol && vof(i, j - 1, k) > tol) { + // VOF is removed from cell j-1 + pdt_lim = vof(i, j - 1, k) / -fy(i, j, k, 0); + } + pdt_fab = amrex::min(pdt_fab, pdt_lim); + }); + return pdt_fab; + }); + const amrex::Real pdt_fz = amrex::ReduceMin( + mf_fz, mf_vof, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, amrex::Array4 const& fz, + amrex::Array4 const& vof) -> amrex::Real { + amrex::Real pdt_fab = 1.0; + amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { + amrex::Real pdt_lim = 1.0; + if (fz(i, j, k, 0) > tol && vof(i, j, k) > tol) { + // VOF is removed from cell k + pdt_lim = vof(i, j, k) / fz(i, j, k, 0); + } else if (fz(i, j, k, 0) < -tol && vof(i, j, k - 1) > tol) { + // VOF is removed from cell k-1 + pdt_lim = vof(i, j, k - 1) / -fz(i, j, k, 0); + } + pdt_fab = amrex::min(pdt_fab, pdt_lim); + }); + return pdt_fab; + }); + const amrex::Real pdt = amrex::min(pdt_fx, amrex::min(pdt_fy, pdt_fz)); + return pdt; +} + +// Apply reinitialization fluxes to modify fields +void apply_fluxes( + amrex::MultiFab& mf_fx, + amrex::MultiFab& mf_fy, + amrex::MultiFab& mf_fz, + amrex::MultiFab& mf_vof, + amrex::MultiFab& mf_dens, + amrex::MultiFab& mf_vel, + amrex::Real ptfac, + amrex::Real vof_tol) +{ + + for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { + const auto& vbx = mfi.validbox(); + const amrex::Array4& fx = mf_fx.array(mfi); + const amrex::Array4& fy = mf_fy.array(mfi); + const amrex::Array4& fz = mf_fz.array(mfi); + const amrex::Array4& vof = mf_vof.array(mfi); + const amrex::Array4& dens = mf_dens.array(mfi); + const amrex::Array4& vel = mf_vel.array(mfi); + amrex::ParallelFor( + vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const amrex::Real olddens = dens(i, j, k); + vof(i, j, k) += ptfac * ((fx(i + 1, j, k, 0) - fx(i, j, k, 0)) + + (fy(i, j + 1, k, 0) - fy(i, j, k, 0)) + + (fz(i, j, k + 1, 0) - fz(i, j, k, 0))); + dens(i, j, k) += + ptfac * ((fx(i + 1, j, k, 1) - fx(i, j, k, 1)) + + (fy(i, j + 1, k, 1) - fy(i, j, k, 1)) + + (fz(i, j, k + 1, 1) - fz(i, j, k, 1))); + vel(i, j, k, 0) = + 1.0 / dens(i, j, k) * + (olddens * vel(i, j, k, 0) + + ptfac * ((fx(i + 1, j, k, 2) - fx(i, j, k, 2)) + + (fy(i, j + 1, k, 2) - fy(i, j, k, 2)) + + (fz(i, j, k + 1, 2) - fz(i, j, k, 2)))); + vel(i, j, k, 1) = + 1.0 / dens(i, j, k) * + (olddens * vel(i, j, k, 1) + + ptfac * ((fx(i + 1, j, k, 3) - fx(i, j, k, 3)) + + (fy(i, j + 1, k, 3) - fy(i, j, k, 3)) + + (fz(i, j, k + 1, 3) - fz(i, j, k, 3)))); + vel(i, j, k, 2) = + 1.0 / dens(i, j, k) * + (olddens * vel(i, j, k, 2) + + ptfac * ((fx(i + 1, j, k, 4) - fx(i, j, k, 4)) + + (fy(i, j + 1, k, 4) - fy(i, j, k, 4)) + + (fz(i, j, k + 1, 4) - fz(i, j, k, 4)))); + + // Ensure vof is bounded + vof(i, j, k) = + vof(i, j, k) < vof_tol + ? 0.0 + : (vof(i, j, k) > 1. - vof_tol ? 1. : vof(i, j, k)); + // Density bounds are enforced elsewhere + }); + } +} + +// Get the size of the smallest VOF flux to quantify convergence +amrex::Real measure_convergence( + amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz) +{ + // Get the maximum flux magnitude, but just for vof fluxes + const amrex::Real err_fx = amrex::ReduceMax( + mf_fx, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& fx) -> amrex::Real { + amrex::Real err_fab = -1.0; + amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { + err_fab = amrex::max(err_fab, std::abs(fx(i, j, k, 0))); + }); + return err_fab; + }); + const amrex::Real err_fy = amrex::ReduceMax( + mf_fy, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& fy) -> amrex::Real { + amrex::Real err_fab = -1.0; + amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { + err_fab = amrex::max(err_fab, std::abs(fy(i, j, k, 0))); + }); + return err_fab; + }); + const amrex::Real err_fz = amrex::ReduceMax( + mf_fz, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& fz) -> amrex::Real { + amrex::Real err_fab = -1.0; + amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { + err_fab = amrex::max(err_fab, std::abs(fz(i, j, k, 0))); + }); + return err_fab; + }); + const amrex::Real err = amrex::max(err_fx, amrex::max(err_fy, err_fz)); + return err; +} + +// Set levelset field to another quantity to view in plotfile for debugging +void equate_field(amrex::MultiFab& mf_dest, amrex::MultiFab& mf_src) +{ + for (amrex::MFIter mfi(mf_dest); mfi.isValid(); ++mfi) { + const auto& vbx = mfi.validbox(); + const amrex::Array4& src = mf_src.const_array(mfi); + const amrex::Array4& dest = mf_dest.array(mfi); + amrex::ParallelFor( + vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + dest(i, j, k) = std::sqrt( + src(i, j, k, 0) * src(i, j, k, 0) + + src(i, j, k, 1) * src(i, j, k, 1) + + src(i, j, k, 2) * src(i, j, k, 2)); + }); + } +} + +// Replace pressure gradient with hydrostatic field in overset regions +void replace_gradp_hs( + amrex::MultiFab& mf_gp, + amrex::MultiFab& mf_density, + amrex::MultiFab& mf_refdens, + amrex::iMultiFab& mf_iblank, + const amrex::Real grav_z, + const bool is_pptb) +{ + for (amrex::MFIter mfi(mf_gp); mfi.isValid(); ++mfi) { + const auto& gbx = mfi.growntilebox(); + const amrex::Array4& gp = mf_gp.array(mfi); + const amrex::Array4& rho = + mf_density.const_array(mfi); + const amrex::Array4& rho0 = + mf_refdens.const_array(mfi); + const amrex::Array4& iblank = mf_iblank.const_array(mfi); + amrex::ParallelFor( + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Replace amr-wind vof values with originals + if (iblank(i, j, k) == -1) { + const amrex::Real dfac = + is_pptb ? rho(i, j, k) - rho0(i, j, k) : rho(i, j, k); + gp(i, j, k, 0) = 0.; + gp(i, j, k, 1) = 0.; + gp(i, j, k, 2) = dfac * grav_z; + } + }); + } +} + +// Swap pressure gradient values in overset region +void replace_gradp( + amrex::MultiFab& mf_gp, + amrex::MultiFab& mf_gp0, + amrex::iMultiFab& mf_iblank) +{ + for (amrex::MFIter mfi(mf_gp); mfi.isValid(); ++mfi) { + const auto& gbx = mfi.growntilebox(); + const amrex::Array4& gp = mf_gp.array(mfi); + const amrex::Array4& gp0 = mf_gp0.const_array(mfi); + const amrex::Array4& iblank = mf_iblank.const_array(mfi); + amrex::ParallelFor( + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Replace amr-wind vof values with originals + if (iblank(i, j, k) == -1) { + gp(i, j, k, 0) = gp0(i, j, k, 0); + gp(i, j, k, 1) = gp0(i, j, k, 1); + gp(i, j, k, 2) = gp0(i, j, k, 2); + } + }); + } +} + +// Apply pressure gradient to velocity field +void apply_pressure_gradient( + amrex::MultiFab& mf_vel, + amrex::MultiFab& mf_density, + amrex::MultiFab& mf_gp, + const amrex::Real scaling_factor) +{ + for (amrex::MFIter mfi(mf_gp); mfi.isValid(); ++mfi) { + const auto& vbx = mfi.validbox(); + const amrex::Array4& vel = mf_vel.array(mfi); + const amrex::Array4& rho = + mf_density.const_array(mfi); + const amrex::Array4& gp = mf_gp.const_array(mfi); + amrex::ParallelFor( + vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const amrex::Real soverrho = scaling_factor / rho(i, j, k); + vel(i, j, k, 0) -= gp(i, j, k, 0) * soverrho; + vel(i, j, k, 1) -= gp(i, j, k, 1) * soverrho; + vel(i, j, k, 2) -= gp(i, j, k, 2) * soverrho; + }); + } +} + +} // namespace amr_wind::overset_ops + +#endif \ No newline at end of file diff --git a/amr-wind/physics/multiphase/MultiPhase.H b/amr-wind/physics/multiphase/MultiPhase.H index 1a3fe62fb2..ca2209f879 100644 --- a/amr-wind/physics/multiphase/MultiPhase.H +++ b/amr-wind/physics/multiphase/MultiPhase.H @@ -3,6 +3,8 @@ #include "amr-wind/core/Physics.H" #include "amr-wind/core/Field.H" +#include "amr-wind/core/IntField.H" +#include "amr-wind/core/ScratchField.H" /** Multiphase physics * @@ -51,6 +53,8 @@ public: void levelset2vof(); + void levelset2vof(IntField& iblank_cell, ScratchField& vof_scr); + void favre_filtering(); amrex::Real volume_fraction_sum(); diff --git a/amr-wind/physics/multiphase/MultiPhase.cpp b/amr-wind/physics/multiphase/MultiPhase.cpp index efa1f1bcb1..fe2fa61ba7 100644 --- a/amr-wind/physics/multiphase/MultiPhase.cpp +++ b/amr-wind/physics/multiphase/MultiPhase.cpp @@ -520,4 +520,83 @@ void MultiPhase::levelset2vof() (*m_vof).fillpatch(0.0); } +// Do levelset2vof with iblank neumann and into supplied scratch field +void MultiPhase::levelset2vof(IntField& iblank_cell, ScratchField& vof_scr) +{ + const int nlevels = m_sim.repo().num_active_levels(); + (*m_levelset).fillpatch(m_sim.time().current_time()); + const auto& geom = m_sim.mesh().Geom(); + + for (int lev = 0; lev < nlevels; ++lev) { + auto& levelset = (*m_levelset)(lev); + auto& vof = vof_scr(lev); + const auto& dx = geom[lev].CellSizeArray(); + + for (amrex::MFIter mfi(levelset); mfi.isValid(); ++mfi) { + const auto& vbx = mfi.validbox(); + const amrex::Array4& phi = levelset.array(mfi); + const amrex::Array4& volfrac = vof.array(mfi); + const amrex::Array4& iblank = + iblank_cell(lev).const_array(mfi); + const amrex::Real eps = 2. * std::cbrt(dx[0] * dx[1] * dx[2]); + amrex::ParallelFor( + vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Neumann of levelset across iblank boundaries + int ibdy = 0; + int jbdy = 0; + int kbdy = 0; + if (iblank(i, j, k) != iblank(i - 1, j, k)) { + ibdy = -1; + } + if (iblank(i, j, k) != iblank(i, j - 1, k)) { + jbdy = -1; + } + if (iblank(i, j, k) != iblank(i, j, k - 1)) { + kbdy = -1; + } + // no cell should be isolated such that -1 and 1 are + // needed + if (iblank(i, j, k) != iblank(i + 1, j, k)) { + ibdy = +1; + } + if (iblank(i, j, k) != iblank(i, j + 1, k)) { + jbdy = +1; + } + if (iblank(i, j, k) != iblank(i, j, k + 1)) { + kbdy = +1; + } + amrex::Real mx, my, mz; + multiphase::youngs_fd_normal_neumann( + i, j, k, ibdy, jbdy, kbdy, phi, mx, my, mz); + mx = std::abs(mx / 32.); + my = std::abs(my / 32.); + mz = std::abs(mz / 32.); + amrex::Real normL1 = (mx + my + mz); + mx = mx / normL1; + my = my / normL1; + mz = mz / normL1; + // Make sure that alpha is negative far away from the + // interface + amrex::Real alpha; + if (phi(i, j, k) < -eps) { + alpha = -1.0; + } else { + alpha = phi(i, j, k) / normL1; + alpha = alpha + 0.5; + } + if (alpha >= 1.0) { + volfrac(i, j, k) = 1.0; + } else if (alpha <= 0.0) { + volfrac(i, j, k) = 0.0; + } else { + volfrac(i, j, k) = + multiphase::cut_volume(mx, my, mz, alpha, 0.0, 1.0); + } + }); + } + } + // Fill ghost and boundary cells before simulation begins + vof_scr.fillpatch(0.0); +} + } // namespace amr_wind From 3324a73c05263ecaccdaf20670a7512dd5da2164 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 17 May 2024 11:45:32 -0600 Subject: [PATCH 02/43] prescribed velocity simulations should not need special treatment --- amr-wind/overset/OversetOps.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index f869fe613a..15f96aa498 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -52,7 +52,6 @@ void OversetOps::initialize(CFDSim& sim) // TODO: // * set up coupling options - // * different behavior depending on prescribe flag } void OversetOps::pre_advance_work() From 6cc387b19e89b0d15796b9223c09de2d22fe850b Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 17 May 2024 12:10:02 -0600 Subject: [PATCH 03/43] provide options for decoupling projections in overset --- amr-wind/equation_systems/icns/icns_advection.cpp | 6 ++++++ amr-wind/overset/OversetOps.cpp | 6 ++---- amr-wind/projection/incflo_apply_nodal_projection.cpp | 11 +++++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 6bffd88035..5659f7d556 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -55,6 +55,12 @@ MacProjOp::MacProjOp( { amrex::ParmParse pp("incflo"); pp.query("density", m_rho_0); + amrex::ParmParse pp_ovst("Overset"); + bool disable_ovst_mac = false; + pp.query("disable_coupled_mac_proj", disable_ovst_mac); + if (m_has_overset && disable_ovst_mac) { + m_has_overset = false; + } } void MacProjOp::init_projector(const MacProjOp::FaceFabPtrVec& beta) noexcept diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 15f96aa498..ee82bc1796 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -22,9 +22,10 @@ void OversetOps::initialize(CFDSim& sim) pp.query("reinit_target_cutoff", m_target_cutoff); // Queries for coupling options + pp.query("replace_gradp_postsolve", m_replace_gp); + // OversetOps does not control these coupling options, merely reports them pp.query("disable_coupled_nodal_proj", m_disable_nodal_proj); pp.query("disable_coupled_mac_proj", m_disable_mac_proj); - pp.query("replace_gradp_postsolve", m_replace_gp); // Verbosity pp.query("verbose", m_verbose); @@ -49,9 +50,6 @@ void OversetOps::initialize(CFDSim& sim) // Output parameters if verbose parameter_output(); - - // TODO: - // * set up coupling options } void OversetOps::pre_advance_work() diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index fe1cac1bfc..4592caaf64 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -358,8 +358,15 @@ void incflo::ApplyProjection( nodal_projector->setCustomRHS(div_vel_rhs->vec_const_ptrs()); } - // Setup masking for overset simulations + // Determine if nodal projection should be coupled for overset + bool disable_ovst_nodal = false; if (sim().has_overset()) { + amrex::ParmParse pp("Overset"); + pp.query("disable_coupled_nodal_proj", disable_ovst_nodal); + } + + // Setup masking for overset simulations + if (sim().has_overset() && !disable_ovst_nodal) { auto& linop = nodal_projector->getLinOp(); const auto& imask_node = repo().get_int_field("mask_node"); for (int lev = 0; lev <= finest_level; ++lev) { @@ -367,7 +374,7 @@ void incflo::ApplyProjection( } } - if (m_sim.has_overset()) { + if (m_sim.has_overset() && !disable_ovst_nodal) { auto phif = m_repo.create_scratch_field(1, 1, amr_wind::FieldLoc::NODE); if (incremental) { for (int lev = 0; lev <= finestLevel(); ++lev) { From 98f417ac1b63f0c4ddca0d41457db1e02b4a7fad Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 17 May 2024 12:16:23 -0600 Subject: [PATCH 04/43] formatting --- amr-wind/equation_systems/vof/volume_fractions.H | 1 - 1 file changed, 1 deletion(-) diff --git a/amr-wind/equation_systems/vof/volume_fractions.H b/amr-wind/equation_systems/vof/volume_fractions.H index 5765840063..120e6bf6a9 100644 --- a/amr-wind/equation_systems/vof/volume_fractions.H +++ b/amr-wind/equation_systems/vof/volume_fractions.H @@ -64,7 +64,6 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal( mz = mm1 - mm2; } - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal_neumann( int i, int j, From dcd5e3688037e73390fcea7506e548109711ed34 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 17 May 2024 13:40:17 -0600 Subject: [PATCH 05/43] CI --- amr-wind/overset/OversetOps.H | 2 +- amr-wind/overset/OversetOps.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/amr-wind/overset/OversetOps.H b/amr-wind/overset/OversetOps.H index c7893c34d8..70b0cbaa85 100644 --- a/amr-wind/overset/OversetOps.H +++ b/amr-wind/overset/OversetOps.H @@ -19,7 +19,7 @@ public: private: // Functions called within public functions - void parameter_output(); + void parameter_output() const; void sharpen_nalu_data(); void set_hydrostatic_gradp(); void replace_masked_gradp(); diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index ee82bc1796..42db2e281f 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -150,7 +150,7 @@ void OversetOps::post_advance_work() /* PUBLIC FUNCTIONS ABOVE, PRIVATE FUNCTIONS BELOW */ /* ----------------------------------------------- */ -void OversetOps::parameter_output() +void OversetOps::parameter_output() const { // Print the details if (m_verbose > 0) { From 998a91e64bbb36aa71772b078ad67de7eb60ef23 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 20 May 2024 13:33:31 -0600 Subject: [PATCH 06/43] testing neumann vof normal calc --- unit_tests/aw_test_utils/iter_tools.H | 28 +++ unit_tests/multiphase/test_vof_plic.cpp | 304 ++++++++++++++++++++++++ 2 files changed, 332 insertions(+) diff --git a/unit_tests/aw_test_utils/iter_tools.H b/unit_tests/aw_test_utils/iter_tools.H index f8a67ea13d..7ef7390f93 100644 --- a/unit_tests/aw_test_utils/iter_tools.H +++ b/unit_tests/aw_test_utils/iter_tools.H @@ -48,6 +48,34 @@ void run_algorithm(amr_wind::Field& field, const Functor& func) } } +template +void run_algorithm( + const int nlevels, + amrex::Vector& intfield, + const Functor& func) +{ + for (int lev = 0; lev < nlevels; ++lev) { + auto& lfab = *intfield[lev]; + + for (amrex::MFIter mfi(lfab); mfi.isValid(); ++mfi) { + func(lev, mfi); + } + } +} + +template +void run_algorithm(amr_wind::IntField& intfield, const Functor& func) +{ + const int nlevels = intfield.repo().num_active_levels(); + for (int lev = 0; lev < nlevels; ++lev) { + auto& lfab = intfield(lev); + + for (amrex::MFIter mfi(lfab); mfi.isValid(); ++mfi) { + func(lev, mfi); + } + } +} + } // namespace amr_wind_tests #endif /* ITER_TOOLS_H */ diff --git a/unit_tests/multiphase/test_vof_plic.cpp b/unit_tests/multiphase/test_vof_plic.cpp index 71cefbbbcd..60f823c21f 100644 --- a/unit_tests/multiphase/test_vof_plic.cpp +++ b/unit_tests/multiphase/test_vof_plic.cpp @@ -98,6 +98,104 @@ void init_vof_h(amr_wind::Field& vof, const amrex::Real vof_val, const int dir) }); } +void initialize_volume_fractions( + const amrex::Box& bx, const amrex::Array4& vof_arr) +{ + amrex::ParallelFor(grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { + vof_arr(i, j, k) = 0.13 * ((amrex::Real)i - 1.5) + + 0.04 * std::pow((amrex::Real)j - 1., 2) + + 0.01 * std::pow((amrex::Real)k - 2., 3) + 0.5; + }); +} + +void init_vof(amr_wind::Field& vof) +{ + run_algorithm(vof, [&](const int lev, const amrex::MFIter& mfi) { + auto vof_arr = vof(lev).array(mfi); + const auto& bx = mfi.validbox(); + initialize_volume_fractions(bx, vof_arr); + }); +} + +void initialize_iblank_distribution( + const int dir, const amrex::Box& bx, const amrex::Array4& iblk_arr) +{ + const int d = dir; + amrex::ParallelFor(grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { + int ii = (d != 0 ? i : 0); + int jj = (d != 1 ? j : 0); + int kk = (d != 2 ? k : 0); + if (ii + jj + kk > 4 || ii + jj + kk < 2) { + iblk_arr(i, j, k) = -1; + } else { + iblk_arr(i, j, k) = 1; + } + }); +} + +void init_iblank(amr_wind::IntField& iblank, const int dir) +{ + run_algorithm(iblank, [&](const int lev, const amrex::MFIter& mfi) { + auto iblk_arr = iblank(lev).array(mfi); + const auto& bx = mfi.validbox(); + initialize_iblank_distribution(dir, bx, iblk_arr); + }); +} + +void initialize_volume_fractions_iblank( + const int dir, + const amrex::Box& bx, + const amrex::Array4& vof_arr, + const amrex::Array4& iblk_arr) +{ + // Does a horizontal interface that is different outside of iblank region + const int d = dir; + amrex::ParallelFor(grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { + // Turns on index tangent to interface + int iit = (d != 0 ? i : 1); + int jjt = (d != 1 ? j : 1); + int kkt = (d != 2 ? k : 1); + // Turns on index normal to interface + int iin = (d == 0 ? i : 0); + int jjn = (d == 1 ? j : 0); + int kkn = (d == 2 ? k : 0); + // Ordinary vof distribution + if (iin + jjn + kkn > 2) { + vof_arr(i, j, k) = 0.0; + } + if (iin + jjn + kkn == 2) { + vof_arr(i, j, k) = 0.5; + } + if (iin + jjn + kkn < 2) { + vof_arr(i, j, k) = 1.0; + } + // iblank distribution + iblk_arr(i, j, k) = 1; + if (iit > 2 || iit < 1 || jjt > 2 || jjt < 1 || kkt > 2 || kkt < 1) { + // Restrict iblank = 1 to central block + iblk_arr(i, j, k) = -1; + // Change vof where iblank = -1 + if (iin + jjn + kkn == 3) { + vof_arr(i, j, k) = 0.25; + } + if (iin + jjn + kkn == 1) { + vof_arr(i, j, k) = 0.75; + } + } + }); +} + +void init_vof_iblank( + amr_wind::Field& vof, amr_wind::IntField& iblank, const int dir) +{ + run_algorithm(vof, [&](const int lev, const amrex::MFIter& mfi) { + auto vof_arr = vof(lev).array(mfi); + auto iblk_arr = iblank(lev).array(mfi); + const auto& bx = mfi.validbox(); + initialize_volume_fractions_iblank(dir, bx, vof_arr, iblk_arr); + }); +} + amrex::Real normal_vector_test_impl(amr_wind::Field& vof, const int dir) { amrex::Real error_total = 0.0; @@ -136,6 +234,153 @@ amrex::Real normal_vector_test_impl(amr_wind::Field& vof, const int dir) return error_total; } +amrex::Real normal_vector_neumann_test_impl( + amr_wind::Field& vof, amr_wind::IntField& iblk_fld) +{ + amrex::Real error_total = 0.0; + + for (int lev = 0; lev < vof.repo().num_active_levels(); ++lev) { + + error_total += amrex::ReduceSum( + vof(lev), iblk_fld(lev), 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& vof_arr, + amrex::Array4 const& iblank) -> amrex::Real { + amrex::Real error = 0.0; + + amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { + int ibdy = 0; + int jbdy = 0; + int kbdy = 0; + if (iblank(i, j, k) != iblank(i - 1, j, k)) { + ibdy = -1; + } + if (iblank(i, j, k) != iblank(i, j - 1, k)) { + jbdy = -1; + } + if (iblank(i, j, k) != iblank(i, j, k - 1)) { + kbdy = -1; + } + if (iblank(i, j, k) != iblank(i + 1, j, k)) { + ibdy = +1; + } + if (iblank(i, j, k) != iblank(i, j + 1, k)) { + jbdy = +1; + } + if (iblank(i, j, k) != iblank(i, j, k + 1)) { + kbdy = +1; + } + amrex::Real mxn, myn, mzn; + amr_wind::multiphase::youngs_fd_normal_neumann( + i, j, k, ibdy, jbdy, kbdy, vof_arr, mxn, myn, mzn); + amrex::Real mx, my, mz; + amr_wind::multiphase::youngs_fd_normal( + i, j, k, vof_arr, mx, my, mz); + + // Use L1 norm, check against non-neumann implementation + // Slope across overset boundary should be different + if (ibdy != 0) { + // x slope should be different + error += std::abs(mx - mxn) > 1e-8 ? 0. : 1.0; + } + if (jbdy != 0) { + // y slope should be different + error += std::abs(my - myn) > 1e-8 ? 0. : 1.0; + } + if (kbdy != 0) { + // z slope should be different + error += std::abs(mz - mzn) > 1e-8 ? 0. : 1.0; + } + // Slope should otherwise be the same + if (ibdy == 0 && jbdy == 0 && kbdy == 0) { + error += std::abs(mx - mxn); + error += std::abs(my - myn); + error += std::abs(mz - mzn); + } + }); + + return error; + }); + } + return error_total; +} + +amrex::Real normal_vector_neumann_test_impl( + amr_wind::Field& vof, amr_wind::IntField& iblk_fld, const int& dir) +{ + const amrex::Real ref_m = 16.0 * (1.0 - 0.0); + amrex::Real error_total = 0.0; + + for (int lev = 0; lev < vof.repo().num_active_levels(); ++lev) { + + error_total += amrex::ReduceSum( + vof(lev), iblk_fld(lev), 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& vof_arr, + amrex::Array4 const& iblank) -> amrex::Real { + amrex::Real error = 0.0; + + amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { + int ibdy = 0; + int jbdy = 0; + int kbdy = 0; + if (iblank(i, j, k) != iblank(i - 1, j, k)) { + ibdy = -1; + } + if (iblank(i, j, k) != iblank(i, j - 1, k)) { + jbdy = -1; + } + if (iblank(i, j, k) != iblank(i, j, k - 1)) { + kbdy = -1; + } + if (iblank(i, j, k) != iblank(i + 1, j, k)) { + ibdy = +1; + } + if (iblank(i, j, k) != iblank(i, j + 1, k)) { + jbdy = +1; + } + if (iblank(i, j, k) != iblank(i, j, k + 1)) { + kbdy = +1; + } + amrex::Real mxn, myn, mzn; + amr_wind::multiphase::youngs_fd_normal_neumann( + i, j, k, ibdy, jbdy, kbdy, vof_arr, mxn, myn, mzn); + + // Use L1 norm, check for 0 + if (iblank(i, j, k) == 1) { + // Only interested in normals from field cells + + // Slope in non-normal directions should be 0 + if (dir != 0) { + error += std::abs(mxn - 0.); + } + if (dir != 1) { + error += std::abs(myn - 0.); + } + if (dir != 2) { + error += std::abs(mzn - 0.); + } + // Slope in normal direction, at center, should be same + if (dir == 0 && i == 2) { + error += std::abs(mxn - ref_m); + } + if (dir == 1 && j == 2) { + error += std::abs(myn - ref_m); + } + if (dir == 2 && k == 2) { + error += std::abs(mzn - ref_m); + } + } + }); + + return error; + }); + } + return error_total; +} + amrex::Real fit_plane_test_impl(amr_wind::Field& vof, const int dir) { amrex::Real error_total = 0.0; @@ -367,4 +612,63 @@ TEST_F(VOFOpTest, interface_plane) } } +TEST_F(VOFOpTest, interface_normal_neumann) +{ + + constexpr double tol = 1.0e-11; + + populate_parameters(); + { + amrex::ParmParse pp("geometry"); + amrex::Vector periodic{{0, 0, 0}}; + pp.addarr("is_periodic", periodic); + } + + initialize_mesh(); + + auto& repo = sim().repo(); + const int ncomp = 1; + const int nghost = 3; + auto& vof = repo.declare_field("vof", ncomp, nghost); + auto& iblank = repo.declare_int_field("iblank", ncomp, 1); + + // Check agreement / disagreement with ordinary calculation + amrex::Real error_total = 0.0; + // iblank constant in x, vof varies in every direction + init_vof(vof); + init_iblank(iblank, 0); + error_total = normal_vector_neumann_test_impl(vof, iblank); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, tol); + // iblank constant in y, vof varies in every direction + init_vof(vof); + init_iblank(iblank, 1); + error_total = normal_vector_neumann_test_impl(vof, iblank); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, tol); + // iblank constant in z, vof varies in every direction + init_vof(vof); + init_iblank(iblank, 2); + error_total = normal_vector_neumann_test_impl(vof, iblank); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, tol); + + // Confirm that neumann gets 0 when it should + // iblank varies in y and z, vof varies in x + init_vof_iblank(vof, iblank, 0); + error_total = normal_vector_neumann_test_impl(vof, iblank, 0); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, tol); + // iblank varies in x and z, vof varies in y + init_vof_iblank(vof, iblank, 1); + error_total = normal_vector_neumann_test_impl(vof, iblank, 1); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, tol); + // iblank varies in x and y, vof varies in z + init_vof_iblank(vof, iblank, 2); + error_total = normal_vector_neumann_test_impl(vof, iblank, 2); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, tol); +} + } // namespace amr_wind_tests From d0bba360182d8ab93fb3ac32fd58e9e5460faca6 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Tue, 21 May 2024 08:33:38 -0600 Subject: [PATCH 07/43] unit test for alpha flux --- unit_tests/multiphase/CMakeLists.txt | 1 + .../multiphase/test_vof_overset_ops.cpp | 286 ++++++++++++++++++ 2 files changed, 287 insertions(+) create mode 100644 unit_tests/multiphase/test_vof_overset_ops.cpp diff --git a/unit_tests/multiphase/CMakeLists.txt b/unit_tests/multiphase/CMakeLists.txt index 7bc2bc6de7..8386a857d8 100644 --- a/unit_tests/multiphase/CMakeLists.txt +++ b/unit_tests/multiphase/CMakeLists.txt @@ -7,4 +7,5 @@ target_sources( test_vof_BCs.cpp test_mflux_schemes.cpp test_reference_fields.cpp + test_vof_overset_ops.cpp ) diff --git a/unit_tests/multiphase/test_vof_overset_ops.cpp b/unit_tests/multiphase/test_vof_overset_ops.cpp new file mode 100644 index 0000000000..7f6d41d022 --- /dev/null +++ b/unit_tests/multiphase/test_vof_overset_ops.cpp @@ -0,0 +1,286 @@ +#include "aw_test_utils/MeshTest.H" +#include "aw_test_utils/iter_tools.H" +#include "aw_test_utils/test_utils.H" +#include "amr-wind/overset/overset_ops_K.H" + +namespace amr_wind_tests { + +class VOFOversetOps : public MeshTest +{ +protected: + void populate_parameters() override + { + MeshTest::populate_parameters(); + + { + amrex::ParmParse pp("amr"); + amrex::Vector ncell{{8, 8, 8}}; + pp.add("max_level", 0); + pp.add("max_grid_size", 4); + pp.addarr("n_cell", ncell); + } + { + amrex::ParmParse pp("geometry"); + amrex::Vector problo{{0.0, 0.0, 0.0}}; + amrex::Vector probhi{{1.0, 1.0, 1.0}}; + + pp.addarr("prob_lo", problo); + pp.addarr("prob_hi", probhi); + } + } +}; + +namespace { +void init_vof_etc( + amr_wind::Field& vof, amr_wind::Field& tg_vof, amr_wind::Field& norm) +{ + run_algorithm(vof, [&](const int lev, const amrex::MFIter& mfi) { + auto vof_arr = vof(lev).array(mfi); + // tgvof and norm values are set randomly to make sure they are involved + auto tgvof_arr = tg_vof(lev).array(mfi); + auto norm_arr = norm(lev).array(mfi); + const auto& bx = mfi.validbox(); + amrex::ParallelFor( + grow(bx, 2), [=] AMREX_GPU_DEVICE(int i, int j, int k) { + vof_arr(i, j, k) = 0.0; + tgvof_arr(i, j, k) = 0.; + norm_arr(i, j, k) = 1.0; + if (i == -1) { + // Within margin + vof_arr(i, j, k) = 0.41; + tgvof_arr(i, j, k) = 0.5; + norm_arr(i, j, k) = -1.0; + } else if (i == 0) { + // Within margin + vof_arr(i, j, k) = 0.55; + tgvof_arr(i, j, k) = 0.5; + norm_arr(i, j, k) = 1.0; + } else if (i == 1) { + // Outside margin, low + vof_arr(i, j, k) = 0.1; + tgvof_arr(i, j, k) = 0.2; + norm_arr(i, j, k) = -1.0; + } else if (i == 2) { + // Also low + vof_arr(i, j, k) = 0.2; + tgvof_arr(i, j, k) = 0.22; + norm_arr(i, j, k) = 1.0; + } else if (i == 3) { + // Above half + vof_arr(i, j, k) = 0.7; + tgvof_arr(i, j, k) = 0.8; + norm_arr(i, j, k) = -1.0; + } else if (i == 4) { + // Also high + vof_arr(i, j, k) = 0.65; + tgvof_arr(i, j, k) = 0.91; + norm_arr(i, j, k) = 1.0; + } else if (i == 5) { + // Also high, positive gradient + vof_arr(i, j, k) = 0.8; + tgvof_arr(i, j, k) = 0.75; + norm_arr(i, j, k) = -1.0; + } else if (i == 6) { + // Within margin + vof_arr(i, j, k) = 0.45; + tgvof_arr(i, j, k) = 0.5; + norm_arr(i, j, k) = 1.0; + } else if (i == 7) { + // Low, negative gradient + vof_arr(i, j, k) = 0.3; + tgvof_arr(i, j, k) = 0.2; + norm_arr(i, j, k) = -1.0; + } else if (i == 8) { + // High + vof_arr(i, j, k) = 0.7; + tgvof_arr(i, j, k) = 0.6; + norm_arr(i, j, k) = 1.0; + } + }); + }); +} + +void init_velocity_etc(amr_wind::Field& velocity) +{ + run_algorithm(velocity, [&](const int lev, const amrex::MFIter& mfi) { + auto vel_arr = velocity(lev).array(mfi); + const auto& bx = mfi.validbox(); + amrex::ParallelFor( + grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { + // Insert velocity distribution + }); + }); +} + +void calc_alpha_flux( + amr_wind::Field& flux, + amr_wind::Field& vof, + amr_wind::Field& tg_vof, + amr_wind::Field& norm, + const int& dir, + const amrex::Real& margin) +{ + run_algorithm(flux, [&](const int lev, const amrex::MFIter& mfi) { + auto f_arr = flux(lev).array(mfi); + auto vof_arr = vof(lev).array(mfi); + auto tgvof_arr = tg_vof(lev).array(mfi); + auto norm_arr = norm(lev).array(mfi); + const auto& bx = mfi.validbox(); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + f_arr(i, j, k) = amr_wind::overset_ops::alpha_flux( + i, j, k, dir, margin, vof_arr, tgvof_arr, norm_arr); + }); + }); +} + +void calc_velocity_face( + amr_wind::Field& flux, + amr_wind::Field& vof, + amr_wind::Field& velocity, + const int& dir) +{ + run_algorithm(flux, [&](const int lev, const amrex::MFIter& mfi) { + auto f_arr = flux(lev).array(mfi); + auto vof_arr = vof(lev).array(mfi); + auto vel_arr = velocity(lev).array(mfi); + const auto& bx = mfi.validbox(); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + amrex::Real u_f, v_f, w_f; + amr_wind::overset_ops::velocity_face( + i, j, k, dir, vof_arr, vel_arr, u_f, v_f, w_f); + f_arr(i, j, k, 0) = u_f; + f_arr(i, j, k, 1) = v_f; + f_arr(i, j, k, 2) = w_f; + }); + }); +} + +amrex::Real check_alpha_flux_impl(amr_wind::Field& flux) +{ + amrex::Real error_total = 0; + + for (int lev = 0; lev < flux.repo().num_active_levels(); ++lev) { + + error_total += amrex::ReduceSum( + flux(lev), 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& f_arr) -> amrex::Real { + amrex::Real error = 0; + + amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { + // Difference between actual and expected + if (i == 0) { + // Both within margin, will average + const amrex::Real flux_answer = + 0.5 * ((0.5 - 0.41) * -1.0 + (0.5 - 0.55) * -1.0); + error += std::abs(f_arr(i, j, k) - flux_answer); + } else if (i == 1) { + // Current is low, neighbor within margin + const amrex::Real flux_answer = (0.2 - 0.1) * 1.0; + error += std::abs(f_arr(i, j, k) - flux_answer); + } else if (i == 2) { + // Low on both sides, gphi > 0 + const amrex::Real flux_answer = (0.2 - 0.1) * -1.0; + error += std::abs(f_arr(i, j, k) - flux_answer); + } else if (i == 3) { + // Opposite sides of margin, no conditional fits + const amrex::Real flux_answer = + 0.5 * ((0.8 - 0.7) * 1.0 + (0.22 - 0.2) * 1.0); + error += std::abs(f_arr(i, j, k) - flux_answer); + } else if (i == 4) { + // High on both sides, gphi < 0 + const amrex::Real flux_answer = (0.8 - 0.7) * -1.0; + error += std::abs(f_arr(i, j, k) - flux_answer); + } else if (i == 5) { + // High on both sides, gphi > 0 + const amrex::Real flux_answer = (0.75 - 0.8) * 1.0; + error += std::abs(f_arr(i, j, k) - flux_answer); + } else if (i == 6) { + // Current is within margin, neighbor is high + const amrex::Real flux_answer = (0.75 - 0.8) * -1.0; + error += std::abs(f_arr(i, j, k) - flux_answer); + } else if (i == 7) { + // Current is low, neighbor is within margin + const amrex::Real flux_answer = (0.2 - 0.3) * 1.0; + error += std::abs(f_arr(i, j, k) - flux_answer); + } else if (i == 8) { + // Opposite sides of margin, no conditional fits + const amrex::Real flux_answer = + 0.5 * ((0.6 - 0.7) * -1.0 + (0.2 - 0.3) * -1.0); + error += std::abs(f_arr(i, j, k) - flux_answer); + } + }); + + return error; + }); + } + return error_total; +} + +/*amrex::Real check_velocity_face_impl(amr_wind::Field& flux) +{ + amrex::Real error_total = 0; + + for (int lev = 0; lev < flux.repo().num_active_levels(); ++lev) { + + error_total += amrex::ReduceSum( + flux(lev), 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& f_arr) -> amrex::Real { + amrex::Real error = 0; + + amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { + // Difference between actual and expected + const amrex::Real flux_answer = ; + error += std::abs(f_arr(i, j, k, 0) - flux_answer); + const amrex::Real flux_answer = ; + error += std::abs(f_arr(i, j, k, 1) - flux_answer); + const amrex::Real flux_answer = ; + error += std::abs(f_arr(i, j, k, 2) - flux_answer); + }); + + return error; + }); + } + return error_total; +}*/ +} // namespace + +TEST_F(VOFOversetOps, alpha_flux) +{ + populate_parameters(); + initialize_mesh(); + + auto& repo = sim().repo(); + const int ncomp = 1; + const int nghost = 3; + auto& vof = repo.declare_field("vof", ncomp, nghost); + auto& tg_vof = repo.declare_field("target_vof", ncomp, nghost); + auto& norm = repo.declare_field("int_normal", ncomp, nghost); + auto& iblank = repo.declare_int_field("iblank", ncomp, nghost); + const int dir = 0; + const amrex::Real margin = 0.1; + + // Not worried about overset boundaries in this test + iblank.setVal(1); + + // Initialize vof and other fields + init_vof_etc(vof, tg_vof, norm); + + // Create flux field + auto& flux_x = + repo.declare_field("flux", 1, 0, 1, amr_wind::FieldLoc::XFACE); + // Populate flux field + calc_alpha_flux(flux_x, vof, tg_vof, norm, dir, margin); + + // Check results + amrex::Real error_total = check_alpha_flux_impl(flux_x); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); +} + +TEST_F(VOFOversetOps, velocity_face) {} + +} // namespace amr_wind_tests \ No newline at end of file From f8685c075fc5a2f035bc10edde6b1e51dda38211 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Tue, 21 May 2024 08:58:01 -0600 Subject: [PATCH 08/43] generalize to test every direction --- .../multiphase/test_vof_overset_ops.cpp | 113 +++++++++++------- 1 file changed, 70 insertions(+), 43 deletions(-) diff --git a/unit_tests/multiphase/test_vof_overset_ops.cpp b/unit_tests/multiphase/test_vof_overset_ops.cpp index 7f6d41d022..00c1e1e8ca 100644 --- a/unit_tests/multiphase/test_vof_overset_ops.cpp +++ b/unit_tests/multiphase/test_vof_overset_ops.cpp @@ -32,7 +32,10 @@ class VOFOversetOps : public MeshTest namespace { void init_vof_etc( - amr_wind::Field& vof, amr_wind::Field& tg_vof, amr_wind::Field& norm) + amr_wind::Field& vof, + amr_wind::Field& tg_vof, + amr_wind::Field& norm, + const int& dir) { run_algorithm(vof, [&](const int lev, const amrex::MFIter& mfi) { auto vof_arr = vof(lev).array(mfi); @@ -44,57 +47,58 @@ void init_vof_etc( grow(bx, 2), [=] AMREX_GPU_DEVICE(int i, int j, int k) { vof_arr(i, j, k) = 0.0; tgvof_arr(i, j, k) = 0.; - norm_arr(i, j, k) = 1.0; - if (i == -1) { + norm_arr(i, j, k, dir) = 1.0; + const int idx = (dir == 0 ? i : (dir == 1 ? j : k)); + if (idx == -1) { // Within margin vof_arr(i, j, k) = 0.41; tgvof_arr(i, j, k) = 0.5; - norm_arr(i, j, k) = -1.0; - } else if (i == 0) { + norm_arr(i, j, k, dir) = -1.0; + } else if (idx == 0) { // Within margin vof_arr(i, j, k) = 0.55; tgvof_arr(i, j, k) = 0.5; - norm_arr(i, j, k) = 1.0; - } else if (i == 1) { + norm_arr(i, j, k, dir) = 1.0; + } else if (idx == 1) { // Outside margin, low vof_arr(i, j, k) = 0.1; tgvof_arr(i, j, k) = 0.2; - norm_arr(i, j, k) = -1.0; - } else if (i == 2) { + norm_arr(i, j, k, dir) = -1.0; + } else if (idx == 2) { // Also low vof_arr(i, j, k) = 0.2; tgvof_arr(i, j, k) = 0.22; - norm_arr(i, j, k) = 1.0; - } else if (i == 3) { + norm_arr(i, j, k, dir) = 1.0; + } else if (idx == 3) { // Above half vof_arr(i, j, k) = 0.7; tgvof_arr(i, j, k) = 0.8; - norm_arr(i, j, k) = -1.0; - } else if (i == 4) { + norm_arr(i, j, k, dir) = -1.0; + } else if (idx == 4) { // Also high vof_arr(i, j, k) = 0.65; tgvof_arr(i, j, k) = 0.91; - norm_arr(i, j, k) = 1.0; - } else if (i == 5) { + norm_arr(i, j, k, dir) = 1.0; + } else if (idx == 5) { // Also high, positive gradient vof_arr(i, j, k) = 0.8; tgvof_arr(i, j, k) = 0.75; - norm_arr(i, j, k) = -1.0; - } else if (i == 6) { + norm_arr(i, j, k, dir) = -1.0; + } else if (idx == 6) { // Within margin vof_arr(i, j, k) = 0.45; tgvof_arr(i, j, k) = 0.5; - norm_arr(i, j, k) = 1.0; - } else if (i == 7) { + norm_arr(i, j, k, dir) = 1.0; + } else if (idx == 7) { // Low, negative gradient vof_arr(i, j, k) = 0.3; tgvof_arr(i, j, k) = 0.2; - norm_arr(i, j, k) = -1.0; - } else if (i == 8) { + norm_arr(i, j, k, dir) = -1.0; + } else if (idx == 8) { // High vof_arr(i, j, k) = 0.7; tgvof_arr(i, j, k) = 0.6; - norm_arr(i, j, k) = 1.0; + norm_arr(i, j, k, dir) = 1.0; } }); }); @@ -155,7 +159,7 @@ void calc_velocity_face( }); } -amrex::Real check_alpha_flux_impl(amr_wind::Field& flux) +amrex::Real check_alpha_flux_impl(amr_wind::Field& flux, const int& dir) { amrex::Real error_total = 0; @@ -169,42 +173,43 @@ amrex::Real check_alpha_flux_impl(amr_wind::Field& flux) amrex::Real error = 0; amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { + const int idx = (dir == 0 ? i : (dir == 1 ? j : k)); // Difference between actual and expected - if (i == 0) { + if (idx == 0) { // Both within margin, will average const amrex::Real flux_answer = 0.5 * ((0.5 - 0.41) * -1.0 + (0.5 - 0.55) * -1.0); error += std::abs(f_arr(i, j, k) - flux_answer); - } else if (i == 1) { + } else if (idx == 1) { // Current is low, neighbor within margin const amrex::Real flux_answer = (0.2 - 0.1) * 1.0; error += std::abs(f_arr(i, j, k) - flux_answer); - } else if (i == 2) { + } else if (idx == 2) { // Low on both sides, gphi > 0 const amrex::Real flux_answer = (0.2 - 0.1) * -1.0; error += std::abs(f_arr(i, j, k) - flux_answer); - } else if (i == 3) { + } else if (idx == 3) { // Opposite sides of margin, no conditional fits const amrex::Real flux_answer = 0.5 * ((0.8 - 0.7) * 1.0 + (0.22 - 0.2) * 1.0); error += std::abs(f_arr(i, j, k) - flux_answer); - } else if (i == 4) { + } else if (idx == 4) { // High on both sides, gphi < 0 const amrex::Real flux_answer = (0.8 - 0.7) * -1.0; error += std::abs(f_arr(i, j, k) - flux_answer); - } else if (i == 5) { + } else if (idx == 5) { // High on both sides, gphi > 0 const amrex::Real flux_answer = (0.75 - 0.8) * 1.0; error += std::abs(f_arr(i, j, k) - flux_answer); - } else if (i == 6) { + } else if (idx == 6) { // Current is within margin, neighbor is high const amrex::Real flux_answer = (0.75 - 0.8) * -1.0; error += std::abs(f_arr(i, j, k) - flux_answer); - } else if (i == 7) { + } else if (idx == 7) { // Current is low, neighbor is within margin const amrex::Real flux_answer = (0.2 - 0.3) * 1.0; error += std::abs(f_arr(i, j, k) - flux_answer); - } else if (i == 8) { + } else if (idx == 8) { // Opposite sides of margin, no conditional fits const amrex::Real flux_answer = 0.5 * ((0.6 - 0.7) * -1.0 + (0.2 - 0.3) * -1.0); @@ -258,25 +263,47 @@ TEST_F(VOFOversetOps, alpha_flux) const int nghost = 3; auto& vof = repo.declare_field("vof", ncomp, nghost); auto& tg_vof = repo.declare_field("target_vof", ncomp, nghost); - auto& norm = repo.declare_field("int_normal", ncomp, nghost); - auto& iblank = repo.declare_int_field("iblank", ncomp, nghost); - const int dir = 0; + auto& norm = repo.declare_field("int_normal", 3, nghost); const amrex::Real margin = 0.1; - // Not worried about overset boundaries in this test - iblank.setVal(1); + // Create flux fields + auto& flux_x = + repo.declare_field("flux_x", 1, 0, 1, amr_wind::FieldLoc::XFACE); + auto& flux_y = + repo.declare_field("flux_y", 1, 0, 1, amr_wind::FieldLoc::YFACE); + auto& flux_z = + repo.declare_field("flux_z", 1, 0, 1, amr_wind::FieldLoc::ZFACE); + // -- Variations in x direction -- // + int dir = 0; // Initialize vof and other fields - init_vof_etc(vof, tg_vof, norm); - - // Create flux field - auto& flux_x = - repo.declare_field("flux", 1, 0, 1, amr_wind::FieldLoc::XFACE); + init_vof_etc(vof, tg_vof, norm, dir); // Populate flux field calc_alpha_flux(flux_x, vof, tg_vof, norm, dir, margin); + // Check results + amrex::Real error_total = check_alpha_flux_impl(flux_x, dir); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); + // -- Variations in y direction -- // + dir = 1; + // Initialize vof and other fields + init_vof_etc(vof, tg_vof, norm, dir); + // Populate flux field + calc_alpha_flux(flux_y, vof, tg_vof, norm, dir, margin); + // Check results + error_total = check_alpha_flux_impl(flux_y, dir); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); + + // -- Variations in z direction -- // + dir = 2; + // Initialize vof and other fields + init_vof_etc(vof, tg_vof, norm, dir); + // Populate flux field + calc_alpha_flux(flux_z, vof, tg_vof, norm, dir, margin); // Check results - amrex::Real error_total = check_alpha_flux_impl(flux_x); + error_total = check_alpha_flux_impl(flux_z, dir); amrex::ParallelDescriptor::ReduceRealSum(error_total); EXPECT_NEAR(error_total, 0.0, 1e-15); } From 8baac513c0499ea07ea3b8bb75f424492e2c0761 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Tue, 21 May 2024 09:27:29 -0600 Subject: [PATCH 09/43] velocity_face unit test --- .../multiphase/test_vof_overset_ops.cpp | 127 +++++++++++++++--- 1 file changed, 110 insertions(+), 17 deletions(-) diff --git a/unit_tests/multiphase/test_vof_overset_ops.cpp b/unit_tests/multiphase/test_vof_overset_ops.cpp index 00c1e1e8ca..82b80083fe 100644 --- a/unit_tests/multiphase/test_vof_overset_ops.cpp +++ b/unit_tests/multiphase/test_vof_overset_ops.cpp @@ -104,14 +104,39 @@ void init_vof_etc( }); } -void init_velocity_etc(amr_wind::Field& velocity) +void init_velocity_etc( + amr_wind::Field& velocity, amr_wind::Field& vof, const int& dir) { run_algorithm(velocity, [&](const int lev, const amrex::MFIter& mfi) { auto vel_arr = velocity(lev).array(mfi); + auto vof_arr = vof(lev).array(mfi); const auto& bx = mfi.validbox(); amrex::ParallelFor( - grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { - // Insert velocity distribution + grow(bx, 2), [=] AMREX_GPU_DEVICE(int i, int j, int k) { + vel_arr(i, j, k) = 0.0; + vof_arr(i, j, k) = 0.0; + const int idx = (dir == 0 ? i : (dir == 1 ? j : k)); + if (idx == -1) { + vof_arr(i, j, k) = 0.41; + vel_arr(i, j, k, 0) = 1.0; + vel_arr(i, j, k, 1) = 2.0; + vel_arr(i, j, k, 2) = 3.0; + } else if (idx == 0) { + vof_arr(i, j, k) = 0.55; + vel_arr(i, j, k, 0) = 1.5; + vel_arr(i, j, k, 1) = 2.5; + vel_arr(i, j, k, 2) = 3.5; + } else if (idx == 1) { + vof_arr(i, j, k) = 0.2; + vel_arr(i, j, k, 0) = 2.0; + vel_arr(i, j, k, 1) = 3.0; + vel_arr(i, j, k, 2) = 4.0; + } else if (idx == 2) { + vof_arr(i, j, k) = 0.2; + vel_arr(i, j, k, 0) = 2.5; + vel_arr(i, j, k, 1) = 3.5; + vel_arr(i, j, k, 2) = 4.5; + } }); }); } @@ -139,8 +164,8 @@ void calc_alpha_flux( void calc_velocity_face( amr_wind::Field& flux, - amr_wind::Field& vof, amr_wind::Field& velocity, + amr_wind::Field& vof, const int& dir) { run_algorithm(flux, [&](const int lev, const amrex::MFIter& mfi) { @@ -223,7 +248,7 @@ amrex::Real check_alpha_flux_impl(amr_wind::Field& flux, const int& dir) return error_total; } -/*amrex::Real check_velocity_face_impl(amr_wind::Field& flux) +amrex::Real check_velocity_face_impl(amr_wind::Field& flux, const int& dir) { amrex::Real error_total = 0; @@ -237,20 +262,39 @@ amrex::Real check_alpha_flux_impl(amr_wind::Field& flux, const int& dir) amrex::Real error = 0; amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { - // Difference between actual and expected - const amrex::Real flux_answer = ; - error += std::abs(f_arr(i, j, k, 0) - flux_answer); - const amrex::Real flux_answer = ; - error += std::abs(f_arr(i, j, k, 1) - flux_answer); - const amrex::Real flux_answer = ; - error += std::abs(f_arr(i, j, k, 2) - flux_answer); + const int idx = (dir == 0 ? i : (dir == 1 ? j : k)); + if (idx == 0) { + // gphi > 0, uwpind from the "left" + amrex::Real flux_answer = 1.0; + error += std::abs(f_arr(i, j, k, 0) - flux_answer); + flux_answer = 2.0; + error += std::abs(f_arr(i, j, k, 1) - flux_answer); + flux_answer = 3.0; + error += std::abs(f_arr(i, j, k, 2) - flux_answer); + } else if (idx == 1) { + // gphi < 0, upwind from the "right" + amrex::Real flux_answer = 2.0; + error += std::abs(f_arr(i, j, k, 0) - flux_answer); + flux_answer = 3.0; + error += std::abs(f_arr(i, j, k, 1) - flux_answer); + flux_answer = 4.0; + error += std::abs(f_arr(i, j, k, 2) - flux_answer); + } else if (idx == 2) { + // gphi = 0, average both sides + amrex::Real flux_answer = 0.5 * (2.5 + 2.0); + error += std::abs(f_arr(i, j, k, 0) - flux_answer); + flux_answer = 0.5 * (3.5 + 3.0); + error += std::abs(f_arr(i, j, k, 1) - flux_answer); + flux_answer = 0.5 * (4.5 + 4.0); + error += std::abs(f_arr(i, j, k, 2) - flux_answer); + } }); return error; }); } return error_total; -}*/ +} } // namespace TEST_F(VOFOversetOps, alpha_flux) @@ -259,10 +303,9 @@ TEST_F(VOFOversetOps, alpha_flux) initialize_mesh(); auto& repo = sim().repo(); - const int ncomp = 1; const int nghost = 3; - auto& vof = repo.declare_field("vof", ncomp, nghost); - auto& tg_vof = repo.declare_field("target_vof", ncomp, nghost); + auto& vof = repo.declare_field("vof", 1, nghost); + auto& tg_vof = repo.declare_field("target_vof", 1, nghost); auto& norm = repo.declare_field("int_normal", 3, nghost); const amrex::Real margin = 0.1; @@ -308,6 +351,56 @@ TEST_F(VOFOversetOps, alpha_flux) EXPECT_NEAR(error_total, 0.0, 1e-15); } -TEST_F(VOFOversetOps, velocity_face) {} +TEST_F(VOFOversetOps, velocity_face) +{ + populate_parameters(); + initialize_mesh(); + + auto& repo = sim().repo(); + const int nghost = 3; + auto& velocity = repo.declare_field("velocity", 3, nghost); + auto& vof = repo.declare_field("vof", 1, nghost); + + // Create flux fields + auto& flux_x = + repo.declare_field("flux_x", 3, 0, 1, amr_wind::FieldLoc::XFACE); + auto& flux_y = + repo.declare_field("flux_y", 3, 0, 1, amr_wind::FieldLoc::YFACE); + auto& flux_z = + repo.declare_field("flux_z", 3, 0, 1, amr_wind::FieldLoc::ZFACE); + + // -- Variations in x direction -- // + int dir = 0; + // Initialize velocity and vof + init_velocity_etc(velocity, vof, dir); + // Populate flux field + calc_velocity_face(flux_x, velocity, vof, dir); + // Check results + amrex::Real error_total = check_velocity_face_impl(flux_x, dir); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); + + // -- Variations in y direction -- // + dir = 1; + // Initialize velocity and vof + init_velocity_etc(velocity, vof, dir); + // Populate flux field + calc_velocity_face(flux_y, velocity, vof, dir); + // Check results + error_total = check_velocity_face_impl(flux_y, dir); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); + + // -- Variations in z direction -- // + dir = 2; + // Initialize velocity and vof + init_velocity_etc(velocity, vof, dir); + // Populate flux field + calc_velocity_face(flux_z, velocity, vof, dir); + // Check results + error_total = check_velocity_face_impl(flux_z, dir); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); +} } // namespace amr_wind_tests \ No newline at end of file From f5bad9e03b90d3ddcf6d088394157975003a8fdd Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Tue, 21 May 2024 11:55:53 -0600 Subject: [PATCH 10/43] verbosity for output on each sharpen step --- amr-wind/overset/OversetOps.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 42db2e281f..4d050dfb74 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -327,8 +327,10 @@ void OversetOps::sharpen_nalu_data() amrex::ParallelDescriptor::ReduceRealMax(err); } - amrex::Print() << "sharpen step " << n << " " << err << " " << m_tol - << std::endl; + if (m_verbose > 0) { + amrex::Print() << "OversetOps: sharpen step " << n << " conv. err " + << err << " tol " << m_tol << std::endl; + } } // Purely for debugging via visualization, should be removed later From 1486fe95c7aace0776630b49d9eecb2f4d8e57d8 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 7 Jun 2024 10:40:31 -0600 Subject: [PATCH 11/43] multiple changes from regression testing * typo in parser argument * simplify hydrostatic ops * fill internal iblank ghost cells * initialize pressure for sloshing tank case --- .../equation_systems/icns/icns_advection.cpp | 2 +- amr-wind/overset/TiogaInterface.cpp | 3 ++ amr-wind/physics/multiphase/SloshingTank.H | 6 +++ amr-wind/physics/multiphase/SloshingTank.cpp | 39 ++++++++++++++++++- amr-wind/physics/multiphase/hydrostatic_ops.H | 15 +++---- 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 5659f7d556..7a87e6afaf 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -57,7 +57,7 @@ MacProjOp::MacProjOp( pp.query("density", m_rho_0); amrex::ParmParse pp_ovst("Overset"); bool disable_ovst_mac = false; - pp.query("disable_coupled_mac_proj", disable_ovst_mac); + pp_ovst.query("disable_coupled_mac_proj", disable_ovst_mac); if (m_has_overset && disable_ovst_mac) { m_has_overset = false; } diff --git a/amr-wind/overset/TiogaInterface.cpp b/amr-wind/overset/TiogaInterface.cpp index 59e42d414d..ec1a75280f 100644 --- a/amr-wind/overset/TiogaInterface.cpp +++ b/amr-wind/overset/TiogaInterface.cpp @@ -127,6 +127,9 @@ void TiogaInterface::post_overset_conn_work() for (int lev = 0; lev < nlevels; ++lev) { htod_memcpy(m_iblank_cell(lev), (*m_iblank_cell_host)(lev), 0, 0, 1); htod_memcpy(m_iblank_node(lev), (*m_iblank_node_host)(lev), 0, 0, 1); + + m_iblank_cell(lev).FillBoundary(m_sim.mesh().Geom()[lev].periodicity()); + m_iblank_node(lev).FillBoundary(m_sim.mesh().Geom()[lev].periodicity()); } iblank_to_mask(m_iblank_cell, m_mask_cell); diff --git a/amr-wind/physics/multiphase/SloshingTank.H b/amr-wind/physics/multiphase/SloshingTank.H index bf7b24b763..c6097798d9 100644 --- a/amr-wind/physics/multiphase/SloshingTank.H +++ b/amr-wind/physics/multiphase/SloshingTank.H @@ -35,6 +35,7 @@ public: private: Field& m_velocity; Field& m_levelset; + Field& m_pressure; //! Initial free surface amplitude magnitude amrex::Real m_amplitude{0.1}; @@ -44,6 +45,11 @@ private: //! Initial zero-level free-surface water depth amrex::Real m_waterlevel{0.0}; + + //! Stuff to get from MultiPhase physics + amrex::Real m_rho1{1000.}; + amrex::Real m_rho2{1.}; + amrex::Vector m_gravity{0.0, 0.0, -9.81}; }; } // namespace amr_wind diff --git a/amr-wind/physics/multiphase/SloshingTank.cpp b/amr-wind/physics/multiphase/SloshingTank.cpp index 7558141c7c..1ab11d7ab6 100644 --- a/amr-wind/physics/multiphase/SloshingTank.cpp +++ b/amr-wind/physics/multiphase/SloshingTank.cpp @@ -1,4 +1,5 @@ #include "amr-wind/physics/multiphase/SloshingTank.H" +#include "amr-wind/physics/multiphase/MultiPhase.H" #include "amr-wind/CFDSim.H" #include "AMReX_ParmParse.H" #include "amr-wind/fvm/gradient.H" @@ -9,16 +10,22 @@ namespace amr_wind { SloshingTank::SloshingTank(CFDSim& sim) : m_velocity(sim.repo().get_field("velocity")) , m_levelset(sim.repo().get_field("levelset")) + , m_pressure(sim.repo().get_field("p")) { amrex::ParmParse pp(identifier()); pp.query("amplitude", m_amplitude); pp.query("peak_enhance", m_kappa); pp.query("water_level", m_waterlevel); + + const auto& mphase = sim.physics_manager().get(); + m_rho1 = mphase.rho1(); + m_rho2 = mphase.rho2(); + m_gravity = mphase.gravity(); } /** Initialize the velocity and levelset fields at the beginning of the - * simulation. - * + * simulation. Initializing pressure has little effect for pure AMR-Wind + * simulations, but it improves initialization in the overset solver. */ void SloshingTank::initialize_fields(int level, const amrex::Geometry& geom) { @@ -26,6 +33,7 @@ void SloshingTank::initialize_fields(int level, const amrex::Geometry& geom) velocity.setVal(0.0); auto& levelset = m_levelset(level); + auto& pressure = m_pressure(level); const auto& dx = geom.CellSizeArray(); const auto& problo = geom.ProbLoArray(); const auto& probhi = geom.ProbHiArray(); @@ -34,10 +42,14 @@ void SloshingTank::initialize_fields(int level, const amrex::Geometry& geom) const amrex::Real water_level = m_waterlevel; const amrex::Real Lx = probhi[0] - problo[0]; const amrex::Real Ly = probhi[1] - problo[1]; + const amrex::Real rho1 = m_rho1; + const amrex::Real rho2 = m_rho2; + const amrex::Real grav_z = m_gravity[2]; for (amrex::MFIter mfi(levelset); mfi.isValid(); ++mfi) { const auto& vbx = mfi.validbox(); auto phi = levelset.array(mfi); + auto p = pressure.array(mfi); amrex::ParallelFor( vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { @@ -51,6 +63,29 @@ void SloshingTank::initialize_fields(int level, const amrex::Geometry& geom) std::pow(y - problo[1] - 0.5 * Ly, 2))); phi(i, j, k) = z0 - z; }); + amrex::Box const& nbx = mfi.grownnodaltilebox(); + amrex::ParallelFor( + nbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // For pressure nodes, no offset + const amrex::Real x = problo[0] + i * dx[0]; + const amrex::Real y = problo[1] + j * dx[1]; + const amrex::Real z = problo[2] + k * dx[2]; + const amrex::Real z0 = + water_level + + Amp * std::exp( + -kappa * (std::pow(x - problo[0] - 0.5 * Lx, 2) + + std::pow(y - problo[1] - 0.5 * Ly, 2))); + // Integrated (top-down in z) phase heights to pressure node + amrex::Real ih_g = + amrex::max(0.0, amrex::min(probhi[2] - z0, probhi[2] - z)); + amrex::Real ih_l = + amrex::max(0.0, amrex::min(z0 - z, z0 - problo[2])); + // Integrated rho at pressure node + const amrex::Real irho = rho1 * ih_l + rho2 * ih_g; + + // Add term to reference pressure + p(i, j, k) = -irho * grav_z; + }); } } diff --git a/amr-wind/physics/multiphase/hydrostatic_ops.H b/amr-wind/physics/multiphase/hydrostatic_ops.H index 4f4eeb1942..ac59e0fe44 100644 --- a/amr-wind/physics/multiphase/hydrostatic_ops.H +++ b/amr-wind/physics/multiphase/hydrostatic_ops.H @@ -10,7 +10,7 @@ static void define_rho0( const amrex::Real rho1, const amrex::Real rho2, const amrex::Real wlev, - const amrex::Vector& geom) + const amrex::Vector geom) { for (int lev = 0; lev < rho0.repo().num_active_levels(); ++lev) { const auto& dx = geom[lev].CellSizeArray(); @@ -35,7 +35,7 @@ static void define_p0( const amrex::Real rho2, const amrex::Real wlev, const amrex::Real grav_z, - const amrex::Vector& geom) + const amrex::Vector geom) { for (int lev = 0; lev < p0.repo().num_active_levels(); ++lev) { const auto& dx = geom[lev].CellSizeArray(); @@ -46,15 +46,12 @@ static void define_p0( auto p0_arr = p0(lev).array(mfi); amrex::ParallelFor( nbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Height of pressure node - const amrex::Real hnode = k * dx[2]; - // Liquid height - const amrex::Real hliq = wlev - problo[2]; + const amrex::Real z = problo[2] + k * dx[2]; // Integrated (top-down in z) phase heights to pressure node amrex::Real ih_g = amrex::max( - 0.0, amrex::min(probhi[2] - hliq, probhi[2] - hnode)); - amrex::Real ih_l = amrex::max( - 0.0, amrex::min(hliq - hnode, hliq - problo[2])); + 0.0, amrex::min(probhi[2] - wlev, probhi[2] - z)); + amrex::Real ih_l = + amrex::max(0.0, amrex::min(wlev - z, wlev - problo[2])); // Integrated rho at pressure node const amrex::Real irho = rho1 * ih_l + rho2 * ih_g; From e18c0b7dd7da7de64ac945ce8af793b9bdbf11ac Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 7 Jun 2024 11:17:56 -0600 Subject: [PATCH 12/43] make hydrostatic_ops unit test harder to pass (fails with old implementation) * remove unintended changes too --- amr-wind/physics/multiphase/hydrostatic_ops.H | 4 ++-- unit_tests/multiphase/test_reference_fields.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/amr-wind/physics/multiphase/hydrostatic_ops.H b/amr-wind/physics/multiphase/hydrostatic_ops.H index ac59e0fe44..0cea6e1af8 100644 --- a/amr-wind/physics/multiphase/hydrostatic_ops.H +++ b/amr-wind/physics/multiphase/hydrostatic_ops.H @@ -10,7 +10,7 @@ static void define_rho0( const amrex::Real rho1, const amrex::Real rho2, const amrex::Real wlev, - const amrex::Vector geom) + const amrex::Vector& geom) { for (int lev = 0; lev < rho0.repo().num_active_levels(); ++lev) { const auto& dx = geom[lev].CellSizeArray(); @@ -35,7 +35,7 @@ static void define_p0( const amrex::Real rho2, const amrex::Real wlev, const amrex::Real grav_z, - const amrex::Vector geom) + const amrex::Vector& geom) { for (int lev = 0; lev < p0.repo().num_active_levels(); ++lev) { const auto& dx = geom[lev].CellSizeArray(); diff --git a/unit_tests/multiphase/test_reference_fields.cpp b/unit_tests/multiphase/test_reference_fields.cpp index 257c024a6e..df80ff4976 100644 --- a/unit_tests/multiphase/test_reference_fields.cpp +++ b/unit_tests/multiphase/test_reference_fields.cpp @@ -108,7 +108,7 @@ class MultiPhaseHydroStatic : public MeshTest } { amrex::ParmParse pp("geometry"); - amrex::Vector problo{{0.0, 0.0, 0.0}}; + amrex::Vector problo{{-1.0, -1.0, -1.0}}; amrex::Vector probhi{{1.0, 1.0, 1.0}}; pp.addarr("prob_lo", problo); @@ -120,7 +120,7 @@ class MultiPhaseHydroStatic : public MeshTest const amrex::Real m_rho2 = 1.0; const amrex::Real m_wlev = 0.5; const amrex::Real m_gz = -9.81; - const int m_nx = 3; + const int m_nx = 16; }; TEST_F(MultiPhaseHydroStatic, reference_density) From a78a35c57962cb0e97e713d5d071a3266cbc8f0e Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 7 Jun 2024 12:49:04 -0600 Subject: [PATCH 13/43] verbosity output tweaks --- amr-wind/overset/OversetOps.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 4d050dfb74..b6b0112679 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -155,7 +155,7 @@ void OversetOps::parameter_output() const // Print the details if (m_verbose > 0) { // Important parameters - amrex::Print() << "Overset Coupling Parameters: \n" + amrex::Print() << "\nOverset Coupling Parameters: \n" << "---- Coupled nodal projection : " << !m_disable_nodal_proj << std::endl << "---- Coupled MAC projection : " @@ -172,14 +172,16 @@ void OversetOps::parameter_output() const << std::endl << "---- Upwinding VOF margin : " << m_margin << std::endl; + if (m_verbose > 1) { + // Less important or less used parameters + amrex::Print() + << "---- Calc. conv. interval : " << m_calcconvint + << std::endl + << "---- Target field cutoff : " << m_target_cutoff + << std::endl; + } } - } - if (m_verbose > 1 && m_vof_exists) { - // Less important or less used parameters - amrex::Print() << "---- Calc. conv. interval : " << m_calcconvint - << std::endl - << "---- Target field cutoff : " << m_target_cutoff - << std::endl; + amrex::Print() << std::endl; } } From 8ebe11e900c30364079448499c4366040c3a9467 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 5 Jul 2024 09:51:09 -0600 Subject: [PATCH 14/43] first set of review edits --- .../equation_systems/icns/icns_advection.cpp | 4 +- .../equation_systems/vof/volume_fractions.H | 71 ++++++++++--------- amr-wind/overset/overset_ops_routines.H | 24 ++----- amr-wind/physics/multiphase/SloshingTank.cpp | 4 +- unit_tests/multiphase/test_vof_plic.cpp | 53 ++++++-------- 5 files changed, 66 insertions(+), 90 deletions(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 7a87e6afaf..931ec66e4d 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -58,9 +58,7 @@ MacProjOp::MacProjOp( amrex::ParmParse pp_ovst("Overset"); bool disable_ovst_mac = false; pp_ovst.query("disable_coupled_mac_proj", disable_ovst_mac); - if (m_has_overset && disable_ovst_mac) { - m_has_overset = false; - } + m_has_overset = !((m_has_overset && disable_ovst_mac)); } void MacProjOp::init_projector(const MacProjOp::FaceFabPtrVec& beta) noexcept diff --git a/amr-wind/equation_systems/vof/volume_fractions.H b/amr-wind/equation_systems/vof/volume_fractions.H index 120e6bf6a9..0cac2336e1 100644 --- a/amr-wind/equation_systems/vof/volume_fractions.H +++ b/amr-wind/equation_systems/vof/volume_fractions.H @@ -17,9 +17,9 @@ namespace amr_wind::multiphase { */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& volfrac, amrex::Real& mx, amrex::Real& my, @@ -65,12 +65,12 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal( } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal_neumann( - int i, - int j, - int k, - int ibdy, - int jbdy, - int kbdy, + const int i, + const int j, + const int k, + const int ibdy, + const int jbdy, + const int kbdy, amrex::Array4 const& volfrac, amrex::Real& mx, amrex::Real& my, @@ -79,12 +79,12 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal_neumann( amrex::Real mm1, mm2; // Do neumann condition via indices - int im1 = ibdy == -1 ? i : i - 1; - int jm1 = jbdy == -1 ? j : j - 1; - int km1 = kbdy == -1 ? k : k - 1; - int ip1 = ibdy == +1 ? i : i + 1; - int jp1 = jbdy == +1 ? j : j + 1; - int kp1 = kbdy == +1 ? k : k + 1; + const int im1 = ibdy == -1 ? i : i - 1; + const int jm1 = jbdy == -1 ? j : j - 1; + const int km1 = kbdy == -1 ? k : k - 1; + const int ip1 = ibdy == +1 ? i : i + 1; + const int jp1 = jbdy == +1 ? j : j + 1; + const int kp1 = kbdy == +1 ? k : k + 1; mm1 = volfrac(im1, jm1, km1) + volfrac(im1, jm1, kp1) + volfrac(im1, jp1, km1) + volfrac(im1, jp1, kp1) + @@ -124,9 +124,9 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal_neumann( } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mixed_youngs_central_normal( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& volfrac, amrex::Real& mx, amrex::Real& my, @@ -273,7 +273,10 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mixed_youngs_central_normal( * given that m1+m2+m3=1 (m1,m2,m3>0) and the volumetric fraction volF */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real volume_intercept( - amrex::Real b1, amrex::Real b2, amrex::Real b3, amrex::Real volF) noexcept + const amrex::Real b1, + const amrex::Real b2, + const amrex::Real b3, + const amrex::Real volF) noexcept { using namespace amrex; @@ -355,15 +358,15 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real volume_intercept( * (5) calculate volume (NOTE: it is assumed:s0=t0=0; ds0=dt0=1.) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real cut_volume( - amrex::Real m1, - amrex::Real m2, - amrex::Real m3, - amrex::Real alpha, - amrex::Real r0, - amrex::Real dr0) noexcept + const amrex::Real m1, + const amrex::Real m2, + const amrex::Real m3, + const amrex::Real alpha, + const amrex::Real r0, + const amrex::Real dr0) noexcept { - amrex::Real const_tiny = std::numeric_limits::epsilon(); + const amrex::Real const_tiny = std::numeric_limits::epsilon(); amrex::Real al; // move origin to x0 @@ -487,11 +490,11 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void fit_plane( } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool interface_band( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& volfrac, - int n_band = 1) noexcept + const int n_band = 1) noexcept { // n_band must be <= number of vof ghost cells (3) constexpr amrex::Real tiny = 1e-12; @@ -515,10 +518,10 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool interface_band( } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real levelset_to_vof( - int i, - int j, - int k, - amrex::Real eps, + const int i, + const int j, + const int k, + const amrex::Real eps, amrex::Array4 const& phi) noexcept { amrex::Real mx, my, mz; diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index bb47c32e42..cf924f1eeb 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -80,25 +80,13 @@ void populate_normal_vector( int ibdy = 0; int jbdy = 0; int kbdy = 0; - if (iblank(i, j, k) != iblank(i - 1, j, k)) { - ibdy = -1; - } - if (iblank(i, j, k) != iblank(i, j - 1, k)) { - jbdy = -1; - } - if (iblank(i, j, k) != iblank(i, j, k - 1)) { - kbdy = -1; - } + ibdy = (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : ibdy; + jbdy = (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : jbdy; + kbdy = (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : kbdy; // no cell should be isolated such that -1 and 1 are needed - if (iblank(i, j, k) != iblank(i + 1, j, k)) { - ibdy = +1; - } - if (iblank(i, j, k) != iblank(i, j + 1, k)) { - jbdy = +1; - } - if (iblank(i, j, k) != iblank(i, j, k + 1)) { - kbdy = +1; - } + ibdy = (iblank(i, j, k) != iblank(i + 1, j, k)) ? +1 : ibdy; + jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; + kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; // Calculate normal amrex::Real mx, my, mz, mmag; multiphase::youngs_fd_normal_neumann( diff --git a/amr-wind/physics/multiphase/SloshingTank.cpp b/amr-wind/physics/multiphase/SloshingTank.cpp index 1ab11d7ab6..167911812f 100644 --- a/amr-wind/physics/multiphase/SloshingTank.cpp +++ b/amr-wind/physics/multiphase/SloshingTank.cpp @@ -76,9 +76,9 @@ void SloshingTank::initialize_fields(int level, const amrex::Geometry& geom) -kappa * (std::pow(x - problo[0] - 0.5 * Lx, 2) + std::pow(y - problo[1] - 0.5 * Ly, 2))); // Integrated (top-down in z) phase heights to pressure node - amrex::Real ih_g = + const amrex::Real ih_g = amrex::max(0.0, amrex::min(probhi[2] - z0, probhi[2] - z)); - amrex::Real ih_l = + const amrex::Real ih_l = amrex::max(0.0, amrex::min(z0 - z, z0 - problo[2])); // Integrated rho at pressure node const amrex::Real irho = rho1 * ih_l + rho2 * ih_g; diff --git a/unit_tests/multiphase/test_vof_plic.cpp b/unit_tests/multiphase/test_vof_plic.cpp index 60f823c21f..257365c7ba 100644 --- a/unit_tests/multiphase/test_vof_plic.cpp +++ b/unit_tests/multiphase/test_vof_plic.cpp @@ -280,17 +280,18 @@ amrex::Real normal_vector_neumann_test_impl( // Use L1 norm, check against non-neumann implementation // Slope across overset boundary should be different + constexpr amrex::Real slp_tol = 1e-8; if (ibdy != 0) { // x slope should be different - error += std::abs(mx - mxn) > 1e-8 ? 0. : 1.0; + error += std::abs(mx - mxn) > slp_tol ? 0. : 1.0; } if (jbdy != 0) { // y slope should be different - error += std::abs(my - myn) > 1e-8 ? 0. : 1.0; + error += std::abs(my - myn) > slp_tol ? 0. : 1.0; } if (kbdy != 0) { // z slope should be different - error += std::abs(mz - mzn) > 1e-8 ? 0. : 1.0; + error += std::abs(mz - mzn) > slp_tol ? 0. : 1.0; } // Slope should otherwise be the same if (ibdy == 0 && jbdy == 0 && kbdy == 0) { @@ -326,24 +327,12 @@ amrex::Real normal_vector_neumann_test_impl( int ibdy = 0; int jbdy = 0; int kbdy = 0; - if (iblank(i, j, k) != iblank(i - 1, j, k)) { - ibdy = -1; - } - if (iblank(i, j, k) != iblank(i, j - 1, k)) { - jbdy = -1; - } - if (iblank(i, j, k) != iblank(i, j, k - 1)) { - kbdy = -1; - } - if (iblank(i, j, k) != iblank(i + 1, j, k)) { - ibdy = +1; - } - if (iblank(i, j, k) != iblank(i, j + 1, k)) { - jbdy = +1; - } - if (iblank(i, j, k) != iblank(i, j, k + 1)) { - kbdy = +1; - } + ibdy = (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : ibdy; + jbdy = (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : jbdy; + kbdy = (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : kbdy; + ibdy = (iblank(i, j, k) != iblank(i + 1, j, k)) ? +1 : ibdy; + jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; + kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; amrex::Real mxn, myn, mzn; amr_wind::multiphase::youngs_fd_normal_neumann( i, j, k, ibdy, jbdy, kbdy, vof_arr, mxn, myn, mzn); @@ -384,7 +373,6 @@ amrex::Real normal_vector_neumann_test_impl( amrex::Real fit_plane_test_impl(amr_wind::Field& vof, const int dir) { amrex::Real error_total = 0.0; - const int d = dir; for (int lev = 0; lev < vof.repo().num_active_levels(); ++lev) { @@ -397,9 +385,9 @@ amrex::Real fit_plane_test_impl(amr_wind::Field& vof, const int dir) amrex::Real error = 0.0; amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { - int ii = (d != 0 ? i : 0); - int jj = (d != 1 ? j : 0); - int kk = (d != 2 ? k : 0); + int ii = (dir != 0 ? i : 0); + int jj = (dir != 1 ? j : 0); + int kk = (dir != 2 ? k : 0); // Check multiphase cells if (ii + jj + kk == 3) { amrex::Real mx, my, mz, alpha; @@ -407,9 +395,9 @@ amrex::Real fit_plane_test_impl(amr_wind::Field& vof, const int dir) i, j, k, vof_arr, mx, my, mz, alpha); // Check slope - error += std::abs(mx - (d != 0 ? 0.5 : 0.0)); - error += std::abs(my - (d != 1 ? 0.5 : 0.0)); - error += std::abs(mz - (d != 2 ? 0.5 : 0.0)); + error += std::abs(mx - (dir != 0 ? 0.5 : 0.0)); + error += std::abs(my - (dir != 1 ? 0.5 : 0.0)); + error += std::abs(mz - (dir != 2 ? 0.5 : 0.0)); // Check intercept error += std::abs(alpha - 0.5); } @@ -425,7 +413,6 @@ amrex::Real fit_plane_test_impl_h( amr_wind::Field& vof, const amrex::Real vof_val, const int dir) { amrex::Real error_total = 0.0; - const int d = dir; const amrex::Real vv = vof_val; for (int lev = 0; lev < vof.repo().num_active_levels(); ++lev) { @@ -439,7 +426,7 @@ amrex::Real fit_plane_test_impl_h( amrex::Real error = 0.0; amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { - int ii = (d == 0 ? i : (d == 1 ? j : k)); + int ii = (dir == 0 ? i : (dir == 1 ? j : k)); // Check multiphase cells if (ii == 1) { amrex::Real mx, my, mz, alpha; @@ -447,9 +434,9 @@ amrex::Real fit_plane_test_impl_h( i, j, k, vof_arr, mx, my, mz, alpha); // Check slope - error += std::abs(mx - (d == 0 ? 1.0 : 0.0)); - error += std::abs(my - (d == 1 ? 1.0 : 0.0)); - error += std::abs(mz - (d == 2 ? 1.0 : 0.0)); + error += std::abs(mx - (dir == 0 ? 1.0 : 0.0)); + error += std::abs(my - (dir == 1 ? 1.0 : 0.0)); + error += std::abs(mz - (dir == 2 ? 1.0 : 0.0)); // Check intercept error += std::abs(alpha - vv); } From 9210f2921ad40a43455976c8b22705ecb3928322 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 5 Jul 2024 16:42:09 -0600 Subject: [PATCH 15/43] correcting logic --- amr-wind/equation_systems/icns/icns_advection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 931ec66e4d..8a715a33ea 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -58,7 +58,7 @@ MacProjOp::MacProjOp( amrex::ParmParse pp_ovst("Overset"); bool disable_ovst_mac = false; pp_ovst.query("disable_coupled_mac_proj", disable_ovst_mac); - m_has_overset = !((m_has_overset && disable_ovst_mac)); + m_has_overset = m_has_overset && !disable_ovst_mac; } void MacProjOp::init_projector(const MacProjOp::FaceFabPtrVec& beta) noexcept From 34a8b660a19891d4cbb8d274375e644b4b618945 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 5 Jul 2024 17:02:56 -0600 Subject: [PATCH 16/43] verbose names, remove comments --- amr-wind/overset/OversetOps.H | 14 ++++---- amr-wind/overset/OversetOps.cpp | 64 +++++++++++++++------------------ 2 files changed, 35 insertions(+), 43 deletions(-) diff --git a/amr-wind/overset/OversetOps.H b/amr-wind/overset/OversetOps.H index 70b0cbaa85..0658019d4d 100644 --- a/amr-wind/overset/OversetOps.H +++ b/amr-wind/overset/OversetOps.H @@ -31,22 +31,22 @@ private: bool m_perturb_p{false}; // This is the only option for now - const bool m_use_hs_pgrad{true}; + const bool m_use_hydrostatic_gradp{true}; // Coupling options bool m_disable_nodal_proj{false}; bool m_disable_mac_proj{false}; - bool m_replace_gp{false}; + bool m_replace_gradp_postsolve{false}; // Verbosity int m_verbose{0}; // Reinitialization parameters - int m_niterations{10}; - int m_calcconvint{1}; // calctolniter, cconv - amrex::Real m_tol = 1e-12; - amrex::Real m_rlscale = 1.5; - amrex::Real m_margin = 0.1; + int m_n_iterations{10}; + int m_calc_convg_interval{1}; // calctolniter, cconv + amrex::Real m_convg_tol = 1e-12; + amrex::Real m_relative_length_scale = 1.5; + amrex::Real m_upw_margin = 0.1; amrex::Real m_target_cutoff = 0.0; // proc_tgvof_tol // Tolerance for VOF-related bound checks diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index b6b0112679..700835afa6 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -14,48 +14,39 @@ void OversetOps::initialize(CFDSim& sim) m_sim_ptr = ∼ // Queries for reinitialization options amrex::ParmParse pp("Overset"); - pp.query("reinit_iterations", m_niterations); - pp.query("reinit_convg_interval", m_calcconvint); - pp.query("reinit_convg_tolerance", m_tol); - pp.query("reinit_rlscale", m_niterations); - pp.query("reinit_upw_margin", m_margin); + pp.query("reinit_iterations", m_n_iterations); + pp.query("reinit_convg_interval", m_calc_convg_interval); + pp.query("reinit_convg_tolerance", m_convg_tol); + pp.query("reinit_rlscale", m_relative_length_scale); + pp.query("reinit_upw_margin", m_upw_margin); pp.query("reinit_target_cutoff", m_target_cutoff); // Queries for coupling options - pp.query("replace_gradp_postsolve", m_replace_gp); + pp.query("replace_gradp_postsolve", m_replace_gradp_postsolve); // OversetOps does not control these coupling options, merely reports them pp.query("disable_coupled_nodal_proj", m_disable_nodal_proj); pp.query("disable_coupled_mac_proj", m_disable_mac_proj); - // Verbosity pp.query("verbose", m_verbose); - // Check for perturbational pressure - // (will be removed soon) amrex::ParmParse pp_icns("ICNS"); pp_icns.query("use_perturb_pressure", m_perturb_p); - // Check for vof to determine if multiphase sim m_vof_exists = (*m_sim_ptr).repo().field_exists("vof"); - - // Set up pointer to MultiPhase physics if (m_vof_exists) { m_mphase = &(*m_sim_ptr).physics_manager().get(); } - - // Set up field to store pressure gradient - if (m_replace_gp) { + if (m_replace_gradp_postsolve) { m_gp_copy = &(*m_sim_ptr).repo().declare_field("gp_copy", 3); } - // Output parameters if verbose parameter_output(); } void OversetOps::pre_advance_work() { // Pressure gradient not updated for current multiphase approach - if (!(m_vof_exists && m_use_hs_pgrad)) { + if (!(m_vof_exists && m_use_hydrostatic_gradp)) { // Update pressure gradient using updated overset pressure field update_gradp(); } @@ -63,14 +54,14 @@ void OversetOps::pre_advance_work() if (m_vof_exists) { // Reinitialize fields sharpen_nalu_data(); - if (m_use_hs_pgrad) { + if (m_use_hydrostatic_gradp) { // Use hydrostatic pressure gradient set_hydrostatic_gradp(); } } // If pressure gradient will be replaced, store current pressure gradient - if (m_replace_gp) { + if (m_replace_gradp_postsolve) { auto& gp = (*m_sim_ptr).repo().get_field("gp"); for (int lev = 0; lev < (*m_sim_ptr).repo().num_active_levels(); ++lev) { @@ -141,7 +132,7 @@ void OversetOps::update_gradp() void OversetOps::post_advance_work() { // Replace and reapply pressure gradient if requested - if (m_replace_gp) { + if (m_replace_gradp_postsolve) { replace_masked_gradp(); } } @@ -160,22 +151,22 @@ void OversetOps::parameter_output() const << !m_disable_nodal_proj << std::endl << "---- Coupled MAC projection : " << !m_disable_mac_proj << std::endl - << "---- Replace overset pres grad: " << m_replace_gp - << std::endl; + << "---- Replace overset pres grad: " + << m_replace_gradp_postsolve << std::endl; if (m_vof_exists) { amrex::Print() << "Overset Reinitialization Parameters:\n" - << "---- Maximum iterations : " << m_niterations - << std::endl - << "---- Convergence tolerance: " << m_tol + << "---- Maximum iterations : " << m_n_iterations << std::endl - << "---- Relative length scale: " << m_rlscale + << "---- Convergence tolerance: " << m_convg_tol << std::endl - << "---- Upwinding VOF margin : " << m_margin + << "---- Relative length scale: " + << m_relative_length_scale << std::endl + << "---- Upwinding VOF margin : " << m_upw_margin << std::endl; if (m_verbose > 1) { // Less important or less used parameters amrex::Print() - << "---- Calc. conv. interval : " << m_calcconvint + << "---- Calc. conv. interval : " << m_calc_convg_interval << std::endl << "---- Target field cutoff : " << m_target_cutoff << std::endl; @@ -215,7 +206,8 @@ void OversetOps::sharpen_nalu_data() for (int lev = 0; lev < nlevels; ++lev) { // Thickness used here is user parameter, whatever works best auto dx = (geom[lev]).CellSizeArray(); - const amrex::Real i_th = m_rlscale * std::cbrt(dx[0] * dx[1] * dx[2]); + const amrex::Real i_th = + m_relative_length_scale * std::cbrt(dx[0] * dx[1] * dx[2]); // Populate approximate signed distance function overset_ops::populate_psi(levelset(lev), vof(lev), i_th, m_asdf_tiny); @@ -246,14 +238,14 @@ void OversetOps::sharpen_nalu_data() } // Pseudo-time loop - amrex::Real err = 100.0 * m_tol; + amrex::Real err = 100.0 * m_convg_tol; int n = 0; - while (n < m_niterations && err > m_tol) { + while (n < m_n_iterations && err > m_convg_tol) { // Increment step counter ++n; // Determine if convergence error is calculated this step - bool cconv = n % m_calcconvint == 0; + bool cconv = n % m_calc_convg_interval == 0; // Zero error if being calculated this step err = cconv ? 0.0 : err; @@ -270,8 +262,8 @@ void OversetOps::sharpen_nalu_data() // Sharpening fluxes for vof, density, and momentum overset_ops::populate_sharpen_fluxes( (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), vof(lev), - (*target_vof)(lev), (*normal_vec)(lev), velocity(lev), m_margin, - m_mphase->rho1(), m_mphase->rho2()); + (*target_vof)(lev), (*normal_vec)(lev), velocity(lev), + m_upw_margin, m_mphase->rho1(), m_mphase->rho2()); // Process fluxes overset_ops::process_fluxes( @@ -302,7 +294,7 @@ void OversetOps::sharpen_nalu_data() // Convergence tolerance determines what size of fluxes matter const amrex::Real ptfac_lev = overset_ops::calculate_pseudo_dt_flux( (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), vof(lev), - m_tol); + m_convg_tol); ptfac = amrex::min(ptfac, ptfac_lev); } amrex::ParallelDescriptor::ReduceRealMin(ptfac); @@ -331,7 +323,7 @@ void OversetOps::sharpen_nalu_data() if (m_verbose > 0) { amrex::Print() << "OversetOps: sharpen step " << n << " conv. err " - << err << " tol " << m_tol << std::endl; + << err << " tol " << m_convg_tol << std::endl; } } From d4890270b50d19efb9448c208fe1f882dbd643df Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 5 Jul 2024 17:26:23 -0600 Subject: [PATCH 17/43] const --- amr-wind/overset/OversetOps.cpp | 35 +++++++++----------- amr-wind/overset/overset_ops_K.H | 16 ++++----- amr-wind/overset/overset_ops_routines.H | 38 +++++++++++----------- amr-wind/physics/multiphase/MultiPhase.H | 2 +- amr-wind/physics/multiphase/MultiPhase.cpp | 3 +- 5 files changed, 46 insertions(+), 48 deletions(-) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 700835afa6..082b68343f 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -178,12 +178,12 @@ void OversetOps::parameter_output() const void OversetOps::sharpen_nalu_data() { - auto& repo = (*m_sim_ptr).repo(); - auto nlevels = repo.num_active_levels(); - auto geom = (*m_sim_ptr).mesh().Geom(); + const auto& repo = (*m_sim_ptr).repo(); + const auto nlevels = repo.num_active_levels(); + const auto geom = (*m_sim_ptr).mesh().Geom(); // Get blanking for cells - auto& iblank_cell = repo.get_int_field("iblank_cell"); + const auto& iblank_cell = repo.get_int_field("iblank_cell"); // Get fields that will be modified auto& vof = repo.get_field("vof"); @@ -241,13 +241,10 @@ void OversetOps::sharpen_nalu_data() amrex::Real err = 100.0 * m_convg_tol; int n = 0; while (n < m_n_iterations && err > m_convg_tol) { - // Increment step counter ++n; - - // Determine if convergence error is calculated this step - bool cconv = n % m_calc_convg_interval == 0; + bool calc_convg = n % m_calc_convg_interval == 0; // Zero error if being calculated this step - err = cconv ? 0.0 : err; + err = calc_convg ? 0.0 : err; // Maximum possible value of pseudo time factor (dtau) amrex::Real ptfac = 1.0; @@ -271,7 +268,7 @@ void OversetOps::sharpen_nalu_data() iblank_cell(lev)); // Measure convergence to determine if loop can stop - if (cconv) { + if (calc_convg) { // Update error at specified interval of steps const amrex::Real err_lev = overset_ops::measure_convergence( (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev)); @@ -317,7 +314,7 @@ void OversetOps::sharpen_nalu_data() m_mphase->set_density_via_vof(); // Ensure that err is same across processors - if (cconv) { + if (calc_convg) { amrex::ParallelDescriptor::ReduceRealMax(err); } @@ -337,12 +334,12 @@ void OversetOps::sharpen_nalu_data() void OversetOps::set_hydrostatic_gradp() { - auto& repo = (*m_sim_ptr).repo(); - auto nlevels = repo.num_active_levels(); - auto geom = (*m_sim_ptr).mesh().Geom(); + const auto& repo = (*m_sim_ptr).repo(); + const auto nlevels = repo.num_active_levels(); + const auto geom = (*m_sim_ptr).mesh().Geom(); // Get blanking for cells - auto& iblank_cell = repo.get_int_field("iblank_cell"); + const auto& iblank_cell = repo.get_int_field("iblank_cell"); // Get fields that will be modified or used Field* rho0{nullptr}; @@ -357,7 +354,7 @@ void OversetOps::set_hydrostatic_gradp() // Replace initial gp with best guess (hydrostatic) for (int lev = 0; lev < nlevels; ++lev) { - overset_ops::replace_gradp_hs( + overset_ops::replace_gradp_hydrostatic( gp(lev), rho(lev), (*rho0)(lev), iblank_cell(lev), m_mphase->gravity()[2], m_perturb_p); } @@ -365,14 +362,14 @@ void OversetOps::set_hydrostatic_gradp() void OversetOps::replace_masked_gradp() { - auto& repo = (*m_sim_ptr).repo(); - auto nlevels = repo.num_active_levels(); + const auto& repo = (*m_sim_ptr).repo(); + const auto nlevels = repo.num_active_levels(); // Get timestep const amrex::Real dt = (*m_sim_ptr).time().deltaT(); // Get blanking for cells - auto& iblank_cell = repo.get_int_field("iblank_cell"); + const auto& iblank_cell = repo.get_int_field("iblank_cell"); // Get fields that will be modified or used auto& vel = repo.get_field("velocity"); diff --git a/amr-wind/overset/overset_ops_K.H b/amr-wind/overset/overset_ops_K.H index 89b0780c7b..5ee5ee0329 100644 --- a/amr-wind/overset/overset_ops_K.H +++ b/amr-wind/overset/overset_ops_K.H @@ -14,10 +14,10 @@ asdf(const amrex::Real a_vof, const amrex::Real i_th, const amrex::Real tiny) } amrex::Real AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE alpha_flux( - int i, - int j, - int k, - int dir, + const int i, + const int j, + const int k, + const int dir, const amrex::Real margin, amrex::Array4 const& vof, amrex::Array4 const& tg_vof, @@ -69,10 +69,10 @@ amrex::Real AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE alpha_flux( } void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE velocity_face( - int i, - int j, - int k, - int dir, + const int i, + const int j, + const int k, + const int dir, amrex::Array4 const& vof, amrex::Array4 const& velocity, amrex::Real& uface, diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index cf924f1eeb..3f567ff555 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -9,7 +9,7 @@ namespace amr_wind::overset_ops { // Populate approximate signed distance function using vof field void populate_psi( amrex::MultiFab& mf_psi, - amrex::MultiFab& mf_vof, + const amrex::MultiFab& mf_vof, const amrex::Real i_th, const amrex::Real asdf_tiny) { @@ -43,8 +43,8 @@ void process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) // Combine overset target vof field with current non-overset vof field void harmonize_vof( amrex::MultiFab& mf_vof_target, - amrex::MultiFab& mf_vof_original, - amrex::iMultiFab& mf_iblank) + const amrex::MultiFab& mf_vof_original, + const amrex::iMultiFab& mf_iblank) { for (amrex::MFIter mfi(mf_vof_target); mfi.isValid(); ++mfi) { const auto& vbx = mfi.validbox(); @@ -65,8 +65,8 @@ void harmonize_vof( // Populate normal vector with special treatment of overset boundary void populate_normal_vector( amrex::MultiFab& mf_normvec, - amrex::MultiFab& mf_vof, - amrex::iMultiFab& mf_iblank) + const amrex::MultiFab& mf_vof, + const amrex::iMultiFab& mf_iblank) { for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { const auto& gbxm1 = grow(mfi.growntilebox(), -1); @@ -107,10 +107,10 @@ void populate_sharpen_fluxes( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, - amrex::MultiFab& mf_vof, - amrex::MultiFab& mf_target_vof, - amrex::MultiFab& mf_norm, - amrex::MultiFab& mf_velocity, + const amrex::MultiFab& mf_vof, + const amrex::MultiFab& mf_target_vof, + const amrex::MultiFab& mf_norm, + const amrex::MultiFab& mf_velocity, const amrex::Real margin, const amrex::Real rho1, const amrex::Real rho2) @@ -179,7 +179,7 @@ void process_fluxes( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, - amrex::iMultiFab& mf_iblank) + const amrex::iMultiFab& mf_iblank) { int ncompx = mf_fx.nComp(); int ncompy = mf_fy.nComp(); @@ -389,7 +389,7 @@ amrex::Real measure_convergence( } // Set levelset field to another quantity to view in plotfile for debugging -void equate_field(amrex::MultiFab& mf_dest, amrex::MultiFab& mf_src) +void equate_field(amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src) { for (amrex::MFIter mfi(mf_dest); mfi.isValid(); ++mfi) { const auto& vbx = mfi.validbox(); @@ -406,11 +406,11 @@ void equate_field(amrex::MultiFab& mf_dest, amrex::MultiFab& mf_src) } // Replace pressure gradient with hydrostatic field in overset regions -void replace_gradp_hs( +void replace_gradp_hydrostatic( amrex::MultiFab& mf_gp, - amrex::MultiFab& mf_density, - amrex::MultiFab& mf_refdens, - amrex::iMultiFab& mf_iblank, + const amrex::MultiFab& mf_density, + const amrex::MultiFab& mf_refdens, + const amrex::iMultiFab& mf_iblank, const amrex::Real grav_z, const bool is_pptb) { @@ -439,8 +439,8 @@ void replace_gradp_hs( // Swap pressure gradient values in overset region void replace_gradp( amrex::MultiFab& mf_gp, - amrex::MultiFab& mf_gp0, - amrex::iMultiFab& mf_iblank) + const amrex::MultiFab& mf_gp0, + const amrex::iMultiFab& mf_iblank) { for (amrex::MFIter mfi(mf_gp); mfi.isValid(); ++mfi) { const auto& gbx = mfi.growntilebox(); @@ -462,8 +462,8 @@ void replace_gradp( // Apply pressure gradient to velocity field void apply_pressure_gradient( amrex::MultiFab& mf_vel, - amrex::MultiFab& mf_density, - amrex::MultiFab& mf_gp, + const amrex::MultiFab& mf_density, + const amrex::MultiFab& mf_gp, const amrex::Real scaling_factor) { for (amrex::MFIter mfi(mf_gp); mfi.isValid(); ++mfi) { diff --git a/amr-wind/physics/multiphase/MultiPhase.H b/amr-wind/physics/multiphase/MultiPhase.H index acb7fbb83f..8b43eef505 100644 --- a/amr-wind/physics/multiphase/MultiPhase.H +++ b/amr-wind/physics/multiphase/MultiPhase.H @@ -53,7 +53,7 @@ public: void levelset2vof(); - void levelset2vof(IntField& iblank_cell, ScratchField& vof_scr); + void levelset2vof(const IntField& iblank_cell, ScratchField& vof_scr); void favre_filtering(); diff --git a/amr-wind/physics/multiphase/MultiPhase.cpp b/amr-wind/physics/multiphase/MultiPhase.cpp index 57bb80e92d..145a3775b8 100644 --- a/amr-wind/physics/multiphase/MultiPhase.cpp +++ b/amr-wind/physics/multiphase/MultiPhase.cpp @@ -521,7 +521,8 @@ void MultiPhase::levelset2vof() } // Do levelset2vof with iblank neumann and into supplied scratch field -void MultiPhase::levelset2vof(IntField& iblank_cell, ScratchField& vof_scr) +void MultiPhase::levelset2vof( + const IntField& iblank_cell, ScratchField& vof_scr) { const int nlevels = m_sim.repo().num_active_levels(); (*m_levelset).fillpatch(m_sim.time().current_time()); From 429d2681f520a4cd7173cf376ddea75611744144 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 06:58:07 -0600 Subject: [PATCH 18/43] remove unnecessary intermediate variables --- unit_tests/multiphase/test_vof_plic.cpp | 43 +++++++++++-------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/unit_tests/multiphase/test_vof_plic.cpp b/unit_tests/multiphase/test_vof_plic.cpp index 257365c7ba..03c640a695 100644 --- a/unit_tests/multiphase/test_vof_plic.cpp +++ b/unit_tests/multiphase/test_vof_plic.cpp @@ -39,11 +39,10 @@ void initialize_volume_fractions( { // grow the box by 1 so that x,y,z go out of bounds and min(max()) corrects // it and it fills the ghosts with wall values - const int d = dir; amrex::ParallelFor(grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { - int ii = (d != 0 ? i : 0); - int jj = (d != 1 ? j : 0); - int kk = (d != 2 ? k : 0); + int ii = (dir != 0 ? i : 0); + int jj = (dir != 1 ? j : 0); + int kk = (dir != 2 ? k : 0); if (ii + jj + kk > 3) { vof_arr(i, j, k) = 0.0; } @@ -73,10 +72,9 @@ void initialize_volume_fractions_horizontal( { // grow the box by 1 so that x,y,z go out of bounds and min(max()) corrects // it and it fills the ghosts with wall values - const int d = dir; const amrex::Real vv = vof_val; amrex::ParallelFor(grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { - int ii = (d == 0 ? i : (d == 1 ? j : k)); + int ii = (dir == 0 ? i : (dir == 1 ? j : k)); if (ii > 1) { vof_arr(i, j, k) = 0.0; } @@ -120,11 +118,10 @@ void init_vof(amr_wind::Field& vof) void initialize_iblank_distribution( const int dir, const amrex::Box& bx, const amrex::Array4& iblk_arr) { - const int d = dir; amrex::ParallelFor(grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { - int ii = (d != 0 ? i : 0); - int jj = (d != 1 ? j : 0); - int kk = (d != 2 ? k : 0); + int ii = (dir != 0 ? i : 0); + int jj = (dir != 1 ? j : 0); + int kk = (dir != 2 ? k : 0); if (ii + jj + kk > 4 || ii + jj + kk < 2) { iblk_arr(i, j, k) = -1; } else { @@ -149,16 +146,15 @@ void initialize_volume_fractions_iblank( const amrex::Array4& iblk_arr) { // Does a horizontal interface that is different outside of iblank region - const int d = dir; amrex::ParallelFor(grow(bx, 1), [=] AMREX_GPU_DEVICE(int i, int j, int k) { // Turns on index tangent to interface - int iit = (d != 0 ? i : 1); - int jjt = (d != 1 ? j : 1); - int kkt = (d != 2 ? k : 1); + int iit = (dir != 0 ? i : 1); + int jjt = (dir != 1 ? j : 1); + int kkt = (dir != 2 ? k : 1); // Turns on index normal to interface - int iin = (d == 0 ? i : 0); - int jjn = (d == 1 ? j : 0); - int kkn = (d == 2 ? k : 0); + int iin = (dir == 0 ? i : 0); + int jjn = (dir == 1 ? j : 0); + int kkn = (dir == 2 ? k : 0); // Ordinary vof distribution if (iin + jjn + kkn > 2) { vof_arr(i, j, k) = 0.0; @@ -199,7 +195,6 @@ void init_vof_iblank( amrex::Real normal_vector_test_impl(amr_wind::Field& vof, const int dir) { amrex::Real error_total = 0.0; - const int d = dir; for (int lev = 0; lev < vof.repo().num_active_levels(); ++lev) { @@ -216,15 +211,15 @@ amrex::Real normal_vector_test_impl(amr_wind::Field& vof, const int dir) amr_wind::multiphase::mixed_youngs_central_normal( i, j, k, vof_arr, mx, my, mz); - int ii = (d != 0 ? i : 0); - int jj = (d != 1 ? j : 0); - int kk = (d != 2 ? k : 0); + int ii = (dir != 0 ? i : 0); + int jj = (dir != 1 ? j : 0); + int kk = (dir != 2 ? k : 0); // Use L1 norm, check cells where slope is known if (ii + jj + kk == 3) { - error += std::abs(mx - (d != 0 ? 0.5 : 0.0)); - error += std::abs(my - (d != 1 ? 0.5 : 0.0)); - error += std::abs(mz - (d != 2 ? 0.5 : 0.0)); + error += std::abs(mx - (dir != 0 ? 0.5 : 0.0)); + error += std::abs(my - (dir != 1 ? 0.5 : 0.0)); + error += std::abs(mz - (dir != 2 ? 0.5 : 0.0)); } }); From 21c865f52cbd43faa61399cfc12a3e102df631af Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 07:10:38 -0600 Subject: [PATCH 19/43] better ibdy code --- amr-wind/overset/overset_ops_routines.H | 9 ++--- amr-wind/physics/multiphase/MultiPhase.cpp | 30 +++++----------- unit_tests/multiphase/test_vof_plic.cpp | 42 ++++++++-------------- 3 files changed, 27 insertions(+), 54 deletions(-) diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index 3f567ff555..ec2318159d 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -77,12 +77,9 @@ void populate_normal_vector( amrex::ParallelFor( gbxm1, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Neumann condition across nalu bdy - int ibdy = 0; - int jbdy = 0; - int kbdy = 0; - ibdy = (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : ibdy; - jbdy = (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : jbdy; - kbdy = (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : kbdy; + int ibdy = (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : 0; + int jbdy = (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : 0; + int kbdy = (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : 0; // no cell should be isolated such that -1 and 1 are needed ibdy = (iblank(i, j, k) != iblank(i + 1, j, k)) ? +1 : ibdy; jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; diff --git a/amr-wind/physics/multiphase/MultiPhase.cpp b/amr-wind/physics/multiphase/MultiPhase.cpp index 145a3775b8..b8ac5faff8 100644 --- a/amr-wind/physics/multiphase/MultiPhase.cpp +++ b/amr-wind/physics/multiphase/MultiPhase.cpp @@ -543,29 +543,17 @@ void MultiPhase::levelset2vof( amrex::ParallelFor( vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Neumann of levelset across iblank boundaries - int ibdy = 0; - int jbdy = 0; - int kbdy = 0; - if (iblank(i, j, k) != iblank(i - 1, j, k)) { - ibdy = -1; - } - if (iblank(i, j, k) != iblank(i, j - 1, k)) { - jbdy = -1; - } - if (iblank(i, j, k) != iblank(i, j, k - 1)) { - kbdy = -1; - } + int ibdy = + (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : 0; + int jbdy = + (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : 0; + int kbdy = + (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : 0; // no cell should be isolated such that -1 and 1 are // needed - if (iblank(i, j, k) != iblank(i + 1, j, k)) { - ibdy = +1; - } - if (iblank(i, j, k) != iblank(i, j + 1, k)) { - jbdy = +1; - } - if (iblank(i, j, k) != iblank(i, j, k + 1)) { - kbdy = +1; - } + ibdy = (iblank(i, j, k) != iblank(i + 1, j, k)) ? +1 : ibdy; + jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; + kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; amrex::Real mx, my, mz; multiphase::youngs_fd_normal_neumann( i, j, k, ibdy, jbdy, kbdy, phi, mx, my, mz); diff --git a/unit_tests/multiphase/test_vof_plic.cpp b/unit_tests/multiphase/test_vof_plic.cpp index 03c640a695..f15d2f718a 100644 --- a/unit_tests/multiphase/test_vof_plic.cpp +++ b/unit_tests/multiphase/test_vof_plic.cpp @@ -245,27 +245,15 @@ amrex::Real normal_vector_neumann_test_impl( amrex::Real error = 0.0; amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { - int ibdy = 0; - int jbdy = 0; - int kbdy = 0; - if (iblank(i, j, k) != iblank(i - 1, j, k)) { - ibdy = -1; - } - if (iblank(i, j, k) != iblank(i, j - 1, k)) { - jbdy = -1; - } - if (iblank(i, j, k) != iblank(i, j, k - 1)) { - kbdy = -1; - } - if (iblank(i, j, k) != iblank(i + 1, j, k)) { - ibdy = +1; - } - if (iblank(i, j, k) != iblank(i, j + 1, k)) { - jbdy = +1; - } - if (iblank(i, j, k) != iblank(i, j, k + 1)) { - kbdy = +1; - } + int ibdy = + (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : 0; + int jbdy = + (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : 0; + int kbdy = + (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : 0; + ibdy = (iblank(i, j, k) != iblank(i + 1, j, k)) ? +1 : ibdy; + jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; + kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; amrex::Real mxn, myn, mzn; amr_wind::multiphase::youngs_fd_normal_neumann( i, j, k, ibdy, jbdy, kbdy, vof_arr, mxn, myn, mzn); @@ -319,12 +307,12 @@ amrex::Real normal_vector_neumann_test_impl( amrex::Real error = 0.0; amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { - int ibdy = 0; - int jbdy = 0; - int kbdy = 0; - ibdy = (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : ibdy; - jbdy = (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : jbdy; - kbdy = (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : kbdy; + int ibdy = + (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : 0; + int jbdy = + (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : 0; + int kbdy = + (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : 0; ibdy = (iblank(i, j, k) != iblank(i + 1, j, k)) ? +1 : ibdy; jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; From 84182bd392296ff62e64146552b075b33fcaf439 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 07:16:53 -0600 Subject: [PATCH 20/43] fd = finite_difference --- amr-wind/equation_systems/vof/volume_fractions.H | 10 ++++++---- amr-wind/overset/overset_ops_routines.H | 2 +- amr-wind/physics/multiphase/MultiPhase.cpp | 5 +++-- unit_tests/multiphase/test_vof_plic.cpp | 12 +++++++----- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/amr-wind/equation_systems/vof/volume_fractions.H b/amr-wind/equation_systems/vof/volume_fractions.H index 0cac2336e1..0815ba83b0 100644 --- a/amr-wind/equation_systems/vof/volume_fractions.H +++ b/amr-wind/equation_systems/vof/volume_fractions.H @@ -16,7 +16,7 @@ namespace amr_wind::multiphase { * to zero. */ -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal( +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_finite_difference_normal( const int i, const int j, const int k, @@ -64,7 +64,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal( mz = mm1 - mm2; } -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_fd_normal_neumann( +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void +youngs_finite_difference_normal_neumann( const int i, const int j, const int k, @@ -239,7 +240,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mixed_youngs_central_normal( } // Youngs-CIAM scheme */ - youngs_fd_normal(i, j, k, volfrac, m(3, 0), m(3, 1), m(3, 2)); + youngs_finite_difference_normal( + i, j, k, volfrac, m(3, 0), m(3, 1), m(3, 2)); // normalize the set (mx,my,mz): |mx|+|my|+|mz| = 1 constexpr amrex::Real tiny = 1e-20; t0 = std::abs(m(3, 0)) + std::abs(m(3, 1)) + std::abs(m(3, 2)) + tiny; @@ -525,7 +527,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real levelset_to_vof( amrex::Array4 const& phi) noexcept { amrex::Real mx, my, mz; - youngs_fd_normal(i, j, k, phi, mx, my, mz); + youngs_finite_difference_normal(i, j, k, phi, mx, my, mz); mx = std::abs(mx / 32.); my = std::abs(my / 32.); mz = std::abs(mz / 32.); diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index ec2318159d..be71c73423 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -86,7 +86,7 @@ void populate_normal_vector( kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; // Calculate normal amrex::Real mx, my, mz, mmag; - multiphase::youngs_fd_normal_neumann( + multiphase::youngs_finite_difference_normal_neumann( i, j, k, ibdy, jbdy, kbdy, vof, mx, my, mz); // Normalize normal mmag = std::sqrt(mx * mx + my * my + mz * mz + 1e-20); diff --git a/amr-wind/physics/multiphase/MultiPhase.cpp b/amr-wind/physics/multiphase/MultiPhase.cpp index b8ac5faff8..de90892e14 100644 --- a/amr-wind/physics/multiphase/MultiPhase.cpp +++ b/amr-wind/physics/multiphase/MultiPhase.cpp @@ -488,7 +488,8 @@ void MultiPhase::levelset2vof() amrex::ParallelFor( vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::Real mx, my, mz; - multiphase::youngs_fd_normal(i, j, k, phi, mx, my, mz); + multiphase::youngs_finite_difference_normal( + i, j, k, phi, mx, my, mz); mx = std::abs(mx / 32.); my = std::abs(my / 32.); mz = std::abs(mz / 32.); @@ -555,7 +556,7 @@ void MultiPhase::levelset2vof( jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; amrex::Real mx, my, mz; - multiphase::youngs_fd_normal_neumann( + multiphase::youngs_finite_difference_normal_neumann( i, j, k, ibdy, jbdy, kbdy, phi, mx, my, mz); mx = std::abs(mx / 32.); my = std::abs(my / 32.); diff --git a/unit_tests/multiphase/test_vof_plic.cpp b/unit_tests/multiphase/test_vof_plic.cpp index f15d2f718a..8d12e54632 100644 --- a/unit_tests/multiphase/test_vof_plic.cpp +++ b/unit_tests/multiphase/test_vof_plic.cpp @@ -255,10 +255,11 @@ amrex::Real normal_vector_neumann_test_impl( jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; amrex::Real mxn, myn, mzn; - amr_wind::multiphase::youngs_fd_normal_neumann( - i, j, k, ibdy, jbdy, kbdy, vof_arr, mxn, myn, mzn); + amr_wind::multiphase:: + youngs_finite_difference_normal_neumann( + i, j, k, ibdy, jbdy, kbdy, vof_arr, mxn, myn, mzn); amrex::Real mx, my, mz; - amr_wind::multiphase::youngs_fd_normal( + amr_wind::multiphase::youngs_finite_difference_normal( i, j, k, vof_arr, mx, my, mz); // Use L1 norm, check against non-neumann implementation @@ -317,8 +318,9 @@ amrex::Real normal_vector_neumann_test_impl( jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; amrex::Real mxn, myn, mzn; - amr_wind::multiphase::youngs_fd_normal_neumann( - i, j, k, ibdy, jbdy, kbdy, vof_arr, mxn, myn, mzn); + amr_wind::multiphase:: + youngs_finite_difference_normal_neumann( + i, j, k, ibdy, jbdy, kbdy, vof_arr, mxn, myn, mzn); // Use L1 norm, check for 0 if (iblank(i, j, k) == 1) { From 7cc5799ecc32a8c3b3dc4921b68ec37e7d1b554d Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 07:20:42 -0600 Subject: [PATCH 21/43] fix weird alpha logic in levelset2vof --- amr-wind/physics/multiphase/MultiPhase.cpp | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/amr-wind/physics/multiphase/MultiPhase.cpp b/amr-wind/physics/multiphase/MultiPhase.cpp index de90892e14..efc390ddfc 100644 --- a/amr-wind/physics/multiphase/MultiPhase.cpp +++ b/amr-wind/physics/multiphase/MultiPhase.cpp @@ -499,13 +499,9 @@ void MultiPhase::levelset2vof() mz = mz / normL1; // Make sure that alpha is negative far away from the // interface - amrex::Real alpha; - if (phi(i, j, k) < -eps) { - alpha = -1.0; - } else { - alpha = phi(i, j, k) / normL1; - alpha = alpha + 0.5; - } + const amrex::Real alpha = (phi(i, j, k) < -eps) + ? -1.0 + : phi(i, j, k) / normL1 + 0.5; if (alpha >= 1.0) { volfrac(i, j, k) = 1.0; } else if (alpha <= 0.0) { @@ -567,13 +563,9 @@ void MultiPhase::levelset2vof( mz = mz / normL1; // Make sure that alpha is negative far away from the // interface - amrex::Real alpha; - if (phi(i, j, k) < -eps) { - alpha = -1.0; - } else { - alpha = phi(i, j, k) / normL1; - alpha = alpha + 0.5; - } + const amrex::Real alpha = (phi(i, j, k) < -eps) + ? -1.0 + : phi(i, j, k) / normL1 + 0.5; if (alpha >= 1.0) { volfrac(i, j, k) = 1.0; } else if (alpha <= 0.0) { From c8fe66d799c92036bbac8c6a8be53d918d6a2c83 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 09:02:37 -0600 Subject: [PATCH 22/43] mm1, mm2 declarations --- .../equation_systems/vof/volume_fractions.H | 59 +++++++++---------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/amr-wind/equation_systems/vof/volume_fractions.H b/amr-wind/equation_systems/vof/volume_fractions.H index 0815ba83b0..26ab361ba9 100644 --- a/amr-wind/equation_systems/vof/volume_fractions.H +++ b/amr-wind/equation_systems/vof/volume_fractions.H @@ -25,18 +25,18 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void youngs_finite_difference_normal( amrex::Real& my, amrex::Real& mz) noexcept { - amrex::Real mm1, mm2; - - mm1 = volfrac(i - 1, j - 1, k - 1) + volfrac(i - 1, j - 1, k + 1) + - volfrac(i - 1, j + 1, k - 1) + volfrac(i - 1, j + 1, k + 1) + - 2.0 * (volfrac(i - 1, j - 1, k) + volfrac(i - 1, j + 1, k) + - volfrac(i - 1, j, k - 1) + volfrac(i - 1, j, k + 1)) + - 4.0 * volfrac(i - 1, j, k); - mm2 = volfrac(i + 1, j - 1, k - 1) + volfrac(i + 1, j - 1, k + 1) + - volfrac(i + 1, j + 1, k - 1) + volfrac(i + 1, j + 1, k + 1) + - 2.0 * (volfrac(i + 1, j - 1, k) + volfrac(i + 1, j + 1, k) + - volfrac(i + 1, j, k - 1) + volfrac(i + 1, j, k + 1)) + - 4.0 * volfrac(i + 1, j, k); + amrex::Real mm1 = + volfrac(i - 1, j - 1, k - 1) + volfrac(i - 1, j - 1, k + 1) + + volfrac(i - 1, j + 1, k - 1) + volfrac(i - 1, j + 1, k + 1) + + 2.0 * (volfrac(i - 1, j - 1, k) + volfrac(i - 1, j + 1, k) + + volfrac(i - 1, j, k - 1) + volfrac(i - 1, j, k + 1)) + + 4.0 * volfrac(i - 1, j, k); + amrex::Real mm2 = + volfrac(i + 1, j - 1, k - 1) + volfrac(i + 1, j - 1, k + 1) + + volfrac(i + 1, j + 1, k - 1) + volfrac(i + 1, j + 1, k + 1) + + 2.0 * (volfrac(i + 1, j - 1, k) + volfrac(i + 1, j + 1, k) + + volfrac(i + 1, j, k - 1) + volfrac(i + 1, j, k + 1)) + + 4.0 * volfrac(i + 1, j, k); mx = mm1 - mm2; mm1 = volfrac(i - 1, j - 1, k - 1) + volfrac(i - 1, j - 1, k + 1) + @@ -77,8 +77,6 @@ youngs_finite_difference_normal_neumann( amrex::Real& my, amrex::Real& mz) noexcept { - amrex::Real mm1, mm2; - // Do neumann condition via indices const int im1 = ibdy == -1 ? i : i - 1; const int jm1 = jbdy == -1 ? j : j - 1; @@ -87,16 +85,16 @@ youngs_finite_difference_normal_neumann( const int jp1 = jbdy == +1 ? j : j + 1; const int kp1 = kbdy == +1 ? k : k + 1; - mm1 = volfrac(im1, jm1, km1) + volfrac(im1, jm1, kp1) + - volfrac(im1, jp1, km1) + volfrac(im1, jp1, kp1) + - 2.0 * (volfrac(im1, jm1, k) + volfrac(im1, jp1, k) + - volfrac(im1, j, km1) + volfrac(im1, j, kp1)) + - 4.0 * volfrac(im1, j, k); - mm2 = volfrac(ip1, jm1, km1) + volfrac(ip1, jm1, kp1) + - volfrac(ip1, jp1, km1) + volfrac(ip1, jp1, kp1) + - 2.0 * (volfrac(ip1, jm1, k) + volfrac(ip1, jp1, k) + - volfrac(ip1, j, km1) + volfrac(ip1, j, kp1)) + - 4.0 * volfrac(ip1, j, k); + amrex::Real mm1 = volfrac(im1, jm1, km1) + volfrac(im1, jm1, kp1) + + volfrac(im1, jp1, km1) + volfrac(im1, jp1, kp1) + + 2.0 * (volfrac(im1, jm1, k) + volfrac(im1, jp1, k) + + volfrac(im1, j, km1) + volfrac(im1, j, kp1)) + + 4.0 * volfrac(im1, j, k); + amrex::Real mm2 = volfrac(ip1, jm1, km1) + volfrac(ip1, jm1, kp1) + + volfrac(ip1, jp1, km1) + volfrac(ip1, jp1, kp1) + + 2.0 * (volfrac(ip1, jm1, k) + volfrac(ip1, jp1, k) + + volfrac(ip1, j, km1) + volfrac(ip1, j, kp1)) + + 4.0 * volfrac(ip1, j, k); mx = mm1 - mm2; mm1 = volfrac(im1, jm1, km1) + volfrac(im1, jm1, kp1) + @@ -134,15 +132,14 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mixed_youngs_central_normal( amrex::Real& mz) noexcept { amrex::Array2D m; - amrex::Real m1, m2; // write the plane as: sgn(mx) X = my Y + mz Z + alpha // m00 X = m01 Y + m02 Z + alpha - m1 = volfrac(i - 1, j, k - 1) + volfrac(i - 1, j, k + 1) + - volfrac(i - 1, j - 1, k) + volfrac(i - 1, j + 1, k) + - volfrac(i - 1, j, k); - m2 = volfrac(i + 1, j, k - 1) + volfrac(i + 1, j, k + 1) + - volfrac(i + 1, j - 1, k) + volfrac(i + 1, j + 1, k) + - volfrac(i + 1, j, k); + amrex::Real m1 = volfrac(i - 1, j, k - 1) + volfrac(i - 1, j, k + 1) + + volfrac(i - 1, j - 1, k) + volfrac(i - 1, j + 1, k) + + volfrac(i - 1, j, k); + amrex::Real m2 = volfrac(i + 1, j, k - 1) + volfrac(i + 1, j, k + 1) + + volfrac(i + 1, j - 1, k) + volfrac(i + 1, j + 1, k) + + volfrac(i + 1, j, k); m(0, 0) = (m1 > m2) ? 1.0 : -1.0; From 81e79144a014f2816e80652f62d0d3288bd6f378 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 09:55:16 -0600 Subject: [PATCH 23/43] some IntVect stuff --- amr-wind/overset/overset_ops_K.H | 69 ++++++++++++++------------------ 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/amr-wind/overset/overset_ops_K.H b/amr-wind/overset/overset_ops_K.H index 5ee5ee0329..739c7b1f4f 100644 --- a/amr-wind/overset/overset_ops_K.H +++ b/amr-wind/overset/overset_ops_K.H @@ -24,45 +24,38 @@ amrex::Real AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE alpha_flux( amrex::Array4 const& normal) { // Set up neighbor indices - int ii = i; - int jj = j; - int kk = k; - ii += (dir == 0) ? -1 : 0; - jj += (dir == 1) ? -1 : 0; - kk += (dir == 2) ? -1 : 0; + const amrex::IntVect iv{i, j, k}; + const amrex::IntVect ivm = + iv - + (amrex::IntVect){(int)(dir == 0), (int)(dir == 1), (int)(dir == 2)}; // Gradient of phi normal to interface - const amrex::Real gphi = (vof(i, j, k) - vof(ii, jj, kk)); + const amrex::Real gphi = (vof(iv) - vof(ivm)); // Normal vector in each cell (already normalized) - const amrex::Real norm_ = normal(i, j, k, dir); - const amrex::Real norm_nb = normal(ii, jj, kk, dir); + const amrex::Real norm_ = normal(iv, dir); + const amrex::Real norm_nb = normal(ivm, dir); // Determine which delta_phi (and multiply by normal) // The sign depends on side of flux face (like upwinding) - const amrex::Real dphi_ = (tg_vof(i, j, k) - vof(i, j, k)) * (-norm_); - const amrex::Real dphi_nb = - (tg_vof(ii, jj, kk) - vof(ii, jj, kk)) * norm_nb; + const amrex::Real dphi_ = (tg_vof(iv) - vof(iv)) * (-norm_); + const amrex::Real dphi_nb = (tg_vof(ivm) - vof(ivm)) * norm_nb; // Average value used across the interface amrex::Real dphi_eval = 0.5 * (dphi_ + dphi_nb); // Upwinding when on the gas side, downwinding on the liquid // Across the interface defined as crossing 0.5 or within margin of 0.5 - if ((std::abs(vof(i, j, k) - 0.5) > margin || - std::abs(vof(ii, jj, kk) - 0.5) > margin)) { + if ((std::abs(vof(iv) - 0.5) > margin || + std::abs(vof(ivm) - 0.5) > margin)) { if (gphi > 0.0) { - dphi_eval = (vof(ii, jj, kk) < 0.5 && vof(i, j, k) <= 0.5 + margin) - ? dphi_nb - : dphi_eval; - dphi_eval = (vof(ii, jj, kk) >= 0.5 - margin && vof(i, j, k) > 0.5) - ? dphi_ - : dphi_eval; + dphi_eval = (vof(ivm) < 0.5 && vof(iv) <= 0.5 + margin) ? dphi_nb + : dphi_eval; + dphi_eval = + (vof(ivm) >= 0.5 - margin && vof(iv) > 0.5) ? dphi_ : dphi_eval; } if (gphi < 0.0) { - dphi_eval = (vof(i, j, k) < 0.5 && vof(ii, jj, kk) <= 0.5 + margin) - ? dphi_ - : dphi_eval; - dphi_eval = (vof(i, j, k) >= 0.5 - margin && vof(ii, jj, kk) > 0.5) - ? dphi_nb - : dphi_eval; + dphi_eval = + (vof(iv) < 0.5 && vof(ivm) <= 0.5 + margin) ? dphi_ : dphi_eval; + dphi_eval = (vof(iv) >= 0.5 - margin && vof(ivm) > 0.5) ? dphi_nb + : dphi_eval; } } return dphi_eval; @@ -80,23 +73,21 @@ void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE velocity_face( amrex::Real& wface) { // Set up neighbor indices - int ii = i; - int jj = j; - int kk = k; - ii += (dir == 0) ? -1 : 0; - jj += (dir == 1) ? -1 : 0; - kk += (dir == 2) ? -1 : 0; + const amrex::IntVect iv{i, j, k}; + const amrex::IntVect ivm = + iv - + (amrex::IntVect){(int)(dir == 0), (int)(dir == 1), (int)(dir == 2)}; // Gradient of phi normal to interface - const amrex::Real gphi = (vof(i, j, k) - vof(ii, jj, kk)); + const amrex::Real gphi = (vof(iv) - vof(ivm)); // Get velocities on both sides - const amrex::Real u_ = velocity(i, j, k, 0); - const amrex::Real v_ = velocity(i, j, k, 1); - const amrex::Real w_ = velocity(i, j, k, 2); - const amrex::Real u_nb = velocity(ii, jj, kk, 0); - const amrex::Real v_nb = velocity(ii, jj, kk, 1); - const amrex::Real w_nb = velocity(ii, jj, kk, 2); + const amrex::Real u_ = velocity(iv, 0); + const amrex::Real v_ = velocity(iv, 1); + const amrex::Real w_ = velocity(iv, 2); + const amrex::Real u_nb = velocity(ivm, 0); + const amrex::Real v_nb = velocity(ivm, 1); + const amrex::Real w_nb = velocity(ivm, 2); // Average value when gphi = 0 uface = 0.5 * (u_ + u_nb); vface = 0.5 * (v_ + v_nb); From dee35409b687972f45a4a6fec44950a573cb6a47 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 11:13:01 -0600 Subject: [PATCH 24/43] correct IntVect declaration --- amr-wind/overset/overset_ops_K.H | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/amr-wind/overset/overset_ops_K.H b/amr-wind/overset/overset_ops_K.H index 739c7b1f4f..a2c36bc113 100644 --- a/amr-wind/overset/overset_ops_K.H +++ b/amr-wind/overset/overset_ops_K.H @@ -25,9 +25,8 @@ amrex::Real AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE alpha_flux( { // Set up neighbor indices const amrex::IntVect iv{i, j, k}; - const amrex::IntVect ivm = - iv - - (amrex::IntVect){(int)(dir == 0), (int)(dir == 1), (int)(dir == 2)}; + const amrex::IntVect v1{dir == 0, dir == 1, dir == 2}; + const amrex::IntVect ivm = iv - v1; // Gradient of phi normal to interface const amrex::Real gphi = (vof(iv) - vof(ivm)); @@ -74,9 +73,8 @@ void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE velocity_face( { // Set up neighbor indices const amrex::IntVect iv{i, j, k}; - const amrex::IntVect ivm = - iv - - (amrex::IntVect){(int)(dir == 0), (int)(dir == 1), (int)(dir == 2)}; + const amrex::IntVect v1{dir == 0, dir == 1, dir == 2}; + const amrex::IntVect ivm = iv - v1; // Gradient of phi normal to interface const amrex::Real gphi = (vof(iv) - vof(ivm)); From aa332f997a18646400975e08cb9d7facdada8648 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 12:04:54 -0600 Subject: [PATCH 25/43] run_algorithm tweaks --- unit_tests/aw_test_utils/iter_tools.H | 39 ++++++++++++++++----------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/unit_tests/aw_test_utils/iter_tools.H b/unit_tests/aw_test_utils/iter_tools.H index 7ef7390f93..908937d1c2 100644 --- a/unit_tests/aw_test_utils/iter_tools.H +++ b/unit_tests/aw_test_utils/iter_tools.H @@ -20,6 +20,25 @@ namespace amr_wind_tests { * @param field MultiFabs for each of the `nlevels` levels * @param func Functor to execute over the MultiFabs */ + +template +void run_algorithm_single_lev( + const int lev, amrex::MultiFab& lfab, const Functor& func) +{ + for (amrex::MFIter mfi(lfab); mfi.isValid(); ++mfi) { + func(lev, mfi); + } +} + +template +void run_algorithm_single_lev( + const int lev, amrex::iMultiFab& lfab, const Functor& func) +{ + for (amrex::MFIter mfi(lfab); mfi.isValid(); ++mfi) { + func(lev, mfi); + } +} + template void run_algorithm( const int nlevels, @@ -28,10 +47,7 @@ void run_algorithm( { for (int lev = 0; lev < nlevels; ++lev) { auto& lfab = *field[lev]; - - for (amrex::MFIter mfi(lfab); mfi.isValid(); ++mfi) { - func(lev, mfi); - } + run_algorithm_single_lev(lev, lfab, func); } } @@ -41,10 +57,7 @@ void run_algorithm(amr_wind::Field& field, const Functor& func) const int nlevels = field.repo().num_active_levels(); for (int lev = 0; lev < nlevels; ++lev) { auto& lfab = field(lev); - - for (amrex::MFIter mfi(lfab); mfi.isValid(); ++mfi) { - func(lev, mfi); - } + run_algorithm_single_lev(lev, lfab, func); } } @@ -56,10 +69,7 @@ void run_algorithm( { for (int lev = 0; lev < nlevels; ++lev) { auto& lfab = *intfield[lev]; - - for (amrex::MFIter mfi(lfab); mfi.isValid(); ++mfi) { - func(lev, mfi); - } + run_algorithm_single_lev(lev, lfab, func); } } @@ -69,10 +79,7 @@ void run_algorithm(amr_wind::IntField& intfield, const Functor& func) const int nlevels = intfield.repo().num_active_levels(); for (int lev = 0; lev < nlevels; ++lev) { auto& lfab = intfield(lev); - - for (amrex::MFIter mfi(lfab); mfi.isValid(); ++mfi) { - func(lev, mfi); - } + run_algorithm_single_lev(lev, lfab, func); } } From e7805c86e0276bce0c502dd0e7d665f8b0a0e7cf Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 12:06:28 -0600 Subject: [PATCH 26/43] remove wrong comments --- amr-wind/overset/overset_ops_routines.H | 2 -- 1 file changed, 2 deletions(-) diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index be71c73423..8d91ec3254 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -421,7 +421,6 @@ void replace_gradp_hydrostatic( const amrex::Array4& iblank = mf_iblank.const_array(mfi); amrex::ParallelFor( gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Replace amr-wind vof values with originals if (iblank(i, j, k) == -1) { const amrex::Real dfac = is_pptb ? rho(i, j, k) - rho0(i, j, k) : rho(i, j, k); @@ -446,7 +445,6 @@ void replace_gradp( const amrex::Array4& iblank = mf_iblank.const_array(mfi); amrex::ParallelFor( gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Replace amr-wind vof values with originals if (iblank(i, j, k) == -1) { gp(i, j, k, 0) = gp0(i, j, k, 0); gp(i, j, k, 1) = gp0(i, j, k, 1); From ba313a0ad912e90e7260f5ff927fab58059bf5ce Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 12:20:28 -0600 Subject: [PATCH 27/43] IntVect tweaks --- amr-wind/overset/overset_ops_K.H | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/amr-wind/overset/overset_ops_K.H b/amr-wind/overset/overset_ops_K.H index a2c36bc113..6b03c6a63e 100644 --- a/amr-wind/overset/overset_ops_K.H +++ b/amr-wind/overset/overset_ops_K.H @@ -25,8 +25,8 @@ amrex::Real AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE alpha_flux( { // Set up neighbor indices const amrex::IntVect iv{i, j, k}; - const amrex::IntVect v1{dir == 0, dir == 1, dir == 2}; - const amrex::IntVect ivm = iv - v1; + const amrex::IntVect dv{(int)(dir == 0), (int)(dir == 1), (int)(dir == 2)}; + const amrex::IntVect ivm = iv - dv; // Gradient of phi normal to interface const amrex::Real gphi = (vof(iv) - vof(ivm)); @@ -73,8 +73,8 @@ void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE velocity_face( { // Set up neighbor indices const amrex::IntVect iv{i, j, k}; - const amrex::IntVect v1{dir == 0, dir == 1, dir == 2}; - const amrex::IntVect ivm = iv - v1; + const amrex::IntVect dv{(int)(dir == 0), (int)(dir == 1), (int)(dir == 2)}; + const amrex::IntVect ivm = iv - dv; // Gradient of phi normal to interface const amrex::Real gphi = (vof(iv) - vof(ivm)); From de635d4c4907713bd6296b82bf8a92b6a923f931 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 12:22:46 -0600 Subject: [PATCH 28/43] initialize things that come from elsewhere with quiet_NaN --- amr-wind/physics/multiphase/SloshingTank.H | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/amr-wind/physics/multiphase/SloshingTank.H b/amr-wind/physics/multiphase/SloshingTank.H index c6097798d9..4d467ac9bc 100644 --- a/amr-wind/physics/multiphase/SloshingTank.H +++ b/amr-wind/physics/multiphase/SloshingTank.H @@ -47,9 +47,12 @@ private: amrex::Real m_waterlevel{0.0}; //! Stuff to get from MultiPhase physics - amrex::Real m_rho1{1000.}; - amrex::Real m_rho2{1.}; - amrex::Vector m_gravity{0.0, 0.0, -9.81}; + amrex::Real m_rho1{std::numeric_limits::quiet_NaN()}; + amrex::Real m_rho2{std::numeric_limits::quiet_NaN()}; + amrex::Vector m_gravity{ + std::numeric_limits::quiet_NaN(), + std::numeric_limits::quiet_NaN(), + std::numeric_limits::quiet_NaN()}; }; } // namespace amr_wind From 1be873420e917386e5d5e2b5c2dfb33761be5865 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 13:26:57 -0600 Subject: [PATCH 29/43] fused mfiters for most loops in overset_ops_routines --- amr-wind/overset/OversetOps.cpp | 6 + amr-wind/overset/overset_ops_routines.H | 327 ++++++++++++------------ 2 files changed, 168 insertions(+), 165 deletions(-) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 082b68343f..3c03ec7498 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -212,6 +212,7 @@ void OversetOps::sharpen_nalu_data() // Populate approximate signed distance function overset_ops::populate_psi(levelset(lev), vof(lev), i_th, m_asdf_tiny); } + amrex::Gpu::synchronize(); // Convert levelset to vof to get target_vof m_mphase->levelset2vof(iblank_cell, *target_vof); @@ -221,12 +222,14 @@ void OversetOps::sharpen_nalu_data() // A tolerance of 0 should do nothing overset_ops::process_vof((*target_vof)(lev), m_target_cutoff); } + amrex::Gpu::synchronize(); // Replace vof with original values in amr domain for (int lev = 0; lev < nlevels; ++lev) { overset_ops::harmonize_vof( (*target_vof)(lev), vof(lev), iblank_cell(lev)); } + amrex::Gpu::synchronize(); // Put fluxes in vector for averaging down during iterations amrex::Vector> fluxes( @@ -275,6 +278,7 @@ void OversetOps::sharpen_nalu_data() err = amrex::max(err, err_lev); } } + amrex::Gpu::synchronize(); // Average down fluxes across levels for consistency for (int lev = nlevels - 1; lev > 0; --lev) { @@ -305,6 +309,7 @@ void OversetOps::sharpen_nalu_data() (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), vof(lev), rho(lev), velocity(lev), ptfac, m_vof_tol); } + amrex::Gpu::synchronize(); // Fillpatch for ghost cells vof.fillpatch((*m_sim_ptr).time().current_time()); @@ -330,6 +335,7 @@ void OversetOps::sharpen_nalu_data() for (int lev = 0; lev < nlevels; ++lev) { overset_ops::equate_field(levelset(lev), velocity(lev)); } + amrex::Gpu::synchronize(); } void OversetOps::set_hydrostatic_gradp() diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index 8d91ec3254..b0f900bd1f 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -13,31 +13,30 @@ void populate_psi( const amrex::Real i_th, const amrex::Real asdf_tiny) { - for (amrex::MFIter mfi(mf_psi); mfi.isValid(); ++mfi) { - const auto& gbx = mfi.growntilebox(); - const amrex::Array4& psi = mf_psi.array(mfi); - const amrex::Array4& vof = mf_vof.const_array(mfi); - amrex::ParallelFor( - gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - psi(i, j, k) = asdf(vof(i, j, k), i_th, asdf_tiny); - }); - } + const auto& psi_arrs = mf_psi.arrays(); + const auto& vof_arrs = mf_vof.const_arrays(); + amrex::ParallelFor( + mf_psi, mf_psi.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + psi_arrs[nbx](i, j, k) = + asdf(vof_arrs[nbx](i, j, k), i_th, asdf_tiny); + }); } // Modify a vof field to not have values that barely differ from 0 or 1 void process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) { - for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { - const auto& gbx = mfi.growntilebox(); - const amrex::Array4& vof = mf_vof.array(mfi); - amrex::ParallelFor( - gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - vof(i, j, k) = - vof(i, j, k) < vof_tol - ? 0.0 - : (vof(i, j, k) > 1. - vof_tol ? 1. : vof(i, j, k)); - }); - } + const auto& vof_arrs = mf_vof.arrays(); + amrex::ParallelFor( + mf_vof, mf_vof.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + vof_arrs[nbx](i, j, k) = + vof_arrs[nbx](i, j, k) < vof_tol + ? 0.0 + : (vof_arrs[nbx](i, j, k) > 1. - vof_tol + ? 1. + : vof_arrs[nbx](i, j, k)); + }); } // Combine overset target vof field with current non-overset vof field @@ -46,20 +45,17 @@ void harmonize_vof( const amrex::MultiFab& mf_vof_original, const amrex::iMultiFab& mf_iblank) { - for (amrex::MFIter mfi(mf_vof_target); mfi.isValid(); ++mfi) { - const auto& vbx = mfi.validbox(); - const amrex::Array4& tg_vof = mf_vof_target.array(mfi); - const amrex::Array4& og_vof = - mf_vof_original.const_array(mfi); - const amrex::Array4& iblank = mf_iblank.const_array(mfi); - amrex::ParallelFor( - vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Replace amr-wind vof values with originals - if (iblank(i, j, k) != -1) { - tg_vof(i, j, k) = og_vof(i, j, k); - } - }); - } + const auto& tg_vof_arrs = mf_vof_target.arrays(); + const auto& og_vof_arrs = mf_vof_original.const_arrays(); + const auto& iblank_arrs = mf_iblank.const_arrays(); + amrex::ParallelFor( + mf_vof_target, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + // Replace amr-wind vof values with originals + if (iblank_arrs[nbx](i, j, k) != -1) { + tg_vof_arrs[nbx](i, j, k) = og_vof_arrs[nbx](i, j, k); + } + }); } // Populate normal vector with special treatment of overset boundary @@ -68,34 +64,47 @@ void populate_normal_vector( const amrex::MultiFab& mf_vof, const amrex::iMultiFab& mf_iblank) { - for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { - const auto& gbxm1 = grow(mfi.growntilebox(), -1); - const amrex::Array4& normvec = mf_normvec.array(mfi); - const amrex::Array4& vof = mf_vof.const_array(mfi); - const amrex::Array4& iblank = mf_iblank.const_array(mfi); - // Calculate gradients in each direction with centered diff - amrex::ParallelFor( - gbxm1, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Neumann condition across nalu bdy - int ibdy = (iblank(i, j, k) != iblank(i - 1, j, k)) ? -1 : 0; - int jbdy = (iblank(i, j, k) != iblank(i, j - 1, k)) ? -1 : 0; - int kbdy = (iblank(i, j, k) != iblank(i, j, k - 1)) ? -1 : 0; - // no cell should be isolated such that -1 and 1 are needed - ibdy = (iblank(i, j, k) != iblank(i + 1, j, k)) ? +1 : ibdy; - jbdy = (iblank(i, j, k) != iblank(i, j + 1, k)) ? +1 : jbdy; - kbdy = (iblank(i, j, k) != iblank(i, j, k + 1)) ? +1 : kbdy; - // Calculate normal - amrex::Real mx, my, mz, mmag; - multiphase::youngs_finite_difference_normal_neumann( - i, j, k, ibdy, jbdy, kbdy, vof, mx, my, mz); - // Normalize normal - mmag = std::sqrt(mx * mx + my * my + mz * mz + 1e-20); - // Save normal - normvec(i, j, k, 0) = mx / mmag; - normvec(i, j, k, 1) = my / mmag; - normvec(i, j, k, 2) = mz / mmag; - }); - } + const auto& normvec_arrs = mf_normvec.arrays(); + const auto& vof_arrs = mf_vof.const_arrays(); + const auto& iblank_arrs = mf_iblank.const_arrays(); + // Calculate gradients in each direction with centered diff + amrex::ParallelFor( + mf_normvec, mf_normvec.n_grow - amrex::IntVect(1), + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + // Neumann condition across nalu bdy + int ibdy = + (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i - 1, j, k)) + ? -1 + : 0; + int jbdy = + (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i, j - 1, k)) + ? -1 + : 0; + int kbdy = + (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i, j, k - 1)) + ? -1 + : 0; + // no cell should be isolated such that -1 and 1 are needed + ibdy = (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i + 1, j, k)) + ? +1 + : ibdy; + jbdy = (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i, j + 1, k)) + ? +1 + : jbdy; + kbdy = (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i, j, k + 1)) + ? +1 + : kbdy; + // Calculate normal + amrex::Real mx, my, mz, mmag; + multiphase::youngs_finite_difference_normal_neumann( + i, j, k, ibdy, jbdy, kbdy, vof_arrs[nbx], mx, my, mz); + // Normalize normal + mmag = std::sqrt(mx * mx + my * my + mz * mz + 1e-20); + // Save normal + normvec_arrs[nbx](i, j, k, 0) = mx / mmag; + normvec_arrs[nbx](i, j, k, 1) = my / mmag; + normvec_arrs[nbx](i, j, k, 2) = mz / mmag; + }); } // Calculate fluxes for reinitialization over entire domain without concern for @@ -295,52 +304,51 @@ void apply_fluxes( amrex::Real ptfac, amrex::Real vof_tol) { + const auto& fx = mf_fx.const_arrays(); + const auto& fy = mf_fy.const_arrays(); + const auto& fz = mf_fz.const_arrays(); + const auto& vof = mf_vof.arrays(); + const auto& dens = mf_dens.arrays(); + const auto& vel = mf_vel.arrays(); - for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { - const auto& vbx = mfi.validbox(); - const amrex::Array4& fx = mf_fx.array(mfi); - const amrex::Array4& fy = mf_fy.array(mfi); - const amrex::Array4& fz = mf_fz.array(mfi); - const amrex::Array4& vof = mf_vof.array(mfi); - const amrex::Array4& dens = mf_dens.array(mfi); - const amrex::Array4& vel = mf_vel.array(mfi); - amrex::ParallelFor( - vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - const amrex::Real olddens = dens(i, j, k); - vof(i, j, k) += ptfac * ((fx(i + 1, j, k, 0) - fx(i, j, k, 0)) + - (fy(i, j + 1, k, 0) - fy(i, j, k, 0)) + - (fz(i, j, k + 1, 0) - fz(i, j, k, 0))); - dens(i, j, k) += - ptfac * ((fx(i + 1, j, k, 1) - fx(i, j, k, 1)) + - (fy(i, j + 1, k, 1) - fy(i, j, k, 1)) + - (fz(i, j, k + 1, 1) - fz(i, j, k, 1))); - vel(i, j, k, 0) = - 1.0 / dens(i, j, k) * - (olddens * vel(i, j, k, 0) + - ptfac * ((fx(i + 1, j, k, 2) - fx(i, j, k, 2)) + - (fy(i, j + 1, k, 2) - fy(i, j, k, 2)) + - (fz(i, j, k + 1, 2) - fz(i, j, k, 2)))); - vel(i, j, k, 1) = - 1.0 / dens(i, j, k) * - (olddens * vel(i, j, k, 1) + - ptfac * ((fx(i + 1, j, k, 3) - fx(i, j, k, 3)) + - (fy(i, j + 1, k, 3) - fy(i, j, k, 3)) + - (fz(i, j, k + 1, 3) - fz(i, j, k, 3)))); - vel(i, j, k, 2) = - 1.0 / dens(i, j, k) * - (olddens * vel(i, j, k, 2) + - ptfac * ((fx(i + 1, j, k, 4) - fx(i, j, k, 4)) + - (fy(i, j + 1, k, 4) - fy(i, j, k, 4)) + - (fz(i, j, k + 1, 4) - fz(i, j, k, 4)))); + amrex::ParallelFor( + mf_vof, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + const amrex::Real olddens = dens[nbx](i, j, k); + vof[nbx](i, j, k) += + ptfac * ((fx[nbx](i + 1, j, k, 0) - fx[nbx](i, j, k, 0)) + + (fy[nbx](i, j + 1, k, 0) - fy[nbx](i, j, k, 0)) + + (fz[nbx](i, j, k + 1, 0) - fz[nbx](i, j, k, 0))); + dens[nbx](i, j, k) += + ptfac * ((fx[nbx](i + 1, j, k, 1) - fx[nbx](i, j, k, 1)) + + (fy[nbx](i, j + 1, k, 1) - fy[nbx](i, j, k, 1)) + + (fz[nbx](i, j, k + 1, 1) - fz[nbx](i, j, k, 1))); + vel[nbx](i, j, k, 0) = + 1.0 / dens[nbx](i, j, k) * + (olddens * vel[nbx](i, j, k, 0) + + ptfac * ((fx[nbx](i + 1, j, k, 2) - fx[nbx](i, j, k, 2)) + + (fy[nbx](i, j + 1, k, 2) - fy[nbx](i, j, k, 2)) + + (fz[nbx](i, j, k + 1, 2) - fz[nbx](i, j, k, 2)))); + vel[nbx](i, j, k, 1) = + 1.0 / dens[nbx](i, j, k) * + (olddens * vel[nbx](i, j, k, 1) + + ptfac * ((fx[nbx](i + 1, j, k, 3) - fx[nbx](i, j, k, 3)) + + (fy[nbx](i, j + 1, k, 3) - fy[nbx](i, j, k, 3)) + + (fz[nbx](i, j, k + 1, 3) - fz[nbx](i, j, k, 3)))); + vel[nbx](i, j, k, 2) = + 1.0 / dens[nbx](i, j, k) * + (olddens * vel[nbx](i, j, k, 2) + + ptfac * ((fx[nbx](i + 1, j, k, 4) - fx[nbx](i, j, k, 4)) + + (fy[nbx](i, j + 1, k, 4) - fy[nbx](i, j, k, 4)) + + (fz[nbx](i, j, k + 1, 4) - fz[nbx](i, j, k, 4)))); - // Ensure vof is bounded - vof(i, j, k) = - vof(i, j, k) < vof_tol - ? 0.0 - : (vof(i, j, k) > 1. - vof_tol ? 1. : vof(i, j, k)); - // Density bounds are enforced elsewhere - }); - } + // Ensure vof is bounded + vof[nbx](i, j, k) = + vof[nbx](i, j, k) < vof_tol + ? 0.0 + : (vof[nbx](i, j, k) > 1. - vof_tol ? 1. + : vof[nbx](i, j, k)); + // Density bounds are enforced elsewhere + }); } // Get the size of the smallest VOF flux to quantify convergence @@ -388,18 +396,15 @@ amrex::Real measure_convergence( // Set levelset field to another quantity to view in plotfile for debugging void equate_field(amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src) { - for (amrex::MFIter mfi(mf_dest); mfi.isValid(); ++mfi) { - const auto& vbx = mfi.validbox(); - const amrex::Array4& src = mf_src.const_array(mfi); - const amrex::Array4& dest = mf_dest.array(mfi); - amrex::ParallelFor( - vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - dest(i, j, k) = std::sqrt( - src(i, j, k, 0) * src(i, j, k, 0) + - src(i, j, k, 1) * src(i, j, k, 1) + - src(i, j, k, 2) * src(i, j, k, 2)); - }); - } + const auto& dest_arrs = mf_dest.arrays(); + const auto& src_arrs = mf_src.const_arrays(); + amrex::ParallelFor( + mf_dest, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + dest_arrs[nbx](i, j, k) = std::sqrt( + src_arrs[nbx](i, j, k, 0) * src_arrs[nbx](i, j, k, 0) + + src_arrs[nbx](i, j, k, 1) * src_arrs[nbx](i, j, k, 1) + + src_arrs[nbx](i, j, k, 2) * src_arrs[nbx](i, j, k, 2)); + }); } // Replace pressure gradient with hydrostatic field in overset regions @@ -411,25 +416,22 @@ void replace_gradp_hydrostatic( const amrex::Real grav_z, const bool is_pptb) { - for (amrex::MFIter mfi(mf_gp); mfi.isValid(); ++mfi) { - const auto& gbx = mfi.growntilebox(); - const amrex::Array4& gp = mf_gp.array(mfi); - const amrex::Array4& rho = - mf_density.const_array(mfi); - const amrex::Array4& rho0 = - mf_refdens.const_array(mfi); - const amrex::Array4& iblank = mf_iblank.const_array(mfi); - amrex::ParallelFor( - gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if (iblank(i, j, k) == -1) { - const amrex::Real dfac = - is_pptb ? rho(i, j, k) - rho0(i, j, k) : rho(i, j, k); - gp(i, j, k, 0) = 0.; - gp(i, j, k, 1) = 0.; - gp(i, j, k, 2) = dfac * grav_z; - } - }); - } + const auto& gp_arrs = mf_gp.arrays(); + const auto& rho_arrs = mf_density.const_arrays(); + const auto& rho0_arrs = mf_refdens.const_arrays(); + const auto& iblank_arrs = mf_iblank.const_arrays(); + amrex::ParallelFor( + mf_gp, mf_gp.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + if (iblank_arrs[nbx](i, j, k) == -1) { + const amrex::Real dfac = + is_pptb ? rho_arrs[nbx](i, j, k) - rho0_arrs[nbx](i, j, k) + : rho_arrs[nbx](i, j, k); + gp_arrs[nbx](i, j, k, 0) = 0.; + gp_arrs[nbx](i, j, k, 1) = 0.; + gp_arrs[nbx](i, j, k, 2) = dfac * grav_z; + } + }); } // Swap pressure gradient values in overset region @@ -438,20 +440,18 @@ void replace_gradp( const amrex::MultiFab& mf_gp0, const amrex::iMultiFab& mf_iblank) { - for (amrex::MFIter mfi(mf_gp); mfi.isValid(); ++mfi) { - const auto& gbx = mfi.growntilebox(); - const amrex::Array4& gp = mf_gp.array(mfi); - const amrex::Array4& gp0 = mf_gp0.const_array(mfi); - const amrex::Array4& iblank = mf_iblank.const_array(mfi); - amrex::ParallelFor( - gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if (iblank(i, j, k) == -1) { - gp(i, j, k, 0) = gp0(i, j, k, 0); - gp(i, j, k, 1) = gp0(i, j, k, 1); - gp(i, j, k, 2) = gp0(i, j, k, 2); - } - }); - } + const auto gp_arrs = mf_gp.arrays(); + const auto gp0_arrs = mf_gp0.const_arrays(); + const auto iblank_arrs = mf_iblank.const_arrays(); + amrex::ParallelFor( + mf_gp, mf_gp.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + if (iblank_arrs[nbx](i, j, k) == -1) { + gp_arrs[nbx](i, j, k, 0) = gp0_arrs[nbx](i, j, k, 0); + gp_arrs[nbx](i, j, k, 1) = gp0_arrs[nbx](i, j, k, 1); + gp_arrs[nbx](i, j, k, 2) = gp0_arrs[nbx](i, j, k, 2); + } + }); } // Apply pressure gradient to velocity field @@ -461,20 +461,17 @@ void apply_pressure_gradient( const amrex::MultiFab& mf_gp, const amrex::Real scaling_factor) { - for (amrex::MFIter mfi(mf_gp); mfi.isValid(); ++mfi) { - const auto& vbx = mfi.validbox(); - const amrex::Array4& vel = mf_vel.array(mfi); - const amrex::Array4& rho = - mf_density.const_array(mfi); - const amrex::Array4& gp = mf_gp.const_array(mfi); - amrex::ParallelFor( - vbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - const amrex::Real soverrho = scaling_factor / rho(i, j, k); - vel(i, j, k, 0) -= gp(i, j, k, 0) * soverrho; - vel(i, j, k, 1) -= gp(i, j, k, 1) * soverrho; - vel(i, j, k, 2) -= gp(i, j, k, 2) * soverrho; - }); - } + const auto& vel_arrs = mf_vel.arrays(); + const auto& rho_arrs = mf_density.const_arrays(); + const auto& gp_arrs = mf_gp.const_arrays(); + amrex::ParallelFor( + mf_vel, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + const amrex::Real soverrho = + scaling_factor / rho_arrs[nbx](i, j, k); + vel_arrs[nbx](i, j, k, 0) -= gp_arrs[nbx](i, j, k, 0) * soverrho; + vel_arrs[nbx](i, j, k, 1) -= gp_arrs[nbx](i, j, k, 1) * soverrho; + vel_arrs[nbx](i, j, k, 2) -= gp_arrs[nbx](i, j, k, 2) * soverrho; + }); } } // namespace amr_wind::overset_ops From e3ba6fb41c6ac1086e1023d5959bf7e3818e3a35 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 15:36:05 -0600 Subject: [PATCH 30/43] more fused MFIter --- amr-wind/overset/overset_ops_routines.H | 168 +++++++++++------------- 1 file changed, 77 insertions(+), 91 deletions(-) diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index b0f900bd1f..e4a94c8110 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -121,63 +121,58 @@ void populate_sharpen_fluxes( const amrex::Real rho1, const amrex::Real rho2) { - for (amrex::MFIter mfi(mf_vof); mfi.isValid(); ++mfi) { - const auto& vbx = mfi.validbox(); - const auto& xbx = amrex::surroundingNodes(vbx, 0); - const auto& ybx = amrex::surroundingNodes(vbx, 1); - const auto& zbx = amrex::surroundingNodes(vbx, 2); - const amrex::Array4& fx = mf_fx.array(mfi); - const amrex::Array4& fy = mf_fy.array(mfi); - const amrex::Array4& fz = mf_fz.array(mfi); - const amrex::Array4& vof = mf_vof.const_array(mfi); - const amrex::Array4& tg_vof = - mf_target_vof.const_array(mfi); - const amrex::Array4& norm = mf_norm.const_array(mfi); - const amrex::Array4& vel = - mf_velocity.const_array(mfi); - // Populate vof and density fluxes for each direction - amrex::ParallelFor( - xbx, ybx, zbx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // vof flux - amrex::Real flux = - alpha_flux(i, j, k, 0, margin, vof, tg_vof, norm); - fx(i, j, k, 0) = flux; - // density flux - flux *= (rho1 - rho2); - fx(i, j, k, 1) = flux; - // momentum fluxes (dens flux * face vel) - amrex::Real uf, vf, wf; - velocity_face(i, j, k, 0, vof, vel, uf, vf, wf); - fx(i, j, k, 2) = flux * uf; - fx(i, j, k, 3) = flux * vf; - fx(i, j, k, 4) = flux * wf; - }, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real flux = - alpha_flux(i, j, k, 1, margin, vof, tg_vof, norm); - fy(i, j, k, 0) = flux; - flux *= (rho1 - rho2); - fy(i, j, k, 1) = flux; - amrex::Real uf, vf, wf; - velocity_face(i, j, k, 1, vof, vel, uf, vf, wf); - fy(i, j, k, 2) = flux * uf; - fy(i, j, k, 3) = flux * vf; - fy(i, j, k, 4) = flux * wf; - }, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real flux = - alpha_flux(i, j, k, 2, margin, vof, tg_vof, norm); - fz(i, j, k, 0) = flux; - flux *= (rho1 - rho2); - fz(i, j, k, 1) = flux; - amrex::Real uf, vf, wf; - velocity_face(i, j, k, 2, vof, vel, uf, vf, wf); - fz(i, j, k, 2) = flux * uf; - fz(i, j, k, 3) = flux * vf; - fz(i, j, k, 4) = flux * wf; - }); - } + const auto& fx = mf_fx.arrays(); + const auto& fy = mf_fy.arrays(); + const auto& fz = mf_fz.arrays(); + const auto& vof = mf_vof.const_arrays(); + const auto& tg_vof = mf_target_vof.const_arrays(); + const auto& norm = mf_norm.const_arrays(); + const auto& vel = mf_velocity.const_arrays(); + amrex::ParallelFor( + mf_fx, mf_fx.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + // vof flux + amrex::Real flux = alpha_flux( + i, j, k, 0, margin, vof[nbx], tg_vof[nbx], norm[nbx]); + fx[nbx](i, j, k, 0) = flux; + // density flux + flux *= (rho1 - rho2); + fx[nbx](i, j, k, 1) = flux; + // momentum fluxes (dens flux * face vel) + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 0, vof[nbx], vel[nbx], uf, vf, wf); + fx[nbx](i, j, k, 2) = flux * uf; + fx[nbx](i, j, k, 3) = flux * vf; + fx[nbx](i, j, k, 4) = flux * wf; + }); + amrex::ParallelFor( + mf_fy, mf_fy.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + amrex::Real flux = alpha_flux( + i, j, k, 1, margin, vof[nbx], tg_vof[nbx], norm[nbx]); + fy[nbx](i, j, k, 0) = flux; + flux *= (rho1 - rho2); + fy[nbx](i, j, k, 1) = flux; + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 1, vof[nbx], vel[nbx], uf, vf, wf); + fy[nbx](i, j, k, 2) = flux * uf; + fy[nbx](i, j, k, 3) = flux * vf; + fy[nbx](i, j, k, 4) = flux * wf; + }); + amrex::ParallelFor( + mf_fz, mf_fz.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + amrex::Real flux = alpha_flux( + i, j, k, 2, margin, vof[nbx], tg_vof[nbx], norm[nbx]); + fz[nbx](i, j, k, 0) = flux; + flux *= (rho1 - rho2); + fz[nbx](i, j, k, 1) = flux; + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 2, vof[nbx], vel[nbx], uf, vf, wf); + fz[nbx](i, j, k, 2) = flux * uf; + fz[nbx](i, j, k, 3) = flux * vf; + fz[nbx](i, j, k, 4) = flux * wf; + }); } // Process reinitialization fluxes - zero non-internal to overset region @@ -187,40 +182,31 @@ void process_fluxes( amrex::MultiFab& mf_fz, const amrex::iMultiFab& mf_iblank) { - int ncompx = mf_fx.nComp(); - int ncompy = mf_fy.nComp(); - int ncompz = mf_fz.nComp(); - for (amrex::MFIter mfi(mf_iblank); mfi.isValid(); ++mfi) { - const auto& vbx = mfi.validbox(); - const auto& xbx = amrex::surroundingNodes(vbx, 0); - const auto& ybx = amrex::surroundingNodes(vbx, 1); - const auto& zbx = amrex::surroundingNodes(vbx, 2); - const amrex::Array4& fx = mf_fx.array(mfi); - const amrex::Array4& fy = mf_fy.array(mfi); - const amrex::Array4& fz = mf_fz.array(mfi); - const amrex::Array4& iblank = mf_iblank.const_array(mfi); - // Only faces with iblank = -1 on both sides can have nonzero flux - amrex::ParallelFor( - xbx, ybx, zbx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - bool zero_all = (iblank(i - 1, j, k) + iblank(i, j, k) > -2); - for (int n = 0; n < ncompx; ++n) { - fx(i, j, k, n) *= zero_all ? 0. : 1.; - } - }, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - bool zero_all = (iblank(i, j - 1, k) + iblank(i, j, k) > -2); - for (int n = 0; n < ncompy; ++n) { - fy(i, j, k, n) *= zero_all ? 0. : 1.; - } - }, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - bool zero_all = (iblank(i, j, k - 1) + iblank(i, j, k) > -2); - for (int n = 0; n < ncompz; ++n) { - fz(i, j, k, n) *= zero_all ? 0. : 1.; - } - }); - } + const auto& fx = mf_fx.arrays(); + const auto& fy = mf_fy.arrays(); + const auto& fz = mf_fz.arrays(); + const auto& iblank = mf_iblank.const_arrays(); + amrex::ParallelFor( + mf_fx, mf_fx.n_grow, mf_fx.n_comp, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { + bool zero_all = + (iblank[nbx](i - 1, j, k) + iblank[nbx](i, j, k) > -2); + fx[nbx](i, j, k, n) *= zero_all ? 0. : 1.; + }); + amrex::ParallelFor( + mf_fy, mf_fy.n_grow, mf_fy.n_comp, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { + bool zero_all = + (iblank[nbx](i, j - 1, k) + iblank[nbx](i, j, k) > -2); + fy[nbx](i, j, k, n) *= zero_all ? 0. : 1.; + }); + amrex::ParallelFor( + mf_fz, mf_fz.n_grow, mf_fz.n_comp, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { + bool zero_all = + (iblank[nbx](i, j, k - 1) + iblank[nbx](i, j, k) > -2); + fz[nbx](i, j, k, n) *= zero_all ? 0. : 1.; + }); } // Calculate a type of CFL by measuring how much % VOF is being removed per cell From e976aff85b5364d9af938dc2f83accec54eb6076 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 8 Jul 2024 15:37:56 -0600 Subject: [PATCH 31/43] remove "_arrs" suffix; seems repetitive and unnecessary for this file --- amr-wind/overset/overset_ops_routines.H | 130 +++++++++++------------- 1 file changed, 59 insertions(+), 71 deletions(-) diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index e4a94c8110..9e289f8ac2 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -13,29 +13,27 @@ void populate_psi( const amrex::Real i_th, const amrex::Real asdf_tiny) { - const auto& psi_arrs = mf_psi.arrays(); - const auto& vof_arrs = mf_vof.const_arrays(); + const auto& psi = mf_psi.arrays(); + const auto& vof = mf_vof.const_arrays(); amrex::ParallelFor( mf_psi, mf_psi.n_grow, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - psi_arrs[nbx](i, j, k) = - asdf(vof_arrs[nbx](i, j, k), i_th, asdf_tiny); + psi[nbx](i, j, k) = asdf(vof[nbx](i, j, k), i_th, asdf_tiny); }); } // Modify a vof field to not have values that barely differ from 0 or 1 void process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) { - const auto& vof_arrs = mf_vof.arrays(); + const auto& vof = mf_vof.arrays(); amrex::ParallelFor( mf_vof, mf_vof.n_grow, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - vof_arrs[nbx](i, j, k) = - vof_arrs[nbx](i, j, k) < vof_tol + vof[nbx](i, j, k) = + vof[nbx](i, j, k) < vof_tol ? 0.0 - : (vof_arrs[nbx](i, j, k) > 1. - vof_tol - ? 1. - : vof_arrs[nbx](i, j, k)); + : (vof[nbx](i, j, k) > 1. - vof_tol ? 1. + : vof[nbx](i, j, k)); }); } @@ -45,15 +43,15 @@ void harmonize_vof( const amrex::MultiFab& mf_vof_original, const amrex::iMultiFab& mf_iblank) { - const auto& tg_vof_arrs = mf_vof_target.arrays(); - const auto& og_vof_arrs = mf_vof_original.const_arrays(); - const auto& iblank_arrs = mf_iblank.const_arrays(); + const auto& tg_vof = mf_vof_target.arrays(); + const auto& og_vof = mf_vof_original.const_arrays(); + const auto& iblank = mf_iblank.const_arrays(); amrex::ParallelFor( mf_vof_target, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { // Replace amr-wind vof values with originals - if (iblank_arrs[nbx](i, j, k) != -1) { - tg_vof_arrs[nbx](i, j, k) = og_vof_arrs[nbx](i, j, k); + if (iblank[nbx](i, j, k) != -1) { + tg_vof[nbx](i, j, k) = og_vof[nbx](i, j, k); } }); } @@ -64,46 +62,37 @@ void populate_normal_vector( const amrex::MultiFab& mf_vof, const amrex::iMultiFab& mf_iblank) { - const auto& normvec_arrs = mf_normvec.arrays(); - const auto& vof_arrs = mf_vof.const_arrays(); - const auto& iblank_arrs = mf_iblank.const_arrays(); + const auto& normvec = mf_normvec.arrays(); + const auto& vof = mf_vof.const_arrays(); + const auto& iblank = mf_iblank.const_arrays(); // Calculate gradients in each direction with centered diff amrex::ParallelFor( mf_normvec, mf_normvec.n_grow - amrex::IntVect(1), [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { // Neumann condition across nalu bdy int ibdy = - (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i - 1, j, k)) - ? -1 - : 0; + (iblank[nbx](i, j, k) != iblank[nbx](i - 1, j, k)) ? -1 : 0; int jbdy = - (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i, j - 1, k)) - ? -1 - : 0; + (iblank[nbx](i, j, k) != iblank[nbx](i, j - 1, k)) ? -1 : 0; int kbdy = - (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i, j, k - 1)) - ? -1 - : 0; + (iblank[nbx](i, j, k) != iblank[nbx](i, j, k - 1)) ? -1 : 0; // no cell should be isolated such that -1 and 1 are needed - ibdy = (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i + 1, j, k)) - ? +1 - : ibdy; - jbdy = (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i, j + 1, k)) - ? +1 - : jbdy; - kbdy = (iblank_arrs[nbx](i, j, k) != iblank_arrs[nbx](i, j, k + 1)) - ? +1 - : kbdy; + ibdy = + (iblank[nbx](i, j, k) != iblank[nbx](i + 1, j, k)) ? +1 : ibdy; + jbdy = + (iblank[nbx](i, j, k) != iblank[nbx](i, j + 1, k)) ? +1 : jbdy; + kbdy = + (iblank[nbx](i, j, k) != iblank[nbx](i, j, k + 1)) ? +1 : kbdy; // Calculate normal amrex::Real mx, my, mz, mmag; multiphase::youngs_finite_difference_normal_neumann( - i, j, k, ibdy, jbdy, kbdy, vof_arrs[nbx], mx, my, mz); + i, j, k, ibdy, jbdy, kbdy, vof[nbx], mx, my, mz); // Normalize normal mmag = std::sqrt(mx * mx + my * my + mz * mz + 1e-20); // Save normal - normvec_arrs[nbx](i, j, k, 0) = mx / mmag; - normvec_arrs[nbx](i, j, k, 1) = my / mmag; - normvec_arrs[nbx](i, j, k, 2) = mz / mmag; + normvec[nbx](i, j, k, 0) = mx / mmag; + normvec[nbx](i, j, k, 1) = my / mmag; + normvec[nbx](i, j, k, 2) = mz / mmag; }); } @@ -382,14 +371,14 @@ amrex::Real measure_convergence( // Set levelset field to another quantity to view in plotfile for debugging void equate_field(amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src) { - const auto& dest_arrs = mf_dest.arrays(); - const auto& src_arrs = mf_src.const_arrays(); + const auto& dest = mf_dest.arrays(); + const auto& src = mf_src.const_arrays(); amrex::ParallelFor( mf_dest, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - dest_arrs[nbx](i, j, k) = std::sqrt( - src_arrs[nbx](i, j, k, 0) * src_arrs[nbx](i, j, k, 0) + - src_arrs[nbx](i, j, k, 1) * src_arrs[nbx](i, j, k, 1) + - src_arrs[nbx](i, j, k, 2) * src_arrs[nbx](i, j, k, 2)); + dest[nbx](i, j, k) = std::sqrt( + src[nbx](i, j, k, 0) * src[nbx](i, j, k, 0) + + src[nbx](i, j, k, 1) * src[nbx](i, j, k, 1) + + src[nbx](i, j, k, 2) * src[nbx](i, j, k, 2)); }); } @@ -402,20 +391,20 @@ void replace_gradp_hydrostatic( const amrex::Real grav_z, const bool is_pptb) { - const auto& gp_arrs = mf_gp.arrays(); - const auto& rho_arrs = mf_density.const_arrays(); - const auto& rho0_arrs = mf_refdens.const_arrays(); - const auto& iblank_arrs = mf_iblank.const_arrays(); + const auto& gp = mf_gp.arrays(); + const auto& rho = mf_density.const_arrays(); + const auto& rho0 = mf_refdens.const_arrays(); + const auto& iblank = mf_iblank.const_arrays(); amrex::ParallelFor( mf_gp, mf_gp.n_grow, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - if (iblank_arrs[nbx](i, j, k) == -1) { + if (iblank[nbx](i, j, k) == -1) { const amrex::Real dfac = - is_pptb ? rho_arrs[nbx](i, j, k) - rho0_arrs[nbx](i, j, k) - : rho_arrs[nbx](i, j, k); - gp_arrs[nbx](i, j, k, 0) = 0.; - gp_arrs[nbx](i, j, k, 1) = 0.; - gp_arrs[nbx](i, j, k, 2) = dfac * grav_z; + is_pptb ? rho[nbx](i, j, k) - rho0[nbx](i, j, k) + : rho[nbx](i, j, k); + gp[nbx](i, j, k, 0) = 0.; + gp[nbx](i, j, k, 1) = 0.; + gp[nbx](i, j, k, 2) = dfac * grav_z; } }); } @@ -426,16 +415,16 @@ void replace_gradp( const amrex::MultiFab& mf_gp0, const amrex::iMultiFab& mf_iblank) { - const auto gp_arrs = mf_gp.arrays(); - const auto gp0_arrs = mf_gp0.const_arrays(); - const auto iblank_arrs = mf_iblank.const_arrays(); + const auto gp = mf_gp.arrays(); + const auto gp0 = mf_gp0.const_arrays(); + const auto iblank = mf_iblank.const_arrays(); amrex::ParallelFor( mf_gp, mf_gp.n_grow, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - if (iblank_arrs[nbx](i, j, k) == -1) { - gp_arrs[nbx](i, j, k, 0) = gp0_arrs[nbx](i, j, k, 0); - gp_arrs[nbx](i, j, k, 1) = gp0_arrs[nbx](i, j, k, 1); - gp_arrs[nbx](i, j, k, 2) = gp0_arrs[nbx](i, j, k, 2); + if (iblank[nbx](i, j, k) == -1) { + gp[nbx](i, j, k, 0) = gp0[nbx](i, j, k, 0); + gp[nbx](i, j, k, 1) = gp0[nbx](i, j, k, 1); + gp[nbx](i, j, k, 2) = gp0[nbx](i, j, k, 2); } }); } @@ -447,16 +436,15 @@ void apply_pressure_gradient( const amrex::MultiFab& mf_gp, const amrex::Real scaling_factor) { - const auto& vel_arrs = mf_vel.arrays(); - const auto& rho_arrs = mf_density.const_arrays(); - const auto& gp_arrs = mf_gp.const_arrays(); + const auto& vel = mf_vel.arrays(); + const auto& rho = mf_density.const_arrays(); + const auto& gp = mf_gp.const_arrays(); amrex::ParallelFor( mf_vel, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - const amrex::Real soverrho = - scaling_factor / rho_arrs[nbx](i, j, k); - vel_arrs[nbx](i, j, k, 0) -= gp_arrs[nbx](i, j, k, 0) * soverrho; - vel_arrs[nbx](i, j, k, 1) -= gp_arrs[nbx](i, j, k, 1) * soverrho; - vel_arrs[nbx](i, j, k, 2) -= gp_arrs[nbx](i, j, k, 2) * soverrho; + const amrex::Real soverrho = scaling_factor / rho[nbx](i, j, k); + vel[nbx](i, j, k, 0) -= gp[nbx](i, j, k, 0) * soverrho; + vel[nbx](i, j, k, 1) -= gp[nbx](i, j, k, 1) * soverrho; + vel[nbx](i, j, k, 2) -= gp[nbx](i, j, k, 2) * soverrho; }); } From 1771c5d3d93f4bce019001dcded635c13bdf0930 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Tue, 11 Jun 2024 15:25:22 -0600 Subject: [PATCH 32/43] changes to include pressure sharpening and treat dx differently --- amr-wind/overset/OversetOps.cpp | 75 +++++++--- amr-wind/overset/overset_ops_K.H | 47 +++++++ amr-wind/overset/overset_ops_routines.H | 175 +++++++++++++++++++++++- 3 files changed, 274 insertions(+), 23 deletions(-) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 3c03ec7498..75c99693f1 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -190,29 +190,53 @@ void OversetOps::sharpen_nalu_data() auto& levelset = repo.get_field("levelset"); auto& rho = repo.get_field("density"); auto& velocity = repo.get_field("velocity"); + auto& gp_noghost = repo.get_field("gp"); + auto& p = repo.get_field("p"); // Create scratch fields for fluxes - // 5 components are vof, density, and 3 of velocity - auto flux_x = repo.create_scratch_field(5, 0, amr_wind::FieldLoc::XFACE); - auto flux_y = repo.create_scratch_field(5, 0, amr_wind::FieldLoc::YFACE); - auto flux_z = repo.create_scratch_field(5, 0, amr_wind::FieldLoc::ZFACE); - // Create scratch field for approximate signed distance function and grad - // (components 0-2 are gradient, 3 is asdf) + // 9 components are vof, density, 3 of velocity, 3 of gp, and psource flag + auto flux_x = repo.create_scratch_field(9, 1, amr_wind::FieldLoc::XFACE); + auto flux_y = repo.create_scratch_field(9, 1, amr_wind::FieldLoc::YFACE); + auto flux_z = repo.create_scratch_field(9, 1, amr_wind::FieldLoc::ZFACE); + // Create scratch field for pressure source term + auto p_src = repo.create_scratch_field(1, 0, amr_wind::FieldLoc::NODE); + // Create scratch fields for normal vector and target vof field auto normal_vec = repo.create_scratch_field(3, vof.num_grow()[0] - 1); - auto target_vof = repo.create_scratch_field(1, vof.num_grow()[0]); - - // Create levelset field + // Create scratch field for pressure gradient that has ghost cells + // Sharpening fluxes (at faces) have 1 ghost, requiring fields to have >= 2 + auto gp_scr = repo.create_scratch_field(3, 2); + auto& gp = *gp_scr; + + // Give initial max possible value of pseudo-velocity scale + auto dx_lev0 = (geom[0]).CellSizeArray(); + const amrex::Real max_pvscale = + std::min(std::min(dx_lev0[0], dx_lev0[1]), dx_lev0[2]); + amrex::Real pvscale = max_pvscale; + + // Prep things that do not change with iterations for (int lev = 0; lev < nlevels; ++lev) { // Thickness used here is user parameter, whatever works best - auto dx = (geom[lev]).CellSizeArray(); + const auto dx = (geom[lev]).CellSizeArray(); const amrex::Real i_th = m_relative_length_scale * std::cbrt(dx[0] * dx[1] * dx[2]); // Populate approximate signed distance function overset_ops::populate_psi(levelset(lev), vof(lev), i_th, m_asdf_tiny); + + // Populate gp scratch field + gp(lev).setVal(0.0); // for external boundaries + amrex::MultiFab::Copy(gp(lev), gp_noghost(lev), 0, 0, 3, 0); // nonghost + gp(lev).FillBoundary(geom[lev].periodicity()); // internal + + // Get pseudo-velocity scale, proportional to smallest dx in iblank + const amrex::Real pvscale_lev = + overset_ops::calculate_pseudo_velocity_scale( + iblank_cell(lev), dx, max_pvscale); + pvscale = std::min(pvscale, pvscale_lev); } amrex::Gpu::synchronize(); + amrex::ParallelDescriptor::ReduceRealMin(pvscale); // Convert levelset to vof to get target_vof m_mphase->levelset2vof(iblank_cell, *target_vof); @@ -262,13 +286,14 @@ void OversetOps::sharpen_nalu_data() // Sharpening fluxes for vof, density, and momentum overset_ops::populate_sharpen_fluxes( (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), vof(lev), - (*target_vof)(lev), (*normal_vec)(lev), velocity(lev), - m_upw_margin, m_mphase->rho1(), m_mphase->rho2()); + (*target_vof)(lev), (*normal_vec)(lev), velocity(lev), gp(lev), + rho(lev), pvscale, m_upw_margin, m_mphase->rho1(), + m_mphase->rho2()); // Process fluxes - overset_ops::process_fluxes( - (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), - iblank_cell(lev)); + overset_ops::process_fluxes_calc_src( + (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), (*p_src)(lev), + vof(lev), iblank_cell(lev)); // Measure convergence to determine if loop can stop if (calc_convg) { @@ -305,15 +330,20 @@ void OversetOps::sharpen_nalu_data() // Apply fluxes for (int lev = 0; lev < nlevels; ++lev) { + auto dx = (geom[lev]).CellSizeArray(); + overset_ops::apply_fluxes( - (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), vof(lev), - rho(lev), velocity(lev), ptfac, m_vof_tol); + (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), (*p_src)(lev), + vof(lev), rho(lev), velocity(lev), gp(lev), p(lev), dx, ptfac, + m_vof_tol); + + // Update ghost cells + vof(lev).FillBoundary(geom[lev].periodicity()); + velocity(lev).FillBoundary(geom[lev].periodicity()); + gp(lev).FillBoundary(geom[lev].periodicity()); } - amrex::Gpu::synchronize(); - // Fillpatch for ghost cells - vof.fillpatch((*m_sim_ptr).time().current_time()); - velocity.fillpatch((*m_sim_ptr).time().current_time()); + amrex::Gpu::synchronize(); // Update density (fillpatch built in) m_mphase->set_density_via_vof(); @@ -329,6 +359,9 @@ void OversetOps::sharpen_nalu_data() } } + // Fillpatch for pressure to make sure pressure stencil has all points + p.fillpatch((*m_sim_ptr).time().current_time()); + // Purely for debugging via visualization, should be removed later // Currently set up to overwrite the levelset field (not used as time // evolves) with the post-sharpening velocity magnitude diff --git a/amr-wind/overset/overset_ops_K.H b/amr-wind/overset/overset_ops_K.H index 6b03c6a63e..eede229031 100644 --- a/amr-wind/overset/overset_ops_K.H +++ b/amr-wind/overset/overset_ops_K.H @@ -103,6 +103,53 @@ void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE velocity_face( } } +void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE gp_rho_face( + int i, + int j, + int k, + int dir, + amrex::Array4 const& vof, + amrex::Array4 const& gp, + amrex::Array4 const& rho, + amrex::Real& uface, + amrex::Real& vface, + amrex::Real& wface) +{ + // Set up neighbor indices + int ii = i; + int jj = j; + int kk = k; + ii += (dir == 0) ? -1 : 0; + jj += (dir == 1) ? -1 : 0; + kk += (dir == 2) ? -1 : 0; + + // Gradient of phi normal to interface + const amrex::Real gphi = (vof(i, j, k) - vof(ii, jj, kk)); + + // Get velocities on both sides + const amrex::Real u_ = gp(i, j, k, 0) / rho(i, j, k); + const amrex::Real v_ = gp(i, j, k, 1) / rho(i, j, k); + const amrex::Real w_ = gp(i, j, k, 2) / rho(i, j, k); + const amrex::Real u_nb = gp(ii, jj, kk, 0) / rho(ii, jj, kk); + const amrex::Real v_nb = gp(ii, jj, kk, 1) / rho(ii, jj, kk); + const amrex::Real w_nb = gp(ii, jj, kk, 2) / rho(ii, jj, kk); + // Average value when gphi = 0 + uface = 0.5 * (u_ + u_nb); + vface = 0.5 * (v_ + v_nb); + wface = 0.5 * (w_ + w_nb); + // Use simple upwinding + if (gphi > 0.0) { + uface = u_nb; + vface = v_nb; + wface = w_nb; + } + if (gphi < 0.0) { + uface = u_; + vface = v_; + wface = w_; + } +} + } // namespace amr_wind::overset_ops #endif \ No newline at end of file diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index 9e289f8ac2..f7232ba203 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -106,6 +106,9 @@ void populate_sharpen_fluxes( const amrex::MultiFab& mf_target_vof, const amrex::MultiFab& mf_norm, const amrex::MultiFab& mf_velocity, + const amrex::MultiFab& mf_gp, + const amrex::MultiFab& mf_density, + const amrex::Real Gamma, const amrex::Real margin, const amrex::Real rho1, const amrex::Real rho2) @@ -117,6 +120,8 @@ void populate_sharpen_fluxes( const auto& tg_vof = mf_target_vof.const_arrays(); const auto& norm = mf_norm.const_arrays(); const auto& vel = mf_velocity.const_arrays(); + const auto& gp = mf_gp.const_arrays(); + const auto& rho = mf_density.const_arrays(); amrex::ParallelFor( mf_fx, mf_fx.n_grow, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { @@ -133,6 +138,14 @@ void populate_sharpen_fluxes( fx[nbx](i, j, k, 2) = flux * uf; fx[nbx](i, j, k, 3) = flux * vf; fx[nbx](i, j, k, 4) = flux * wf; + // pressure gradient fluxes + gp_rho_face(i, j, k, 0, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); + fx[nbx](i, j, k, 5) = flux * uf; + fx[nbx](i, j, k, 6) = flux * vf; + fx[nbx](i, j, k, 7) = flux * wf; + // Turn "on" all flux faces, later modified in + // process_fluxes_calc_src + fx[nbx](i, j, k, 8) = 1.0; }); amrex::ParallelFor( mf_fy, mf_fy.n_grow, @@ -147,6 +160,11 @@ void populate_sharpen_fluxes( fy[nbx](i, j, k, 2) = flux * uf; fy[nbx](i, j, k, 3) = flux * vf; fy[nbx](i, j, k, 4) = flux * wf; + gp_rho_face(i, j, k, 1, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); + fy[nbx](i, j, k, 5) = flux * uf; + fy[nbx](i, j, k, 6) = flux * vf; + fy[nbx](i, j, k, 7) = flux * wf; + fy[nbx](i, j, k, 8) = 1.0; }); amrex::ParallelFor( mf_fz, mf_fz.n_grow, @@ -161,20 +179,32 @@ void populate_sharpen_fluxes( fz[nbx](i, j, k, 2) = flux * uf; fz[nbx](i, j, k, 3) = flux * vf; fz[nbx](i, j, k, 4) = flux * wf; + gp_rho_face(i, j, k, 2, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); + fz[nbx](i, j, k, 5) = flux * uf; + fz[nbx](i, j, k, 6) = flux * vf; + fz[nbx](i, j, k, 7) = flux * wf; + fz[nbx](i, j, k, 8) = 1.0; }); } -// Process reinitialization fluxes - zero non-internal to overset region -void process_fluxes( +// Process reinitialization fluxes - zero non-internal to overset region; +// also calculate pressure source / sink term as a function of fluxes +void process_fluxes_calc_src( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, + amrex::MultiFab& mf_psource, + const amrex::MultiFab& mf_vof, const amrex::iMultiFab& mf_iblank) { const auto& fx = mf_fx.arrays(); const auto& fy = mf_fy.arrays(); const auto& fz = mf_fz.arrays(); + const auto& sp = mf_psource.arrays(); + const auto& vof = mf_vof.const_arrays(); const auto& iblank = mf_iblank.const_arrays(); + constexpr amrex::Real tiny = std::numeric_limits::epsilon(); + // Zero fluxes based on iblank array amrex::ParallelFor( mf_fx, mf_fx.n_grow, mf_fx.n_comp, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { @@ -196,6 +226,120 @@ void process_fluxes( (iblank[nbx](i, j, k - 1) + iblank[nbx](i, j, k) > -2); fz[nbx](i, j, k, n) *= zero_all ? 0. : 1.; }); + // With knowledge of fluxes, compute pressure source term + amrex::ParallelFor( + mf_psource, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + // Get components of outer product between f and gradp/rho + // from interpolating between values at neighboring faces + const amrex::Real fxgpx_flux = + (fx[nbx](i, j, k, 5) + fx[nbx](i, j, k - 1, 5) + + fx[nbx](i, j - 1, k, 5) + fx[nbx](i, j - 1, k - 1, 5)) / + (fx[nbx](i, j, k, 8) + fx[nbx](i, j, k - 1, 8) + + fx[nbx](i, j - 1, k, 8) + fx[nbx](i, j - 1, k - 1, 8) + tiny); + const amrex::Real fxgpy_flux = + (fx[nbx](i, j, k, 6) + fx[nbx](i, j, k - 1, 6) + + fx[nbx](i, j - 1, k, 6) + fx[nbx](i, j - 1, k - 1, 6)) / + (fx[nbx](i, j, k, 8) + fx[nbx](i, j, k - 1, 8) + + fx[nbx](i, j - 1, k, 8) + fx[nbx](i, j - 1, k - 1, 8) + tiny); + const amrex::Real fxgpz_flux = + (fx[nbx](i, j, k, 7) + fx[nbx](i, j, k - 1, 7) + + fx[nbx](i, j - 1, k, 7) + fx[nbx](i, j - 1, k - 1, 7)) / + (fx[nbx](i, j, k, 8) + fx[nbx](i, j, k - 1, 8) + + fx[nbx](i, j - 1, k, 8) + fx[nbx](i, j - 1, k - 1, 8) + tiny); + + const amrex::Real fygpx_flux = + (fy[nbx](i, j, k, 5) + fy[nbx](i - 1, j, k, 5) + + fy[nbx](i, j, k - 1, 5) + fy[nbx](i - 1, j, k - 1, 5)) / + (fy[nbx](i, j, k, 8) + fy[nbx](i - 1, j, k, 8) + + fy[nbx](i, j, k - 1, 8) + fy[nbx](i - 1, j, k - 1, 8) + tiny); + const amrex::Real fygpy_flux = + (fy[nbx](i, j, k, 6) + fy[nbx](i - 1, j, k, 6) + + fy[nbx](i, j, k - 1, 6) + fy[nbx](i - 1, j, k - 1, 6)) / + (fy[nbx](i, j, k, 8) + fy[nbx](i - 1, j, k, 8) + + fy[nbx](i, j, k - 1, 8) + fy[nbx](i - 1, j, k - 1, 8) + tiny); + const amrex::Real fygpz_flux = + (fy[nbx](i, j, k, 7) + fy[nbx](i - 1, j, k, 7) + + fy[nbx](i, j, k - 1, 7) + fy[nbx](i - 1, j, k - 1, 7)) / + (fy[nbx](i, j, k, 8) + fy[nbx](i - 1, j, k, 8) + + fy[nbx](i, j, k - 1, 8) + fy[nbx](i - 1, j, k - 1, 8) + tiny); + + const amrex::Real fzgpx_flux = + (fz[nbx](i, j, k, 5) + fz[nbx](i - 1, j, k, 5) + + fz[nbx](i, j - 1, k, 5) + fz[nbx](i - 1, j - 1, k, 5)) / + (fz[nbx](i, j, k, 8) + fz[nbx](i - 1, j, k, 8) + + fz[nbx](i, j - 1, k, 8) + fz[nbx](i - 1, j - 1, k, 8) + tiny); + const amrex::Real fzgpy_flux = + (fz[nbx](i, j, k, 6) + fz[nbx](i - 1, j, k, 6) + + fz[nbx](i, j - 1, k, 6) + fz[nbx](i - 1, j - 1, k, 6)) / + (fz[nbx](i, j, k, 8) + fz[nbx](i - 1, j, k, 8) + + fz[nbx](i, j - 1, k, 8) + fz[nbx](i - 1, j - 1, k, 8) + tiny); + const amrex::Real fzgpz_flux = + (fz[nbx](i, j, k, 7) + fz[nbx](i - 1, j, k, 7) + + fz[nbx](i, j - 1, k, 7) + fz[nbx](i - 1, j - 1, k, 7)) / + (fz[nbx](i, j, k, 8) + fz[nbx](i - 1, j, k, 8) + + fz[nbx](i, j - 1, k, 8) + fz[nbx](i - 1, j - 1, k, 8) + tiny); + + // Calculate interface normal at pressure node + amrex::Real n0 = + ((vof[nbx](i, j, k) - vof[nbx](i - 1, j, k)) * + fx[nbx](i, j, k, 8) + + (vof[nbx](i, j - 1, k) - vof[nbx](i - 1, j - 1, k)) * + fx[nbx](i, j - 1, k, 8) + + (vof[nbx](i, j, k - 1) - vof[nbx](i - 1, j, k - 1)) * + fx[nbx](i, j, k - 1, 8) + + (vof[nbx](i, j - 1, k - 1) - vof[nbx](i - 1, j - 1, k - 1)) * + fx[nbx](i, j - 1, k - 1, 8)) / + (fx[nbx](i, j, k, 8) + fx[nbx](i, j, k - 1, 8) + + fx[nbx](i, j - 1, k, 8) + fx[nbx](i, j - 1, k - 1, 8) + tiny); + amrex::Real n1 = + ((vof[nbx](i, j, k) - vof[nbx](i, j - 1, k)) * + fy[nbx](i, j, k, 8) + + (vof[nbx](i - 1, j, k) - vof[nbx](i - 1, j - 1, k)) * + fy[nbx](i - 1, j, k, 8) + + (vof[nbx](i, j, k - 1) - vof[nbx](i, j - 1, k - 1)) * + fy[nbx](i, j, k - 1, 8) + + (vof[nbx](i - 1, j, k - 1) - vof[nbx](i - 1, j - 1, k - 1)) * + fy[nbx](i - 1, j, k - 1, 8)) / + (fy[nbx](i, j, k, 8) + fy[nbx](i, j, k - 1, 8) + + fy[nbx](i - 1, j, k, 8) + fy[nbx](i - 1, j, k - 1, 8) + tiny); + amrex::Real n2 = + ((vof[nbx](i, j, k) - vof[nbx](i, j, k - 1)) * + fz[nbx](i, j, k, 8) + + (vof[nbx](i - 1, j, k) - vof[nbx](i - 1, j, k - 1)) * + fz[nbx](i - 1, j, k, 8) + + (vof[nbx](i, j - 1, k) - vof[nbx](i, j - 1, k - 1)) * + fz[nbx](i, j - 1, k, 8) + + (vof[nbx](i - 1, j - 1, k) - vof[nbx](i - 1, j - 1, k - 1)) * + fz[nbx](i - 1, j - 1, k, 8)) / + (fz[nbx](i, j, k, 8) + fz[nbx](i, j - 1, k, 8) + + fz[nbx](i - 1, j, k, 8) + fz[nbx](i - 1, j - 1, k, 8) + tiny); + // Normalize vector + amrex::Real nmag = std::sqrt(n0 * n0 + n1 * n1 + n2 * n2) + tiny; + n0 /= nmag; + n1 /= nmag; + n2 /= nmag; + + // Perform double contraction + sp[nbx](i, j, k) = fxgpx_flux * n0 * n0 + fxgpy_flux * n0 * n1 + + fxgpz_flux * n0 * n2 + fygpx_flux * n1 * n0 + + fygpy_flux * n1 * n1 + fygpz_flux * n1 * n2 + + fzgpx_flux * n2 * n0 + fzgpy_flux * n2 * n1 + + fzgpz_flux * n2 * n2; + }); +} + +amrex::Real calculate_pseudo_velocity_scale( + const amrex::iMultiFab& mf_iblank, + const amrex::GpuArray dx, + const amrex::Real pvmax) +{ + // Get minimum length scale from dx + const amrex::Real dx_min = std::min(std::min(dx[0], dx[1]), dx[2]); + // The dx of this level should be considered if iblank = -1 here + // Otherwise, set to max possible value for this mesh (level 0 values) + const amrex::Real pvscale = (mf_iblank.min(0) == -1) ? dx_min : pvmax; + return pvscale; } // Calculate a type of CFL by measuring how much % VOF is being removed per cell @@ -273,18 +417,25 @@ void apply_fluxes( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, + amrex::MultiFab& mf_psource, amrex::MultiFab& mf_vof, amrex::MultiFab& mf_dens, amrex::MultiFab& mf_vel, + amrex::MultiFab& mf_gp, + amrex::MultiFab& mf_pressure, + amrex::GpuArray dx, amrex::Real ptfac, amrex::Real vof_tol) { const auto& fx = mf_fx.const_arrays(); const auto& fy = mf_fy.const_arrays(); const auto& fz = mf_fz.const_arrays(); + const auto& sp = mf_psource.const_arrays(); const auto& vof = mf_vof.arrays(); const auto& dens = mf_dens.arrays(); const auto& vel = mf_vel.arrays(); + const auto& gp = mf_gp.arrays(); + const auto& p = mf_pressure.arrays(); amrex::ParallelFor( mf_vof, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { @@ -315,6 +466,21 @@ void apply_fluxes( ptfac * ((fx[nbx](i + 1, j, k, 4) - fx[nbx](i, j, k, 4)) + (fy[nbx](i, j + 1, k, 4) - fy[nbx](i, j, k, 4)) + (fz[nbx](i, j, k + 1, 4) - fz[nbx](i, j, k, 4)))); + gp[nbx](i, j, k, 0) += + ptfac * + ((fx[nbx](i + 1, j, k, 5) - fx[nbx](i, j, k, 5)) / dx[0] + + (fy[nbx](i, j + 1, k, 5) - fy[nbx](i, j, k, 5)) / dx[1] + + (fz[nbx](i, j, k + 1, 5) - fz[nbx](i, j, k, 5)) / dx[2]); + gp[nbx](i, j, k, 1) += + ptfac * + ((fx[nbx](i + 1, j, k, 6) - fx[nbx](i, j, k, 6)) / dx[0] + + (fy[nbx](i, j + 1, k, 6) - fy[nbx](i, j, k, 6)) / dx[1] + + (fz[nbx](i, j, k + 1, 6) - fz[nbx](i, j, k, 6)) / dx[2]); + gp[nbx](i, j, k, 2) += + ptfac * + ((fx[nbx](i + 1, j, k, 7) - fx[nbx](i, j, k, 7)) / dx[0] + + (fy[nbx](i, j + 1, k, 7) - fy[nbx](i, j, k, 7)) / dx[1] + + (fz[nbx](i, j, k + 1, 7) - fz[nbx](i, j, k, 7)) / dx[2]); // Ensure vof is bounded vof[nbx](i, j, k) = @@ -324,6 +490,11 @@ void apply_fluxes( : vof[nbx](i, j, k)); // Density bounds are enforced elsewhere }); + amrex::ParallelFor( + mf_pressure, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + p[nbx](i, j, k) += ptfac * sp[nbx](i, j, k); + }); } // Get the size of the smallest VOF flux to quantify convergence From 2d44311d3a11b9aac461b27e1b3e28eb14f4a846 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Wed, 10 Jul 2024 09:09:21 -0600 Subject: [PATCH 33/43] const --- amr-wind/overset/OversetOps.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 75c99693f1..e697fd5c5f 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -209,7 +209,7 @@ void OversetOps::sharpen_nalu_data() auto& gp = *gp_scr; // Give initial max possible value of pseudo-velocity scale - auto dx_lev0 = (geom[0]).CellSizeArray(); + const auto dx_lev0 = (geom[0]).CellSizeArray(); const amrex::Real max_pvscale = std::min(std::min(dx_lev0[0], dx_lev0[1]), dx_lev0[2]); amrex::Real pvscale = max_pvscale; From 4de5cc971ce6b56f5a3237f52c42f56bf412ea1d Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Wed, 10 Jul 2024 09:23:20 -0600 Subject: [PATCH 34/43] put Gamma back in --- amr-wind/overset/overset_ops_routines.H | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index f7232ba203..c450a19fbf 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -126,8 +126,9 @@ void populate_sharpen_fluxes( mf_fx, mf_fx.n_grow, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { // vof flux - amrex::Real flux = alpha_flux( - i, j, k, 0, margin, vof[nbx], tg_vof[nbx], norm[nbx]); + amrex::Real flux = Gamma * alpha_flux( + i, j, k, 0, margin, vof[nbx], + tg_vof[nbx], norm[nbx]); fx[nbx](i, j, k, 0) = flux; // density flux flux *= (rho1 - rho2); @@ -150,8 +151,9 @@ void populate_sharpen_fluxes( amrex::ParallelFor( mf_fy, mf_fy.n_grow, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - amrex::Real flux = alpha_flux( - i, j, k, 1, margin, vof[nbx], tg_vof[nbx], norm[nbx]); + amrex::Real flux = Gamma * alpha_flux( + i, j, k, 1, margin, vof[nbx], + tg_vof[nbx], norm[nbx]); fy[nbx](i, j, k, 0) = flux; flux *= (rho1 - rho2); fy[nbx](i, j, k, 1) = flux; @@ -169,8 +171,9 @@ void populate_sharpen_fluxes( amrex::ParallelFor( mf_fz, mf_fz.n_grow, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - amrex::Real flux = alpha_flux( - i, j, k, 2, margin, vof[nbx], tg_vof[nbx], norm[nbx]); + amrex::Real flux = Gamma * alpha_flux( + i, j, k, 2, margin, vof[nbx], + tg_vof[nbx], norm[nbx]); fz[nbx](i, j, k, 0) = flux; flux *= (rho1 - rho2); fz[nbx](i, j, k, 1) = flux; From d0a06027092141b23a7f4bec622f90d644b58977 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Wed, 10 Jul 2024 11:33:01 -0600 Subject: [PATCH 35/43] divide by dx as intended, add some consts --- amr-wind/overset/overset_ops_routines.H | 49 ++++++++++++++----------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index c450a19fbf..cea69c2b64 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -417,18 +417,18 @@ amrex::Real calculate_pseudo_dt_flux( // Apply reinitialization fluxes to modify fields void apply_fluxes( - amrex::MultiFab& mf_fx, - amrex::MultiFab& mf_fy, - amrex::MultiFab& mf_fz, - amrex::MultiFab& mf_psource, + const amrex::MultiFab& mf_fx, + const amrex::MultiFab& mf_fy, + const amrex::MultiFab& mf_fz, + const amrex::MultiFab& mf_psource, amrex::MultiFab& mf_vof, amrex::MultiFab& mf_dens, amrex::MultiFab& mf_vel, amrex::MultiFab& mf_gp, amrex::MultiFab& mf_pressure, - amrex::GpuArray dx, - amrex::Real ptfac, - amrex::Real vof_tol) + const amrex::GpuArray dx, + const amrex::Real ptfac, + const amrex::Real vof_tol) { const auto& fx = mf_fx.const_arrays(); const auto& fy = mf_fy.const_arrays(); @@ -444,31 +444,36 @@ void apply_fluxes( mf_vof, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { const amrex::Real olddens = dens[nbx](i, j, k); vof[nbx](i, j, k) += - ptfac * ((fx[nbx](i + 1, j, k, 0) - fx[nbx](i, j, k, 0)) + - (fy[nbx](i, j + 1, k, 0) - fy[nbx](i, j, k, 0)) + - (fz[nbx](i, j, k + 1, 0) - fz[nbx](i, j, k, 0))); + ptfac * + ((fx[nbx](i + 1, j, k, 0) - fx[nbx](i, j, k, 0)) / dx[0] + + (fy[nbx](i, j + 1, k, 0) - fy[nbx](i, j, k, 0)) / dx[1] + + (fz[nbx](i, j, k + 1, 0) - fz[nbx](i, j, k, 0)) / dx[2]); dens[nbx](i, j, k) += - ptfac * ((fx[nbx](i + 1, j, k, 1) - fx[nbx](i, j, k, 1)) + - (fy[nbx](i, j + 1, k, 1) - fy[nbx](i, j, k, 1)) + - (fz[nbx](i, j, k + 1, 1) - fz[nbx](i, j, k, 1))); + ptfac * + ((fx[nbx](i + 1, j, k, 1) - fx[nbx](i, j, k, 1)) / dx[0] + + (fy[nbx](i, j + 1, k, 1) - fy[nbx](i, j, k, 1)) / dx[1] + + (fz[nbx](i, j, k + 1, 1) - fz[nbx](i, j, k, 1)) / dx[2]); vel[nbx](i, j, k, 0) = 1.0 / dens[nbx](i, j, k) * (olddens * vel[nbx](i, j, k, 0) + - ptfac * ((fx[nbx](i + 1, j, k, 2) - fx[nbx](i, j, k, 2)) + - (fy[nbx](i, j + 1, k, 2) - fy[nbx](i, j, k, 2)) + - (fz[nbx](i, j, k + 1, 2) - fz[nbx](i, j, k, 2)))); + ptfac * + ((fx[nbx](i + 1, j, k, 2) - fx[nbx](i, j, k, 2)) / dx[0] + + (fy[nbx](i, j + 1, k, 2) - fy[nbx](i, j, k, 2)) / dx[1] + + (fz[nbx](i, j, k + 1, 2) - fz[nbx](i, j, k, 2)) / dx[2])); vel[nbx](i, j, k, 1) = 1.0 / dens[nbx](i, j, k) * (olddens * vel[nbx](i, j, k, 1) + - ptfac * ((fx[nbx](i + 1, j, k, 3) - fx[nbx](i, j, k, 3)) + - (fy[nbx](i, j + 1, k, 3) - fy[nbx](i, j, k, 3)) + - (fz[nbx](i, j, k + 1, 3) - fz[nbx](i, j, k, 3)))); + ptfac * + ((fx[nbx](i + 1, j, k, 3) - fx[nbx](i, j, k, 3)) / dx[0] + + (fy[nbx](i, j + 1, k, 3) - fy[nbx](i, j, k, 3)) / dx[1] + + (fz[nbx](i, j, k + 1, 3) - fz[nbx](i, j, k, 3)) / dx[2])); vel[nbx](i, j, k, 2) = 1.0 / dens[nbx](i, j, k) * (olddens * vel[nbx](i, j, k, 2) + - ptfac * ((fx[nbx](i + 1, j, k, 4) - fx[nbx](i, j, k, 4)) + - (fy[nbx](i, j + 1, k, 4) - fy[nbx](i, j, k, 4)) + - (fz[nbx](i, j, k + 1, 4) - fz[nbx](i, j, k, 4)))); + ptfac * + ((fx[nbx](i + 1, j, k, 4) - fx[nbx](i, j, k, 4)) / dx[0] + + (fy[nbx](i, j + 1, k, 4) - fy[nbx](i, j, k, 4)) / dx[1] + + (fz[nbx](i, j, k + 1, 4) - fz[nbx](i, j, k, 4)) / dx[2])); gp[nbx](i, j, k, 0) += ptfac * ((fx[nbx](i + 1, j, k, 5) - fx[nbx](i, j, k, 5)) / dx[0] + From af88592cb8d908c0763b0a138d0135824e751762 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Wed, 10 Jul 2024 14:37:28 -0600 Subject: [PATCH 36/43] changing ops, adding consts --- amr-wind/overset/OversetOps.H | 4 ++-- amr-wind/overset/OversetOps.cpp | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/amr-wind/overset/OversetOps.H b/amr-wind/overset/OversetOps.H index 0658019d4d..9d6dcb254e 100644 --- a/amr-wind/overset/OversetOps.H +++ b/amr-wind/overset/OversetOps.H @@ -30,8 +30,8 @@ private: // (will be removed soon) bool m_perturb_p{false}; - // This is the only option for now - const bool m_use_hydrostatic_gradp{true}; + // To avoid using sharpened pressure, use hydrostatic pressure + bool m_use_hydrostatic_gradp{false}; // Coupling options bool m_disable_nodal_proj{false}; diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index e697fd5c5f..1bde730b42 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -22,6 +22,7 @@ void OversetOps::initialize(CFDSim& sim) pp.query("reinit_target_cutoff", m_target_cutoff); // Queries for coupling options + pp.query("use_hydrostatic_gradp", m_use_hydrostatic_gradp); pp.query("replace_gradp_postsolve", m_replace_gradp_postsolve); // OversetOps does not control these coupling options, merely reports them pp.query("disable_coupled_nodal_proj", m_disable_nodal_proj); @@ -62,7 +63,7 @@ void OversetOps::pre_advance_work() // If pressure gradient will be replaced, store current pressure gradient if (m_replace_gradp_postsolve) { - auto& gp = (*m_sim_ptr).repo().get_field("gp"); + const auto& gp = (*m_sim_ptr).repo().get_field("gp"); for (int lev = 0; lev < (*m_sim_ptr).repo().num_active_levels(); ++lev) { amrex::MultiFab::Copy( @@ -307,7 +308,7 @@ void OversetOps::sharpen_nalu_data() // Average down fluxes across levels for consistency for (int lev = nlevels - 1; lev > 0; --lev) { - amrex::IntVect rr = + const amrex::IntVect rr = geom[lev].Domain().size() / geom[lev - 1].Domain().size(); amrex::average_down_faces( GetArrOfConstPtrs(fluxes[lev]), fluxes[lev - 1], rr, @@ -330,7 +331,7 @@ void OversetOps::sharpen_nalu_data() // Apply fluxes for (int lev = 0; lev < nlevels; ++lev) { - auto dx = (geom[lev]).CellSizeArray(); + const auto dx = (geom[lev]).CellSizeArray(); overset_ops::apply_fluxes( (*flux_x)(lev), (*flux_y)(lev), (*flux_z)(lev), (*p_src)(lev), From 1f45a584173ed072fbf3787974a69dde2c516510 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Wed, 10 Jul 2024 14:37:40 -0600 Subject: [PATCH 37/43] gp_rho unit test --- .../multiphase/test_vof_overset_ops.cpp | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/unit_tests/multiphase/test_vof_overset_ops.cpp b/unit_tests/multiphase/test_vof_overset_ops.cpp index 82b80083fe..897c73664f 100644 --- a/unit_tests/multiphase/test_vof_overset_ops.cpp +++ b/unit_tests/multiphase/test_vof_overset_ops.cpp @@ -141,6 +141,51 @@ void init_velocity_etc( }); } +void init_gp_rho_etc( + amr_wind::Field& gp, + amr_wind::Field& rho, + amr_wind::Field& vof, + const int& dir, + const amrex::Real& rho1, + const amrex::Real& rho2) +{ + run_algorithm(gp, [&](const int lev, const amrex::MFIter& mfi) { + auto gp_arr = gp(lev).array(mfi); + auto rho_arr = rho(lev).array(mfi); + auto vof_arr = vof(lev).array(mfi); + const auto& bx = mfi.validbox(); + amrex::ParallelFor( + grow(bx, 2), [=] AMREX_GPU_DEVICE(int i, int j, int k) { + gp_arr(i, j, k) = 0.0; + vof_arr(i, j, k) = 0.0; + const int idx = (dir == 0 ? i : (dir == 1 ? j : k)); + if (idx == -1) { + vof_arr(i, j, k) = 0.41; + gp_arr(i, j, k, 0) = 1.0; + gp_arr(i, j, k, 1) = 2.0; + gp_arr(i, j, k, 2) = 3.0; + } else if (idx == 0) { + vof_arr(i, j, k) = 0.55; + gp_arr(i, j, k, 0) = 1.5; + gp_arr(i, j, k, 1) = 2.5; + gp_arr(i, j, k, 2) = 3.5; + } else if (idx == 1) { + vof_arr(i, j, k) = 0.2; + gp_arr(i, j, k, 0) = 2.0; + gp_arr(i, j, k, 1) = 3.0; + gp_arr(i, j, k, 2) = 4.0; + } else if (idx == 2) { + vof_arr(i, j, k) = 0.2; + gp_arr(i, j, k, 0) = 2.5; + gp_arr(i, j, k, 1) = 3.5; + gp_arr(i, j, k, 2) = 4.5; + } + rho_arr(i, j, k) = + rho1 * vof_arr(i, j, k) + rho2 * (1. - vof_arr(i, j, k)); + }); + }); +} + void calc_alpha_flux( amr_wind::Field& flux, amr_wind::Field& vof, @@ -184,6 +229,30 @@ void calc_velocity_face( }); } +void calc_gp_rho_face( + amr_wind::Field& flux, + amr_wind::Field& gp, + amr_wind::Field& rho, + amr_wind::Field& vof, + const int& dir) +{ + run_algorithm(flux, [&](const int lev, const amrex::MFIter& mfi) { + auto f_arr = flux(lev).array(mfi); + auto vof_arr = vof(lev).array(mfi); + auto gp_arr = gp(lev).array(mfi); + auto rho_arr = rho(lev).array(mfi); + const auto& bx = mfi.validbox(); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + amrex::Real u_f, v_f, w_f; + amr_wind::overset_ops::gp_rho_face( + i, j, k, dir, vof_arr, gp_arr, rho_arr, u_f, v_f, w_f); + f_arr(i, j, k, 0) = u_f; + f_arr(i, j, k, 1) = v_f; + f_arr(i, j, k, 2) = w_f; + }); + }); +} + amrex::Real check_alpha_flux_impl(amr_wind::Field& flux, const int& dir) { amrex::Real error_total = 0; @@ -295,6 +364,67 @@ amrex::Real check_velocity_face_impl(amr_wind::Field& flux, const int& dir) } return error_total; } + +amrex::Real check_gp_rho_face_impl( + amr_wind::Field& flux, + const int& dir, + const amrex::Real& rho1, + const amrex::Real& rho2) +{ + amrex::Real error_total = 0; + + for (int lev = 0; lev < flux.repo().num_active_levels(); ++lev) { + + error_total += amrex::ReduceSum( + flux(lev), 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& f_arr) -> amrex::Real { + amrex::Real error = 0; + + amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { + const int idx = (dir == 0 ? i : (dir == 1 ? j : k)); + if (idx == 0) { + // gphi > 0, uwpind from the "left" + const amrex::Real rho_answer = + rho1 * 0.41 + rho2 * (1. - 0.41); + amrex::Real flux_answer = 1.0 / rho_answer; + error += std::abs(f_arr(i, j, k, 0) - flux_answer); + flux_answer = 2.0 / rho_answer; + error += std::abs(f_arr(i, j, k, 1) - flux_answer); + flux_answer = 3.0 / rho_answer; + error += std::abs(f_arr(i, j, k, 2) - flux_answer); + } else if (idx == 1) { + // gphi < 0, upwind from the "right" + const amrex::Real rho_answer = + rho1 * 0.2 + rho2 * (1. - 0.2); + amrex::Real flux_answer = 2.0 / rho_answer; + error += std::abs(f_arr(i, j, k, 0) - flux_answer); + flux_answer = 3.0 / rho_answer; + error += std::abs(f_arr(i, j, k, 1) - flux_answer); + flux_answer = 4.0 / rho_answer; + error += std::abs(f_arr(i, j, k, 2) - flux_answer); + } else if (idx == 2) { + // gphi = 0, average both sides + const amrex::Real rho_r = + rho1 * 0.2 + rho2 * (1. - 0.2); + const amrex::Real rho_l = + rho1 * 0.2 + rho2 * (1. - 0.2); + amrex::Real flux_answer = + 0.5 * (2.5 / rho_r + 2.0 / rho_l); + error += std::abs(f_arr(i, j, k, 0) - flux_answer); + flux_answer = 0.5 * (3.5 / rho_r + 3.0 / rho_l); + error += std::abs(f_arr(i, j, k, 1) - flux_answer); + flux_answer = 0.5 * (4.5 / rho_r + 4.0 / rho_l); + error += std::abs(f_arr(i, j, k, 2) - flux_answer); + } + }); + + return error; + }); + } + return error_total; +} } // namespace TEST_F(VOFOversetOps, alpha_flux) @@ -403,4 +533,61 @@ TEST_F(VOFOversetOps, velocity_face) EXPECT_NEAR(error_total, 0.0, 1e-15); } +TEST_F(VOFOversetOps, gp_rho_face) +{ + populate_parameters(); + initialize_mesh(); + + const amrex::Real rho_liq = 1000.; + const amrex::Real rho_gas = 1.; + + auto& repo = sim().repo(); + const int nghost = 3; + auto& gp = repo.declare_field("gp", 3, nghost); + auto& rho = repo.declare_field("density", 1, nghost); + auto& vof = repo.declare_field("vof", 1, nghost); + + // Create flux fields + auto& flux_x = + repo.declare_field("flux_x", 3, 0, 1, amr_wind::FieldLoc::XFACE); + auto& flux_y = + repo.declare_field("flux_y", 3, 0, 1, amr_wind::FieldLoc::YFACE); + auto& flux_z = + repo.declare_field("flux_z", 3, 0, 1, amr_wind::FieldLoc::ZFACE); + + // -- Variations in x direction -- // + int dir = 0; + // Initialize gp, rho, and vof + init_gp_rho_etc(gp, rho, vof, dir, rho_liq, rho_gas); + // Populate flux field + calc_gp_rho_face(flux_x, gp, rho, vof, dir); + // Check results + amrex::Real error_total = + check_gp_rho_face_impl(flux_x, dir, rho_liq, rho_gas); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); + + // -- Variations in y direction -- // + dir = 1; + // Initialize gp, rho, and vof + init_gp_rho_etc(gp, rho, vof, dir, rho_liq, rho_gas); + // Populate flux field + calc_gp_rho_face(flux_y, gp, rho, vof, dir); + // Check results + error_total = check_gp_rho_face_impl(flux_y, dir, rho_liq, rho_gas); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); + + // -- Variations in z direction -- // + dir = 2; + // Initialize gp, rho, and vof + init_gp_rho_etc(gp, rho, vof, dir, rho_liq, rho_gas); + // Populate flux field + calc_gp_rho_face(flux_z, gp, rho, vof, dir); + // Check results + error_total = check_gp_rho_face_impl(flux_z, dir, rho_liq, rho_gas); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-15); +} + } // namespace amr_wind_tests \ No newline at end of file From 49d1505c611206b4b72318be91338d9bcd47cd6a Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Thu, 18 Jul 2024 11:19:59 -0600 Subject: [PATCH 38/43] more unit tests --- amr-wind/overset/overset_ops_routines.H | 29 ++--- .../multiphase/test_vof_overset_ops.cpp | 105 +++++++++++++++++- 2 files changed, 119 insertions(+), 15 deletions(-) diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index cea69c2b64..4e97a2b88c 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -7,7 +7,7 @@ namespace amr_wind::overset_ops { // Populate approximate signed distance function using vof field -void populate_psi( +void static populate_psi( amrex::MultiFab& mf_psi, const amrex::MultiFab& mf_vof, const amrex::Real i_th, @@ -23,7 +23,7 @@ void populate_psi( } // Modify a vof field to not have values that barely differ from 0 or 1 -void process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) +void static process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) { const auto& vof = mf_vof.arrays(); amrex::ParallelFor( @@ -38,7 +38,7 @@ void process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) } // Combine overset target vof field with current non-overset vof field -void harmonize_vof( +void static harmonize_vof( amrex::MultiFab& mf_vof_target, const amrex::MultiFab& mf_vof_original, const amrex::iMultiFab& mf_iblank) @@ -57,7 +57,7 @@ void harmonize_vof( } // Populate normal vector with special treatment of overset boundary -void populate_normal_vector( +void static populate_normal_vector( amrex::MultiFab& mf_normvec, const amrex::MultiFab& mf_vof, const amrex::iMultiFab& mf_iblank) @@ -98,7 +98,7 @@ void populate_normal_vector( // Calculate fluxes for reinitialization over entire domain without concern for // overset bdy -void populate_sharpen_fluxes( +void static populate_sharpen_fluxes( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, @@ -192,7 +192,7 @@ void populate_sharpen_fluxes( // Process reinitialization fluxes - zero non-internal to overset region; // also calculate pressure source / sink term as a function of fluxes -void process_fluxes_calc_src( +void static process_fluxes_calc_src( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, @@ -332,7 +332,7 @@ void process_fluxes_calc_src( }); } -amrex::Real calculate_pseudo_velocity_scale( +amrex::Real static calculate_pseudo_velocity_scale( const amrex::iMultiFab& mf_iblank, const amrex::GpuArray dx, const amrex::Real pvmax) @@ -346,7 +346,7 @@ amrex::Real calculate_pseudo_velocity_scale( } // Calculate a type of CFL by measuring how much % VOF is being removed per cell -amrex::Real calculate_pseudo_dt_flux( +amrex::Real static calculate_pseudo_dt_flux( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, @@ -416,7 +416,7 @@ amrex::Real calculate_pseudo_dt_flux( } // Apply reinitialization fluxes to modify fields -void apply_fluxes( +void static apply_fluxes( const amrex::MultiFab& mf_fx, const amrex::MultiFab& mf_fy, const amrex::MultiFab& mf_fz, @@ -506,7 +506,7 @@ void apply_fluxes( } // Get the size of the smallest VOF flux to quantify convergence -amrex::Real measure_convergence( +amrex::Real static measure_convergence( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz) { // Get the maximum flux magnitude, but just for vof fluxes @@ -548,7 +548,8 @@ amrex::Real measure_convergence( } // Set levelset field to another quantity to view in plotfile for debugging -void equate_field(amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src) +void static equate_field( + amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src) { const auto& dest = mf_dest.arrays(); const auto& src = mf_src.const_arrays(); @@ -562,7 +563,7 @@ void equate_field(amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src) } // Replace pressure gradient with hydrostatic field in overset regions -void replace_gradp_hydrostatic( +void static replace_gradp_hydrostatic( amrex::MultiFab& mf_gp, const amrex::MultiFab& mf_density, const amrex::MultiFab& mf_refdens, @@ -589,7 +590,7 @@ void replace_gradp_hydrostatic( } // Swap pressure gradient values in overset region -void replace_gradp( +void static replace_gradp( amrex::MultiFab& mf_gp, const amrex::MultiFab& mf_gp0, const amrex::iMultiFab& mf_iblank) @@ -609,7 +610,7 @@ void replace_gradp( } // Apply pressure gradient to velocity field -void apply_pressure_gradient( +void static apply_pressure_gradient( amrex::MultiFab& mf_vel, const amrex::MultiFab& mf_density, const amrex::MultiFab& mf_gp, diff --git a/unit_tests/multiphase/test_vof_overset_ops.cpp b/unit_tests/multiphase/test_vof_overset_ops.cpp index 897c73664f..12331d24fc 100644 --- a/unit_tests/multiphase/test_vof_overset_ops.cpp +++ b/unit_tests/multiphase/test_vof_overset_ops.cpp @@ -1,7 +1,7 @@ #include "aw_test_utils/MeshTest.H" #include "aw_test_utils/iter_tools.H" #include "aw_test_utils/test_utils.H" -#include "amr-wind/overset/overset_ops_K.H" +#include "amr-wind/overset/overset_ops_routines.H" namespace amr_wind_tests { @@ -590,4 +590,107 @@ TEST_F(VOFOversetOps, gp_rho_face) EXPECT_NEAR(error_total, 0.0, 1e-15); } +TEST_F(VOFOversetOps, pseudo_vscale_dt) +{ + populate_parameters(); + initialize_mesh(); + + auto& repo = sim().repo(); + const int nghost = 3; + auto& vof = repo.declare_field("vof", 1, nghost); + auto& tg_vof = repo.declare_field("target_vof", 1, nghost); + auto& norm = repo.declare_field("int_normal", 3, nghost); + auto& iblank = repo.declare_int_field("iblank_cell", 1, nghost); + iblank.setVal(-1); + constexpr amrex::Real margin = 0.1; + constexpr amrex::Real convg_tol = 1e-8; + // With simple fluxes, pseudo dt should be 100% + constexpr amrex::Real pdt_answer = 1.0; + // With a single level, pseudo velocity scale should be dx of lev 0 + const auto dx_lev0 = repo.mesh().Geom(0).CellSizeArray(); + const amrex::Real pvs_answer = + std::min(std::min(dx_lev0[0], dx_lev0[1]), dx_lev0[2]); + + // Create flux fields + auto& flux_x = + repo.declare_field("flux_x", 1, 0, 1, amr_wind::FieldLoc::XFACE); + auto& flux_y = + repo.declare_field("flux_y", 1, 0, 1, amr_wind::FieldLoc::YFACE); + auto& flux_z = + repo.declare_field("flux_z", 1, 0, 1, amr_wind::FieldLoc::ZFACE); + + // -- Variations in x direction -- // + int dir = 0; + // Initialize vof and other fields + init_vof_etc(vof, tg_vof, norm, dir); + // Populate flux field + calc_alpha_flux(flux_x, vof, tg_vof, norm, dir, margin); + // Calculate pseudo dt + amrex::Real ptfac = 100.0; + for (int lev = 0; lev < repo.num_active_levels(); ++lev) { + const amrex::Real ptfac_lev = + amr_wind::overset_ops::calculate_pseudo_dt_flux( + flux_x(lev), flux_y(lev), flux_z(lev), vof(lev), convg_tol); + ptfac = amrex::min(ptfac, ptfac_lev); + } + amrex::ParallelDescriptor::ReduceRealMin(ptfac); + EXPECT_DOUBLE_EQ(ptfac, pdt_answer); + // Zero flux for subsequent tests + flux_x.setVal(0.0); + + // -- Variations in y direction -- // + dir = 1; + // Initialize vof and other fields + init_vof_etc(vof, tg_vof, norm, dir); + // Populate flux field + calc_alpha_flux(flux_y, vof, tg_vof, norm, dir, margin); + // Calculate pseudo dt + ptfac = 100.0; + for (int lev = 0; lev < repo.num_active_levels(); ++lev) { + const amrex::Real ptfac_lev = + amr_wind::overset_ops::calculate_pseudo_dt_flux( + flux_x(lev), flux_y(lev), flux_z(lev), vof(lev), convg_tol); + ptfac = amrex::min(ptfac, ptfac_lev); + } + amrex::ParallelDescriptor::ReduceRealMin(ptfac); + EXPECT_DOUBLE_EQ(ptfac, pdt_answer); + // Zero flux for subsequent test + flux_y.setVal(0.0); + + // -- Variations in z direction -- // + dir = 2; + // Initialize vof and other fields + init_vof_etc(vof, tg_vof, norm, dir); + // Populate flux field + calc_alpha_flux(flux_z, vof, tg_vof, norm, dir, margin); + // Calculate pseudo dt + ptfac = 100.0; + for (int lev = 0; lev < repo.num_active_levels(); ++lev) { + const amrex::Real ptfac_lev = + amr_wind::overset_ops::calculate_pseudo_dt_flux( + flux_x(lev), flux_y(lev), flux_z(lev), vof(lev), convg_tol); + ptfac = amrex::min(ptfac, ptfac_lev); + } + amrex::ParallelDescriptor::ReduceRealMin(ptfac); + EXPECT_DOUBLE_EQ(ptfac, pdt_answer); + + // Pseudo-velocity scale, should be the smallest dx in iblank region + const auto& iblank_cell = repo.get_int_field("iblank_cell"); + const amrex::Real max_pvscale = 100.; + amrex::Real pvscale = max_pvscale; + for (int lev = 0; lev < repo.num_active_levels(); ++lev) { + const amrex::Real pvscale_lev = + amr_wind::overset_ops::calculate_pseudo_velocity_scale( + iblank_cell(lev), repo.mesh().Geom(lev).CellSizeArray(), + max_pvscale); + pvscale = std::min(pvscale, pvscale_lev); + } + amrex::ParallelDescriptor::ReduceRealMin(pvscale); + EXPECT_DOUBLE_EQ(pvscale, pvs_answer); + + /* + pseudo velocity scale! + */ +} + } // namespace amr_wind_tests \ No newline at end of file From eaaea07291dbdd5e4ecbc267d8938bcb3ffa8ccd Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Thu, 18 Jul 2024 14:56:30 -0600 Subject: [PATCH 39/43] use tensors for double contraction, add unit test --- amr-wind/overset/overset_ops_K.H | 115 ++++++++++ amr-wind/overset/overset_ops_routines.H | 100 +------- .../multiphase/test_vof_overset_ops.cpp | 214 +++++++++++++++++- 3 files changed, 323 insertions(+), 106 deletions(-) diff --git a/amr-wind/overset/overset_ops_K.H b/amr-wind/overset/overset_ops_K.H index eede229031..a06f4c97e1 100644 --- a/amr-wind/overset/overset_ops_K.H +++ b/amr-wind/overset/overset_ops_K.H @@ -1,6 +1,7 @@ #ifndef OVERSET_OPS_K_H_ #define OVERSET_OPS_K_H_ +#include "amr-wind/core/vs/vector_space.H" #include namespace amr_wind::overset_ops { @@ -150,6 +151,120 @@ void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE gp_rho_face( } } +vs::Tensor AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE gp_flux_tensor( + const int i, + const int j, + const int k, + amrex::Array4 const& fx, + amrex::Array4 const& fy, + amrex::Array4 const& fz, + const amrex::Real tiny) +{ + // Averaging depends on iblank array, encoded in 8th component + const amrex::Real avg_fx = fx(i, j, k, 8) + fx(i, j, k - 1, 8) + + fx(i, j - 1, k, 8) + fx(i, j - 1, k - 1, 8) + + tiny; + const amrex::Real avg_fy = fy(i, j, k, 8) + fy(i - 1, j, k, 8) + + fy(i, j, k - 1, 8) + fy(i - 1, j, k - 1, 8) + + tiny; + const amrex::Real avg_fz = fz(i, j, k, 8) + fz(i - 1, j, k, 8) + + fz(i, j - 1, k, 8) + fz(i - 1, j - 1, k, 8) + + tiny; + + auto f_otimes_gradp = vs::Tensor::I(); + // Average fluxes (faces) to nodes where pressure exists + f_otimes_gradp.xx() = (fx(i, j, k, 5) + fx(i, j, k - 1, 5) + + fx(i, j - 1, k, 5) + fx(i, j - 1, k - 1, 5)) / + avg_fx; + f_otimes_gradp.xy() = (fx(i, j, k, 6) + fx(i, j, k - 1, 6) + + fx(i, j - 1, k, 6) + fx(i, j - 1, k - 1, 6)) / + avg_fx; + f_otimes_gradp.xz() = (fx(i, j, k, 7) + fx(i, j, k - 1, 7) + + fx(i, j - 1, k, 7) + fx(i, j - 1, k - 1, 7)) / + avg_fx; + f_otimes_gradp.yx() = (fy(i, j, k, 5) + fy(i - 1, j, k, 5) + + fy(i, j, k - 1, 5) + fy(i - 1, j, k - 1, 5)) / + avg_fy; + f_otimes_gradp.yy() = (fy(i, j, k, 6) + fy(i - 1, j, k, 6) + + fy(i, j, k - 1, 6) + fy(i - 1, j, k - 1, 6)) / + avg_fy; + f_otimes_gradp.yz() = (fy(i, j, k, 7) + fy(i - 1, j, k, 7) + + fy(i, j, k - 1, 7) + fy(i - 1, j, k - 1, 7)) / + avg_fy; + f_otimes_gradp.zx() = (fz(i, j, k, 5) + fz(i - 1, j, k, 5) + + fz(i, j - 1, k, 5) + fz(i - 1, j - 1, k, 5)) / + avg_fz; + f_otimes_gradp.zy() = (fz(i, j, k, 6) + fz(i - 1, j, k, 6) + + fz(i, j - 1, k, 6) + fz(i - 1, j - 1, k, 6)) / + avg_fz; + f_otimes_gradp.zz() = (fz(i, j, k, 7) + fz(i - 1, j, k, 7) + + fz(i, j - 1, k, 7) + fz(i - 1, j - 1, k, 7)) / + avg_fz; + + return f_otimes_gradp; +} + +vs::Tensor AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE normal_reinit_tensor( + const int i, + const int j, + const int k, + amrex::Array4 const& fx, + amrex::Array4 const& fy, + amrex::Array4 const& fz, + amrex::Array4 const& vof, + const amrex::Real tiny) +{ + // Averaging depends on iblank array, encoded in 8th component + const amrex::Real avg_fx = fx(i, j, k, 8) + fx(i, j, k - 1, 8) + + fx(i, j - 1, k, 8) + fx(i, j - 1, k - 1, 8) + + tiny; + const amrex::Real avg_fy = fy(i, j, k, 8) + fy(i - 1, j, k, 8) + + fy(i, j, k - 1, 8) + fy(i - 1, j, k - 1, 8) + + tiny; + const amrex::Real avg_fz = fz(i, j, k, 8) + fz(i - 1, j, k, 8) + + fz(i, j - 1, k, 8) + fz(i - 1, j - 1, k, 8) + + tiny; + + auto n_zeta = vs::Vector::one(); + // Get components of normal in each position + n_zeta.x() = + ((vof(i, j, k) - vof(i - 1, j, k)) * fx(i, j, k, 8) + + (vof(i, j - 1, k) - vof(i - 1, j - 1, k)) * fx(i, j - 1, k, 8) + + (vof(i, j, k - 1) - vof(i - 1, j, k - 1)) * fx(i, j, k - 1, 8) + + (vof(i, j - 1, k - 1) - vof(i - 1, j - 1, k - 1)) * + fx(i, j - 1, k - 1, 8)) / + avg_fx; + n_zeta.y() = + ((vof(i, j, k) - vof(i, j - 1, k)) * fy(i, j, k, 8) + + (vof(i - 1, j, k) - vof(i - 1, j - 1, k)) * fy(i - 1, j, k, 8) + + (vof(i, j, k - 1) - vof(i, j - 1, k - 1)) * fy(i, j, k - 1, 8) + + (vof(i - 1, j, k - 1) - vof(i - 1, j - 1, k - 1)) * + fy(i - 1, j, k - 1, 8)) / + avg_fy; + n_zeta.z() = + ((vof(i, j, k) - vof(i, j, k - 1)) * fz(i, j, k, 8) + + (vof(i - 1, j, k) - vof(i - 1, j, k - 1)) * fz(i - 1, j, k, 8) + + (vof(i, j - 1, k) - vof(i, j - 1, k - 1)) * fz(i, j - 1, k, 8) + + (vof(i - 1, j - 1, k) - vof(i - 1, j - 1, k - 1)) * + fz(i - 1, j - 1, k, 8)) / + avg_fz; + + n_zeta.normalize(); + + // To do outer product between vectors, set up tensors + auto n_tensor = vs::Tensor::I(); + auto n_tensor_T = vs::Tensor::I(); + n_tensor.rows(n_zeta, n_zeta, n_zeta); + n_tensor_T.cols(n_zeta, n_zeta, n_zeta); + // Multiply tensors elementwise + auto n_otimes_n = vs::Tensor::I(); + n_otimes_n.rows( + n_tensor.x() * n_tensor_T.x(), n_tensor.y() * n_tensor_T.y(), + n_tensor.z() * n_tensor_T.z()); + + return n_otimes_n; +} + } // namespace amr_wind::overset_ops #endif \ No newline at end of file diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index 4e97a2b88c..90cc9fe2da 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -233,102 +233,10 @@ void static process_fluxes_calc_src( amrex::ParallelFor( mf_psource, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - // Get components of outer product between f and gradp/rho - // from interpolating between values at neighboring faces - const amrex::Real fxgpx_flux = - (fx[nbx](i, j, k, 5) + fx[nbx](i, j, k - 1, 5) + - fx[nbx](i, j - 1, k, 5) + fx[nbx](i, j - 1, k - 1, 5)) / - (fx[nbx](i, j, k, 8) + fx[nbx](i, j, k - 1, 8) + - fx[nbx](i, j - 1, k, 8) + fx[nbx](i, j - 1, k - 1, 8) + tiny); - const amrex::Real fxgpy_flux = - (fx[nbx](i, j, k, 6) + fx[nbx](i, j, k - 1, 6) + - fx[nbx](i, j - 1, k, 6) + fx[nbx](i, j - 1, k - 1, 6)) / - (fx[nbx](i, j, k, 8) + fx[nbx](i, j, k - 1, 8) + - fx[nbx](i, j - 1, k, 8) + fx[nbx](i, j - 1, k - 1, 8) + tiny); - const amrex::Real fxgpz_flux = - (fx[nbx](i, j, k, 7) + fx[nbx](i, j, k - 1, 7) + - fx[nbx](i, j - 1, k, 7) + fx[nbx](i, j - 1, k - 1, 7)) / - (fx[nbx](i, j, k, 8) + fx[nbx](i, j, k - 1, 8) + - fx[nbx](i, j - 1, k, 8) + fx[nbx](i, j - 1, k - 1, 8) + tiny); - - const amrex::Real fygpx_flux = - (fy[nbx](i, j, k, 5) + fy[nbx](i - 1, j, k, 5) + - fy[nbx](i, j, k - 1, 5) + fy[nbx](i - 1, j, k - 1, 5)) / - (fy[nbx](i, j, k, 8) + fy[nbx](i - 1, j, k, 8) + - fy[nbx](i, j, k - 1, 8) + fy[nbx](i - 1, j, k - 1, 8) + tiny); - const amrex::Real fygpy_flux = - (fy[nbx](i, j, k, 6) + fy[nbx](i - 1, j, k, 6) + - fy[nbx](i, j, k - 1, 6) + fy[nbx](i - 1, j, k - 1, 6)) / - (fy[nbx](i, j, k, 8) + fy[nbx](i - 1, j, k, 8) + - fy[nbx](i, j, k - 1, 8) + fy[nbx](i - 1, j, k - 1, 8) + tiny); - const amrex::Real fygpz_flux = - (fy[nbx](i, j, k, 7) + fy[nbx](i - 1, j, k, 7) + - fy[nbx](i, j, k - 1, 7) + fy[nbx](i - 1, j, k - 1, 7)) / - (fy[nbx](i, j, k, 8) + fy[nbx](i - 1, j, k, 8) + - fy[nbx](i, j, k - 1, 8) + fy[nbx](i - 1, j, k - 1, 8) + tiny); - - const amrex::Real fzgpx_flux = - (fz[nbx](i, j, k, 5) + fz[nbx](i - 1, j, k, 5) + - fz[nbx](i, j - 1, k, 5) + fz[nbx](i - 1, j - 1, k, 5)) / - (fz[nbx](i, j, k, 8) + fz[nbx](i - 1, j, k, 8) + - fz[nbx](i, j - 1, k, 8) + fz[nbx](i - 1, j - 1, k, 8) + tiny); - const amrex::Real fzgpy_flux = - (fz[nbx](i, j, k, 6) + fz[nbx](i - 1, j, k, 6) + - fz[nbx](i, j - 1, k, 6) + fz[nbx](i - 1, j - 1, k, 6)) / - (fz[nbx](i, j, k, 8) + fz[nbx](i - 1, j, k, 8) + - fz[nbx](i, j - 1, k, 8) + fz[nbx](i - 1, j - 1, k, 8) + tiny); - const amrex::Real fzgpz_flux = - (fz[nbx](i, j, k, 7) + fz[nbx](i - 1, j, k, 7) + - fz[nbx](i, j - 1, k, 7) + fz[nbx](i - 1, j - 1, k, 7)) / - (fz[nbx](i, j, k, 8) + fz[nbx](i - 1, j, k, 8) + - fz[nbx](i, j - 1, k, 8) + fz[nbx](i - 1, j - 1, k, 8) + tiny); - - // Calculate interface normal at pressure node - amrex::Real n0 = - ((vof[nbx](i, j, k) - vof[nbx](i - 1, j, k)) * - fx[nbx](i, j, k, 8) + - (vof[nbx](i, j - 1, k) - vof[nbx](i - 1, j - 1, k)) * - fx[nbx](i, j - 1, k, 8) + - (vof[nbx](i, j, k - 1) - vof[nbx](i - 1, j, k - 1)) * - fx[nbx](i, j, k - 1, 8) + - (vof[nbx](i, j - 1, k - 1) - vof[nbx](i - 1, j - 1, k - 1)) * - fx[nbx](i, j - 1, k - 1, 8)) / - (fx[nbx](i, j, k, 8) + fx[nbx](i, j, k - 1, 8) + - fx[nbx](i, j - 1, k, 8) + fx[nbx](i, j - 1, k - 1, 8) + tiny); - amrex::Real n1 = - ((vof[nbx](i, j, k) - vof[nbx](i, j - 1, k)) * - fy[nbx](i, j, k, 8) + - (vof[nbx](i - 1, j, k) - vof[nbx](i - 1, j - 1, k)) * - fy[nbx](i - 1, j, k, 8) + - (vof[nbx](i, j, k - 1) - vof[nbx](i, j - 1, k - 1)) * - fy[nbx](i, j, k - 1, 8) + - (vof[nbx](i - 1, j, k - 1) - vof[nbx](i - 1, j - 1, k - 1)) * - fy[nbx](i - 1, j, k - 1, 8)) / - (fy[nbx](i, j, k, 8) + fy[nbx](i, j, k - 1, 8) + - fy[nbx](i - 1, j, k, 8) + fy[nbx](i - 1, j, k - 1, 8) + tiny); - amrex::Real n2 = - ((vof[nbx](i, j, k) - vof[nbx](i, j, k - 1)) * - fz[nbx](i, j, k, 8) + - (vof[nbx](i - 1, j, k) - vof[nbx](i - 1, j, k - 1)) * - fz[nbx](i - 1, j, k, 8) + - (vof[nbx](i, j - 1, k) - vof[nbx](i, j - 1, k - 1)) * - fz[nbx](i, j - 1, k, 8) + - (vof[nbx](i - 1, j - 1, k) - vof[nbx](i - 1, j - 1, k - 1)) * - fz[nbx](i - 1, j - 1, k, 8)) / - (fz[nbx](i, j, k, 8) + fz[nbx](i, j - 1, k, 8) + - fz[nbx](i - 1, j, k, 8) + fz[nbx](i - 1, j - 1, k, 8) + tiny); - // Normalize vector - amrex::Real nmag = std::sqrt(n0 * n0 + n1 * n1 + n2 * n2) + tiny; - n0 /= nmag; - n1 /= nmag; - n2 /= nmag; - - // Perform double contraction - sp[nbx](i, j, k) = fxgpx_flux * n0 * n0 + fxgpy_flux * n0 * n1 + - fxgpz_flux * n0 * n2 + fygpx_flux * n1 * n0 + - fygpy_flux * n1 * n1 + fygpz_flux * n1 * n2 + - fzgpx_flux * n2 * n0 + fzgpy_flux * n2 * n1 + - fzgpz_flux * n2 * n2; + sp[nbx](i, j, k) = + gp_flux_tensor(i, j, k, fx[nbx], fy[nbx], fz[nbx], tiny) && + normal_reinit_tensor( + i, j, k, fx[nbx], fy[nbx], fz[nbx], vof[nbx], tiny); }); } diff --git a/unit_tests/multiphase/test_vof_overset_ops.cpp b/unit_tests/multiphase/test_vof_overset_ops.cpp index 12331d24fc..34dffcec04 100644 --- a/unit_tests/multiphase/test_vof_overset_ops.cpp +++ b/unit_tests/multiphase/test_vof_overset_ops.cpp @@ -234,7 +234,9 @@ void calc_gp_rho_face( amr_wind::Field& gp, amr_wind::Field& rho, amr_wind::Field& vof, - const int& dir) + const int& dir, + const int& ioff = 0, + bool random_fflag = false) { run_algorithm(flux, [&](const int lev, const amrex::MFIter& mfi) { auto f_arr = flux(lev).array(mfi); @@ -242,13 +244,138 @@ void calc_gp_rho_face( auto gp_arr = gp(lev).array(mfi); auto rho_arr = rho(lev).array(mfi); const auto& bx = mfi.validbox(); + amrex::ParallelFor( + grow(bx, flux.num_grow()), + [=] AMREX_GPU_DEVICE(int i, int j, int k) { + amrex::Real u_f, v_f, w_f; + amrex::Real fac = 1.0; + if (random_fflag) { + fac = ((i + j + k) % 8 == 0) ? 0. : fac; + f_arr(i, j, k, 3 + ioff) = fac; + } + amr_wind::overset_ops::gp_rho_face( + i, j, k, dir, vof_arr, gp_arr, rho_arr, u_f, v_f, w_f); + f_arr(i, j, k, 0 + ioff) = u_f * fac; + f_arr(i, j, k, 1 + ioff) = v_f * fac; + f_arr(i, j, k, 2 + ioff) = w_f * fac; + }); + }); +} + +void calc_psrc( + amr_wind::Field& f_fx, + amr_wind::Field& f_fy, + amr_wind::Field& f_fz, + amr_wind::Field& f_vof, + amr_wind::Field& f_psrc, + amr_wind::Field& f_psrc_manual) +{ + constexpr amrex::Real tiny = std::numeric_limits::epsilon(); + run_algorithm(f_vof, [&](const int lev, const amrex::MFIter& mfi) { + auto fx = f_fx(lev).const_array(mfi); + auto fy = f_fy(lev).const_array(mfi); + auto fz = f_fz(lev).const_array(mfi); + auto vof = f_vof(lev).const_array(mfi); + auto psrc = f_psrc(lev).array(mfi); + auto psmn = f_psrc_manual(lev).array(mfi); + const auto& bx = mfi.validbox(); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - amrex::Real u_f, v_f, w_f; - amr_wind::overset_ops::gp_rho_face( - i, j, k, dir, vof_arr, gp_arr, rho_arr, u_f, v_f, w_f); - f_arr(i, j, k, 0) = u_f; - f_arr(i, j, k, 1) = v_f; - f_arr(i, j, k, 2) = w_f; + psrc(i, j, k) = amr_wind::overset_ops::gp_flux_tensor( + i, j, k, fx, fy, fz, tiny) && + amr_wind::overset_ops::normal_reinit_tensor( + i, j, k, fx, fy, fz, vof, tiny); + + const amrex::Real fxgpx_flux = + (fx(i, j, k, 5) + fx(i, j, k - 1, 5) + fx(i, j - 1, k, 5) + + fx(i, j - 1, k - 1, 5)) / + (fx(i, j, k, 8) + fx(i, j, k - 1, 8) + fx(i, j - 1, k, 8) + + fx(i, j - 1, k - 1, 8) + tiny); + const amrex::Real fxgpy_flux = + (fx(i, j, k, 6) + fx(i, j, k - 1, 6) + fx(i, j - 1, k, 6) + + fx(i, j - 1, k - 1, 6)) / + (fx(i, j, k, 8) + fx(i, j, k - 1, 8) + fx(i, j - 1, k, 8) + + fx(i, j - 1, k - 1, 8) + tiny); + const amrex::Real fxgpz_flux = + (fx(i, j, k, 7) + fx(i, j, k - 1, 7) + fx(i, j - 1, k, 7) + + fx(i, j - 1, k - 1, 7)) / + (fx(i, j, k, 8) + fx(i, j, k - 1, 8) + fx(i, j - 1, k, 8) + + fx(i, j - 1, k - 1, 8) + tiny); + + const amrex::Real fygpx_flux = + (fy(i, j, k, 5) + fy(i - 1, j, k, 5) + fy(i, j, k - 1, 5) + + fy(i - 1, j, k - 1, 5)) / + (fy(i, j, k, 8) + fy(i - 1, j, k, 8) + fy(i, j, k - 1, 8) + + fy(i - 1, j, k - 1, 8) + tiny); + const amrex::Real fygpy_flux = + (fy(i, j, k, 6) + fy(i - 1, j, k, 6) + fy(i, j, k - 1, 6) + + fy(i - 1, j, k - 1, 6)) / + (fy(i, j, k, 8) + fy(i - 1, j, k, 8) + fy(i, j, k - 1, 8) + + fy(i - 1, j, k - 1, 8) + tiny); + const amrex::Real fygpz_flux = + (fy(i, j, k, 7) + fy(i - 1, j, k, 7) + fy(i, j, k - 1, 7) + + fy(i - 1, j, k - 1, 7)) / + (fy(i, j, k, 8) + fy(i - 1, j, k, 8) + fy(i, j, k - 1, 8) + + fy(i - 1, j, k - 1, 8) + tiny); + + const amrex::Real fzgpx_flux = + (fz(i, j, k, 5) + fz(i - 1, j, k, 5) + fz(i, j - 1, k, 5) + + fz(i - 1, j - 1, k, 5)) / + (fz(i, j, k, 8) + fz(i - 1, j, k, 8) + fz(i, j - 1, k, 8) + + fz(i - 1, j - 1, k, 8) + tiny); + const amrex::Real fzgpy_flux = + (fz(i, j, k, 6) + fz(i - 1, j, k, 6) + fz(i, j - 1, k, 6) + + fz(i - 1, j - 1, k, 6)) / + (fz(i, j, k, 8) + fz(i - 1, j, k, 8) + fz(i, j - 1, k, 8) + + fz(i - 1, j - 1, k, 8) + tiny); + const amrex::Real fzgpz_flux = + (fz(i, j, k, 7) + fz(i - 1, j, k, 7) + fz(i, j - 1, k, 7) + + fz(i - 1, j - 1, k, 7)) / + (fz(i, j, k, 8) + fz(i - 1, j, k, 8) + fz(i, j - 1, k, 8) + + fz(i - 1, j - 1, k, 8) + tiny); + + // Calculate interface normal at pressure node + amrex::Real n0 = + ((vof(i, j, k) - vof(i - 1, j, k)) * fx(i, j, k, 8) + + (vof(i, j - 1, k) - vof(i - 1, j - 1, k)) * + fx(i, j - 1, k, 8) + + (vof(i, j, k - 1) - vof(i - 1, j, k - 1)) * + fx(i, j, k - 1, 8) + + (vof(i, j - 1, k - 1) - vof(i - 1, j - 1, k - 1)) * + fx(i, j - 1, k - 1, 8)) / + (fx(i, j, k, 8) + fx(i, j, k - 1, 8) + fx(i, j - 1, k, 8) + + fx(i, j - 1, k - 1, 8) + tiny); + amrex::Real n1 = + ((vof(i, j, k) - vof(i, j - 1, k)) * fy(i, j, k, 8) + + (vof(i - 1, j, k) - vof(i - 1, j - 1, k)) * + fy(i - 1, j, k, 8) + + (vof(i, j, k - 1) - vof(i, j - 1, k - 1)) * + fy(i, j, k - 1, 8) + + (vof(i - 1, j, k - 1) - vof(i - 1, j - 1, k - 1)) * + fy(i - 1, j, k - 1, 8)) / + (fy(i, j, k, 8) + fy(i, j, k - 1, 8) + fy(i - 1, j, k, 8) + + fy(i - 1, j, k - 1, 8) + tiny); + amrex::Real n2 = + ((vof(i, j, k) - vof(i, j, k - 1)) * fz(i, j, k, 8) + + (vof(i - 1, j, k) - vof(i - 1, j, k - 1)) * + fz(i - 1, j, k, 8) + + (vof(i, j - 1, k) - vof(i, j - 1, k - 1)) * + fz(i, j - 1, k, 8) + + (vof(i - 1, j - 1, k) - vof(i - 1, j - 1, k - 1)) * + fz(i - 1, j - 1, k, 8)) / + (fz(i, j, k, 8) + fz(i, j - 1, k, 8) + fz(i - 1, j, k, 8) + + fz(i - 1, j - 1, k, 8) + tiny); + // Normalize vector + amrex::Real nmag = std::sqrt(n0 * n0 + n1 * n1 + n2 * n2) + tiny; + n0 /= nmag; + n1 /= nmag; + n2 /= nmag; + + // Perform double contraction + psmn(i, j, k) = fxgpx_flux * n0 * n0 + fxgpy_flux * n0 * n1 + + fxgpz_flux * n0 * n2 + fygpx_flux * n1 * n0 + + fygpy_flux * n1 * n1 + fygpz_flux * n1 * n2 + + fzgpx_flux * n2 * n0 + fzgpy_flux * n2 * n1 + + fzgpz_flux * n2 * n2; }); }); } @@ -425,6 +552,31 @@ amrex::Real check_gp_rho_face_impl( } return error_total; } + +amrex::Real +check_psrc_manual_impl(amr_wind::Field& psrc, amr_wind::Field& psrc_manual) +{ + amrex::Real error_total = 0; + + for (int lev = 0; lev < psrc.repo().num_active_levels(); ++lev) { + + error_total += amrex::ReduceSum( + psrc(lev), psrc_manual(lev), 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& f_arr, + amrex::Array4 const& fm_arr) -> amrex::Real { + amrex::Real error = 0; + + amrex::Loop(bx, [=, &error](int i, int j, int k) noexcept { + error += std::abs(f_arr(i, j, k) - fm_arr(i, j, k)); + }); + + return error; + }); + } + return error_total; +} } // namespace TEST_F(VOFOversetOps, alpha_flux) @@ -687,10 +839,52 @@ TEST_F(VOFOversetOps, pseudo_vscale_dt) } amrex::ParallelDescriptor::ReduceRealMin(pvscale); EXPECT_DOUBLE_EQ(pvscale, pvs_answer); +} + +TEST_F(VOFOversetOps, psource_manual) +{ + populate_parameters(); + initialize_mesh(); + + const amrex::Real rho_liq = 1000.; + const amrex::Real rho_gas = 1.; + + auto& repo = sim().repo(); + const int nghost = 3; + auto& gp = repo.declare_field("gp", 3, nghost); + auto& rho = repo.declare_field("density", 1, nghost); + auto& vof = repo.declare_field("vof", 1, nghost); + + auto& psrc = repo.declare_field("psrc", 1, 0, 1, amr_wind::FieldLoc::NODE); + auto& psmn = + repo.declare_field("psrc_manual", 1, 0, 1, amr_wind::FieldLoc::NODE); + + // Create flux fields + auto& flux_x = + repo.declare_field("flux_x", 9, 1, 1, amr_wind::FieldLoc::XFACE); + auto& flux_y = + repo.declare_field("flux_y", 9, 1, 1, amr_wind::FieldLoc::YFACE); + auto& flux_z = + repo.declare_field("flux_z", 9, 1, 1, amr_wind::FieldLoc::ZFACE); - /* - pseudo velocity scale! - */ + // Initialize gp_rho fluxes in every direction + int dir = 0; + init_gp_rho_etc(gp, rho, vof, dir, rho_liq, rho_gas); + calc_gp_rho_face(flux_x, gp, rho, vof, dir, 5, true); + dir = 1; + init_gp_rho_etc(gp, rho, vof, dir, rho_liq, rho_gas); + calc_gp_rho_face(flux_y, gp, rho, vof, dir, 5, true); + dir = 3; + init_gp_rho_etc(gp, rho, vof, dir, rho_liq, rho_gas); + calc_gp_rho_face(flux_z, gp, rho, vof, dir, 5, true); + + // Calculate psrc with vector/tensor ops and manually + calc_psrc(flux_x, flux_y, flux_z, vof, psrc, psmn); + + // Check difference + amrex::Real error_total = check_psrc_manual_impl(psrc, psmn); + amrex::ParallelDescriptor::ReduceRealSum(error_total); + EXPECT_NEAR(error_total, 0.0, 1e-10); } } // namespace amr_wind_tests \ No newline at end of file From 5f94cddb3ba9d334becfb577e34a7b5da9046cd6 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Thu, 18 Jul 2024 15:32:10 -0600 Subject: [PATCH 40/43] use a header file the way it's intended --- amr-wind/overset/CMakeLists.txt | 1 + amr-wind/overset/overset_ops_routines.H | 478 ++----------------- amr-wind/overset/overset_ops_routines.cpp | 533 ++++++++++++++++++++++ 3 files changed, 562 insertions(+), 450 deletions(-) create mode 100644 amr-wind/overset/overset_ops_routines.cpp diff --git a/amr-wind/overset/CMakeLists.txt b/amr-wind/overset/CMakeLists.txt index 757b857a1b..8874741902 100644 --- a/amr-wind/overset/CMakeLists.txt +++ b/amr-wind/overset/CMakeLists.txt @@ -2,4 +2,5 @@ target_sources(${amr_wind_lib_name} PRIVATE TiogaInterface.cpp OversetOps.cpp + overset_ops_routines.cpp ) diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index 90cc9fe2da..8f828f54bf 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -1,104 +1,38 @@ #ifndef OVERSET_OPS_ROUTINES_H_ #define OVERSET_OPS_ROUTINES_H_ +#include "AMReX_iMultiFab.H" +#include "AMReX_MultiFab.H" #include "amr-wind/equation_systems/vof/volume_fractions.H" #include "amr-wind/overset/overset_ops_K.H" namespace amr_wind::overset_ops { // Populate approximate signed distance function using vof field -void static populate_psi( +void populate_psi( amrex::MultiFab& mf_psi, const amrex::MultiFab& mf_vof, const amrex::Real i_th, - const amrex::Real asdf_tiny) -{ - const auto& psi = mf_psi.arrays(); - const auto& vof = mf_vof.const_arrays(); - amrex::ParallelFor( - mf_psi, mf_psi.n_grow, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - psi[nbx](i, j, k) = asdf(vof[nbx](i, j, k), i_th, asdf_tiny); - }); -} + const amrex::Real asdf_tiny); // Modify a vof field to not have values that barely differ from 0 or 1 -void static process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) -{ - const auto& vof = mf_vof.arrays(); - amrex::ParallelFor( - mf_vof, mf_vof.n_grow, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - vof[nbx](i, j, k) = - vof[nbx](i, j, k) < vof_tol - ? 0.0 - : (vof[nbx](i, j, k) > 1. - vof_tol ? 1. - : vof[nbx](i, j, k)); - }); -} +void process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol); // Combine overset target vof field with current non-overset vof field -void static harmonize_vof( +void harmonize_vof( amrex::MultiFab& mf_vof_target, const amrex::MultiFab& mf_vof_original, - const amrex::iMultiFab& mf_iblank) -{ - const auto& tg_vof = mf_vof_target.arrays(); - const auto& og_vof = mf_vof_original.const_arrays(); - const auto& iblank = mf_iblank.const_arrays(); - amrex::ParallelFor( - mf_vof_target, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - // Replace amr-wind vof values with originals - if (iblank[nbx](i, j, k) != -1) { - tg_vof[nbx](i, j, k) = og_vof[nbx](i, j, k); - } - }); -} + const amrex::iMultiFab& mf_iblank); // Populate normal vector with special treatment of overset boundary -void static populate_normal_vector( +void populate_normal_vector( amrex::MultiFab& mf_normvec, const amrex::MultiFab& mf_vof, - const amrex::iMultiFab& mf_iblank) -{ - const auto& normvec = mf_normvec.arrays(); - const auto& vof = mf_vof.const_arrays(); - const auto& iblank = mf_iblank.const_arrays(); - // Calculate gradients in each direction with centered diff - amrex::ParallelFor( - mf_normvec, mf_normvec.n_grow - amrex::IntVect(1), - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - // Neumann condition across nalu bdy - int ibdy = - (iblank[nbx](i, j, k) != iblank[nbx](i - 1, j, k)) ? -1 : 0; - int jbdy = - (iblank[nbx](i, j, k) != iblank[nbx](i, j - 1, k)) ? -1 : 0; - int kbdy = - (iblank[nbx](i, j, k) != iblank[nbx](i, j, k - 1)) ? -1 : 0; - // no cell should be isolated such that -1 and 1 are needed - ibdy = - (iblank[nbx](i, j, k) != iblank[nbx](i + 1, j, k)) ? +1 : ibdy; - jbdy = - (iblank[nbx](i, j, k) != iblank[nbx](i, j + 1, k)) ? +1 : jbdy; - kbdy = - (iblank[nbx](i, j, k) != iblank[nbx](i, j, k + 1)) ? +1 : kbdy; - // Calculate normal - amrex::Real mx, my, mz, mmag; - multiphase::youngs_finite_difference_normal_neumann( - i, j, k, ibdy, jbdy, kbdy, vof[nbx], mx, my, mz); - // Normalize normal - mmag = std::sqrt(mx * mx + my * my + mz * mz + 1e-20); - // Save normal - normvec[nbx](i, j, k, 0) = mx / mmag; - normvec[nbx](i, j, k, 1) = my / mmag; - normvec[nbx](i, j, k, 2) = mz / mmag; - }); -} + const amrex::iMultiFab& mf_iblank); // Calculate fluxes for reinitialization over entire domain without concern for // overset bdy -void static populate_sharpen_fluxes( +void populate_sharpen_fluxes( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, @@ -111,220 +45,33 @@ void static populate_sharpen_fluxes( const amrex::Real Gamma, const amrex::Real margin, const amrex::Real rho1, - const amrex::Real rho2) -{ - const auto& fx = mf_fx.arrays(); - const auto& fy = mf_fy.arrays(); - const auto& fz = mf_fz.arrays(); - const auto& vof = mf_vof.const_arrays(); - const auto& tg_vof = mf_target_vof.const_arrays(); - const auto& norm = mf_norm.const_arrays(); - const auto& vel = mf_velocity.const_arrays(); - const auto& gp = mf_gp.const_arrays(); - const auto& rho = mf_density.const_arrays(); - amrex::ParallelFor( - mf_fx, mf_fx.n_grow, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - // vof flux - amrex::Real flux = Gamma * alpha_flux( - i, j, k, 0, margin, vof[nbx], - tg_vof[nbx], norm[nbx]); - fx[nbx](i, j, k, 0) = flux; - // density flux - flux *= (rho1 - rho2); - fx[nbx](i, j, k, 1) = flux; - // momentum fluxes (dens flux * face vel) - amrex::Real uf, vf, wf; - velocity_face(i, j, k, 0, vof[nbx], vel[nbx], uf, vf, wf); - fx[nbx](i, j, k, 2) = flux * uf; - fx[nbx](i, j, k, 3) = flux * vf; - fx[nbx](i, j, k, 4) = flux * wf; - // pressure gradient fluxes - gp_rho_face(i, j, k, 0, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); - fx[nbx](i, j, k, 5) = flux * uf; - fx[nbx](i, j, k, 6) = flux * vf; - fx[nbx](i, j, k, 7) = flux * wf; - // Turn "on" all flux faces, later modified in - // process_fluxes_calc_src - fx[nbx](i, j, k, 8) = 1.0; - }); - amrex::ParallelFor( - mf_fy, mf_fy.n_grow, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - amrex::Real flux = Gamma * alpha_flux( - i, j, k, 1, margin, vof[nbx], - tg_vof[nbx], norm[nbx]); - fy[nbx](i, j, k, 0) = flux; - flux *= (rho1 - rho2); - fy[nbx](i, j, k, 1) = flux; - amrex::Real uf, vf, wf; - velocity_face(i, j, k, 1, vof[nbx], vel[nbx], uf, vf, wf); - fy[nbx](i, j, k, 2) = flux * uf; - fy[nbx](i, j, k, 3) = flux * vf; - fy[nbx](i, j, k, 4) = flux * wf; - gp_rho_face(i, j, k, 1, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); - fy[nbx](i, j, k, 5) = flux * uf; - fy[nbx](i, j, k, 6) = flux * vf; - fy[nbx](i, j, k, 7) = flux * wf; - fy[nbx](i, j, k, 8) = 1.0; - }); - amrex::ParallelFor( - mf_fz, mf_fz.n_grow, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - amrex::Real flux = Gamma * alpha_flux( - i, j, k, 2, margin, vof[nbx], - tg_vof[nbx], norm[nbx]); - fz[nbx](i, j, k, 0) = flux; - flux *= (rho1 - rho2); - fz[nbx](i, j, k, 1) = flux; - amrex::Real uf, vf, wf; - velocity_face(i, j, k, 2, vof[nbx], vel[nbx], uf, vf, wf); - fz[nbx](i, j, k, 2) = flux * uf; - fz[nbx](i, j, k, 3) = flux * vf; - fz[nbx](i, j, k, 4) = flux * wf; - gp_rho_face(i, j, k, 2, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); - fz[nbx](i, j, k, 5) = flux * uf; - fz[nbx](i, j, k, 6) = flux * vf; - fz[nbx](i, j, k, 7) = flux * wf; - fz[nbx](i, j, k, 8) = 1.0; - }); -} + const amrex::Real rho2); // Process reinitialization fluxes - zero non-internal to overset region; // also calculate pressure source / sink term as a function of fluxes -void static process_fluxes_calc_src( +void process_fluxes_calc_src( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, amrex::MultiFab& mf_psource, const amrex::MultiFab& mf_vof, - const amrex::iMultiFab& mf_iblank) -{ - const auto& fx = mf_fx.arrays(); - const auto& fy = mf_fy.arrays(); - const auto& fz = mf_fz.arrays(); - const auto& sp = mf_psource.arrays(); - const auto& vof = mf_vof.const_arrays(); - const auto& iblank = mf_iblank.const_arrays(); - constexpr amrex::Real tiny = std::numeric_limits::epsilon(); - // Zero fluxes based on iblank array - amrex::ParallelFor( - mf_fx, mf_fx.n_grow, mf_fx.n_comp, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { - bool zero_all = - (iblank[nbx](i - 1, j, k) + iblank[nbx](i, j, k) > -2); - fx[nbx](i, j, k, n) *= zero_all ? 0. : 1.; - }); - amrex::ParallelFor( - mf_fy, mf_fy.n_grow, mf_fy.n_comp, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { - bool zero_all = - (iblank[nbx](i, j - 1, k) + iblank[nbx](i, j, k) > -2); - fy[nbx](i, j, k, n) *= zero_all ? 0. : 1.; - }); - amrex::ParallelFor( - mf_fz, mf_fz.n_grow, mf_fz.n_comp, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { - bool zero_all = - (iblank[nbx](i, j, k - 1) + iblank[nbx](i, j, k) > -2); - fz[nbx](i, j, k, n) *= zero_all ? 0. : 1.; - }); - // With knowledge of fluxes, compute pressure source term - amrex::ParallelFor( - mf_psource, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - sp[nbx](i, j, k) = - gp_flux_tensor(i, j, k, fx[nbx], fy[nbx], fz[nbx], tiny) && - normal_reinit_tensor( - i, j, k, fx[nbx], fy[nbx], fz[nbx], vof[nbx], tiny); - }); -} + const amrex::iMultiFab& mf_iblank); -amrex::Real static calculate_pseudo_velocity_scale( +amrex::Real calculate_pseudo_velocity_scale( const amrex::iMultiFab& mf_iblank, const amrex::GpuArray dx, - const amrex::Real pvmax) -{ - // Get minimum length scale from dx - const amrex::Real dx_min = std::min(std::min(dx[0], dx[1]), dx[2]); - // The dx of this level should be considered if iblank = -1 here - // Otherwise, set to max possible value for this mesh (level 0 values) - const amrex::Real pvscale = (mf_iblank.min(0) == -1) ? dx_min : pvmax; - return pvscale; -} + const amrex::Real pvmax); // Calculate a type of CFL by measuring how much % VOF is being removed per cell -amrex::Real static calculate_pseudo_dt_flux( +amrex::Real calculate_pseudo_dt_flux( amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz, amrex::MultiFab& mf_vof, - amrex::Real tol) -{ - // Get the maximum flux magnitude, but just for vof fluxes - const amrex::Real pdt_fx = amrex::ReduceMin( - mf_fx, mf_vof, 0, - [=] AMREX_GPU_HOST_DEVICE( - amrex::Box const& bx, amrex::Array4 const& fx, - amrex::Array4 const& vof) -> amrex::Real { - amrex::Real pdt_fab = 1.0; - amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { - amrex::Real pdt_lim = 1.0; - if (fx(i, j, k, 0) > tol && vof(i, j, k) > tol) { - // VOF is removed from cell i - pdt_lim = vof(i, j, k) / fx(i, j, k, 0); - } else if (fx(i, j, k, 0) < -tol && vof(i - 1, j, k) > tol) { - // VOF is removed from cell i-1 - pdt_lim = vof(i - 1, j, k) / -fx(i, j, k, 0); - } - pdt_fab = amrex::min(pdt_fab, pdt_lim); - }); - return pdt_fab; - }); - const amrex::Real pdt_fy = amrex::ReduceMin( - mf_fy, mf_vof, 0, - [=] AMREX_GPU_HOST_DEVICE( - amrex::Box const& bx, amrex::Array4 const& fy, - amrex::Array4 const& vof) -> amrex::Real { - amrex::Real pdt_fab = 1.0; - amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { - amrex::Real pdt_lim = 1.0; - if (fy(i, j, k, 0) > tol && vof(i, j, k) > tol) { - // VOF is removed from cell j - pdt_lim = vof(i, j, k) / fy(i, j, k, 0); - } else if (fy(i, j, k, 0) < -tol && vof(i, j - 1, k) > tol) { - // VOF is removed from cell j-1 - pdt_lim = vof(i, j - 1, k) / -fy(i, j, k, 0); - } - pdt_fab = amrex::min(pdt_fab, pdt_lim); - }); - return pdt_fab; - }); - const amrex::Real pdt_fz = amrex::ReduceMin( - mf_fz, mf_vof, 0, - [=] AMREX_GPU_HOST_DEVICE( - amrex::Box const& bx, amrex::Array4 const& fz, - amrex::Array4 const& vof) -> amrex::Real { - amrex::Real pdt_fab = 1.0; - amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { - amrex::Real pdt_lim = 1.0; - if (fz(i, j, k, 0) > tol && vof(i, j, k) > tol) { - // VOF is removed from cell k - pdt_lim = vof(i, j, k) / fz(i, j, k, 0); - } else if (fz(i, j, k, 0) < -tol && vof(i, j, k - 1) > tol) { - // VOF is removed from cell k-1 - pdt_lim = vof(i, j, k - 1) / -fz(i, j, k, 0); - } - pdt_fab = amrex::min(pdt_fab, pdt_lim); - }); - return pdt_fab; - }); - const amrex::Real pdt = amrex::min(pdt_fx, amrex::min(pdt_fy, pdt_fz)); - return pdt; -} + amrex::Real tol); // Apply reinitialization fluxes to modify fields -void static apply_fluxes( +void apply_fluxes( const amrex::MultiFab& mf_fx, const amrex::MultiFab& mf_fy, const amrex::MultiFab& mf_fz, @@ -336,205 +83,36 @@ void static apply_fluxes( amrex::MultiFab& mf_pressure, const amrex::GpuArray dx, const amrex::Real ptfac, - const amrex::Real vof_tol) -{ - const auto& fx = mf_fx.const_arrays(); - const auto& fy = mf_fy.const_arrays(); - const auto& fz = mf_fz.const_arrays(); - const auto& sp = mf_psource.const_arrays(); - const auto& vof = mf_vof.arrays(); - const auto& dens = mf_dens.arrays(); - const auto& vel = mf_vel.arrays(); - const auto& gp = mf_gp.arrays(); - const auto& p = mf_pressure.arrays(); - - amrex::ParallelFor( - mf_vof, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - const amrex::Real olddens = dens[nbx](i, j, k); - vof[nbx](i, j, k) += - ptfac * - ((fx[nbx](i + 1, j, k, 0) - fx[nbx](i, j, k, 0)) / dx[0] + - (fy[nbx](i, j + 1, k, 0) - fy[nbx](i, j, k, 0)) / dx[1] + - (fz[nbx](i, j, k + 1, 0) - fz[nbx](i, j, k, 0)) / dx[2]); - dens[nbx](i, j, k) += - ptfac * - ((fx[nbx](i + 1, j, k, 1) - fx[nbx](i, j, k, 1)) / dx[0] + - (fy[nbx](i, j + 1, k, 1) - fy[nbx](i, j, k, 1)) / dx[1] + - (fz[nbx](i, j, k + 1, 1) - fz[nbx](i, j, k, 1)) / dx[2]); - vel[nbx](i, j, k, 0) = - 1.0 / dens[nbx](i, j, k) * - (olddens * vel[nbx](i, j, k, 0) + - ptfac * - ((fx[nbx](i + 1, j, k, 2) - fx[nbx](i, j, k, 2)) / dx[0] + - (fy[nbx](i, j + 1, k, 2) - fy[nbx](i, j, k, 2)) / dx[1] + - (fz[nbx](i, j, k + 1, 2) - fz[nbx](i, j, k, 2)) / dx[2])); - vel[nbx](i, j, k, 1) = - 1.0 / dens[nbx](i, j, k) * - (olddens * vel[nbx](i, j, k, 1) + - ptfac * - ((fx[nbx](i + 1, j, k, 3) - fx[nbx](i, j, k, 3)) / dx[0] + - (fy[nbx](i, j + 1, k, 3) - fy[nbx](i, j, k, 3)) / dx[1] + - (fz[nbx](i, j, k + 1, 3) - fz[nbx](i, j, k, 3)) / dx[2])); - vel[nbx](i, j, k, 2) = - 1.0 / dens[nbx](i, j, k) * - (olddens * vel[nbx](i, j, k, 2) + - ptfac * - ((fx[nbx](i + 1, j, k, 4) - fx[nbx](i, j, k, 4)) / dx[0] + - (fy[nbx](i, j + 1, k, 4) - fy[nbx](i, j, k, 4)) / dx[1] + - (fz[nbx](i, j, k + 1, 4) - fz[nbx](i, j, k, 4)) / dx[2])); - gp[nbx](i, j, k, 0) += - ptfac * - ((fx[nbx](i + 1, j, k, 5) - fx[nbx](i, j, k, 5)) / dx[0] + - (fy[nbx](i, j + 1, k, 5) - fy[nbx](i, j, k, 5)) / dx[1] + - (fz[nbx](i, j, k + 1, 5) - fz[nbx](i, j, k, 5)) / dx[2]); - gp[nbx](i, j, k, 1) += - ptfac * - ((fx[nbx](i + 1, j, k, 6) - fx[nbx](i, j, k, 6)) / dx[0] + - (fy[nbx](i, j + 1, k, 6) - fy[nbx](i, j, k, 6)) / dx[1] + - (fz[nbx](i, j, k + 1, 6) - fz[nbx](i, j, k, 6)) / dx[2]); - gp[nbx](i, j, k, 2) += - ptfac * - ((fx[nbx](i + 1, j, k, 7) - fx[nbx](i, j, k, 7)) / dx[0] + - (fy[nbx](i, j + 1, k, 7) - fy[nbx](i, j, k, 7)) / dx[1] + - (fz[nbx](i, j, k + 1, 7) - fz[nbx](i, j, k, 7)) / dx[2]); - - // Ensure vof is bounded - vof[nbx](i, j, k) = - vof[nbx](i, j, k) < vof_tol - ? 0.0 - : (vof[nbx](i, j, k) > 1. - vof_tol ? 1. - : vof[nbx](i, j, k)); - // Density bounds are enforced elsewhere - }); - amrex::ParallelFor( - mf_pressure, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - p[nbx](i, j, k) += ptfac * sp[nbx](i, j, k); - }); -} + const amrex::Real vof_tol); // Get the size of the smallest VOF flux to quantify convergence -amrex::Real static measure_convergence( - amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz) -{ - // Get the maximum flux magnitude, but just for vof fluxes - const amrex::Real err_fx = amrex::ReduceMax( - mf_fx, 0, - [=] AMREX_GPU_HOST_DEVICE( - amrex::Box const& bx, - amrex::Array4 const& fx) -> amrex::Real { - amrex::Real err_fab = -1.0; - amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { - err_fab = amrex::max(err_fab, std::abs(fx(i, j, k, 0))); - }); - return err_fab; - }); - const amrex::Real err_fy = amrex::ReduceMax( - mf_fy, 0, - [=] AMREX_GPU_HOST_DEVICE( - amrex::Box const& bx, - amrex::Array4 const& fy) -> amrex::Real { - amrex::Real err_fab = -1.0; - amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { - err_fab = amrex::max(err_fab, std::abs(fy(i, j, k, 0))); - }); - return err_fab; - }); - const amrex::Real err_fz = amrex::ReduceMax( - mf_fz, 0, - [=] AMREX_GPU_HOST_DEVICE( - amrex::Box const& bx, - amrex::Array4 const& fz) -> amrex::Real { - amrex::Real err_fab = -1.0; - amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { - err_fab = amrex::max(err_fab, std::abs(fz(i, j, k, 0))); - }); - return err_fab; - }); - const amrex::Real err = amrex::max(err_fx, amrex::max(err_fy, err_fz)); - return err; -} +amrex::Real measure_convergence( + amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz); // Set levelset field to another quantity to view in plotfile for debugging -void static equate_field( - amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src) -{ - const auto& dest = mf_dest.arrays(); - const auto& src = mf_src.const_arrays(); - amrex::ParallelFor( - mf_dest, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - dest[nbx](i, j, k) = std::sqrt( - src[nbx](i, j, k, 0) * src[nbx](i, j, k, 0) + - src[nbx](i, j, k, 1) * src[nbx](i, j, k, 1) + - src[nbx](i, j, k, 2) * src[nbx](i, j, k, 2)); - }); -} +void equate_field(amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src); // Replace pressure gradient with hydrostatic field in overset regions -void static replace_gradp_hydrostatic( +void replace_gradp_hydrostatic( amrex::MultiFab& mf_gp, const amrex::MultiFab& mf_density, const amrex::MultiFab& mf_refdens, const amrex::iMultiFab& mf_iblank, const amrex::Real grav_z, - const bool is_pptb) -{ - const auto& gp = mf_gp.arrays(); - const auto& rho = mf_density.const_arrays(); - const auto& rho0 = mf_refdens.const_arrays(); - const auto& iblank = mf_iblank.const_arrays(); - amrex::ParallelFor( - mf_gp, mf_gp.n_grow, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - if (iblank[nbx](i, j, k) == -1) { - const amrex::Real dfac = - is_pptb ? rho[nbx](i, j, k) - rho0[nbx](i, j, k) - : rho[nbx](i, j, k); - gp[nbx](i, j, k, 0) = 0.; - gp[nbx](i, j, k, 1) = 0.; - gp[nbx](i, j, k, 2) = dfac * grav_z; - } - }); -} + const bool is_pptb); // Swap pressure gradient values in overset region -void static replace_gradp( +void replace_gradp( amrex::MultiFab& mf_gp, const amrex::MultiFab& mf_gp0, - const amrex::iMultiFab& mf_iblank) -{ - const auto gp = mf_gp.arrays(); - const auto gp0 = mf_gp0.const_arrays(); - const auto iblank = mf_iblank.const_arrays(); - amrex::ParallelFor( - mf_gp, mf_gp.n_grow, - [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - if (iblank[nbx](i, j, k) == -1) { - gp[nbx](i, j, k, 0) = gp0[nbx](i, j, k, 0); - gp[nbx](i, j, k, 1) = gp0[nbx](i, j, k, 1); - gp[nbx](i, j, k, 2) = gp0[nbx](i, j, k, 2); - } - }); -} + const amrex::iMultiFab& mf_iblank); // Apply pressure gradient to velocity field -void static apply_pressure_gradient( +void apply_pressure_gradient( amrex::MultiFab& mf_vel, const amrex::MultiFab& mf_density, const amrex::MultiFab& mf_gp, - const amrex::Real scaling_factor) -{ - const auto& vel = mf_vel.arrays(); - const auto& rho = mf_density.const_arrays(); - const auto& gp = mf_gp.const_arrays(); - amrex::ParallelFor( - mf_vel, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { - const amrex::Real soverrho = scaling_factor / rho[nbx](i, j, k); - vel[nbx](i, j, k, 0) -= gp[nbx](i, j, k, 0) * soverrho; - vel[nbx](i, j, k, 1) -= gp[nbx](i, j, k, 1) * soverrho; - vel[nbx](i, j, k, 2) -= gp[nbx](i, j, k, 2) * soverrho; - }); -} + const amrex::Real scaling_factor); } // namespace amr_wind::overset_ops diff --git a/amr-wind/overset/overset_ops_routines.cpp b/amr-wind/overset/overset_ops_routines.cpp new file mode 100644 index 0000000000..e63b4e8b80 --- /dev/null +++ b/amr-wind/overset/overset_ops_routines.cpp @@ -0,0 +1,533 @@ +#include "amr-wind/overset/overset_ops_routines.H" + +namespace amr_wind::overset_ops { +// Populate approximate signed distance function using vof field +void populate_psi( + amrex::MultiFab& mf_psi, + const amrex::MultiFab& mf_vof, + const amrex::Real i_th, + const amrex::Real asdf_tiny) +{ + const auto& psi = mf_psi.arrays(); + const auto& vof = mf_vof.const_arrays(); + amrex::ParallelFor( + mf_psi, mf_psi.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + psi[nbx](i, j, k) = asdf(vof[nbx](i, j, k), i_th, asdf_tiny); + }); +} + +// Modify a vof field to not have values that barely differ from 0 or 1 +void process_vof(amrex::MultiFab& mf_vof, const amrex::Real vof_tol) +{ + const auto& vof = mf_vof.arrays(); + amrex::ParallelFor( + mf_vof, mf_vof.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + vof[nbx](i, j, k) = + vof[nbx](i, j, k) < vof_tol + ? 0.0 + : (vof[nbx](i, j, k) > 1. - vof_tol ? 1. + : vof[nbx](i, j, k)); + }); +} + +// Combine overset target vof field with current non-overset vof field +void harmonize_vof( + amrex::MultiFab& mf_vof_target, + const amrex::MultiFab& mf_vof_original, + const amrex::iMultiFab& mf_iblank) +{ + const auto& tg_vof = mf_vof_target.arrays(); + const auto& og_vof = mf_vof_original.const_arrays(); + const auto& iblank = mf_iblank.const_arrays(); + amrex::ParallelFor( + mf_vof_target, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + // Replace amr-wind vof values with originals + if (iblank[nbx](i, j, k) != -1) { + tg_vof[nbx](i, j, k) = og_vof[nbx](i, j, k); + } + }); +} + +// Populate normal vector with special treatment of overset boundary +void populate_normal_vector( + amrex::MultiFab& mf_normvec, + const amrex::MultiFab& mf_vof, + const amrex::iMultiFab& mf_iblank) +{ + const auto& normvec = mf_normvec.arrays(); + const auto& vof = mf_vof.const_arrays(); + const auto& iblank = mf_iblank.const_arrays(); + // Calculate gradients in each direction with centered diff + amrex::ParallelFor( + mf_normvec, mf_normvec.n_grow - amrex::IntVect(1), + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + // Neumann condition across nalu bdy + int ibdy = + (iblank[nbx](i, j, k) != iblank[nbx](i - 1, j, k)) ? -1 : 0; + int jbdy = + (iblank[nbx](i, j, k) != iblank[nbx](i, j - 1, k)) ? -1 : 0; + int kbdy = + (iblank[nbx](i, j, k) != iblank[nbx](i, j, k - 1)) ? -1 : 0; + // no cell should be isolated such that -1 and 1 are needed + ibdy = + (iblank[nbx](i, j, k) != iblank[nbx](i + 1, j, k)) ? +1 : ibdy; + jbdy = + (iblank[nbx](i, j, k) != iblank[nbx](i, j + 1, k)) ? +1 : jbdy; + kbdy = + (iblank[nbx](i, j, k) != iblank[nbx](i, j, k + 1)) ? +1 : kbdy; + // Calculate normal + amrex::Real mx, my, mz, mmag; + multiphase::youngs_finite_difference_normal_neumann( + i, j, k, ibdy, jbdy, kbdy, vof[nbx], mx, my, mz); + // Normalize normal + mmag = std::sqrt(mx * mx + my * my + mz * mz + 1e-20); + // Save normal + normvec[nbx](i, j, k, 0) = mx / mmag; + normvec[nbx](i, j, k, 1) = my / mmag; + normvec[nbx](i, j, k, 2) = mz / mmag; + }); +} + +// Calculate fluxes for reinitialization over entire domain without concern for +// overset bdy +void populate_sharpen_fluxes( + amrex::MultiFab& mf_fx, + amrex::MultiFab& mf_fy, + amrex::MultiFab& mf_fz, + const amrex::MultiFab& mf_vof, + const amrex::MultiFab& mf_target_vof, + const amrex::MultiFab& mf_norm, + const amrex::MultiFab& mf_velocity, + const amrex::MultiFab& mf_gp, + const amrex::MultiFab& mf_density, + const amrex::Real Gamma, + const amrex::Real margin, + const amrex::Real rho1, + const amrex::Real rho2) +{ + const auto& fx = mf_fx.arrays(); + const auto& fy = mf_fy.arrays(); + const auto& fz = mf_fz.arrays(); + const auto& vof = mf_vof.const_arrays(); + const auto& tg_vof = mf_target_vof.const_arrays(); + const auto& norm = mf_norm.const_arrays(); + const auto& vel = mf_velocity.const_arrays(); + const auto& gp = mf_gp.const_arrays(); + const auto& rho = mf_density.const_arrays(); + amrex::ParallelFor( + mf_fx, mf_fx.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + // vof flux + amrex::Real flux = Gamma * alpha_flux( + i, j, k, 0, margin, vof[nbx], + tg_vof[nbx], norm[nbx]); + fx[nbx](i, j, k, 0) = flux; + // density flux + flux *= (rho1 - rho2); + fx[nbx](i, j, k, 1) = flux; + // momentum fluxes (dens flux * face vel) + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 0, vof[nbx], vel[nbx], uf, vf, wf); + fx[nbx](i, j, k, 2) = flux * uf; + fx[nbx](i, j, k, 3) = flux * vf; + fx[nbx](i, j, k, 4) = flux * wf; + // pressure gradient fluxes + gp_rho_face(i, j, k, 0, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); + fx[nbx](i, j, k, 5) = flux * uf; + fx[nbx](i, j, k, 6) = flux * vf; + fx[nbx](i, j, k, 7) = flux * wf; + // Turn "on" all flux faces, later modified in + // process_fluxes_calc_src + fx[nbx](i, j, k, 8) = 1.0; + }); + amrex::ParallelFor( + mf_fy, mf_fy.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + amrex::Real flux = Gamma * alpha_flux( + i, j, k, 1, margin, vof[nbx], + tg_vof[nbx], norm[nbx]); + fy[nbx](i, j, k, 0) = flux; + flux *= (rho1 - rho2); + fy[nbx](i, j, k, 1) = flux; + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 1, vof[nbx], vel[nbx], uf, vf, wf); + fy[nbx](i, j, k, 2) = flux * uf; + fy[nbx](i, j, k, 3) = flux * vf; + fy[nbx](i, j, k, 4) = flux * wf; + gp_rho_face(i, j, k, 1, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); + fy[nbx](i, j, k, 5) = flux * uf; + fy[nbx](i, j, k, 6) = flux * vf; + fy[nbx](i, j, k, 7) = flux * wf; + fy[nbx](i, j, k, 8) = 1.0; + }); + amrex::ParallelFor( + mf_fz, mf_fz.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + amrex::Real flux = Gamma * alpha_flux( + i, j, k, 2, margin, vof[nbx], + tg_vof[nbx], norm[nbx]); + fz[nbx](i, j, k, 0) = flux; + flux *= (rho1 - rho2); + fz[nbx](i, j, k, 1) = flux; + amrex::Real uf, vf, wf; + velocity_face(i, j, k, 2, vof[nbx], vel[nbx], uf, vf, wf); + fz[nbx](i, j, k, 2) = flux * uf; + fz[nbx](i, j, k, 3) = flux * vf; + fz[nbx](i, j, k, 4) = flux * wf; + gp_rho_face(i, j, k, 2, vof[nbx], gp[nbx], rho[nbx], uf, vf, wf); + fz[nbx](i, j, k, 5) = flux * uf; + fz[nbx](i, j, k, 6) = flux * vf; + fz[nbx](i, j, k, 7) = flux * wf; + fz[nbx](i, j, k, 8) = 1.0; + }); +} + +// Process reinitialization fluxes - zero non-internal to overset region; +// also calculate pressure source / sink term as a function of fluxes +void process_fluxes_calc_src( + amrex::MultiFab& mf_fx, + amrex::MultiFab& mf_fy, + amrex::MultiFab& mf_fz, + amrex::MultiFab& mf_psource, + const amrex::MultiFab& mf_vof, + const amrex::iMultiFab& mf_iblank) +{ + const auto& fx = mf_fx.arrays(); + const auto& fy = mf_fy.arrays(); + const auto& fz = mf_fz.arrays(); + const auto& sp = mf_psource.arrays(); + const auto& vof = mf_vof.const_arrays(); + const auto& iblank = mf_iblank.const_arrays(); + constexpr amrex::Real tiny = std::numeric_limits::epsilon(); + // Zero fluxes based on iblank array + amrex::ParallelFor( + mf_fx, mf_fx.n_grow, mf_fx.n_comp, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { + bool zero_all = + (iblank[nbx](i - 1, j, k) + iblank[nbx](i, j, k) > -2); + fx[nbx](i, j, k, n) *= zero_all ? 0. : 1.; + }); + amrex::ParallelFor( + mf_fy, mf_fy.n_grow, mf_fy.n_comp, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { + bool zero_all = + (iblank[nbx](i, j - 1, k) + iblank[nbx](i, j, k) > -2); + fy[nbx](i, j, k, n) *= zero_all ? 0. : 1.; + }); + amrex::ParallelFor( + mf_fz, mf_fz.n_grow, mf_fz.n_comp, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { + bool zero_all = + (iblank[nbx](i, j, k - 1) + iblank[nbx](i, j, k) > -2); + fz[nbx](i, j, k, n) *= zero_all ? 0. : 1.; + }); + // With knowledge of fluxes, compute pressure source term + amrex::ParallelFor( + mf_psource, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + sp[nbx](i, j, k) = + gp_flux_tensor(i, j, k, fx[nbx], fy[nbx], fz[nbx], tiny) && + normal_reinit_tensor( + i, j, k, fx[nbx], fy[nbx], fz[nbx], vof[nbx], tiny); + }); +} + +amrex::Real calculate_pseudo_velocity_scale( + const amrex::iMultiFab& mf_iblank, + const amrex::GpuArray dx, + const amrex::Real pvmax) +{ + // Get minimum length scale from dx + const amrex::Real dx_min = std::min(std::min(dx[0], dx[1]), dx[2]); + // The dx of this level should be considered if iblank = -1 here + // Otherwise, set to max possible value for this mesh (level 0 values) + const amrex::Real pvscale = (mf_iblank.min(0) == -1) ? dx_min : pvmax; + return pvscale; +} + +// Calculate a type of CFL by measuring how much % VOF is being removed per cell +amrex::Real calculate_pseudo_dt_flux( + amrex::MultiFab& mf_fx, + amrex::MultiFab& mf_fy, + amrex::MultiFab& mf_fz, + amrex::MultiFab& mf_vof, + amrex::Real tol) +{ + // Get the maximum flux magnitude, but just for vof fluxes + const amrex::Real pdt_fx = amrex::ReduceMin( + mf_fx, mf_vof, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, amrex::Array4 const& fx, + amrex::Array4 const& vof) -> amrex::Real { + amrex::Real pdt_fab = 1.0; + amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { + amrex::Real pdt_lim = 1.0; + if (fx(i, j, k, 0) > tol && vof(i, j, k) > tol) { + // VOF is removed from cell i + pdt_lim = vof(i, j, k) / fx(i, j, k, 0); + } else if (fx(i, j, k, 0) < -tol && vof(i - 1, j, k) > tol) { + // VOF is removed from cell i-1 + pdt_lim = vof(i - 1, j, k) / -fx(i, j, k, 0); + } + pdt_fab = amrex::min(pdt_fab, pdt_lim); + }); + return pdt_fab; + }); + const amrex::Real pdt_fy = amrex::ReduceMin( + mf_fy, mf_vof, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, amrex::Array4 const& fy, + amrex::Array4 const& vof) -> amrex::Real { + amrex::Real pdt_fab = 1.0; + amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { + amrex::Real pdt_lim = 1.0; + if (fy(i, j, k, 0) > tol && vof(i, j, k) > tol) { + // VOF is removed from cell j + pdt_lim = vof(i, j, k) / fy(i, j, k, 0); + } else if (fy(i, j, k, 0) < -tol && vof(i, j - 1, k) > tol) { + // VOF is removed from cell j-1 + pdt_lim = vof(i, j - 1, k) / -fy(i, j, k, 0); + } + pdt_fab = amrex::min(pdt_fab, pdt_lim); + }); + return pdt_fab; + }); + const amrex::Real pdt_fz = amrex::ReduceMin( + mf_fz, mf_vof, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, amrex::Array4 const& fz, + amrex::Array4 const& vof) -> amrex::Real { + amrex::Real pdt_fab = 1.0; + amrex::Loop(bx, [=, &pdt_fab](int i, int j, int k) noexcept { + amrex::Real pdt_lim = 1.0; + if (fz(i, j, k, 0) > tol && vof(i, j, k) > tol) { + // VOF is removed from cell k + pdt_lim = vof(i, j, k) / fz(i, j, k, 0); + } else if (fz(i, j, k, 0) < -tol && vof(i, j, k - 1) > tol) { + // VOF is removed from cell k-1 + pdt_lim = vof(i, j, k - 1) / -fz(i, j, k, 0); + } + pdt_fab = amrex::min(pdt_fab, pdt_lim); + }); + return pdt_fab; + }); + const amrex::Real pdt = amrex::min(pdt_fx, amrex::min(pdt_fy, pdt_fz)); + return pdt; +} + +// Apply reinitialization fluxes to modify fields +void apply_fluxes( + const amrex::MultiFab& mf_fx, + const amrex::MultiFab& mf_fy, + const amrex::MultiFab& mf_fz, + const amrex::MultiFab& mf_psource, + amrex::MultiFab& mf_vof, + amrex::MultiFab& mf_dens, + amrex::MultiFab& mf_vel, + amrex::MultiFab& mf_gp, + amrex::MultiFab& mf_pressure, + const amrex::GpuArray dx, + const amrex::Real ptfac, + const amrex::Real vof_tol) +{ + const auto& fx = mf_fx.const_arrays(); + const auto& fy = mf_fy.const_arrays(); + const auto& fz = mf_fz.const_arrays(); + const auto& sp = mf_psource.const_arrays(); + const auto& vof = mf_vof.arrays(); + const auto& dens = mf_dens.arrays(); + const auto& vel = mf_vel.arrays(); + const auto& gp = mf_gp.arrays(); + const auto& p = mf_pressure.arrays(); + + amrex::ParallelFor( + mf_vof, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + const amrex::Real olddens = dens[nbx](i, j, k); + vof[nbx](i, j, k) += + ptfac * + ((fx[nbx](i + 1, j, k, 0) - fx[nbx](i, j, k, 0)) / dx[0] + + (fy[nbx](i, j + 1, k, 0) - fy[nbx](i, j, k, 0)) / dx[1] + + (fz[nbx](i, j, k + 1, 0) - fz[nbx](i, j, k, 0)) / dx[2]); + dens[nbx](i, j, k) += + ptfac * + ((fx[nbx](i + 1, j, k, 1) - fx[nbx](i, j, k, 1)) / dx[0] + + (fy[nbx](i, j + 1, k, 1) - fy[nbx](i, j, k, 1)) / dx[1] + + (fz[nbx](i, j, k + 1, 1) - fz[nbx](i, j, k, 1)) / dx[2]); + vel[nbx](i, j, k, 0) = + 1.0 / dens[nbx](i, j, k) * + (olddens * vel[nbx](i, j, k, 0) + + ptfac * + ((fx[nbx](i + 1, j, k, 2) - fx[nbx](i, j, k, 2)) / dx[0] + + (fy[nbx](i, j + 1, k, 2) - fy[nbx](i, j, k, 2)) / dx[1] + + (fz[nbx](i, j, k + 1, 2) - fz[nbx](i, j, k, 2)) / dx[2])); + vel[nbx](i, j, k, 1) = + 1.0 / dens[nbx](i, j, k) * + (olddens * vel[nbx](i, j, k, 1) + + ptfac * + ((fx[nbx](i + 1, j, k, 3) - fx[nbx](i, j, k, 3)) / dx[0] + + (fy[nbx](i, j + 1, k, 3) - fy[nbx](i, j, k, 3)) / dx[1] + + (fz[nbx](i, j, k + 1, 3) - fz[nbx](i, j, k, 3)) / dx[2])); + vel[nbx](i, j, k, 2) = + 1.0 / dens[nbx](i, j, k) * + (olddens * vel[nbx](i, j, k, 2) + + ptfac * + ((fx[nbx](i + 1, j, k, 4) - fx[nbx](i, j, k, 4)) / dx[0] + + (fy[nbx](i, j + 1, k, 4) - fy[nbx](i, j, k, 4)) / dx[1] + + (fz[nbx](i, j, k + 1, 4) - fz[nbx](i, j, k, 4)) / dx[2])); + gp[nbx](i, j, k, 0) += + ptfac * + ((fx[nbx](i + 1, j, k, 5) - fx[nbx](i, j, k, 5)) / dx[0] + + (fy[nbx](i, j + 1, k, 5) - fy[nbx](i, j, k, 5)) / dx[1] + + (fz[nbx](i, j, k + 1, 5) - fz[nbx](i, j, k, 5)) / dx[2]); + gp[nbx](i, j, k, 1) += + ptfac * + ((fx[nbx](i + 1, j, k, 6) - fx[nbx](i, j, k, 6)) / dx[0] + + (fy[nbx](i, j + 1, k, 6) - fy[nbx](i, j, k, 6)) / dx[1] + + (fz[nbx](i, j, k + 1, 6) - fz[nbx](i, j, k, 6)) / dx[2]); + gp[nbx](i, j, k, 2) += + ptfac * + ((fx[nbx](i + 1, j, k, 7) - fx[nbx](i, j, k, 7)) / dx[0] + + (fy[nbx](i, j + 1, k, 7) - fy[nbx](i, j, k, 7)) / dx[1] + + (fz[nbx](i, j, k + 1, 7) - fz[nbx](i, j, k, 7)) / dx[2]); + + // Ensure vof is bounded + vof[nbx](i, j, k) = + vof[nbx](i, j, k) < vof_tol + ? 0.0 + : (vof[nbx](i, j, k) > 1. - vof_tol ? 1. + : vof[nbx](i, j, k)); + // Density bounds are enforced elsewhere + }); + amrex::ParallelFor( + mf_pressure, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + p[nbx](i, j, k) += ptfac * sp[nbx](i, j, k); + }); +} + +// Get the size of the smallest VOF flux to quantify convergence +amrex::Real measure_convergence( + amrex::MultiFab& mf_fx, amrex::MultiFab& mf_fy, amrex::MultiFab& mf_fz) +{ + // Get the maximum flux magnitude, but just for vof fluxes + const amrex::Real err_fx = amrex::ReduceMax( + mf_fx, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& fx) -> amrex::Real { + amrex::Real err_fab = -1.0; + amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { + err_fab = amrex::max(err_fab, std::abs(fx(i, j, k, 0))); + }); + return err_fab; + }); + const amrex::Real err_fy = amrex::ReduceMax( + mf_fy, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& fy) -> amrex::Real { + amrex::Real err_fab = -1.0; + amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { + err_fab = amrex::max(err_fab, std::abs(fy(i, j, k, 0))); + }); + return err_fab; + }); + const amrex::Real err_fz = amrex::ReduceMax( + mf_fz, 0, + [=] AMREX_GPU_HOST_DEVICE( + amrex::Box const& bx, + amrex::Array4 const& fz) -> amrex::Real { + amrex::Real err_fab = -1.0; + amrex::Loop(bx, [=, &err_fab](int i, int j, int k) noexcept { + err_fab = amrex::max(err_fab, std::abs(fz(i, j, k, 0))); + }); + return err_fab; + }); + const amrex::Real err = amrex::max(err_fx, amrex::max(err_fy, err_fz)); + return err; +} + +// Set levelset field to another quantity to view in plotfile for debugging +void equate_field(amrex::MultiFab& mf_dest, const amrex::MultiFab& mf_src) +{ + const auto& dest = mf_dest.arrays(); + const auto& src = mf_src.const_arrays(); + amrex::ParallelFor( + mf_dest, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + dest[nbx](i, j, k) = std::sqrt( + src[nbx](i, j, k, 0) * src[nbx](i, j, k, 0) + + src[nbx](i, j, k, 1) * src[nbx](i, j, k, 1) + + src[nbx](i, j, k, 2) * src[nbx](i, j, k, 2)); + }); +} + +// Replace pressure gradient with hydrostatic field in overset regions +void replace_gradp_hydrostatic( + amrex::MultiFab& mf_gp, + const amrex::MultiFab& mf_density, + const amrex::MultiFab& mf_refdens, + const amrex::iMultiFab& mf_iblank, + const amrex::Real grav_z, + const bool is_pptb) +{ + const auto& gp = mf_gp.arrays(); + const auto& rho = mf_density.const_arrays(); + const auto& rho0 = mf_refdens.const_arrays(); + const auto& iblank = mf_iblank.const_arrays(); + amrex::ParallelFor( + mf_gp, mf_gp.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + if (iblank[nbx](i, j, k) == -1) { + const amrex::Real dfac = + is_pptb ? rho[nbx](i, j, k) - rho0[nbx](i, j, k) + : rho[nbx](i, j, k); + gp[nbx](i, j, k, 0) = 0.; + gp[nbx](i, j, k, 1) = 0.; + gp[nbx](i, j, k, 2) = dfac * grav_z; + } + }); +} + +// Swap pressure gradient values in overset region +void replace_gradp( + amrex::MultiFab& mf_gp, + const amrex::MultiFab& mf_gp0, + const amrex::iMultiFab& mf_iblank) +{ + const auto gp = mf_gp.arrays(); + const auto gp0 = mf_gp0.const_arrays(); + const auto iblank = mf_iblank.const_arrays(); + amrex::ParallelFor( + mf_gp, mf_gp.n_grow, + [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + if (iblank[nbx](i, j, k) == -1) { + gp[nbx](i, j, k, 0) = gp0[nbx](i, j, k, 0); + gp[nbx](i, j, k, 1) = gp0[nbx](i, j, k, 1); + gp[nbx](i, j, k, 2) = gp0[nbx](i, j, k, 2); + } + }); +} + +// Apply pressure gradient to velocity field +void apply_pressure_gradient( + amrex::MultiFab& mf_vel, + const amrex::MultiFab& mf_density, + const amrex::MultiFab& mf_gp, + const amrex::Real scaling_factor) +{ + const auto& vel = mf_vel.arrays(); + const auto& rho = mf_density.const_arrays(); + const auto& gp = mf_gp.const_arrays(); + amrex::ParallelFor( + mf_vel, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept { + const amrex::Real soverrho = scaling_factor / rho[nbx](i, j, k); + vel[nbx](i, j, k, 0) -= gp[nbx](i, j, k, 0) * soverrho; + vel[nbx](i, j, k, 1) -= gp[nbx](i, j, k, 1) * soverrho; + vel[nbx](i, j, k, 2) -= gp[nbx](i, j, k, 2) * soverrho; + }); +} + +} // namespace amr_wind::overset_ops \ No newline at end of file From b7bfc1ad8d6c3e8414d16ff744c521dbc0721e59 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 19 Jul 2024 09:47:57 -0600 Subject: [PATCH 41/43] calculate gradp after p has been modified --- amr-wind/overset/OversetOps.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index 1bde730b42..ac2e4edd3a 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -58,6 +58,9 @@ void OversetOps::pre_advance_work() if (m_use_hydrostatic_gradp) { // Use hydrostatic pressure gradient set_hydrostatic_gradp(); + } else { + // Update pressure gradient using sharpened pressure field + update_gradp(); } } From 725e03b5fbe5c06ffd02074def3f3949627b3ccc Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 19 Jul 2024 10:55:58 -0600 Subject: [PATCH 42/43] stuff for perturbational pressure --- amr-wind/overset/OversetOps.H | 4 +-- amr-wind/overset/OversetOps.cpp | 43 ++++++++++++++++++++---- amr-wind/physics/multiphase/MultiPhase.H | 7 ++++ 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/amr-wind/overset/OversetOps.H b/amr-wind/overset/OversetOps.H index 9d6dcb254e..ae935202eb 100644 --- a/amr-wind/overset/OversetOps.H +++ b/amr-wind/overset/OversetOps.H @@ -21,14 +21,12 @@ private: // Functions called within public functions void parameter_output() const; void sharpen_nalu_data(); + void form_perturb_pressure(); void set_hydrostatic_gradp(); void replace_masked_gradp(); // Check for multiphase sim bool m_vof_exists{false}; - // Check for perturbational pressure - // (will be removed soon) - bool m_perturb_p{false}; // To avoid using sharpened pressure, use hydrostatic pressure bool m_use_hydrostatic_gradp{false}; diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index ac2e4edd3a..d34fb1ed87 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -30,12 +30,25 @@ void OversetOps::initialize(CFDSim& sim) pp.query("verbose", m_verbose); - amrex::ParmParse pp_icns("ICNS"); - pp_icns.query("use_perturb_pressure", m_perturb_p); - m_vof_exists = (*m_sim_ptr).repo().field_exists("vof"); if (m_vof_exists) { m_mphase = &(*m_sim_ptr).physics_manager().get(); + + // Check combination of pressure options + if (m_mphase->perturb_pressure() && + !m_mphase->reconstruct_true_pressure()) { + amrex::Abort( + "OversetOps: perturbational pressure is turned on, but true " + "pressure reconstruction is turned off. This approach will be " + "incorrect when coupling with Nalu-Wind."); + } + if (!m_mphase->perturb_pressure() && + m_mphase->reconstruct_true_pressure()) { + amrex::Print() + << "WARNING (OversetOps): true pressure reconstruction is " + "turned on, but it will remain inactive because " + "perturbational pressure is turned off.\n"; + } } if (m_replace_gradp_postsolve) { m_gp_copy = &(*m_sim_ptr).repo().declare_field("gp_copy", 3); @@ -46,7 +59,6 @@ void OversetOps::initialize(CFDSim& sim) void OversetOps::pre_advance_work() { - // Pressure gradient not updated for current multiphase approach if (!(m_vof_exists && m_use_hydrostatic_gradp)) { // Update pressure gradient using updated overset pressure field update_gradp(); @@ -59,6 +71,10 @@ void OversetOps::pre_advance_work() // Use hydrostatic pressure gradient set_hydrostatic_gradp(); } else { + if (m_mphase->perturb_pressure()) { + // Modify to be consistent with internal source terms + form_perturb_pressure(); + } // Update pressure gradient using sharpened pressure field update_gradp(); } @@ -158,6 +174,11 @@ void OversetOps::parameter_output() const << "---- Replace overset pres grad: " << m_replace_gradp_postsolve << std::endl; if (m_vof_exists) { + amrex::Print() << "---- Perturbational pressure : " + << m_mphase->perturb_pressure() << std::endl + << "---- Reconstruct true pressure: " + << m_mphase->reconstruct_true_pressure() + << std::endl; amrex::Print() << "Overset Reinitialization Parameters:\n" << "---- Maximum iterations : " << m_n_iterations << std::endl @@ -375,6 +396,16 @@ void OversetOps::sharpen_nalu_data() amrex::Gpu::synchronize(); } +void OversetOps::form_perturb_pressure() +{ + auto& pressure = (*m_sim_ptr).repo().get_field("p"); + const auto& p0 = (*m_sim_ptr).repo().get_field("reference_pressure"); + for (int lev = 0; lev < (*m_sim_ptr).repo().num_active_levels(); lev++) { + amrex::MultiFab::Subtract( + pressure(lev), p0(lev), 0, 0, 1, pressure.num_grow()[0]); + } +} + void OversetOps::set_hydrostatic_gradp() { const auto& repo = (*m_sim_ptr).repo(); @@ -388,7 +419,7 @@ void OversetOps::set_hydrostatic_gradp() Field* rho0{nullptr}; auto& rho = repo.get_field("density"); auto& gp = repo.get_field("gp"); - if (m_perturb_p) { + if (m_mphase->perturb_pressure()) { rho0 = &((*m_sim_ptr).repo().get_field("reference_density")); } else { // Point to existing field, won't be used @@ -399,7 +430,7 @@ void OversetOps::set_hydrostatic_gradp() for (int lev = 0; lev < nlevels; ++lev) { overset_ops::replace_gradp_hydrostatic( gp(lev), rho(lev), (*rho0)(lev), iblank_cell(lev), - m_mphase->gravity()[2], m_perturb_p); + m_mphase->gravity()[2], m_mphase->perturb_pressure()); } } diff --git a/amr-wind/physics/multiphase/MultiPhase.H b/amr-wind/physics/multiphase/MultiPhase.H index 8b43eef505..e127bfbc6f 100644 --- a/amr-wind/physics/multiphase/MultiPhase.H +++ b/amr-wind/physics/multiphase/MultiPhase.H @@ -71,6 +71,13 @@ public: amrex::Real water_level() const { return water_level0; } + bool perturb_pressure() const { return m_use_perturb_pressure; } + + bool reconstruct_true_pressure() const + { + return m_reconstruct_true_pressure; + } + private: const CFDSim& m_sim; From 86accdcdf57457c282c28f74fb983b7d7449bd40 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Fri, 19 Jul 2024 11:16:03 -0600 Subject: [PATCH 43/43] code review --- amr-wind/overset/OversetOps.cpp | 20 +++++------- amr-wind/overset/overset_ops_K.H | 31 +++++++++---------- amr-wind/overset/overset_ops_routines.H | 10 +++--- amr-wind/overset/overset_ops_routines.cpp | 22 ++++++------- .../multiphase/test_vof_overset_ops.cpp | 2 +- 5 files changed, 37 insertions(+), 48 deletions(-) diff --git a/amr-wind/overset/OversetOps.cpp b/amr-wind/overset/OversetOps.cpp index d34fb1ed87..361a97014d 100644 --- a/amr-wind/overset/OversetOps.cpp +++ b/amr-wind/overset/OversetOps.cpp @@ -218,17 +218,15 @@ void OversetOps::sharpen_nalu_data() auto& gp_noghost = repo.get_field("gp"); auto& p = repo.get_field("p"); - // Create scratch fields for fluxes // 9 components are vof, density, 3 of velocity, 3 of gp, and psource flag auto flux_x = repo.create_scratch_field(9, 1, amr_wind::FieldLoc::XFACE); auto flux_y = repo.create_scratch_field(9, 1, amr_wind::FieldLoc::YFACE); auto flux_z = repo.create_scratch_field(9, 1, amr_wind::FieldLoc::ZFACE); - // Create scratch field for pressure source term + auto p_src = repo.create_scratch_field(1, 0, amr_wind::FieldLoc::NODE); - // Create scratch fields for normal vector and target vof field auto normal_vec = repo.create_scratch_field(3, vof.num_grow()[0] - 1); auto target_vof = repo.create_scratch_field(1, vof.num_grow()[0]); - // Create scratch field for pressure gradient that has ghost cells + // Sharpening fluxes (at faces) have 1 ghost, requiring fields to have >= 2 auto gp_scr = repo.create_scratch_field(3, 2); auto& gp = *gp_scr; @@ -249,10 +247,9 @@ void OversetOps::sharpen_nalu_data() // Populate approximate signed distance function overset_ops::populate_psi(levelset(lev), vof(lev), i_th, m_asdf_tiny); - // Populate gp scratch field - gp(lev).setVal(0.0); // for external boundaries - amrex::MultiFab::Copy(gp(lev), gp_noghost(lev), 0, 0, 3, 0); // nonghost - gp(lev).FillBoundary(geom[lev].periodicity()); // internal + gp(lev).setVal(0.0); + amrex::MultiFab::Copy(gp(lev), gp_noghost(lev), 0, 0, 3, 0); + gp(lev).FillBoundary(geom[lev].periodicity()); // Get pseudo-velocity scale, proportional to smallest dx in iblank const amrex::Real pvscale_lev = @@ -332,11 +329,9 @@ void OversetOps::sharpen_nalu_data() // Average down fluxes across levels for consistency for (int lev = nlevels - 1; lev > 0; --lev) { - const amrex::IntVect rr = - geom[lev].Domain().size() / geom[lev - 1].Domain().size(); amrex::average_down_faces( - GetArrOfConstPtrs(fluxes[lev]), fluxes[lev - 1], rr, - geom[lev - 1]); + GetArrOfConstPtrs(fluxes[lev]), fluxes[lev - 1], + repo.mesh().refRatio(lev), geom[lev - 1]); } // Get pseudo dt (dtau) @@ -362,7 +357,6 @@ void OversetOps::sharpen_nalu_data() vof(lev), rho(lev), velocity(lev), gp(lev), p(lev), dx, ptfac, m_vof_tol); - // Update ghost cells vof(lev).FillBoundary(geom[lev].periodicity()); velocity(lev).FillBoundary(geom[lev].periodicity()); gp(lev).FillBoundary(geom[lev].periodicity()); diff --git a/amr-wind/overset/overset_ops_K.H b/amr-wind/overset/overset_ops_K.H index a06f4c97e1..143731ad36 100644 --- a/amr-wind/overset/overset_ops_K.H +++ b/amr-wind/overset/overset_ops_K.H @@ -105,10 +105,10 @@ void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE velocity_face( } void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE gp_rho_face( - int i, - int j, - int k, - int dir, + const int i, + const int j, + const int k, + const int dir, amrex::Array4 const& vof, amrex::Array4 const& gp, amrex::Array4 const& rho, @@ -117,23 +117,20 @@ void AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE gp_rho_face( amrex::Real& wface) { // Set up neighbor indices - int ii = i; - int jj = j; - int kk = k; - ii += (dir == 0) ? -1 : 0; - jj += (dir == 1) ? -1 : 0; - kk += (dir == 2) ? -1 : 0; + const amrex::IntVect iv{i, j, k}; + const amrex::IntVect dv{(int)(dir == 0), (int)(dir == 1), (int)(dir == 2)}; + const amrex::IntVect ivm = iv - dv; // Gradient of phi normal to interface - const amrex::Real gphi = (vof(i, j, k) - vof(ii, jj, kk)); + const amrex::Real gphi = (vof(iv) - vof(ivm)); // Get velocities on both sides - const amrex::Real u_ = gp(i, j, k, 0) / rho(i, j, k); - const amrex::Real v_ = gp(i, j, k, 1) / rho(i, j, k); - const amrex::Real w_ = gp(i, j, k, 2) / rho(i, j, k); - const amrex::Real u_nb = gp(ii, jj, kk, 0) / rho(ii, jj, kk); - const amrex::Real v_nb = gp(ii, jj, kk, 1) / rho(ii, jj, kk); - const amrex::Real w_nb = gp(ii, jj, kk, 2) / rho(ii, jj, kk); + const amrex::Real u_ = gp(iv, 0) / rho(iv); + const amrex::Real v_ = gp(iv, 1) / rho(iv); + const amrex::Real w_ = gp(iv, 2) / rho(iv); + const amrex::Real u_nb = gp(ivm, 0) / rho(ivm); + const amrex::Real v_nb = gp(ivm, 1) / rho(ivm); + const amrex::Real w_nb = gp(ivm, 2) / rho(ivm); // Average value when gphi = 0 uface = 0.5 * (u_ + u_nb); vface = 0.5 * (v_ + v_nb); diff --git a/amr-wind/overset/overset_ops_routines.H b/amr-wind/overset/overset_ops_routines.H index 8f828f54bf..de9afbdff8 100644 --- a/amr-wind/overset/overset_ops_routines.H +++ b/amr-wind/overset/overset_ops_routines.H @@ -64,11 +64,11 @@ amrex::Real calculate_pseudo_velocity_scale( // Calculate a type of CFL by measuring how much % VOF is being removed per cell amrex::Real calculate_pseudo_dt_flux( - amrex::MultiFab& mf_fx, - amrex::MultiFab& mf_fy, - amrex::MultiFab& mf_fz, - amrex::MultiFab& mf_vof, - amrex::Real tol); + const amrex::MultiFab& mf_fx, + const amrex::MultiFab& mf_fy, + const amrex::MultiFab& mf_fz, + const amrex::MultiFab& mf_vof, + const amrex::Real tol); // Apply reinitialization fluxes to modify fields void apply_fluxes( diff --git a/amr-wind/overset/overset_ops_routines.cpp b/amr-wind/overset/overset_ops_routines.cpp index e63b4e8b80..57becc49ec 100644 --- a/amr-wind/overset/overset_ops_routines.cpp +++ b/amr-wind/overset/overset_ops_routines.cpp @@ -206,21 +206,21 @@ void process_fluxes_calc_src( amrex::ParallelFor( mf_fx, mf_fx.n_grow, mf_fx.n_comp, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { - bool zero_all = + const bool zero_all = (iblank[nbx](i - 1, j, k) + iblank[nbx](i, j, k) > -2); fx[nbx](i, j, k, n) *= zero_all ? 0. : 1.; }); amrex::ParallelFor( mf_fy, mf_fy.n_grow, mf_fy.n_comp, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { - bool zero_all = + const bool zero_all = (iblank[nbx](i, j - 1, k) + iblank[nbx](i, j, k) > -2); fy[nbx](i, j, k, n) *= zero_all ? 0. : 1.; }); amrex::ParallelFor( mf_fz, mf_fz.n_grow, mf_fz.n_comp, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept { - bool zero_all = + const bool zero_all = (iblank[nbx](i, j, k - 1) + iblank[nbx](i, j, k) > -2); fz[nbx](i, j, k, n) *= zero_all ? 0. : 1.; }); @@ -240,21 +240,19 @@ amrex::Real calculate_pseudo_velocity_scale( const amrex::GpuArray dx, const amrex::Real pvmax) { - // Get minimum length scale from dx - const amrex::Real dx_min = std::min(std::min(dx[0], dx[1]), dx[2]); // The dx of this level should be considered if iblank = -1 here // Otherwise, set to max possible value for this mesh (level 0 values) - const amrex::Real pvscale = (mf_iblank.min(0) == -1) ? dx_min : pvmax; - return pvscale; + return (mf_iblank.min(0) == -1) ? std::min(std::min(dx[0], dx[1]), dx[2]) + : pvmax; } // Calculate a type of CFL by measuring how much % VOF is being removed per cell amrex::Real calculate_pseudo_dt_flux( - amrex::MultiFab& mf_fx, - amrex::MultiFab& mf_fy, - amrex::MultiFab& mf_fz, - amrex::MultiFab& mf_vof, - amrex::Real tol) + const amrex::MultiFab& mf_fx, + const amrex::MultiFab& mf_fy, + const amrex::MultiFab& mf_fz, + const amrex::MultiFab& mf_vof, + const amrex::Real tol) { // Get the maximum flux magnitude, but just for vof fluxes const amrex::Real pdt_fx = amrex::ReduceMin( diff --git a/unit_tests/multiphase/test_vof_overset_ops.cpp b/unit_tests/multiphase/test_vof_overset_ops.cpp index 34dffcec04..fa27e9d538 100644 --- a/unit_tests/multiphase/test_vof_overset_ops.cpp +++ b/unit_tests/multiphase/test_vof_overset_ops.cpp @@ -874,7 +874,7 @@ TEST_F(VOFOversetOps, psource_manual) dir = 1; init_gp_rho_etc(gp, rho, vof, dir, rho_liq, rho_gas); calc_gp_rho_face(flux_y, gp, rho, vof, dir, 5, true); - dir = 3; + dir = 2; init_gp_rho_etc(gp, rho, vof, dir, rho_liq, rho_gas); calc_gp_rho_face(flux_z, gp, rho, vof, dir, 5, true);