Skip to content

Commit

Permalink
Merge branch 'Exawind:main' into spinner-final
Browse files Browse the repository at this point in the history
  • Loading branch information
ndevelder authored Jul 28, 2023
2 parents 2913050 + 9cf678b commit 835f571
Show file tree
Hide file tree
Showing 142 changed files with 5,066 additions and 453 deletions.
24 changes: 12 additions & 12 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ jobs:
- name: Clone
uses: actions/checkout@v3
- name: Check formatting
uses: DoozyX/clang-format-lint-action@v0.14
uses: DoozyX/clang-format-lint-action@v0.16.2
with:
source: './amr-wind ./unit_tests ./tools/utilities'
exclude: '.'
extensions: 'H,h,cpp'
clangFormatVersion: 14
clangFormatVersion: 16
CPU:
needs: Formatting
runs-on: ${{matrix.os}}
Expand Down Expand Up @@ -163,7 +163,7 @@ jobs:
${{github.workspace}}
cmake --build build -- -j $(nproc)
GPU-Intel:
name: GPU-DPCPP
name: GPU-SYCL
needs: Formatting
runs-on: ubuntu-20.04
steps:
Expand All @@ -175,27 +175,27 @@ jobs:
uses: actions/checkout@v3
with:
submodules: true
- name: Prepare DPC++ environment
- name: Prepare SyCL environment
run: |
export DEBIAN_FRONTEND=noninteractive
sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
sudo apt-get update
sudo apt-get install -y --no-install-recommends ninja-build intel-oneapi-dpcpp-cpp-compiler intel-oneapi-mkl-devel
sudo apt-get install -y --no-install-recommends ninja-build intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel
- name: Configure and build
run: |
set +e
source /opt/intel/oneapi/setvars.sh
set -e
cmake -G Ninja -B build-DPCPP \
cmake -G Ninja -B build-SYCL \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DCMAKE_CXX_COMPILER:STRING=$(which dpcpp) \
-DCMAKE_C_COMPILER:STRING=$(which clang) \
-DCMAKE_CXX_COMPILER:STRING=$(which icpx) \
-DCMAKE_C_COMPILER:STRING=$(which icx) \
-DAMR_WIND_ENABLE_MPI:BOOL=OFF \
-DAMR_WIND_ENABLE_DPCPP:BOOL=ON \
-DAMR_WIND_ENABLE_SYCL:BOOL=ON \
${{github.workspace}}
cmake --build build-DPCPP -- -j $(nproc)
cmake --build build-SYCL -- -j $(nproc)
Lint-cppcheck:
needs: Formatting
runs-on: macos-latest
Expand Down
20 changes: 20 additions & 0 deletions .github/workflows/stale.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
name: 'Managing stale issues and pull requests'
on:
schedule:
- cron: '30 1 * * *'

jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v8
with:
days-before-stale: 30
stale-issue-message: 'This issue is stale because it has been open 30 days with no activity.'
close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.'
stale-issue-label: 'no-issue-activity'
days-before-issue-close: -1
stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity.'
stale-pr-label: 'no-pr-activity'
days-before-pr-close: 7
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ option(AMR_WIND_ENABLE_MPI "Enable MPI" OFF)
option(AMR_WIND_ENABLE_OPENMP "Enable OpenMP" OFF)
option(AMR_WIND_ENABLE_CUDA "Enable CUDA" OFF)
option(AMR_WIND_ENABLE_ROCM "Enable ROCm/HIP" OFF)
option(AMR_WIND_ENABLE_DPCPP "Enable Intel OneAPI DPC++" OFF)
option(AMR_WIND_ENABLE_SYCL "Enable Intel OneAPI SyCL" OFF)
option(AMR_WIND_ENABLE_TINY_PROFILE "Enable AMReX TinyProfile support" OFF)
set(AMR_WIND_PRECISION "DOUBLE" CACHE STRING "Floating point precision SINGLE or DOUBLE")

Expand Down
13 changes: 7 additions & 6 deletions amr-wind/CFDSim.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,19 @@ CFDSim::~CFDSim() = default;

void CFDSim::create_turbulence_model()
{
std::string transport_model = "ConstTransport";
std::string turbulence_model = "Laminar";
std::string transport_model_name = "ConstTransport";
std::string turbulence_model_name = "Laminar";
{
amrex::ParmParse pp("transport");
pp.query("model", transport_model);
pp.query("model", transport_model_name);
}
{
amrex::ParmParse pp("turbulence");
pp.query("model", turbulence_model);
pp.query("model", turbulence_model_name);
}

const std::string identifier = turbulence_model + "-" + transport_model;
const std::string identifier =
turbulence_model_name + "-" + transport_model_name;
m_turbulence = turbulence::TurbulenceModel::create(identifier, *this);
m_turbulence->parse_model_coeffs();
}
Expand All @@ -45,7 +46,7 @@ void CFDSim::init_physics()
amrex::Vector<std::string> phys_names;
pp.queryarr("physics", phys_names);

for (auto& phy : phys_names) {
for (const auto& phy : phys_names) {
m_physics_mgr.create(phy, *this);
}
}
Expand Down
18 changes: 18 additions & 0 deletions amr-wind/boundary_conditions/BCInterface.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ void BCIface::read_bctype()
ibctype[ori] = BC::fixed_gradient;
} else if ((bcstr == "wave_generation") || (bcstr == "wg")) {
ibctype[ori] = BC::wave_generation;
} else if ((bcstr == "symmetric_wall") || (bcstr == "symw")) {
ibctype[ori] = BC::symmetric_wall;
} else {
ibctype[ori] = BC::undefined;
}
Expand Down Expand Up @@ -129,6 +131,7 @@ std::pair<const std::string, const std::string> BCIface::get_dirichlet_udfs()
"faces");
} else {
inflow_udf = val;
has_inflow_udf = true;
}
}
}
Expand All @@ -144,6 +147,7 @@ std::pair<const std::string, const std::string> BCIface::get_dirichlet_udfs()
"faces");
} else {
wall_udf = val;
has_wall_udf = true;
}
}
}
Expand Down Expand Up @@ -206,6 +210,19 @@ void BCVelocity::set_bcrec()
bcrec[dir].setHi(dir, amrex::BCType::ext_dir);
}
break;
case BC::symmetric_wall:
if (side == amrex::Orientation::low) {
// Tangential directions use first-order extrapolation
set_bcrec_lo(dir, amrex::BCType::foextrap);
// Normal direction uses dirichlet (ext-dir)
bcrec[dir].setLo(dir, amrex::BCType::ext_dir);
} else {
// Tangential directions use first-order extrapolation
set_bcrec_hi(dir, amrex::BCType::foextrap);
// Normal direction uses dirichlet (ext-dir)
bcrec[dir].setHi(dir, amrex::BCType::ext_dir);
}
break;

default:
amrex::Abort("Invalid incflo BC type encountered");
Expand Down Expand Up @@ -254,6 +271,7 @@ void BCScalar::set_bcrec()
case BC::pressure_inflow:
case BC::pressure_outflow:
case BC::zero_gradient:
case BC::symmetric_wall:
if (side == amrex::Orientation::low) {
set_bcrec_lo(dir, amrex::BCType::foextrap);
} else {
Expand Down
2 changes: 2 additions & 0 deletions amr-wind/boundary_conditions/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,5 @@ target_sources(${amr_wind_lib_name}
BCInterface.cpp
FixedGradientBC.cpp
)

add_subdirectory(wall_models)
13 changes: 10 additions & 3 deletions amr-wind/boundary_conditions/README.org
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,17 @@ The boundary conditions that can be specified in the input file are:
if the user does not specify any inputs in the input file.

- =zero_gradient= :: Sets =BCRec= to =foextrap= and =LinOpBCType= to Neumann.
Like =no_slip_wall=, the normal component of velocity is automatically reset
to zero. The value on the boundary face is the same as the value at the cell
Note, unlike the =no_slip_wall=, the normal component of velocity is also zero
Neumann. The value on the boundary face is the same as the value at the cell
center. No additional inputs are necessary when this BC is specified.

- =slip_wall= :: =BCRec = hoextrap= and =LinOpBCType = Neumann=. The value on
the boundary face is /extrapolated/ from the two interior cells using a
one-sided stencil. No additional inputs are necessary when this BC is specified.

- =symmetric_wall= :: Sets =BCRec= to =foextrap= and =LinOpBCType= to Neumann.
Similar to =zero_gradient= but the normal component of velocity is set to 0.

- =fixed_gradient= :: =BCRec = hoextrap= and =LinOpBCType = inhomogNeumann=.
Like =slip_wall= the value on the face is extrapolated (during fillpatch)
using a one-sided stencil. However, for linear solvers the value on the face
Expand Down Expand Up @@ -86,7 +89,8 @@ possible for scalars and tangential component of velocity.
| pressure_outflow | po | foextrap | foextrap | foextrap | foextrap |
| mass_inflow | mi | ext_dir | ext_dir | ext_dir | foextrap |
| no_slip_wall | nsw | ext_dir (0) | ext_dir | ext_dir | foextrap |
| zero_gradient | zg | ext_dir (0) | foextrap | foextrap | foextrap |
| symmetric_wall | symw | ext_dir (0) | foextrap | foextrap | foextrap |
| zero_gradient | zg | foextrap | foextrap | foextrap | foextrap |
| slip_wall | sw | ext_dir (0) | hoextrap | hoextrap | foextrap |
| wall_model | wm | ext_dir (0) | hoextrap | hoextrap | foextrap |
| fixed_gradient | fg | hoextrap | hoextrap | hoextrap | foextrap |
Expand All @@ -103,6 +107,7 @@ projection (pressure) and MAC projections.
| pressure_outflow | po | Dirichlet |
| mass_inflow | mi | Neumann |
| zero_gradient | zg | Neumann |
| symmetric_wall | symw | Neumann |
| no_slip_wall | nsw | Neumann |
| slip_wall | sw | Neumann |
| wall_model | wm | Neumann |
Expand All @@ -118,6 +123,7 @@ projection (pressure) and MAC projections.
| mass_inflow | mi | Dirichlet | Dirichlet |
| zero_gradient | zg | Dirichlet (0) | Neumann |
| no_slip_wall | nsw | Dirichlet (0) | Dirichlet |
| symmetric_wall | symw | Dirichlet (0) | Neumann |
| slip_wall | sw | Dirichlet (0) | Neumann |
| wall_model | wm | Dirichlet (0) | Inhomog Neumann |
| fixed_gradient | fg | Inhomog Neumann | Inhomog Neumann |
Expand All @@ -132,6 +138,7 @@ projection (pressure) and MAC projections.
| pressure_outflow | po | Neumann |
| mass_inflow | mi | Dirichlet |
| zero_gradient | zg | Neumann |
| symmetric_wall | symw | Neumann |
| no_slip_wall | nsw | Dirichlet |
| slip_wall | sw | Neumann |
| wall_model | wm | Inhomog Neumann |
Expand Down
5 changes: 5 additions & 0 deletions amr-wind/boundary_conditions/wall_models/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
target_sources(${amr_wind_lib_name}
PRIVATE
#C++
WallFunction.cpp
)
59 changes: 59 additions & 0 deletions amr-wind/boundary_conditions/wall_models/LogLaw.H
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#ifndef LogLaw_H
#define LogLaw_H

#include "AMReX_AmrCore.H"
#include <AMReX.H>
namespace amr_wind {
struct LogLaw
{
/*
* A simple wall model that sets the wall-shear stress
* based on computing u_tau given the horizontal velocity
* magnitude at a zref. This is akin to an explicit non-linear
* Robin boundary condition at the wall.
*/

// Log law constants from Lee & Moser 2015
// https://doi.org/10.1017/jfm.2015.268.
amrex::Real B{4.27};
amrex::Real kappa{0.384};
int max_iters = 25; // Max iterations for u_tau Newton-Raphson solve
// Reference height for log law
amrex::Real zref;
int ref_index{0};
amrex::Real nu; // molecular viscosity
// u_tau state variable, gets updated in update_utau depending on
// the type of wall model used
amrex::Real utau_mean{1.0};
amrex::Real wspd_mean; // mean horizontal velocity magnitude

void update_utau_mean() { utau_mean = get_utau(wspd_mean); }

AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real
get_utau(amrex::Real wspd) const
{
amrex::Real utau_iter = -1;
amrex::Real wspd_pred;
amrex::Real wspd_deriv;
amrex::Real zplus;
amrex::Real utau = utau_mean;
int iter = 0;
while ((std::abs(utau_iter - utau) > 1e-5) && iter <= max_iters) {
utau_iter = utau;
zplus = zref * utau / nu;
// Get wspd for a given utau from log-law
wspd_pred = utau * (std::log(zplus) / kappa + B);
wspd_deriv = (1 + std::log(zplus)) / kappa + B; // d(wspd)/d(utau)
utau =
utau - (wspd_pred - wspd) / wspd_deriv; // Newton-Raphson update
++iter;
}
if (iter == max_iters) {
amrex::Abort();
}
return utau;
}
};
} /* namespace amr_wind */

#endif /* LogLaw_H */
40 changes: 40 additions & 0 deletions amr-wind/boundary_conditions/wall_models/ShearStressSimple.H
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#ifndef SHEARSTRESSSIMPLE_H
#define SHEARSTRESSSIMPLE_H

#include "amr-wind/boundary_conditions/wall_models/LogLaw.H"
#include "amr-wind/wind_energy/ShearStress.H"

namespace amr_wind {

struct SimpleShearSchumann
{
explicit SimpleShearSchumann(const amr_wind::LogLaw& ll)
: utau2(ll.utau_mean * ll.utau_mean), wspd_mean(ll.wspd_mean), m_ll(ll)
{}

AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real
get_shear(amrex::Real u, amrex::Real /* wspd */) const
{
return u / wspd_mean * utau2;
};

amrex::Real utau2;
amrex::Real wspd_mean;
const amr_wind::LogLaw m_ll;
};
struct SimpleShearLogLaw
{
explicit SimpleShearLogLaw(const amr_wind::LogLaw& ll) : m_ll(ll) {}

AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real
get_shear(amrex::Real u, amrex::Real wspd) const
{
amrex::Real utau = m_ll.get_utau(wspd);
return utau * utau * u / wspd;
};

const amr_wind::LogLaw m_ll;
};
} // namespace amr_wind

#endif /* SHEARSTRESSSIMPLE_H */
Loading

0 comments on commit 835f571

Please sign in to comment.