is this any faster?

Exawind · Oct 10, 2024 · d433dcf · d433dcf
1 parent 171b62a
commit d433dcf
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 27 deletions.
diff --git a/amr-wind/utilities/sampling/PlaneSampler.cpp b/amr-wind/utilities/sampling/PlaneSampler.cpp
@@ -107,12 +107,12 @@ void PlaneSampler::sampling_locations(
     for (int k = 0; k < nplanes; ++k) {
         for (int j = 0; j < m_npts_dir[1]; ++j) {
             for (int i = 0; i < m_npts_dir[0]; ++i) {
-                amrex::RealVect loc;
-                for (int d = 0; d < AMREX_SPACEDIM; ++d) {
-                    loc[d] = m_origin[d] + dx[d] * i + dy[d] * j +
-                             m_poffsets[k] * m_offset_vector[d];
-                }
                 for (const auto& box : boxes) {
+                    amrex::RealVect loc;
+                    for (int d = 0; d < AMREX_SPACEDIM; ++d) {
+                        loc[d] = m_origin[d] + dx[d] * i + dy[d] * j +
+                                 m_poffsets[k] * m_offset_vector[d];
+                    }
                     if (utils::contains(box, loc, plo, dxinv)) {
                         sample_locs.push_back(loc, idx);
                         break;

diff --git a/amr-wind/utilities/sampling/SamplingContainer.cpp b/amr-wind/utilities/sampling/SamplingContainer.cpp
@@ -58,20 +58,22 @@ void SamplingContainer::initialize_particles(
     for (int iprobe = 0; iprobe < nprobes; iprobe++) {
         const auto& probe = samplers[iprobe];
         const auto total_num_points = probe->num_points();
-        SampleLocType sample_locs;
-        probe->sampling_locations(sample_locs, owned_boxes);
-        const auto& locs = sample_locs.locations();
-        const int npts = static_cast<int>(locs.size());
-        amrex::Gpu::DeviceVector<amrex::RealVect> dlocs(npts);
-
-        amrex::Gpu::copy(
-            amrex::Gpu::hostToDevice, locs.begin(), locs.end(), dlocs.begin());
-        const auto* dpos = dlocs.data();
 
         // don't use openmp
         for (amrex::MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) {
             const amrex::Box& box = mfi.tilebox();
 
+            SampleLocType sample_locs;
+            probe->sampling_locations(sample_locs, {box});
+            const auto& locs = sample_locs.locations();
+            const int npts = static_cast<int>(locs.size());
+            amrex::Gpu::DeviceVector<amrex::RealVect> dlocs(npts);
+
+            amrex::Gpu::copy(
+                amrex::Gpu::hostToDevice, locs.begin(), locs.end(),
+                dlocs.begin());
+            const auto* dpos = dlocs.data();
+
             // count the number of particles in each cell
             const auto& np_arr = particle_counts[mfi].array();
             amrex::ParallelFor(
@@ -140,19 +142,6 @@ void SamplingContainer::initialize_particles(
         const auto& probe = samplers[iprobe];
         const auto total_num_points = probe->num_points();
         const auto probe_id = probe->id();
-        SampleLocType sample_locs;
-        probe->sampling_locations(sample_locs, owned_boxes);
-        const auto& locs = sample_locs.locations();
-        const int npts = static_cast<int>(locs.size());
-        amrex::Gpu::DeviceVector<amrex::RealVect> dlocs(npts);
-        amrex::Gpu::copy(
-            amrex::Gpu::hostToDevice, locs.begin(), locs.end(), dlocs.begin());
-        const auto* p_dlocs = dlocs.data();
-        const auto& ids = sample_locs.ids();
-        amrex::Gpu::DeviceVector<amrex::Long> dids(npts);
-        amrex::Gpu::copy(
-            amrex::Gpu::hostToDevice, ids.begin(), ids.end(), dids.begin());
-        const auto* p_dids = dids.data();
 
         // don't use openmp
         for (amrex::MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) {
@@ -176,6 +165,21 @@ void SamplingContainer::initialize_particles(
             const amrex::Long nextid = ParticleType::NextID();
             ParticleType::NextID(nextid + np_probe);
 
+            SampleLocType sample_locs;
+            probe->sampling_locations(sample_locs, {box});
+            const auto& locs = sample_locs.locations();
+            const int npts = static_cast<int>(locs.size());
+            amrex::Gpu::DeviceVector<amrex::RealVect> dlocs(npts);
+            amrex::Gpu::copy(
+                amrex::Gpu::hostToDevice, locs.begin(), locs.end(),
+                dlocs.begin());
+            const auto* p_dlocs = dlocs.data();
+            const auto& ids = sample_locs.ids();
+            amrex::Gpu::DeviceVector<amrex::Long> dids(npts);
+            amrex::Gpu::copy(
+                amrex::Gpu::hostToDevice, ids.begin(), ids.end(), dids.begin());
+            const auto* p_dids = dids.data();
+
             auto* pstruct = ptile.GetArrayOfStructs()().data();
             amrex::ParallelFor(
                 box, [=] AMREX_GPU_DEVICE(