Remove lock/unlock test, add API for active target communication

Author: hmenke

c++/mpi/group.hpp

@@ -0,0 +1,125 @@
+// Copyright (c) 2024 Simons Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0.txt
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Authors: Thomas Hahn, Alexander Hampel, Olivier Parcollet, Nils Wentzell
+
+/**
+ * @file
+ * @brief Provides a C++ wrapper class for an @p MPI_Group object.
+ */
+
+#pragma once
+
+#include "./communicator.hpp"
+#include "./environment.hpp"
+
+#include <mpi.h>
+
+#include <cstdlib>
+#include <unistd.h>
+
+namespace mpi {
+
+  /**
+   * @ingroup mpi_essentials
+   * @brief C++ wrapper around @p MPI_Group providing various convenience functions.
+   *
+   * @details It stores an @p MPI_Group object as its only member which by default is set to @p MPI_GROUP_NULL.
+   */
+  class group {
+    // Wrapped `MPI_Comm` object.
+    MPI_Group _grp = MPI_GROUP_NULL;
+
+    public:
+    /// Construct a communicator with @p MPI_GROUP_NULL.
+    group() = default;
+
+    /// Deleted copy constructor.
+    group(group const&) = delete;
+
+    /// Deleted copy assignment operator.
+    group& operator=(group const&) = delete;
+
+    /// Move constructor leaves moved-from object with @p MPI_GROUP_NULL.
+    group(group &&other) noexcept : _grp{std::exchange(other._grp, MPI_GROUP_NULL)} {}
+
+    /// Move assignment operator leaves moved-from object with @p MPI_GROUP_NULL.
+    group& operator=(group &&rhs) noexcept {
+      if (this != std::addressof(rhs)) {
+        this->free();
+        this->_grp = std::exchange(rhs._grp, MPI_GROUP_NULL);
+      }
+      return *this;
+    }
+
+    /**
+     * @brief Take ownership of an existing @p MPI_Group object.
+     * @param grp The group to be handled.
+     */
+    explicit group(MPI_Group grp) : _grp(grp) {}
+
+    /**
+     * @brief Create a group from a communicator.
+     * @param c The communicator from which to create a group.
+     */
+    explicit group(communicator c) {
+      if (has_env) {
+        MPI_Comm_group(c.get(), &_grp);
+      }
+    }
+
+    /// Get the wrapped @p MPI_Group object.
+    [[nodiscard]] MPI_Group get() const noexcept { return _grp; }
+
+    /// Check if the contained @p MPI_Group is @p MPI_GROUP_NULL.
+    [[nodiscard]] bool is_null() const noexcept { return _grp == MPI_GROUP_NULL; }
+
+    /// Rank of the calling process in the given group.
+    [[nodiscard]] int rank() const {
+      int rank = 0;
+      if (has_env) {
+        MPI_Group_rank(_grp, &rank);
+      }
+      return rank;
+    }
+
+    /// Size of a group.
+    [[nodiscard]] int size() const {
+      int size = 1;
+      if (has_env) {
+        MPI_Group_size(_grp, &size);
+      }
+      return size;
+    }
+
+    /// Produces a group by reordering an existing group and taking only listed members.
+    group incl(std::vector<int> const &ranks) const {
+      MPI_Group newgroup = MPI_GROUP_NULL;
+      if (has_env) {
+        MPI_Group_incl(_grp, ranks.size(), ranks.data(), &newgroup);
+      }
+      return group(newgroup);
+    }
+
+    /// Free the group.
+    void free() {
+      if (has_env) {
+        if (_grp != MPI_GROUP_NULL) {
+          MPI_Group_free(&_grp);
+        }
+      }
+    }
+  };
+
+} // namespace mpi

c++/mpi/window.hpp

@@ -22,6 +22,7 @@
 #pragma once
 
 #include "./communicator.hpp"
+#include "./group.hpp"
 #include "./macros.hpp"
 
 #include <mpi.h>
@@ -86,7 +87,6 @@ namespace mpi {
     * @param size The number of elements of type @p BaseType in the buffer. (default @p 0)
     * @param info Additional MPI information. (default @p MPI_INFO_NULL)
     */
-
     explicit window(communicator &c, BaseType *base, MPI_Aint size = 0, MPI_Info info = MPI_INFO_NULL) noexcept(false) {
       ASSERT(size >= 0)
       ASSERT(!(base == nullptr && size > 0))
@@ -241,6 +241,48 @@ namespace mpi {
       }
     }
 
+    /**
+    * @brief Starts an RMA access epoch.
+    *
+    * @param grp The group of target processes.
+    * @param assert An assertion flag providing optimization hints to MPI.
+    */
+    void start(group const &grp, int assert = 0) const noexcept {
+      if (has_env) {
+        MPI_Win_start(grp.get(), assert, win);
+      }
+    }
+
+    /**
+    * @brief Completes an RMA access epoch on win started by a call to @p start.
+    */
+    void complete() const noexcept {
+      if (has_env) {
+        MPI_Win_complete(win);
+      }
+    }
+
+    /**
+    * @brief Starts an RMA exposure epoch for the local window.
+    *
+    * @param grp The group of origin processes.
+    * @param assert An assertion flag providing optimization hints to MPI.
+    */
+    void post(group const &grp, int assert = 0) const noexcept {
+      if (has_env) {
+        MPI_Win_post(grp.get(), assert, win);
+      }
+    }
+
+    /**
+    * @brief Completes an RMA exposure epoch started by a call to @p post.
+    */
+    void wait() const noexcept {
+      if (has_env) {
+        MPI_Win_wait(win);
+      }
+    }
+
     /**
     * @brief Reads data from a remote memory window.
     *

test/c++/mpi_window.cpp

@@ -58,61 +58,63 @@ TEST(MPI_Window, WindowAllocate) {
   EXPECT_EQ(rcv, rank); 
 } 
 
-TEST(MPI_Window, LockUnlock) {
+TEST(MPI_Window, PassiveTargetCommunication) {
   mpi::communicator world;
   int rank = world.rank();
-  int target_rank = 1;
-  int origin_count = 1;
 
-  if (target_rank >= world.size()) return; 
+  auto win_comm = world.split(rank == 0 || rank == 1 ? 0 : MPI_UNDEFINED);
 
-  mpi::window<int> win{world, 1};
-
-  *(win.base()) = 0;
-
-  win.fence();
+  if (rank == 0 || rank == 1) {
+    mpi::window<int> win{win_comm, 1};
+    *(win.base()) = -1;
 
-  if (rank == 0) {
-    int origin_addr = 42;
-    win.lock(MPI_LOCK_EXCLUSIVE, target_rank);
-    win.put(&origin_addr, origin_count, target_rank);
-    win.unlock(target_rank);
-  }
+    win.fence();
+    if (rank == 0) {
+      int val = 42;
+      win.put(&val, 1, 1);
+    }
+    win.fence();
 
-  win.fence();
-  
-  if (rank == target_rank) {
-    EXPECT_EQ(*(win.base()), 42);
+    if (rank == 1) {
+      EXPECT_EQ(*(win.base()), 42);
+    }
   }
 }
 
-TEST(MPI_Window, RoyalFlush) {
+TEST(MPI_Window, ActiveTargetCommunication) {
   mpi::communicator world;
   int rank = world.rank();
-  int size = world.size();
+  int target_rank = 1;
+  int origin_count = 1;
 
-  int* buf;
-  mpi::window<int> win{world, 1};
+  if (target_rank >= world.size()) return; 
 
-  buf = win.base();
+  mpi::window<int> win{world, 1};
+  *(win.base()) = -1;
 
-  int next = (rank + 1) % size;
-  int val = rank;
+  int origin_rank = 0;
+  int target_rank = 1;
 
-  win.lock(next); 
-  win.put(&val, 1, next);
-  win.flush(next); 
-  win.unlock(next);
+  // Only the origin and target ranks will participate in the communication.
+  mpi::group world_group(world);
+  auto origin_group = world_group.incl({origin_rank});
+  auto target_group = world_group.incl({target_rank});
 
-  win.fence();
+  if (rank == target_rank) {
+    win.post(origin_group);
+    win.wait(); // blocks until origin_rank calls complete()
+    EXPECT_EQ(*(win.base()), 42);
+  }
 
-  if (rank == (size - 1) && mpi::has_env) {
-    EXPECT_EQ(*buf, size - 2); 
+  if (rank == origin_rank) {
+    win.start(target_group); // blocks until target_rank calls post()
+    int origin_addr[] = { 42 };
+    int origin_count = 1;
+    win.put(origin_addr, origin_count, target_rank);
+    win.complete();
   }
 }
 
-
-
 TEST(MPI_Window, GetAttrSize) {
   mpi::communicator world;
   int buffer;