Port MPI replicas communication from LAMMPS interface to library

lammps/src/COLVARS/colvarproxy_lammps.cpp

@@ -41,9 +41,6 @@ colvarproxy_lammps::colvarproxy_lammps(LAMMPS_NS::LAMMPS *lmp)
   previous_step = -1;
   do_exit = false;
 
-  inter_me = 0;
-  inter_num = 1;
-
   engine_ready_ = false;
 }
 
@@ -91,17 +88,6 @@ void colvarproxy_lammps::set_random_seed(int seed)
 }
 
 
-void colvarproxy_lammps::set_replicas_communicator(MPI_Comm root2root)
-{
-  inter_comm = root2root;
-  // initialize multi-replica support, if available
-  if (replica_enabled() == COLVARS_OK) {
-    MPI_Comm_rank(inter_comm, &inter_me);
-    MPI_Comm_size(inter_comm, &inter_num);
-  }
-}
-
-
 // re-initialize data where needed
 int colvarproxy_lammps::setup()
 {
@@ -260,58 +246,6 @@ int colvarproxy_lammps::set_unit_system(std::string const &units_in, bool /*chec
 }
 
 
-// multi-replica support
-
-int colvarproxy_lammps::replica_enabled()
-{
-  return (inter_comm != MPI_COMM_NULL) ? COLVARS_OK : COLVARS_NOT_IMPLEMENTED;
-}
-
-
-int colvarproxy_lammps::replica_index()
-{
-  return inter_me;
-}
-
-
-int colvarproxy_lammps::num_replicas()
-{
-  return inter_num;
-}
-
-
-void colvarproxy_lammps::replica_comm_barrier()
-{
-  MPI_Barrier(inter_comm);
-}
-
-
-int colvarproxy_lammps::replica_comm_recv(char* msg_data,
-                                          int buf_len, int src_rep)
-{
-  MPI_Status status;
-  int retval;
-
-  retval = MPI_Recv(msg_data,buf_len,MPI_CHAR,src_rep,0,inter_comm,&status);
-  if (retval == MPI_SUCCESS) {
-    MPI_Get_count(&status, MPI_CHAR, &retval);
-  } else retval = 0;
-  return retval;
-}
-
-
-int colvarproxy_lammps::replica_comm_send(char* msg_data,
-                                          int msg_len, int dest_rep)
-{
-  int retval;
-  retval = MPI_Send(msg_data,msg_len,MPI_CHAR,dest_rep,0,inter_comm);
-  if (retval == MPI_SUCCESS) {
-    retval = msg_len;
-  } else retval = 0;
-  return retval;
-}
-
-
 
 int colvarproxy_lammps::check_atom_id(int atom_number)
 {

lammps/src/COLVARS/colvarproxy_lammps.h

@@ -41,9 +41,6 @@ class colvarproxy_lammps : public colvarproxy {
 
   std::vector<int> atoms_types;
 
-  MPI_Comm inter_comm;        // MPI comm with 1 root proc from each world
-  int inter_me, inter_num;    // rank for the inter replica comm
-
  public:
   friend class cvm::atom;
 
@@ -56,9 +53,6 @@ class colvarproxy_lammps : public colvarproxy {
   /// Set the internal seed used by \link rand_gaussian() \endlink
   void set_random_seed(int seed);
 
-  /// Set the multiple replicas communicator
-  void set_replicas_communicator(MPI_Comm root2root);
-
   int setup() override;
 
   // disable default and copy constructor
@@ -99,14 +93,6 @@ class colvarproxy_lammps : public colvarproxy {
   int check_atom_id(int atom_number) override;
 
   inline std::vector<int> *modify_atom_types() { return &atoms_types; }
-
-  int replica_enabled() override;
-  int replica_index() override;
-  int num_replicas() override;
-
-  void replica_comm_barrier() override;
-  int replica_comm_recv(char *msg_data, int buf_len, int src_rep) override;
-  int replica_comm_send(char *msg_data, int msg_len, int dest_rep) override;
 };
 
 #endif

lammps/src/COLVARS/fix_colvars.cpp

@@ -286,7 +286,7 @@ void FixColvars::init()
     }
     MPI_Comm_split(universe->uworld, color, universe->iworld, &root2root);
     if (me == 0) {
-      proxy->set_replicas_communicator(root2root);
+      proxy->set_replicas_mpi_communicator(root2root);
     }
   }
 }

src/colvarproxy_replicas.cpp

@@ -7,50 +7,103 @@
 // If you wish to distribute your changes, please submit them to the
 // Colvars repository at GitHub.
 
+
+#ifdef COLVARS_MPI
+#include <mpi.h>
+#endif
+
 #include "colvarmodule.h"
-#include "colvarproxy.h"
+#include "colvarproxy_replicas.h"
 
 
-colvarproxy_replicas::colvarproxy_replicas() {}
+colvarproxy_replicas::colvarproxy_replicas()
+{
+#ifdef COLVARS_MPI
+  replicas_mpi_comm = MPI_COMM_NULL;
+#endif
+}
 
 
 colvarproxy_replicas::~colvarproxy_replicas() {}
 
 
-int colvarproxy_replicas::replica_enabled()
+void colvarproxy_replicas::set_replicas_mpi_communicator(MPI_Comm comm)
+{
+#ifdef COLVARS_MPI
+  replicas_mpi_comm = comm;
+  if (comm != MPI_COMM_NULL) {
+    MPI_Comm_rank(comm, &replicas_mpi_rank);
+    MPI_Comm_size(comm, &replicas_mpi_num);
+  }
+#endif
+}
+
+
+int colvarproxy_replicas::check_replicas_enabled()
 {
+#ifdef COLVARS_MPI
+  // Return "not implemented" when the simulation does not support partitions
+  return (replicas_mpi_comm != MPI_COMM_NULL) ? COLVARS_OK : COLVARS_NOT_IMPLEMENTED;
+#else
   return COLVARS_NOT_IMPLEMENTED;
+#endif
 }
 
 
 int colvarproxy_replicas::replica_index()
 {
-  return 0;
+  return replicas_mpi_rank;
 }
 
 
 int colvarproxy_replicas::num_replicas()
 {
-  return 1;
+  return replicas_mpi_num;
 }
 
 
-void colvarproxy_replicas::replica_comm_barrier() {}
+void colvarproxy_replicas::replica_comm_barrier()
+{
+#ifdef COLVARS_MPI
+  MPI_Barrier(replicas_mpi_comm);
+#endif
+}
 
 
-int colvarproxy_replicas::replica_comm_recv(char* /* msg_data */,
-                                            int /* buf_len */,
-                                            int /* src_rep */)
+int colvarproxy_replicas::replica_comm_recv(char *buffer, int buffer_length, int source_rank)
 {
+#ifdef COLVARS_MPI
+  MPI_Status status;
+  int retval = MPI_Recv(buffer, buffer_length, MPI_CHAR, source_rank, 0, replicas_mpi_comm, &status);
+  if (retval == MPI_SUCCESS) {
+    MPI_Get_count(&status, MPI_CHAR, &retval);
+  } else {
+    retval = 0;
+  }
+  return retval;
+#else
+  (void)buffer;
+  (void)buffer_length;
+  (void)source_rank;
   return COLVARS_NOT_IMPLEMENTED;
+#endif
 }
 
 
-int colvarproxy_replicas::replica_comm_send(char* /* msg_data */,
-                                            int /* msg_len */,
-                                            int /* dest_rep */)
+int colvarproxy_replicas::replica_comm_send(char *buffer, int buffer_length, int destination_rank)
 {
+#ifdef COLVARS_MPI
+  int retval = MPI_Send(buffer, buffer_length, MPI_CHAR, destination_rank, 0, replicas_mpi_comm);
+  if (retval == MPI_SUCCESS) {
+    retval = buffer_length;
+  } else {
+    retval = 0;
+  }
+  return retval;
+#else
+  (void)buffer;
+  (void)buffer_length;
+  (void)destination_rank;
   return COLVARS_NOT_IMPLEMENTED;
+#endif
 }
-
-

src/colvarproxy_replicas.h

@@ -11,6 +11,18 @@
 #define COLVARPROXY_REPLICAS_H
 
 
+#if defined(COLVARS_LAMMPS)
+// TODO Set this directly from the LAMMPS build system once GNU Make support is removed
+#define COLVARS_MPI
+#endif
+
+#ifdef COLVARS_MPI
+#include <mpi.h>
+#else
+typedef void* MPI_Comm;
+#endif
+
+
 /// \brief Methods for multiple-replica communication
 class colvarproxy_replicas {
 
@@ -22,6 +34,9 @@ class colvarproxy_replicas {
   /// Destructor
   virtual ~colvarproxy_replicas();
 
+  /// Set the multiple replicas communicator
+  virtual void set_replicas_mpi_communicator(MPI_Comm comm);
+
   /// Indicate if multi-replica support is available and active
   virtual int check_replicas_enabled();
 
@@ -39,6 +54,17 @@ class colvarproxy_replicas {
 
   /// Send data to other replica
   virtual int replica_comm_send(char* msg_data, int msg_len, int dest_rep);
+
+protected:
+
+  /// MPI communicator containint 1 root proc from each world
+  MPI_Comm replicas_mpi_comm;
+
+  /// Index (rank) of this replica in the MPI implementation
+  int replicas_mpi_rank = 0;
+
+  /// Number of replicas in the MPI implementation
+  int replicas_mpi_num = 1;
 };
 
 #endif