Switch to dual heartbeat and add new application -- perfSimulator

applications/Makefile

 CC=mpiicpc
-CFLAGS += -g -Wall -std=c++11 -DCOMPARE_PROGRESS
+CFLAGS += -g -Wall -std=c++11
 LFLAGS += -L../lib -ltmpi
 
 
-TARGETS = bin/MasterWorker bin/VectorAdd bin/Latency
+TARGETS = bin/MasterWorker bin/VectorAdd bin/Latency bin/PerfSimulator
 LIB = ../lib/libtmpi.so
 
 
@@ -22,6 +22,10 @@ bin/VectorAdd : bin/VectorAdd.o
 	
 bin/Latency : bin/Latency.o
 	$(CC) $(CFLAGS) $^ -o $@ $(LFLAGS)
+
+bin/PerfSimulator : bin/PerfSimulator.o
+	$(CC) $(CFLAGS) $^ -o $@ $(LFLAGS)
+
 	
 bin/MasterWorker.o : MasterWorker/MasterWorker.cpp
 	$(CC) $(CFLAGS) -c $^ -o $@ $(LFLAGS)
@@ -32,6 +36,9 @@ bin/VectorAdd.o : VectorAdd/VectorAdd.cpp
 bin/Latency.o : Latency/Latency.cpp
 	$(CC) $(CFLAGS) -c $^ -o $@ $(LFLAGS)
 
+bin/PerfSimulator.o : PerfSimulator/PerfSimulator.cpp
+	$(CC) $(CFLAGS) -c $^ -o $@ $(LFLAGS)
+
 .PHONY : $(LIB)
 
 $(LIB):

applications/PerfSimulator/PerfSimulator.cpp

+/*
+ * PerfSimulator.cpp
+ *
+ *  Created on: 5 Jun 2018
+ *      Author: Ben Hazelwood
+ */
+
+#include <mpi.h>
+#include <iostream>
+#include <assert.h>
+#include <cstdlib>
+#include <math.h>  
+
+const int NUM_TRIALS = 100;
+const int NUM_COMPUTATIONS = 5e7;
+
+int main(int argc, char *argv[])
+{
+
+  MPI_Init(&argc, &argv);
+
+  // if (argc != 3)
+  // {
+  //   std::cout << "Usage: mpirun Latency <NUM TRIALS> <NUM COMPUTATIONS>\n";
+  //   MPI_Abort(MPI_COMM_WORLD, 1);
+  // }
+
+  int rank;
+  int size;
+
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+  MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+  const int source = 0;
+  const int dest = size - 1;
+
+  char m = '0';
+
+  for (int t = 0; t < NUM_TRIALS; t++)
+  {
+    MPI_Barrier(MPI_COMM_WORLD);
+#ifdef COMPARE_PROGRESS
+    MPI_Sendrecv(MPI_IN_PLACE, 0, MPI_BYTE, MPI_PROC_NULL, 0, MPI_IN_PLACE, 0, MPI_BYTE, MPI_PROC_NULL, 0, MPI_COMM_SELF, MPI_STATUS_IGNORE);
+#endif
+
+    for (int i = 0; i < NUM_COMPUTATIONS; i++) {
+      sin(1.0/3.0);
+    }
+
+#ifdef COMPARE_PROGRESS
+    MPI_Sendrecv(MPI_IN_PLACE, 0, MPI_BYTE, MPI_PROC_NULL, 0, MPI_IN_PLACE, 0, MPI_BYTE, MPI_PROC_NULL, 0, MPI_COMM_SELF, MPI_STATUS_IGNORE);
+#endif
+    MPI_Barrier(MPI_COMM_WORLD);
+    // if (rank == source)
+    // {
+    //   MPI_Recv(&m, 1, MPI_CHAR, source, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+    // }
+    // else
+    // {
+    //   MPI_Send(&m, 1, MPI_CHAR, dest, 0, MPI_COMM_WORLD);
+    // }
+  }
+
+  MPI_Finalize();
+  return 0;
+}

lib/Timing.cpp

@@ -58,13 +58,15 @@ void Timing::finaliseTiming() {
   timer.endTime = PMPI_Wtime();
 }
 
-void Timing::markTimeline() {
+void Timing::markTimeline(int tag) {
     timer.heartbeatTimes.at(getTeam()).push_back(PMPI_Wtime());
-    compareProgressWithReplicas();
+    if (tag > 0) {
+      compareProgressWithReplicas();
+    }
 }
 
-void Timing::markTimeline(const void *sendbuf, int sendcount, MPI_Datatype sendtype) {
-  markTimeline();
+void Timing::markTimeline(int tag, const void *sendbuf, int sendcount, MPI_Datatype sendtype) {
+  markTimeline(tag);
   compareBufferWithReplicas(sendbuf, sendcount, sendtype);
 }
 
@@ -73,15 +75,16 @@ void Timing::compareProgressWithReplicas() {
     if (r != getTeam()) {
       // Send out this replica's times
       MPI_Request request;
-      PMPI_Isend(&timer.heartbeatTimes.at(getTeam()).back(), 1, MPI_DOUBLE,
+      PMPI_Isend(timer.heartbeatTimes.at(getTeam()).data()+timer.heartbeatTimes.at(getTeam()).size() - 2, 2, MPI_DOUBLE,
                 mapTeamToWorldRank(getTeamRank(), r), getTeam(),
                 getLibComm(), &request);
       MPI_Request_free(&request);
 
       // Receive times from other replicas
       timer.heartbeatTimes.at(r).push_back(0.0);
+      timer.heartbeatTimes.at(r).push_back(0.0);
       timer.heartbeatTimeRequests.at(r).push_back(MPI_Request());
-      PMPI_Irecv(&timer.heartbeatTimes.at(r).back(), 1, MPI_DOUBLE,
+      PMPI_Irecv(timer.heartbeatTimes.at(getTeam()).data()+timer.heartbeatTimes.at(getTeam()).size() - 2, 2, MPI_DOUBLE,
                  mapTeamToWorldRank(getTeamRank(), r), r, getLibComm(), &timer.heartbeatTimeRequests.at(r).back());
 
       // Test for completion of Irecv's

lib/Timing.h

@@ -14,9 +14,9 @@
 namespace Timing {
 
 // Mark time only for this heartbeat
-void markTimeline();
+void markTimeline(int tag);
 // Also mark the hash for the heartbeat buffer
-void markTimeline(const void *sendbuf, int sendcount, MPI_Datatype sendtype);
+void markTimeline(int tag, const void *sendbuf, int sendcount, MPI_Datatype sendtype);
 
 void initialiseTiming();
 void finaliseTiming();

lib/Wrapper.cpp

@@ -288,9 +288,9 @@ int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                 MPI_Comm comm, MPI_Status *status) {
   if (comm == MPI_COMM_SELF) {
     if (sendcount == 0) {
-      Timing::markTimeline();
+      Timing::markTimeline(sendtag);
     } else {
-      Timing::markTimeline(sendbuf, sendcount, sendtype);
+      Timing::markTimeline(sendtag, sendbuf, sendcount, sendtype);
     }
   } else {
     assert(comm == MPI_COMM_WORLD);