Timing.cpp 3.56 KB
Newer Older
1
2
3
4
/*
 * Timing.cpp
 *
 *  Created on: 2 Mar 2018
5
 *      Author: Ben Hazelwood
6
7
8
9
 */

#include "Timing.h"

Ben Hazelwood's avatar
Ben Hazelwood committed
10
11
#include <fstream>
#include <map>
12
#include <set>
Ben Hazelwood's avatar
Ben Hazelwood committed
13
14
15
#include <sstream>
#include <string>
#include <utility>
16
#include <stddef.h>
Ben Hazelwood's avatar
Ben Hazelwood committed
17
18

#include "Logging.h"
Ben Hazelwood's avatar
Ben Hazelwood committed
19
#include "Rank.h"
20
21

struct Timer {
Ben Hazelwood's avatar
Ben Hazelwood committed
22
23
  double startTime;
  double endTime;
24
25
26

  std::map< int, std::vector<double> > syncPoints;
  std::map< int, std::vector<MPI_Request> > requests;
Ben Hazelwood's avatar
Ben Hazelwood committed
27
28
} timer;

29

30

31
void Timing::initialiseTiming() {
Ben Hazelwood's avatar
Ben Hazelwood committed
32
  synchroniseRanksInTeam();
33
  timer.startTime = PMPI_Wtime();
Ben Hazelwood's avatar
Ben Hazelwood committed
34
  for (int i=0; i < getNumberOfTeams(); i++) {
35
36
37
38
39
40
    timer.syncPoints.insert(std::make_pair(i,std::vector<double>()));
    timer.requests.insert(std::make_pair(i,std::vector<MPI_Request>()));
  }
}

void Timing::finaliseTiming() {
Ben Hazelwood's avatar
Ben Hazelwood committed
41
  synchroniseRanksInTeam();
42
43
44
45
  timer.endTime = PMPI_Wtime();
}

void Timing::markTimeline() {
Ben Hazelwood's avatar
Ben Hazelwood committed
46
    timer.syncPoints.at(getTeam()).push_back(PMPI_Wtime());
47
    compareProgressWithReplicas();
48
49
50
}

void Timing::compareProgressWithReplicas() {
Ben Hazelwood's avatar
Ben Hazelwood committed
51
52
  for (int r=0; r < getNumberOfTeams(); r++) {
    if (r != getTeam()) {
Ben Hazelwood's avatar
Ben Hazelwood committed
53
      // Send out this replica's times
54
      MPI_Request request;
Ben Hazelwood's avatar
Ben Hazelwood committed
55
56
57
      PMPI_Isend(&timer.syncPoints.at(getTeam()).back(), 1, MPI_DOUBLE,
                mapTeamToWorldRank(getTeamRank(), r), getTeam(),
                getLibComm(), &request);
58
59
60
61
62
63
      MPI_Request_free(&request);

      // Receive times from other replicas
      timer.syncPoints.at(r).push_back(0.0);
      timer.requests.at(r).push_back(MPI_Request());
      PMPI_Irecv(&timer.syncPoints.at(r).back(), 1, MPI_DOUBLE,
Ben Hazelwood's avatar
Ben Hazelwood committed
64
                 mapTeamToWorldRank(getTeamRank(), r), r, getLibComm(), &timer.requests.at(r).back());
65
66
67
68
69
70
71
72
73
74

      // Test for completion of Irecv's
      int numPending = 0;
      for (int i=0; i < timer.requests.at(r).size(); i++) {
        int flag = 0;
        PMPI_Test(&timer.requests.at(r).at(i), &flag, MPI_STATUS_IGNORE);
        numPending += 1 - flag;
      }
    }
  }
Ben Hazelwood's avatar
Ben Hazelwood committed
75
76
77
}

void Timing::outputTiming() {
78
79
  std::cout.flush();
  PMPI_Barrier(MPI_COMM_WORLD);
Ben Hazelwood's avatar
Ben Hazelwood committed
80

81
  // Output simple replica timings
82
  if ((getTeamRank() == MASTER) && (getWorldRank() != MASTER)) {
Ben Hazelwood's avatar
Ben Hazelwood committed
83
    PMPI_Send(&timer.endTime, 1, MPI_DOUBLE, MASTER, 0, getLibComm());
Ben Hazelwood's avatar
Ben Hazelwood committed
84
85
  }

86

87
88
89
90
91
92
93
94
  if (getWorldRank() == MASTER) {
    std::cout << std::endl;
    std::cout << "----------TMPI_TIMING----------\n";
    std::cout << "timing_file=";
#ifdef TMPI_TIMING
    std::cout << "tmpi_filename.csv";
#else
    std::cout << "timing_not_enabled";
Ben Hazelwood's avatar
Ben Hazelwood committed
95
#endif
96
    std::cout << "\n";
Ben Hazelwood's avatar
Ben Hazelwood committed
97
98
    std::cout << "num_replicas=" << getNumberOfTeams() << "\n";
    for (int i=0; i < getNumberOfTeams(); i++) {
99
100
101
102
      double rEndTime = 0.0;
      if (i == MASTER) {
        rEndTime = timer.endTime;
      } else {
Ben Hazelwood's avatar
Ben Hazelwood committed
103
        PMPI_Recv(&rEndTime, 1, MPI_DOUBLE, mapTeamToWorldRank(MASTER, i), 0, getLibComm(), MPI_STATUS_IGNORE);
104
105
106
107
108
109
110
111
      }

      std::cout << "replica_" << i << "=" << rEndTime << "\n";
    }
    std::cout << "-------------------------------\n";
  }
  std::cout.flush();
  PMPI_Barrier(MPI_COMM_WORLD);
112
113
114
115
116
117
118
119
120

  // Write Generic Sync points to files
  char sep = ',';
  std::ostringstream filename;
  std::string outputFolder("tmpi-timings");
  filename << outputFolder << "/"
      << "timings" << "-"
      << getWorldRank() << "-"
      << getTeamRank() << "-"
Ben Hazelwood's avatar
Ben Hazelwood committed
121
      << getTeam()
122
123
124
125
126
127
      << ".csv";
  std::ofstream f;
  f.open(filename.str().c_str());

  logInfo("Writing timings to " << filename);

128
  f << "endTime" << sep << timer.endTime - timer.startTime << "\n";
129
130

  f << "syncPoints";
Ben Hazelwood's avatar
Ben Hazelwood committed
131
  for (const double& t : timer.syncPoints.at(getTeam())) {
132
    f << sep << t - timer.startTime;
133
134
135
136
137
138
  }
  f << "\n";

  f.close();

  PMPI_Barrier(MPI_COMM_WORLD);
139
140
}