Rank.cpp 3.97 KB
Newer Older
1
2
3
4
/*
 * RankOperations.cpp
 *
 *  Created on: 2 Mar 2018
5
 *      Author: Ben Hazelwood
6
7
 */

Ben Hazelwood's avatar
Ben Hazelwood committed
8
#include "Rank.h"
9
10

#include <cassert>
Ben Hazelwood's avatar
Ben Hazelwood committed
11
#include <cstdlib>
12
#include <iostream>
Ben Hazelwood's avatar
Ben Hazelwood committed
13

14
#include "RankControl.h"
Ben Hazelwood's avatar
Ben Hazelwood committed
15
16
#include "Logging.h"
#include "Timing.h"
17

Ben Hazelwood's avatar
Ben Hazelwood committed
18
19
20
21
22
23
24
25
static int worldRank;
static int worldSize;
static int teamRank;
static int teamSize;
static int numTeams;

static MPI_Comm TMPI_COMM_TEAM;
static MPI_Comm TMPI_COMM_DUP;
26
static MPI_Comm TMPI_COMM_INTER_TEAM;
Ben Hazelwood's avatar
Ben Hazelwood committed
27

28
int initialiseTMPI() {
Ben Hazelwood's avatar
Ben Hazelwood committed
29
30
31
32
33
  /**
   * The application should have no knowledge of the world_size or world_rank
   */
  setEnvironment();

Ben Hazelwood's avatar
Ben Hazelwood committed
34
35
36
  PMPI_Comm_size(MPI_COMM_WORLD, &worldSize);
  PMPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
  teamSize = worldSize / numTeams;
Ben Hazelwood's avatar
Ben Hazelwood committed
37

Ben Hazelwood's avatar
Ben Hazelwood committed
38
  int color = worldRank / teamSize;
Ben Hazelwood's avatar
Ben Hazelwood committed
39

Ben Hazelwood's avatar
Ben Hazelwood committed
40
  PMPI_Comm_dup(MPI_COMM_WORLD, &TMPI_COMM_DUP);
Ben Hazelwood's avatar
Ben Hazelwood committed
41

Ben Hazelwood's avatar
Ben Hazelwood committed
42
  PMPI_Comm_split(MPI_COMM_WORLD, color, worldRank, &TMPI_COMM_TEAM);
Ben Hazelwood's avatar
Ben Hazelwood committed
43

Ben Hazelwood's avatar
Ben Hazelwood committed
44
  PMPI_Comm_rank(TMPI_COMM_TEAM, &teamRank);
Ben Hazelwood's avatar
Ben Hazelwood committed
45

Ben Hazelwood's avatar
Ben Hazelwood committed
46
  PMPI_Comm_size(TMPI_COMM_TEAM, &teamSize);
Ben Hazelwood's avatar
Ben Hazelwood committed
47

48
49
50
  // Todo: free 
  PMPI_Comm_split(MPI_COMM_WORLD, teamRank, worldRank, &TMPI_COMM_INTER_TEAM);

Ben Hazelwood's avatar
Ben Hazelwood committed
51
  assert(teamSize == (worldSize / numTeams));
Ben Hazelwood's avatar
Ben Hazelwood committed
52

Ben Hazelwood's avatar
Ben Hazelwood committed
53
  registerSignalHandler();
Ben Hazelwood's avatar
Ben Hazelwood committed
54
55
56
57
58
  outputEnvironment();

#ifndef REPLICAS_OUTPUT
  // Disable output for all but master replica (0)
  if (getTeam() > 0) {
59
//    disableLogging();
Ben Hazelwood's avatar
Ben Hazelwood committed
60
61
62
63
64
  }
#endif

  Timing::initialiseTiming();

Ben Hazelwood's avatar
Ben Hazelwood committed
65
  synchroniseRanksGlobally();
Ben Hazelwood's avatar
Ben Hazelwood committed
66
67
68
69
70

  return MPI_SUCCESS;
}


71
int getWorldRank() {
Ben Hazelwood's avatar
Ben Hazelwood committed
72
  return worldRank;
73
74
75
}

int getWorldSize() {
Ben Hazelwood's avatar
Ben Hazelwood committed
76
  return worldSize;
77
78
79
}

int getTeamRank() {
Ben Hazelwood's avatar
Ben Hazelwood committed
80
  return teamRank;
81
82
83
}

int getTeamSize() {
Ben Hazelwood's avatar
Ben Hazelwood committed
84
  return teamSize;
85
86
}

Ben Hazelwood's avatar
Ben Hazelwood committed
87
88
89
90
91
92
int getTeam() {
  return getWorldRank() / getTeamSize();
}

int getNumberOfTeams() {
  return getWorldSize() / getTeamSize();
93
94
}

95
96
MPI_Comm getTeamComm(MPI_Comm comm) {
  return (comm==MPI_COMM_WORLD) ? TMPI_COMM_TEAM : comm;
97
98
}

99
100
101
102
MPI_Comm getTeamInterComm() {
  return TMPI_COMM_INTER_TEAM;
}

Ben Hazelwood's avatar
Ben Hazelwood committed
103
int freeTeamComm() {
Ben Hazelwood's avatar
Ben Hazelwood committed
104
  return MPI_Comm_free(&TMPI_COMM_TEAM);
105
106
}

Ben Hazelwood's avatar
Ben Hazelwood committed
107
MPI_Comm getLibComm() {
Ben Hazelwood's avatar
Ben Hazelwood committed
108
  return TMPI_COMM_DUP;
109
110
}

Ben Hazelwood's avatar
Ben Hazelwood committed
111
int freeLibComm() {
Ben Hazelwood's avatar
Ben Hazelwood committed
112
  return MPI_Comm_free(&TMPI_COMM_DUP);
113
114
}

115
116
117
118
119
120
std::string getEnvString(std::string const& key)
{
    char const* val = std::getenv(key.c_str());
    return val == nullptr ? std::string() : std::string(val);
}

Ben Hazelwood's avatar
Ben Hazelwood committed
121
void outputEnvironment(){
Ben Hazelwood's avatar
Ben Hazelwood committed
122
  assert(worldSize % numTeams == 0);
123

124
  PMPI_Barrier(MPI_COMM_WORLD);
125
  double my_time = MPI_Wtime();
126
  PMPI_Barrier(MPI_COMM_WORLD);
Ben Hazelwood's avatar
Ben Hazelwood committed
127
  double times[worldSize];
128

129
  PMPI_Gather(&my_time, 1, MPI_DOUBLE, times, 1, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);
130

131
  PMPI_Barrier(MPI_COMM_WORLD);
132

Ben Hazelwood's avatar
Ben Hazelwood committed
133
  if (worldRank == MASTER) {
134
    std::cout << "------------TMPI SETTINGS------------\n";
135
    std::cout << "Number of teams: " << numTeams << "\n";
136

Ben Hazelwood's avatar
Ben Hazelwood committed
137
138
    std::cout << "Team size: " << teamSize << "\n";
    std::cout << "Total ranks: " << worldSize << "\n\n";
139
140


Ben Hazelwood's avatar
Ben Hazelwood committed
141
142
    if (worldRank == MASTER) {
      for (int i=0; i < worldSize; i++) {
Ben Hazelwood's avatar
Ben Hazelwood committed
143
        std::cout << "Tshift(" << i << "->" << mapWorldToTeamRank(i) << "->" << mapTeamToWorldRank(mapWorldToTeamRank(i),mapRankToTeamNumber(i)) << ") = " << times[i] - times[0] << "\n";
144
145
      }
    }
Ben Hazelwood's avatar
Ben Hazelwood committed
146
    std::cout << "---------------------------------------\n\n";
147
148
  }

149
  PMPI_Barrier(MPI_COMM_WORLD);
150
151
}

Ben Hazelwood's avatar
Ben Hazelwood committed
152
void setEnvironment() {
Ben Hazelwood's avatar
Ben Hazelwood committed
153
154
  std::string env(getEnvString("TEAMS"));
  numTeams = env.empty() ? 2 : std::stoi(env);
155
156
}

Ben Hazelwood's avatar
Ben Hazelwood committed
157
158
int mapRankToTeamNumber(int rank) {
  return rank / getTeamSize();
159
160
}

Ben Hazelwood's avatar
Ben Hazelwood committed
161
int mapWorldToTeamRank(int rank) {
162
163
164
  if (rank == MPI_ANY_SOURCE) {
    return MPI_ANY_SOURCE;
  } else {
Ben Hazelwood's avatar
Ben Hazelwood committed
165
    return rank % getTeamSize();
166
167
168
  }
}

Ben Hazelwood's avatar
Ben Hazelwood committed
169
int mapTeamToWorldRank(int rank, int r) {
170
171
172
  if (rank == MPI_ANY_SOURCE) {
    return MPI_ANY_SOURCE;
  }
173

Ben Hazelwood's avatar
Ben Hazelwood committed
174
  return rank + r * getTeamSize();
175
176
}

Ben Hazelwood's avatar
Ben Hazelwood committed
177
178
179
180
181
void remapStatus(MPI_Status *status) {
  if ((status != MPI_STATUS_IGNORE ) && (status != MPI_STATUSES_IGNORE )) {
    status->MPI_SOURCE = mapWorldToTeamRank(status->MPI_SOURCE);
    logInfo(
        "remap status source " << status->MPI_SOURCE << " to " << mapWorldToTeamRank(status->MPI_SOURCE));
182
183
184
  }
}

Ben Hazelwood's avatar
Ben Hazelwood committed
185
int synchroniseRanksInTeam() {
186
  return PMPI_Barrier(getTeamComm(MPI_COMM_WORLD));
Ben Hazelwood's avatar
Ben Hazelwood committed
187
188
189
190
191
192
193
}

int synchroniseRanksGlobally() {
  return PMPI_Barrier(getLibComm());
}