Rank.cpp 3.72 KB
Newer Older
1
2
3
4
/*
 * RankOperations.cpp
 *
 *  Created on: 2 Mar 2018
5
 *      Author: Ben Hazelwood
6
7
 */

Ben Hazelwood's avatar
Ben Hazelwood committed
8
#include "Rank.h"
9
10

#include <cassert>
Ben Hazelwood's avatar
Ben Hazelwood committed
11
#include <cstdlib>
12
#include <iostream>
Ben Hazelwood's avatar
Ben Hazelwood committed
13

14
#include "RankControl.h"
Ben Hazelwood's avatar
Ben Hazelwood committed
15
16
#include "Logging.h"
#include "Timing.h"
17
18


19

Ben Hazelwood's avatar
Ben Hazelwood committed
20
21
22
23
24
25
26
27
28
static int worldRank;
static int worldSize;
static int teamRank;
static int teamSize;
static int numTeams;

static MPI_Comm TMPI_COMM_TEAM;
static MPI_Comm TMPI_COMM_DUP;

29
int initialiseTMPI() {
Ben Hazelwood's avatar
Ben Hazelwood committed
30
31
32
  /**
   * The application should have no knowledge of the world_size or world_rank
   */
33
  registerSignalHandler();
Ben Hazelwood's avatar
Ben Hazelwood committed
34
35
  setEnvironment();

Ben Hazelwood's avatar
Ben Hazelwood committed
36
37
38
  PMPI_Comm_size(MPI_COMM_WORLD, &worldSize);
  PMPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
  teamSize = worldSize / numTeams;
Ben Hazelwood's avatar
Ben Hazelwood committed
39

Ben Hazelwood's avatar
Ben Hazelwood committed
40
  int color = worldRank / teamSize;
Ben Hazelwood's avatar
Ben Hazelwood committed
41

Ben Hazelwood's avatar
Ben Hazelwood committed
42
  PMPI_Comm_dup(MPI_COMM_WORLD, &TMPI_COMM_DUP);
Ben Hazelwood's avatar
Ben Hazelwood committed
43

Ben Hazelwood's avatar
Ben Hazelwood committed
44
  PMPI_Comm_split(MPI_COMM_WORLD, color, worldRank, &TMPI_COMM_TEAM);
Ben Hazelwood's avatar
Ben Hazelwood committed
45

Ben Hazelwood's avatar
Ben Hazelwood committed
46
  PMPI_Comm_rank(TMPI_COMM_TEAM, &teamRank);
Ben Hazelwood's avatar
Ben Hazelwood committed
47

Ben Hazelwood's avatar
Ben Hazelwood committed
48
  PMPI_Comm_size(TMPI_COMM_TEAM, &teamSize);
Ben Hazelwood's avatar
Ben Hazelwood committed
49

Ben Hazelwood's avatar
Ben Hazelwood committed
50
  assert(teamSize == (worldSize / numTeams));
Ben Hazelwood's avatar
Ben Hazelwood committed
51
52
53
54
55
56

  outputEnvironment();

#ifndef REPLICAS_OUTPUT
  // Disable output for all but master replica (0)
  if (getTeam() > 0) {
57
//    disableLogging();
Ben Hazelwood's avatar
Ben Hazelwood committed
58
59
60
61
62
  }
#endif

  Timing::initialiseTiming();

Ben Hazelwood's avatar
Ben Hazelwood committed
63
  synchroniseRanksGlobally();
Ben Hazelwood's avatar
Ben Hazelwood committed
64
65
66
67
68

  return MPI_SUCCESS;
}


69
int getWorldRank() {
Ben Hazelwood's avatar
Ben Hazelwood committed
70
  return worldRank;
71
72
73
}

int getWorldSize() {
Ben Hazelwood's avatar
Ben Hazelwood committed
74
  return worldSize;
75
76
77
}

int getTeamRank() {
Ben Hazelwood's avatar
Ben Hazelwood committed
78
  return teamRank;
79
80
81
}

int getTeamSize() {
Ben Hazelwood's avatar
Ben Hazelwood committed
82
  return teamSize;
83
84
}

Ben Hazelwood's avatar
Ben Hazelwood committed
85
86
87
88
89
90
int getTeam() {
  return getWorldRank() / getTeamSize();
}

int getNumberOfTeams() {
  return getWorldSize() / getTeamSize();
91
92
}

Ben Hazelwood's avatar
Ben Hazelwood committed
93
MPI_Comm getTeamComm() {
Ben Hazelwood's avatar
Ben Hazelwood committed
94
  return TMPI_COMM_TEAM;
95
96
}

Ben Hazelwood's avatar
Ben Hazelwood committed
97
int freeTeamComm() {
Ben Hazelwood's avatar
Ben Hazelwood committed
98
  return MPI_Comm_free(&TMPI_COMM_TEAM);
99
100
}

Ben Hazelwood's avatar
Ben Hazelwood committed
101
MPI_Comm getLibComm() {
Ben Hazelwood's avatar
Ben Hazelwood committed
102
  return TMPI_COMM_DUP;
103
104
}

Ben Hazelwood's avatar
Ben Hazelwood committed
105
int freeLibComm() {
Ben Hazelwood's avatar
Ben Hazelwood committed
106
  return MPI_Comm_free(&TMPI_COMM_DUP);
107
108
}

109
110
111
112
113
114
std::string getEnvString(std::string const& key)
{
    char const* val = std::getenv(key.c_str());
    return val == nullptr ? std::string() : std::string(val);
}

Ben Hazelwood's avatar
Ben Hazelwood committed
115
void outputEnvironment(){
Ben Hazelwood's avatar
Ben Hazelwood committed
116
  assert(worldSize % numTeams == 0);
117

118
  PMPI_Barrier(MPI_COMM_WORLD);
119
  double my_time = MPI_Wtime();
120
  PMPI_Barrier(MPI_COMM_WORLD);
Ben Hazelwood's avatar
Ben Hazelwood committed
121
  double times[worldSize];
122

123
  PMPI_Gather(&my_time, 1, MPI_DOUBLE, times, 1, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);
124

125
  PMPI_Barrier(MPI_COMM_WORLD);
126

Ben Hazelwood's avatar
Ben Hazelwood committed
127
  if (worldRank == MASTER) {
128
    std::cout << "------------TMPI SETTINGS------------\n";
Ben Hazelwood's avatar
Ben Hazelwood committed
129
    std::cout << "R_FACTOR = " << numTeams << "\n";
130

Ben Hazelwood's avatar
Ben Hazelwood committed
131
132
    std::cout << "Team size: " << teamSize << "\n";
    std::cout << "Total ranks: " << worldSize << "\n\n";
133
134


Ben Hazelwood's avatar
Ben Hazelwood committed
135
136
    if (worldRank == MASTER) {
      for (int i=0; i < worldSize; i++) {
Ben Hazelwood's avatar
Ben Hazelwood committed
137
        std::cout << "Tshift(" << i << "->" << mapWorldToTeamRank(i) << "->" << mapTeamToWorldRank(mapWorldToTeamRank(i),mapRankToTeamNumber(i)) << ") = " << times[i] - times[0] << "\n";
138
139
      }
    }
Ben Hazelwood's avatar
Ben Hazelwood committed
140
    std::cout << "---------------------------------------\n\n";
141
142
  }

143
  PMPI_Barrier(MPI_COMM_WORLD);
144
145
}

Ben Hazelwood's avatar
Ben Hazelwood committed
146
void setEnvironment() {
Ben Hazelwood's avatar
Ben Hazelwood committed
147
148
  std::string env(getEnvString("TEAMS"));
  numTeams = env.empty() ? 2 : std::stoi(env);
149
150
}

Ben Hazelwood's avatar
Ben Hazelwood committed
151
152
int mapRankToTeamNumber(int rank) {
  return rank / getTeamSize();
153
154
}

Ben Hazelwood's avatar
Ben Hazelwood committed
155
int mapWorldToTeamRank(int rank) {
156
157
158
  if (rank == MPI_ANY_SOURCE) {
    return MPI_ANY_SOURCE;
  } else {
Ben Hazelwood's avatar
Ben Hazelwood committed
159
    return rank % getTeamSize();
160
161
162
  }
}

Ben Hazelwood's avatar
Ben Hazelwood committed
163
int mapTeamToWorldRank(int rank, int r) {
164
165
166
  if (rank == MPI_ANY_SOURCE) {
    return MPI_ANY_SOURCE;
  }
167

Ben Hazelwood's avatar
Ben Hazelwood committed
168
  return rank + r * getTeamSize();
169
170
}

Ben Hazelwood's avatar
Ben Hazelwood committed
171
172
173
174
175
void remapStatus(MPI_Status *status) {
  if ((status != MPI_STATUS_IGNORE ) && (status != MPI_STATUSES_IGNORE )) {
    status->MPI_SOURCE = mapWorldToTeamRank(status->MPI_SOURCE);
    logInfo(
        "remap status source " << status->MPI_SOURCE << " to " << mapWorldToTeamRank(status->MPI_SOURCE));
176
177
178
  }
}

Ben Hazelwood's avatar
Ben Hazelwood committed
179
180
181
182
183
184
185
186
187
int synchroniseRanksInTeam() {
  return PMPI_Barrier(getTeamComm());
}

int synchroniseRanksGlobally() {
  return PMPI_Barrier(getLibComm());
}