Rank.cpp 3.92 KB
Newer Older
1
2
3
4
/*
 * RankOperations.cpp
 *
 *  Created on: 2 Mar 2018
5
 *      Author: Ben Hazelwood
6
7
 */

Ben Hazelwood's avatar
Ben Hazelwood committed
8
#include "Rank.h"
9
10

#include <cassert>
Ben Hazelwood's avatar
Ben Hazelwood committed
11
#include <cstdlib>
12
#include <iostream>
13
14
#include <csignal>
#include <unistd.h>
Ben Hazelwood's avatar
Ben Hazelwood committed
15
16
17

#include "Logging.h"
#include "Timing.h"
18
19
20


static int R_FACTOR;
21
22
static MPI_Comm TMPI_COMM_REP;
static MPI_Comm TMPI_COMM_WORLD;
23
24
25
26
27
static int world_rank;
static int world_size;
static int team_rank;
static int team_size;

Ben Hazelwood's avatar
Ben Hazelwood committed
28

29
int initialiseTMPI() {
Ben Hazelwood's avatar
Ben Hazelwood committed
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
  /**
   * The application should have no knowledge of the world_size or world_rank
   */

  signal(SIGUSR1, pauseThisRankSignalHandler);
  setEnvironment();

  PMPI_Comm_size(MPI_COMM_WORLD, &world_size);
  PMPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
  team_size = world_size / R_FACTOR;

  int color = world_rank / team_size;

  PMPI_Comm_dup(MPI_COMM_WORLD, &TMPI_COMM_WORLD);

  PMPI_Comm_split(MPI_COMM_WORLD, color, world_rank, &TMPI_COMM_REP);

  PMPI_Comm_rank(TMPI_COMM_REP, &team_rank);

  PMPI_Comm_size(TMPI_COMM_REP, &team_size);

  assert(team_size == (world_size / R_FACTOR));

  outputEnvironment();

#ifndef REPLICAS_OUTPUT
  // Disable output for all but master replica (0)
  if (getTeam() > 0) {
58
    disableLogging();
Ben Hazelwood's avatar
Ben Hazelwood committed
59
60
61
62
63
64
65
66
67
68
69
  }
#endif

  Timing::initialiseTiming();

  PMPI_Barrier(getLibComm());

  return MPI_SUCCESS;
}


70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
int getWorldRank() {
  return world_rank;
}

int getWorldSize() {
  return world_size;
}

int getTeamRank() {
  return team_rank;
}

int getTeamSize() {
  return team_size;
}

Ben Hazelwood's avatar
Ben Hazelwood committed
86
87
88
89
90
91
int getTeam() {
  return getWorldRank() / getTeamSize();
}

int getNumberOfTeams() {
  return getWorldSize() / getTeamSize();
92
93
}

Ben Hazelwood's avatar
Ben Hazelwood committed
94
MPI_Comm getTeamComm() {
95
96
97
  return TMPI_COMM_REP;
}

Ben Hazelwood's avatar
Ben Hazelwood committed
98
int freeTeamComm() {
99
100
101
  return MPI_Comm_free(&TMPI_COMM_REP);
}

Ben Hazelwood's avatar
Ben Hazelwood committed
102
MPI_Comm getLibComm() {
103
104
105
  return TMPI_COMM_WORLD;
}

Ben Hazelwood's avatar
Ben Hazelwood committed
106
int freeLibComm() {
107
  return MPI_Comm_free(&TMPI_COMM_WORLD);
108
109
}

Ben Hazelwood's avatar
Ben Hazelwood committed
110
111
112



113
114
115
116
117
118
std::string getEnvString(std::string const& key)
{
    char const* val = std::getenv(key.c_str());
    return val == nullptr ? std::string() : std::string(val);
}

Ben Hazelwood's avatar
Ben Hazelwood committed
119
void outputEnvironment(){
120
121
  assert(world_size % R_FACTOR == 0);

122
  PMPI_Barrier(MPI_COMM_WORLD);
123
  double my_time = MPI_Wtime();
124
  PMPI_Barrier(MPI_COMM_WORLD);
125
126
  double times[world_size];

127
  PMPI_Gather(&my_time, 1, MPI_DOUBLE, times, 1, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);
128

129
  PMPI_Barrier(MPI_COMM_WORLD);
130
131
132
133
134
135
136
137
138
139
140

  if (world_rank == MASTER) {
    std::cout << "------------TMPI SETTINGS------------\n";
    std::cout << "R_FACTOR = " << R_FACTOR << "\n";

    std::cout << "Team size: " << team_size << "\n";
    std::cout << "Total ranks: " << world_size << "\n\n";


    if (world_rank == MASTER) {
      for (int i=0; i < world_size; i++) {
Ben Hazelwood's avatar
Ben Hazelwood committed
141
        std::cout << "Tshift(" << i << "->" << mapWorldToTeamRank(i) << "->" << mapTeamToWorldRank(mapWorldToTeamRank(i),mapRankToTeamNumber(i)) << ") = " << times[i] - times[0] << "\n";
142
143
      }
    }
Ben Hazelwood's avatar
Ben Hazelwood committed
144
    std::cout << "---------------------------------------\n\n";
145
146
  }

147
  PMPI_Barrier(MPI_COMM_WORLD);
148
149
}

Ben Hazelwood's avatar
Ben Hazelwood committed
150
void setEnvironment() {
151
152
153
154
155
156
157
  std::string env;

  env = getEnvString("R_FACTOR");
  R_FACTOR = env.empty() ? 2 : std::stoi(env);

}

Ben Hazelwood's avatar
Ben Hazelwood committed
158
void pauseThisRankSignalHandler( int signum ) {
159
160
161
162
163
164
  const int startValue = 1e4;
  const int multiplier = 2;
  static int sleepLength = startValue;
  logDebug( "Signal received: sleep for " << (double)sleepLength / 1e6 << "s");
  usleep(sleepLength);
  sleepLength *= multiplier;
165
166
}

Ben Hazelwood's avatar
Ben Hazelwood committed
167
168
int mapRankToTeamNumber(int rank) {
  return rank / getTeamSize();
169
170
}

Ben Hazelwood's avatar
Ben Hazelwood committed
171
int mapWorldToTeamRank(int rank) {
172
173
174
  if (rank == MPI_ANY_SOURCE) {
    return MPI_ANY_SOURCE;
  } else {
Ben Hazelwood's avatar
Ben Hazelwood committed
175
    return rank % getTeamSize();
176
177
178
  }
}

Ben Hazelwood's avatar
Ben Hazelwood committed
179
int mapTeamToWorldRank(int rank, int r) {
180
181
182
  if (rank == MPI_ANY_SOURCE) {
    return MPI_ANY_SOURCE;
  }
183

Ben Hazelwood's avatar
Ben Hazelwood committed
184
  return rank + r * getTeamSize();
185
186
}

Ben Hazelwood's avatar
Ben Hazelwood committed
187
188
189
190
191
void remapStatus(MPI_Status *status) {
  if ((status != MPI_STATUS_IGNORE ) && (status != MPI_STATUSES_IGNORE )) {
    status->MPI_SOURCE = mapWorldToTeamRank(status->MPI_SOURCE);
    logInfo(
        "remap status source " << status->MPI_SOURCE << " to " << mapWorldToTeamRank(status->MPI_SOURCE));
192
193
194
  }
}