Zoltan2
Loading...
Searching...
No Matches
Zoltan2_MachineTorusTopoMgrForTesting.hpp
Go to the documentation of this file.
1#ifndef _ZOLTAN2_MACHINE_TORUS_TOPOMANAGERTEST_HPP_
2#define _ZOLTAN2_MACHINE_TORUS_TOPOMANAGERTEST_HPP_
3
4#include <Teuchos_Comm.hpp>
5#include <Teuchos_CommHelpers.hpp>
6#include <Zoltan2_Machine.hpp>
7
8
9namespace Zoltan2{
10
15template <typename pcoord_t, typename part_t>
16class MachineTorusBGQTest : public Machine <pcoord_t, part_t> {
17
18public:
23 MachineTorusBGQTest(const Teuchos::Comm<int> &comm ):
24 Machine<pcoord_t,part_t>(comm),
25 networkDim(6),
26 procCoords(NULL),machine_extent(NULL),
27 delete_transformed_coords(false),
28 transformed_network_dim(0),
29 transformed_coordinates (NULL), pl(NULL)
30 {
31 transformed_network_dim = networkDim - 1;
32 transformed_coordinates = procCoords;
33
34 machine_extent = new int[networkDim];
35 this->getMachineExtent(this->machine_extent);
36 machine_extent[5] = 1;
37
38 // Allocate memory for processor coordinates.
39 procCoords = new pcoord_t *[networkDim];
40 for (int i = 0; i < networkDim; ++i) {
41 procCoords[i] = new pcoord_t[this->numRanks];
42 memset(procCoords[i], 0, sizeof(pcoord_t) * this->numRanks);
43 }
44
45 // Obtain the coordinate of the processor.
46 pcoord_t *xyz = new pcoord_t[networkDim];
48 for (int i = 0; i < networkDim; i++)
49 procCoords[i][this->myRank] = xyz[i];
50 delete [] xyz;
51
52 // reduceAll the coordinates of each processor.
53 gatherMachineCoordinates(comm);
54
55 }
56
57 MachineTorusBGQTest(const Teuchos::Comm<int> &comm,
58 const Teuchos::ParameterList &pl_):
59 Machine<pcoord_t,part_t>(comm),
60 networkDim(6),
61 procCoords(NULL),machine_extent(NULL),
62 delete_transformed_coords(false),
63 transformed_network_dim(0),
64 transformed_coordinates (NULL),
65 pl(&pl_)
66 {
67 transformed_network_dim = networkDim - 1;
68 transformed_coordinates = procCoords;
69 machine_extent = new int[networkDim];
70
71 this->getMachineExtent(this->machine_extent);
72 machine_extent[5] = 1;
73
74 // Allocate memory for processor coordinates.
75 procCoords = new pcoord_t *[networkDim];
76 for (int i = 0; i < networkDim; ++i) {
77 procCoords[i] = new pcoord_t[this->numRanks];
78 memset(procCoords[i], 0, sizeof(pcoord_t) * this->numRanks);
79 }
80
81 // Obtain the coordinate of the processor.
82 pcoord_t *xyz = new pcoord_t[networkDim];
84 for (int i = 0; i < networkDim; i++)
85 procCoords[i][this->myRank] = xyz[i];
86 delete [] xyz;
87
88 // reduceAll the coordinates of each processor.
89 gatherMachineCoordinates(comm);
90
91 const Teuchos::ParameterEntry *pe =
92 this->pl->getEntryPtr("Machine_Optimization_Level");
93
94 if (pe) {
95 int optimization_level = 0;
96
97 optimization_level = pe->getValue<int>(&optimization_level);
98
99 if (optimization_level == 0) {
100 transformed_network_dim = networkDim - 1;
101 transformed_coordinates = procCoords;
102 }
103 else if (optimization_level >= 1) {
104 transformed_network_dim = networkDim - 2;
105 transformed_coordinates = procCoords;
106 }
107 }
108 }
109
110
112 for (int i = 0; i < networkDim; i++) {
113 delete [] procCoords[i];
114 }
115 delete [] procCoords;
116
117 delete [] machine_extent;
118 if (delete_transformed_coords) {
119 for (int i = 0; i < transformed_network_dim; i++) {
120 delete [] transformed_coordinates[i];
121 }
122 delete [] transformed_coordinates;
123 }
124 }
125
126 bool hasMachineCoordinates() const { return true; }
127
128 int getMachineDim() const { return transformed_network_dim; }
129
130 bool getMachineExtentWrapArounds(bool *wrap_around) const {
131 int dim = 0;
132 if (dim < transformed_network_dim)
133 wrap_around[dim++] = true;
134
135 if (dim < transformed_network_dim)
136 wrap_around[dim++] = true;
137
138 if (dim < transformed_network_dim)
139 wrap_around[dim++] = true;
140
141 if (dim < transformed_network_dim)
142 wrap_around[dim++] = true;
143
144 if (dim < transformed_network_dim)
145 wrap_around[dim++] = true;
146
147 if (dim < transformed_network_dim)
148 wrap_around[dim++] = true;
149 return true;
150 }
151
152
153 bool getMachineExtentWrapArounds(part_t *wrap_around) const {
154 int dim = 0;
155 if (dim < transformed_network_dim)
156 wrap_around[dim++] = true;
157
158 if (dim < transformed_network_dim)
159 wrap_around[dim++] = true;
160
161 if (dim < transformed_network_dim)
162 wrap_around[dim++] = true;
163
164 if (dim < transformed_network_dim)
165 wrap_around[dim++] = true;
166
167 if (dim < transformed_network_dim)
168 wrap_around[dim++] = true;
169
170 if (dim < transformed_network_dim)
171 wrap_around[dim++] = true;
172 return true;
173 }
174
175 bool getMachineExtent(part_t *extent) const {
176 part_t nxyz[6];
177
178 nxyz[0] = 1;
179 nxyz[1] = 1;
180 nxyz[2] = 1;
181 nxyz[3] = 1;
182 nxyz[4] = 1;
183 nxyz[5] = 1;
184 const int rank_per_node = 1;
185 const int num_nodes = this->numRanks / rank_per_node;
186
187 if (num_nodes <= 1) {
188 nxyz[0] = nxyz[1] = nxyz[2] = nxyz[3] = 1;
189 nxyz[4] = 1;
190 nxyz[5] = 1;
191 }
192 else if (num_nodes <= 2) {
193 nxyz[0] = nxyz[1] = nxyz[2] = nxyz[3] = 1;
194 nxyz[4] = 2;
195 nxyz[5] = 1;
196 }
197 else if (num_nodes <= 4) {
198 nxyz[0] = nxyz[1] = nxyz[2] = 1;
199 nxyz[3] = 2;
200 nxyz[4] = 2;
201 nxyz[5] = 1;
202 }
203 else if (num_nodes <= 8) {
204 nxyz[0] = nxyz[1] = 1;
205 nxyz[2] = 2;
206 nxyz[3] = 2;
207 nxyz[4] = 2;
208 nxyz[5] = 1;
209 }
210 else if (num_nodes <= 16) {
211 nxyz[0] = 1;
212 nxyz[1] = 2;
213 nxyz[2] = 2;
214 nxyz[3] = 2;
215 nxyz[4] = 2;
216 nxyz[5] = 1;
217 }
218 else if (num_nodes <= 32) {
219 nxyz[0] = 1;
220 nxyz[1] = 2;
221 nxyz[2] = 2;
222 nxyz[3] = 4;
223 nxyz[4] = 2;
224 nxyz[5] = 1;
225 }
226 else if (num_nodes <= 64) {
227 nxyz[0] = 1;
228 nxyz[1] = 2;
229 nxyz[2] = 4;
230 nxyz[3] = 4;
231 nxyz[4] = 2;
232 nxyz[5] = 1;
233 }
234 else if (num_nodes <= 128) {
235 nxyz[0] = 2;
236 nxyz[1] = 2;
237 nxyz[2] = 4;
238 nxyz[3] = 4;
239 nxyz[4] = 2;
240 nxyz[5] = 1;
241 }
242 else if (num_nodes <= 256) {
243 nxyz[0] = 4;
244 nxyz[1] = 2;
245 nxyz[2] = 4;
246 nxyz[3] = 4;
247 nxyz[4] = 2;
248 nxyz[5] = 1;
249 }
250 else if (num_nodes <= 512) {
251 nxyz[0] = 4;
252 nxyz[1] = 4;
253 nxyz[2] = 4;
254 nxyz[3] = 4;
255 nxyz[4] = 2;
256 nxyz[5] = 1;
257 }
258 else if (num_nodes <= 1024) {
259 nxyz[0] = 4;
260 nxyz[1] = 4;
261 nxyz[2] = 4;
262 nxyz[3] = 8;
263 nxyz[4] = 2;
264 nxyz[5] = 1;
265 }
266 else if (num_nodes <= 2048) {
267 nxyz[0] = 4;
268 nxyz[1] = 4;
269 nxyz[2] = 4;
270 nxyz[3] = 16;
271 nxyz[4] = 2;
272 nxyz[5] = 1;
273 }
274 else if (num_nodes <= 4096) {
275 nxyz[0] = 8;
276 nxyz[1] = 4;
277 nxyz[2] = 4;
278 nxyz[3] = 16;
279 nxyz[4] = 2;
280 nxyz[5] = 1;
281 }
282 else {
283 std::cerr << "Too many ranks to test" << std::endl;
284 }
285 nxyz[5] = rank_per_node;
286
287 int dim = 0;
288 if (dim < transformed_network_dim)
289 extent[dim] = nxyz[dim];
290 ++dim;
291 if (dim < transformed_network_dim)
292 extent[dim] = nxyz[dim];
293 ++dim;
294 if (dim < transformed_network_dim)
295 extent[dim] = nxyz[dim];
296 ++dim;
297 if (dim < transformed_network_dim)
298 extent[dim] = nxyz[dim];
299 ++dim;
300 if (dim < transformed_network_dim)
301 extent[dim] = nxyz[dim];
302 ++dim;
303 if (dim < transformed_network_dim)
304 extent[dim] = nxyz[dim];
305
306 return true;
307 }
308
309
310 bool getMyMachineCoordinate(pcoord_t *xyz) {
311 for (int i = 0; i < this->transformed_network_dim; ++i) {
312 xyz[i] = transformed_coordinates[i][this->myRank];
313 }
314 return true;
315 }
316
317 bool getMyActualMachineCoordinate(pcoord_t *xyz) {
318 int a,b,c,d,e,t;
319
320 int me = this->myRank;
321 t = me % machine_extent[5];
322
323 me = me / machine_extent[5];
324 e = me % machine_extent[4];
325
326 me = me / machine_extent[4];
327 d = me % machine_extent[3];
328
329 me = me / machine_extent[3];
330 c = me % machine_extent[2];
331
332 me = me / machine_extent[2];
333 b = me % machine_extent[1];
334
335 me = me / machine_extent[1];
336 a = me % machine_extent[0];
337
338 xyz[0] = a; xyz[1] = b; xyz[2] = c; xyz[3] = d; xyz[4] = e; xyz[5] = t;
339
340// std::cout << "me:" << this->myRank << " " << a << " " << b
341// << " " << c << " " << d << " " << e << " " << t << std::endl;
342 return true;
343 }
344
345
346 inline bool getMachineCoordinate(const int rank,
347 pcoord_t *xyz) const {
348 return false;
349 }
350 bool getMachineCoordinate(const char *nodename, pcoord_t *xyz) {
351 return false; // cannot yet return from nodename
352 }
353
354 bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const {
355 allCoords = procCoords;
356 return true;
357 }
358
359 virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops) const override {
360
361 hops = 0;
362 for (int i = 0; i < networkDim - 1; ++i) {
363 pcoord_t distance = procCoords[i][rank1] - procCoords[i][rank2];
364 if (distance < 0 )
365 distance = -distance;
366 if (machine_extent[i] - distance < distance)
367 distance = machine_extent[i] - distance;
368 hops += distance;
369 }
370
371/*
372 if (this->myRank == 0) {
373 std::cout << "rank1:" << rank1 << " rank2:" << rank2
374 << " hops:" << hops << std::endl;
375 }
376*/
377
378 return true;
379 }
380
381
382private:
383
384 int networkDim;
385 pcoord_t **procCoords; // KDD Maybe should be RCP?
386 part_t *machine_extent;
387
388 bool delete_transformed_coords;
389 int transformed_network_dim;
390 pcoord_t **transformed_coordinates;
391 const Teuchos::ParameterList *pl;
392
393 void gatherMachineCoordinates(const Teuchos::Comm<int> &comm) {
394 // Reduces and stores all machine coordinates.
395 pcoord_t *tmpVect = new pcoord_t [this->numRanks];
396
397 for (int i = 0; i < networkDim; i++) {
398 Teuchos::reduceAll<int, pcoord_t>(comm, Teuchos::REDUCE_SUM,
399 this->numRanks,
400 procCoords[i], tmpVect);
401 pcoord_t *tmp = tmpVect;
402 tmpVect = procCoords[i];
403 procCoords[i] = tmp;
404 }
405 delete [] tmpVect;
406 }
407};
408}
409#endif
A TopoManager Machine Class (Torus Networks) for testing only A more realistic machine should be used...
MachineTorusBGQTest(const Teuchos::Comm< int > &comm)
Constructor: A BlueGeneQ network machine description;.
bool getMachineCoordinate(const int rank, pcoord_t *xyz) const
bool getMachineCoordinate(const char *nodename, pcoord_t *xyz)
virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops) const override
getHopCount function set hops between rank1 and rank2 return true if coordinates are available
bool getMachineExtentWrapArounds(bool *wrap_around) const
MachineTorusBGQTest(const Teuchos::Comm< int > &comm, const Teuchos::ParameterList &pl_)
bool getMachineExtentWrapArounds(part_t *wrap_around) const
bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const
MachineClass Base class for representing machine coordinates, networks, etc.
Created by mbenlioglu on Aug 31, 2020.
SparseMatrixAdapter_t::part_t part_t