52#include "Zoltan2_config.h"
56#define MEMORY_CHECK(iPrint, msg) \
58 long kb = Zoltan2::getProcessKilobytes(); \
59 std::cout.width(10); \
60 std::cout.fill('*'); \
61 std::cout << kb << " KB, " << msg << std::endl; \
63 std::cout.fill(' '); \
66#include <Teuchos_RCP.hpp>
67#include <Teuchos_ArrayView.hpp>
68#include <Teuchos_ParameterList.hpp>
69#include <Teuchos_DefaultComm.hpp>
70#include <Teuchos_Comm.hpp>
71#include <Teuchos_CommHelpers.hpp>
74#include <Tpetra_MultiVector.hpp>
75#include <KokkosCompat_DefaultNode.hpp>
89using Teuchos::ArrayView;
90using Teuchos::CommandLineProcessor;
92typedef Tpetra::MultiVector<zscalar_t, zlno_t, zgno_t, znode_t>
tMVector_t;
93typedef Tpetra::Map<zlno_t, zgno_t, znode_t>
tMap_t;
107 const string& delimiters =
" \f\n\r\t\v" )
109 return s.substr( 0, s.find_last_not_of( delimiters ) + 1 );
114 const string& delimiters =
" \f\n\r\t\v" )
116 return s.substr( s.find_first_not_of( delimiters ) );
121 const string& delimiters =
" \f\n\r\t\v" )
126void readGeoGenParams(
string paramFileName, Teuchos::ParameterList &geoparams,
const RCP<
const Teuchos::Comm<int> > & comm){
127 std::string input =
"";
129 for(
int i = 0; i < 25000; ++i){
134 if(comm->getRank() == 0){
136 std::fstream inParam(paramFileName.c_str());
143 std::string tmp =
"";
144 getline (inParam,tmp);
145 while (!inParam.eof()){
152 getline (inParam,tmp);
155 for (
size_t i = 0; i < input.size(); ++i){
163 int size = input.size();
167 comm->broadcast(0,
sizeof(
int), (
char*) &size);
169 throw "File " + paramFileName +
" cannot be opened.";
171 comm->broadcast(0, size, inp);
172 std::istringstream inParam(inp);
174 getline (inParam,str);
175 while (!inParam.eof()){
177 size_t pos = str.find(
'=');
178 if(pos == string::npos){
179 throw "Invalid Line:" + str +
" in parameter file";
181 string paramname =
trim_copy(str.substr(0,pos));
182 string paramvalue =
trim_copy(str.substr(pos + 1));
183 geoparams.set(paramname, paramvalue);
185 getline (inParam,str);
212 ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
213 int num_wgts,
float *obj_wgts,
int *ierr)
222 if (
sizeof(ZOLTAN_ID_TYPE) ==
sizeof(
zgno_t))
223 memcpy(gids, ids,
sizeof(ZOLTAN_ID_TYPE) * localLen);
225 for (
size_t i=0; i < localLen; i++)
226 gids[i] =
static_cast<ZOLTAN_ID_TYPE
>(ids[i]);
229 float *wgts = obj_wgts;
230 for (
size_t i=0; i < localLen; i++)
231 for (
int w=0; w < num_wgts; w++)
238 int numObj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
239 int dim,
double *coords,
int *ierr)
245 double *val = coords;
249 for (
int i=0; i < numObj; i++){
250 *val++ =
static_cast<double>(x[i]);
251 *val++ =
static_cast<double>(y[i]);
252 *val++ =
static_cast<double>(z[i]);
258 double *val = coords;
261 for (
int i=0; i < numObj; i++){
262 *val++ =
static_cast<double>(x[i]);
263 *val++ =
static_cast<double>(y[i]);
281 const RCP<
const Teuchos::Comm<int> > & comm,
282 vector<float> &wgts,
weightTypes how,
float scale,
int rank)
286 float val = scale + rank%2;
287 for (
zlno_t i=0; i < len; i++)
291 for (
int i=0; i < 10; i++){
292 float val = (i + 10)*scale;
293 for (
zlno_t j=i; j < len; j += 10)
298 float val = scale + rank;
299 for (
zlno_t i=0; i < len; i++)
310 const RCP<
const Teuchos::Comm<int> > & comm,
313 int rank = comm->getRank();
314 int nprocs = comm->getSize();
316 double k = log(numGlobalCoords) / 3;
317 double xdimf = exp(k) + 0.5;
318 ssize_t xdim =
static_cast<ssize_t
>(floor(xdimf));
320 ssize_t zdim = numGlobalCoords / (xdim*ydim);
321 ssize_t num=xdim*ydim*zdim;
322 ssize_t diff = numGlobalCoords - num;
326 if (zdim > xdim && zdim > ydim){
328 newdiff = diff - (xdim*ydim);
333 else if (ydim > xdim && ydim > zdim){
335 newdiff = diff - (xdim*zdim);
342 newdiff = diff - (ydim*zdim);
352 diff = numGlobalCoords - num;
354 diff /= -numGlobalCoords;
356 diff /= numGlobalCoords;
360 std::cout <<
"Warning: Difference " << diff*100 <<
" percent" << std::endl;
361 std::cout <<
"Mesh size: " << xdim <<
"x" << ydim <<
"x" <<
362 zdim <<
", " << num <<
" vertices." << std::endl;
367 ssize_t numLocalCoords = num / nprocs;
368 ssize_t leftOver = num % nprocs;
371 if (rank <= leftOver)
372 gid0 = rank * (numLocalCoords+1);
374 gid0 = (leftOver * (numLocalCoords+1)) +
375 ((rank - leftOver) * numLocalCoords);
380 ssize_t gid1 = gid0 + numLocalCoords;
385 ArrayView<zgno_t> idArray(ids, numLocalCoords);
388 for (ssize_t i=gid0; i < gid1; i++)
391 RCP<const tMap_t> idMap = rcp(
new tMap_t(num, idArray, 0, comm));
398 if (!x)
throw bad_alloc();
405 zgno_t xyPlane = xdim*ydim;
406 zgno_t zStart = gid0 / xyPlane;
407 zgno_t rem = gid0 % xyPlane;
414 for (
zscalar_t zval=zStart; next < numLocalCoords && zval < zdim; zval+=1.){
415 for (
zscalar_t yval=yStart; next < numLocalCoords && yval < ydim; yval+=1.){
416 for (
zscalar_t xval=xStart; next < numLocalCoords && xval < xdim;xval+=1.){
427 ArrayView<const zscalar_t> xArray(x, numLocalCoords*3);
439 Tpetra::ScopeGuard tscope(&narg, &arg);
440 Teuchos::RCP<const Teuchos::Comm<int> > comm = Tpetra::getDefaultComm();
441 int rank = comm->getRank();
442 int nprocs = comm->getSize();
445 MEMORY_CHECK(rank==0 || rank==nprocs-1,
"After initializing MPI");
448 std::cout <<
"Number of processes: " << nprocs << std::endl;
451 zgno_t numGlobalCoords = 1000;
454 string memoryOn(
"memoryOn");
455 string memoryOff(
"memoryOff");
457 int numGlobalParts = nprocs;
461 string balanceCount(
"balance_object_count");
462 string balanceWeight(
"balance_object_weight");
463 string mcnorm1(
"multicriteria_minimize_total_weight");
464 string mcnorm2(
"multicriteria_balance_total_maximum");
465 string mcnorm3(
"multicriteria_minimize_maximum_weight");
466 string objective(balanceWeight);
469 CommandLineProcessor commandLine(
false,
true);
472 int input_option = 0;
473 commandLine.setOption(
"input_option", &input_option,
474 "whether to use mesh creation, geometric generator, or file input");
475 string inputFile =
"";
477 commandLine.setOption(
"input_file", &inputFile,
478 "the input file for geometric generator or file input");
481 commandLine.setOption(
"size", &numGlobalCoords,
482 "Approximate number of global coordinates.");
483 commandLine.setOption(
"numParts", &numGlobalParts,
484 "Number of parts (default is one per proc).");
485 commandLine.setOption(
"nWeights", &nWeights,
486 "Number of weights per coordinate, zero implies uniform weights.");
487 commandLine.setOption(
"debug", &debugLevel,
"Zoltan1 debug level");
488 commandLine.setOption(
"remap",
"no-remap", &remap,
489 "Zoltan1 REMAP parameter; disabled by default for scalability testing");
490 commandLine.setOption(
"timers", &dummyTimer,
"ignored");
491 commandLine.setOption(memoryOn.c_str(), memoryOff.c_str(), &doMemory,
492 "do memory profiling");
494 string doc(balanceCount);
495 doc.append(
": ignore weights\n");
496 doc.append(balanceWeight);
497 doc.append(
": balance on first weight\n");
499 doc.append(
": given multiple weights, balance their total.\n");
501 doc.append(
": given multiple weights, "
502 "balance the maximum for each coordinate.\n");
504 doc.append(
": given multiple weights, balance the L2 norm of the weights.\n");
505 commandLine.setOption(
"objective", &objective, doc.c_str());
507 CommandLineProcessor::EParseCommandLineReturn rc =
508 commandLine.parse(narg, arg);
512 if (rc != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
513 if (rc == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED) {
514 if (rank==0) std::cout <<
"PASS" << std::endl;
518 if (rank==0) std::cout <<
"FAIL" << std::endl;
526 size_t numLocalCoords = 0;
527 if (input_option == 0){
533 for (
int p=0; p < nprocs; p++){
535 std::cout <<
"Rank " << rank <<
", " << numLocalCoords <<
"coords" << std::endl;
539 for (
zlno_t i=0; i < numLocalCoords; i++)
540 std::cout <<
" " << x[i] <<
" " << y[i] <<
" " << z[i] << std::endl;
553 for (
int i=0; i < nWeights; i++){
564 else if(input_option == 1){
565 Teuchos::ParameterList geoparams(
"geo params");
574 for(
int i = 0; i < coord_dim; ++i){
575 coords[i] =
new zscalar_t[numLocalCoords];
581 for(
int i = 0; i < nWeights; ++i){
582 weight[i] =
new zscalar_t[numLocalCoords];
589 RCP<Tpetra::Map<zlno_t, zgno_t, znode_t> > mp = rcp(
590 new Tpetra::Map<zlno_t, zgno_t, znode_t> (numGlobalCoords, numLocalCoords, 0, comm));
592 Teuchos::Array<Teuchos::ArrayView<const zscalar_t> > coordView(coord_dim);
593 for (
int i=0; i < coord_dim; i++){
594 if(numLocalCoords > 0){
595 Teuchos::ArrayView<const zscalar_t> a(coords[i], numLocalCoords);
598 Teuchos::ArrayView<const zscalar_t> a;
609 for (
int i = 0; i < nWeights;++i){
616 for(
int i = 0; i < nWeights; ++i)
627 numLocalCoords = coords->getLocalLength();
628 numGlobalCoords = coords->getGlobalLength();
631 MEMORY_CHECK(doMemory && rank==0,
"After creating input");
636 int aok = Zoltan_Initialize(narg, arg, &ver);
639 printf(
"Zoltan_Initialize failed\n");
643 struct Zoltan_Struct *zz;
644 zz = Zoltan_Create(MPI_COMM_WORLD);
646 Zoltan_Set_Param(zz,
"LB_METHOD",
"RCB");
647 Zoltan_Set_Param(zz,
"LB_APPROACH",
"PARTITION");
648 Zoltan_Set_Param(zz,
"CHECK_GEOM",
"0");
649 Zoltan_Set_Param(zz,
"NUM_GID_ENTRIES",
"1");
650 Zoltan_Set_Param(zz,
"NUM_LID_ENTRIES",
"0");
651 Zoltan_Set_Param(zz,
"RETURN_LISTS",
"PART");
652 std::ostringstream oss;
653 oss << numGlobalParts;
654 Zoltan_Set_Param(zz,
"NUM_GLOBAL_PARTS", oss.str().c_str());
657 Zoltan_Set_Param(zz,
"DEBUG_LEVEL", oss.str().c_str());
660 Zoltan_Set_Param(zz,
"REMAP",
"1");
662 Zoltan_Set_Param(zz,
"REMAP",
"0");
664 if (objective != balanceCount){
667 Zoltan_Set_Param(zz,
"OBJ_WEIGHT_DIM", oss.str().c_str());
669 if (objective == mcnorm1)
670 Zoltan_Set_Param(zz,
"RCB_MULTICRITERIA_NORM",
"1");
671 else if (objective == mcnorm2)
672 Zoltan_Set_Param(zz,
"RCB_MULTICRITERIA_NORM",
"2");
673 else if (objective == mcnorm3)
674 Zoltan_Set_Param(zz,
"RCB_MULTICRITERIA_NORM",
"3");
677 Zoltan_Set_Param(zz,
"OBJ_WEIGHT_DIM",
"0");
685 int changes, numGidEntries, numLidEntries, numImport, numExport;
686 ZOLTAN_ID_PTR importGlobalGids, importLocalGids;
687 ZOLTAN_ID_PTR exportGlobalGids, exportLocalGids;
688 int *importProcs, *importToPart, *exportProcs, *exportToPart;
690 MEMORY_CHECK(doMemory && rank==0,
"Before Zoltan_LB_Partition");
692 if (rank == 0) std::cout <<
"Calling Zoltan_LB_Partition" << std::endl;
693 aok = Zoltan_LB_Partition(zz, &changes, &numGidEntries, &numLidEntries,
694 &numImport, &importGlobalGids, &importLocalGids,
695 &importProcs, &importToPart,
696 &numExport, &exportGlobalGids, &exportLocalGids,
697 &exportProcs, &exportToPart);
698 if (rank == 0) std::cout <<
"Returned from Zoltan_LB_Partition" << std::endl;
700 MEMORY_CHECK(doMemory && rank==0,
"After Zoltan_LB_Partition");
704 float *sumWgtPerPart =
new float[numGlobalParts];
705 float *gsumWgtPerPart =
new float[numGlobalParts];
706 for (
int i = 0; i < numGlobalParts; i++) sumWgtPerPart[i] = 0.;
708 for (
size_t i = 0; i < numLocalCoords; i++)
709 sumWgtPerPart[exportToPart[i]] += (nWeights ?
dots.
weights[0][i]: 1.);
711 Teuchos::reduceAll<int, float>(*comm, Teuchos::REDUCE_SUM, numGlobalParts,
712 sumWgtPerPart, gsumWgtPerPart);
714 float maxSumWgtPerPart = 0.;
715 float minSumWgtPerPart = std::numeric_limits<float>::max();
717 int maxSumWgtPart=0, minSumWgtPart=0;
718 for (
int i = 0; i < numGlobalParts; i++) {
719 if (gsumWgtPerPart[i] > maxSumWgtPerPart) {
720 maxSumWgtPerPart = gsumWgtPerPart[i];
723 if (gsumWgtPerPart[i] < minSumWgtPerPart) {
724 minSumWgtPerPart = gsumWgtPerPart[i];
727 totWgt += gsumWgtPerPart[i];
731 std::cout << std::endl << std::endl
732 <<
"Part loads (per part for " << numGlobalParts <<
" parts):"
734 <<
" min = " << minSumWgtPerPart
735 <<
" in part " << minSumWgtPart << std::endl
736 <<
" max = " << maxSumWgtPerPart
737 <<
" in part " << maxSumWgtPart << std::endl
738 <<
" tot = " << totWgt << std::endl
739 <<
" avg = " << totWgt / numGlobalParts
740 << std::endl << std::endl << std::endl;
742 delete [] sumWgtPerPart;
743 delete [] gsumWgtPerPart;
746 MEMORY_CHECK(doMemory && rank==0,
"After Zoltan_Destroy");
749 for (
int i = 0; i < nWeights; i++)
753 MEMORY_CHECK(doMemory && rank==0,
"After destroying input");
757 std::cout <<
"FAIL" << std::endl;
759 std::cout <<
"PASS" << std::endl;
common code used by tests
Tpetra::Map ::local_ordinal_type zlno_t
std::string testDataFilePath(".")
Tpetra::Map ::global_ordinal_type zgno_t
A gathering of useful namespace methods.
int getCoordinateDimension()
lno_t getNumLocalCoords()
void getLocalWeightsCopy(scalar_t **w)
gno_t getNumGlobalCoords()
void getLocalCoordinatesCopy(scalar_t **c)
static const std::string fail
vector< vector< float > > weights
Tpetra::MultiVector< zscalar_t, zlno_t, zgno_t, znode_t > tMVector_t