MueLu Version of the Day
Loading...
Searching...
No Matches
MueLu_AMGXOperator_decl.hpp
Go to the documentation of this file.
1// @HEADER
2//
3// ***********************************************************************
4//
5// MueLu: A package for multigrid based preconditioning
6// Copyright 2012 Sandia Corporation
7//
8// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9// the U.S. Government retains certain rights in this software.
10//
11// Redistribution and use in source and binary forms, with or without
12// modification, are permitted provided that the following conditions are
13// met:
14//
15// 1. Redistributions of source code must retain the above copyright
16// notice, this list of conditions and the following disclaimer.
17//
18// 2. Redistributions in binary form must reproduce the above copyright
19// notice, this list of conditions and the following disclaimer in the
20// documentation and/or other materials provided with the distribution.
21//
22// 3. Neither the name of the Corporation nor the names of the
23// contributors may be used to endorse or promote products derived from
24// this software without specific prior written permission.
25//
26// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37//
38// Questions? Contact
39// Jonathan Hu (jhu@sandia.gov)
40// Andrey Prokopenko (aprokop@sandia.gov)
41// Ray Tuminaro (rstumin@sandia.gov)
42//
43// ***********************************************************************
44//
45// @HEADER
46#ifndef MUELU_AMGXOPERATOR_DECL_HPP
47#define MUELU_AMGXOPERATOR_DECL_HPP
48
49#if defined (HAVE_MUELU_AMGX)
50#include <Teuchos_ParameterList.hpp>
51
52#include <Tpetra_Operator.hpp>
53#include <Tpetra_CrsMatrix.hpp>
54#include <Tpetra_MultiVector.hpp>
55#include <Tpetra_Distributor.hpp>
56#include <Tpetra_HashTable.hpp>
57#include <Tpetra_Import.hpp>
58#include <Tpetra_Import_Util.hpp>
59
60#include "MueLu_Exceptions.hpp"
61#include "MueLu_TimeMonitor.hpp"
62#include "MueLu_TpetraOperator.hpp"
64
65#include <cuda_runtime.h>
66#include <amgx_c.h>
67
68namespace MueLu {
69
70
77 template <class Scalar,
78 class LocalOrdinal,
79 class GlobalOrdinal,
80 class Node>
81 class AMGXOperator : public TpetraOperator<Scalar, LocalOrdinal, GlobalOrdinal, Node>, public BaseClass {
82 private:
83 typedef Scalar SC;
86 typedef Node NO;
87
88 typedef Tpetra::Map<LO,GO,NO> Map;
89 typedef Tpetra::MultiVector<SC,LO,GO,NO> MultiVector;
90
91 public:
92
94
95
97 AMGXOperator(const Teuchos::RCP<Tpetra::CrsMatrix<SC,LO,GO,NO> > &InA, Teuchos::ParameterList &paramListIn) { }
98
100 virtual ~AMGXOperator() {}
101
103
105 Teuchos::RCP<const Map> getDomainMap() const{
106 throw Exceptions::RuntimeError("Cannot use AMGXOperator with scalar != double and/or global ordinal != int \n");
107 }
108
110 Teuchos::RCP<const Map> getRangeMap() const{
111 throw Exceptions::RuntimeError("Cannot use AMGXOperator with scalar != double and/or global ordinal != int \n");
112 }
113
115
119 void apply(const MultiVector& X, MultiVector& Y, Teuchos::ETransp mode = Teuchos::NO_TRANS,
120 Scalar alpha = Teuchos::ScalarTraits<Scalar>::one(), Scalar beta = Teuchos::ScalarTraits<Scalar>::zero()) const {
121 throw Exceptions::RuntimeError("Cannot use AMGXOperator with scalar != double and/or global ordinal != int \n");
122 }
123
125 bool hasTransposeApply() const{
126 throw Exceptions::RuntimeError("Cannot use AMGXOperator with scalar != double and/or global ordinal != int \n");
127 }
128
129 RCP<MueLu::Hierarchy<SC,LO,GO,NO> > GetHierarchy() const {
130 throw Exceptions::RuntimeError("AMGXOperator does not hold a MueLu::Hierarchy object \n");
131 }
132
133 private:
134 };
135
142 template<class Node>
143 class AMGXOperator<double, int, int, Node> : public TpetraOperator<double, int, int, Node> {
144 private:
145 typedef double SC;
146 typedef int LO;
147 typedef int GO;
148 typedef Node NO;
149
150 typedef Tpetra::Map<LO,GO,NO> Map;
151 typedef Tpetra::MultiVector<SC,LO,GO,NO> MultiVector;
152
153
154 void printMaps(Teuchos::RCP<const Teuchos::Comm<int> >& comm, const std::vector<std::vector<int> >& vec, const std::vector<int>& perm,
155 const int* nbrs, const Map& map, const std::string& label) {
156 for (int p = 0; p < comm->getSize(); p++) {
157 if (comm->getRank() == p) {
158 std::cout << "========\n" << label << ", lid (gid), PID " << p << "\n========" << std::endl;
159
160 for (size_t i = 0; i < vec.size(); ++i) {
161 std::cout << " neighbor " << nbrs[i] << " :";
162 for (size_t j = 0; j < vec[i].size(); ++j)
163 std::cout << " " << vec[i][j] << " (" << map.getGlobalElement(perm[vec[i][j]]) << ")";
164 std::cout << std::endl;
165 }
166 std::cout << std::endl;
167 } else {
168 sleep(1);
169 }
170 comm->barrier();
171 }
172 }
173
174 public:
175
177
178 AMGXOperator(const Teuchos::RCP<Tpetra::CrsMatrix<SC,LO,GO,NO> > &inA, Teuchos::ParameterList &paramListIn) {
179 RCP<const Teuchos::Comm<int> > comm = inA->getRowMap()->getComm();
180 int numProcs = comm->getSize();
181 int myRank = comm->getRank();
182
183
184 RCP<Teuchos::Time> amgxTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: initialize");
185 amgxTimer->start();
186 // Initialize
187 //AMGX_SAFE_CALL(AMGX_initialize());
188 //AMGX_SAFE_CALL(AMGX_initialize_plugins());
189
190
191 /*system*/
192 //AMGX_SAFE_CALL(AMGX_register_print_callback(&print_callback));
193 AMGX_SAFE_CALL(AMGX_install_signal_handler());
194 Teuchos::ParameterList configs = paramListIn.sublist("amgx:params", true);
195 if (configs.isParameter("json file")) {
196 AMGX_SAFE_CALL(AMGX_config_create_from_file(&Config_, (const char *) &configs.get<std::string>("json file")[0]));
197 } else {
198 std::ostringstream oss;
199 oss << "";
200 ParameterList::ConstIterator itr;
201 for (itr = configs.begin(); itr != configs.end(); ++itr) {
202 const std::string& name = configs.name(itr);
203 const ParameterEntry& entry = configs.entry(itr);
204 oss << name << "=" << filterValueToString(entry) << ", ";
205 }
206 oss << "\0";
207 std::string configString = oss.str();
208 if (configString == "") {
209 //print msg that using defaults
210 //GetOStream(Warnings0) << "Warning: No configuration parameters specified, using default AMGX configuration parameters. \n";
211 }
212 AMGX_SAFE_CALL(AMGX_config_create(&Config_, configString.c_str()));
213 }
214
215 // TODO: we probably need to add "exception_handling=1" to the parameter list
216 // to switch on internal error handling (with no need for AMGX_SAFE_CALL)
217
218 //AMGX_SAFE_CALL(AMGX_config_add_parameters(&Config_, "exception_handling=1"))
219
220#define NEW_COMM
221#ifdef NEW_COMM
222 // NOTE: MPI communicator used in AMGX_resources_create must exist in the scope of AMGX_matrix_comm_from_maps_one_ring
223 // FIXME: fix for serial comm
224 RCP<const Teuchos::MpiComm<int> > tmpic = Teuchos::rcp_dynamic_cast<const Teuchos::MpiComm<int> >(comm->duplicate());
225 TEUCHOS_TEST_FOR_EXCEPTION(tmpic.is_null(), Exceptions::RuntimeError, "Communicator is not MpiComm");
226
227 RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > rawMpiComm = tmpic->getRawMpiComm();
228 MPI_Comm mpiComm = *rawMpiComm;
229#endif
230
231 // Construct AMGX resources
232 if (numProcs == 1) {
233 AMGX_resources_create_simple(&Resources_, Config_);
234
235 } else {
236 int numGPUDevices;
237 cudaGetDeviceCount(&numGPUDevices);
238 int device[] = {(comm->getRank() % numGPUDevices)};
239
240 AMGX_config_add_parameters(&Config_, "communicator=MPI");
241#ifdef NEW_COMM
242 AMGX_resources_create(&Resources_, Config_, &mpiComm, 1/* number of GPU devices utilized by this rank */, device);
243#else
244 AMGX_resources_create(&Resources_, Config_, MPI_COMM_WORLD, 1/* number of GPU devices utilized by this rank */, device);
245#endif
246 }
247
248 AMGX_Mode mode = AMGX_mode_dDDI;
249 AMGX_solver_create(&Solver_, Resources_, mode, Config_);
250 AMGX_matrix_create(&A_, Resources_, mode);
251 AMGX_vector_create(&X_, Resources_, mode);
252 AMGX_vector_create(&Y_, Resources_, mode);
253
254 amgxTimer->stop();
255 amgxTimer->incrementNumCalls();
256
257 std::vector<int> amgx2muelu;
258
259 // Construct AMGX communication pattern
260 if (numProcs > 1) {
261 RCP<const Tpetra::Import<LO,GO,NO> > importer = inA->getCrsGraph()->getImporter();
262
263 TEUCHOS_TEST_FOR_EXCEPTION(importer.is_null(), MueLu::Exceptions::RuntimeError, "The matrix A has no Import object.");
264
265 Tpetra::Distributor distributor = importer->getDistributor();
266
267 Array<int> sendRanks = distributor.getProcsTo();
268 Array<int> recvRanks = distributor.getProcsFrom();
269
270 std::sort(sendRanks.begin(), sendRanks.end());
271 std::sort(recvRanks.begin(), recvRanks.end());
272
273 bool match = true;
274 if (sendRanks.size() != recvRanks.size()) {
275 match = false;
276 } else {
277 for (int i = 0; i < sendRanks.size(); i++) {
278 if (recvRanks[i] != sendRanks[i])
279 match = false;
280 break;
281 }
282 }
283 TEUCHOS_TEST_FOR_EXCEPTION(!match, MueLu::Exceptions::RuntimeError, "AMGX requires that the processors that we send to and receive from are the same. "
284 "This is not the case: we send to {" << sendRanks << "} and receive from {" << recvRanks << "}");
285
286 int num_neighbors = sendRanks.size(); // does not include the calling process
287 const int* neighbors = &sendRanks[0];
288
289 // Later on, we'll have to organize the send and recv data by PIDs,
290 // i.e, a vector V of vectors, where V[i] is PID i's vector of data.
291 // Hence we need to be able to quickly look up an array index
292 // associated with each PID.
293 Tpetra::Details::HashTable<int,int> hashTable(3*num_neighbors);
294 for (int i = 0; i < num_neighbors; i++)
295 hashTable.add(neighbors[i], i);
296
297 // Get some information out
298 ArrayView<const int> exportLIDs = importer->getExportLIDs();
299 ArrayView<const int> exportPIDs = importer->getExportPIDs();
300 Array<int> importPIDs;
301 Tpetra::Import_Util::getPids(*importer, importPIDs, true/* make local -1 */);
302
303 // Construct the reordering for AMGX as in AMGX_matrix_upload_all documentation
304 RCP<const Map> rowMap = inA->getRowMap();
305 RCP<const Map> colMap = inA->getColMap();
306
307 int N = rowMap->getLocalNumElements(), Nc = colMap->getLocalNumElements();
308 muelu2amgx_.resize(Nc, -1);
309
310 int numUniqExports = 0;
311 for (int i = 0; i < exportLIDs.size(); i++)
312 if (muelu2amgx_[exportLIDs[i]] == -1) {
313 numUniqExports++;
314 muelu2amgx_[exportLIDs[i]] = -2;
315 }
316
317 int localOffset = 0, exportOffset = N - numUniqExports;
318 // Go through exported LIDs and put them at the end of LIDs
319 for (int i = 0; i < exportLIDs.size(); i++)
320 if (muelu2amgx_[exportLIDs[i]] < 0) // exportLIDs are not unique
321 muelu2amgx_[exportLIDs[i]] = exportOffset++;
322 // Go through all non-export LIDs, and put them at the beginning of LIDs
323 for (int i = 0; i < N; i++)
324 if (muelu2amgx_[i] == -1)
325 muelu2amgx_[i] = localOffset++;
326 // Go through the tail (imported LIDs), and order those by neighbors
327 int importOffset = N;
328 for (int k = 0; k < num_neighbors; k++)
329 for (int i = 0; i < importPIDs.size(); i++)
330 if (importPIDs[i] != -1 && hashTable.get(importPIDs[i]) == k)
331 muelu2amgx_[i] = importOffset++;
332
333 amgx2muelu.resize(muelu2amgx_.size());
334 for (int i = 0; i < (int)muelu2amgx_.size(); i++)
335 amgx2muelu[muelu2amgx_[i]] = i;
336
337 // Construct send arrays
338 std::vector<std::vector<int> > sendDatas (num_neighbors);
339 std::vector<int> send_sizes(num_neighbors, 0);
340 for (int i = 0; i < exportPIDs.size(); i++) {
341 int index = hashTable.get(exportPIDs[i]);
342 sendDatas [index].push_back(muelu2amgx_[exportLIDs[i]]);
343 send_sizes[index]++;
344 }
345 // FIXME: sendDatas must be sorted (based on GIDs)
346
347 std::vector<const int*> send_maps(num_neighbors);
348 for (int i = 0; i < num_neighbors; i++)
349 send_maps[i] = &(sendDatas[i][0]);
350
351 // Debugging
352 // printMaps(comm, sendDatas, amgx2muelu, neighbors, *importer->getTargetMap(), "send_map_vector");
353
354 // Construct recv arrays
355 std::vector<std::vector<int> > recvDatas (num_neighbors);
356 std::vector<int> recv_sizes(num_neighbors, 0);
357 for (int i = 0; i < importPIDs.size(); i++)
358 if (importPIDs[i] != -1) {
359 int index = hashTable.get(importPIDs[i]);
360 recvDatas [index].push_back(muelu2amgx_[i]);
361 recv_sizes[index]++;
362 }
363 // FIXME: recvDatas must be sorted (based on GIDs)
364
365 std::vector<const int*> recv_maps(num_neighbors);
366 for (int i = 0; i < num_neighbors; i++)
367 recv_maps[i] = &(recvDatas[i][0]);
368
369 // Debugging
370 // printMaps(comm, recvDatas, amgx2muelu, neighbors, *importer->getTargetMap(), "recv_map_vector");
371
372 AMGX_SAFE_CALL(AMGX_matrix_comm_from_maps_one_ring(A_, 1, num_neighbors, neighbors, &send_sizes[0], &send_maps[0], &recv_sizes[0], &recv_maps[0]));
373
374 AMGX_vector_bind(X_, A_);
375 AMGX_vector_bind(Y_, A_);
376 }
377
378 RCP<Teuchos::Time> matrixTransformTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transform matrix");
379 matrixTransformTimer->start();
380
381 ArrayRCP<const size_t> ia_s;
382 ArrayRCP<const int> ja;
383 ArrayRCP<const double> a;
384 inA->getAllValues(ia_s, ja, a);
385
386 ArrayRCP<int> ia(ia_s.size());
387 for (int i = 0; i < ia.size(); i++)
388 ia[i] = Teuchos::as<int>(ia_s[i]);
389
390 N_ = inA->getLocalNumRows();
391 int nnz = inA->getLocalNumEntries();
392
393 matrixTransformTimer->stop();
394 matrixTransformTimer->incrementNumCalls();
395
396
397 // Upload matrix
398 // TODO Do we need to pin memory here through AMGX_pin_memory?
399 RCP<Teuchos::Time> matrixTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer matrix CPU->GPU");
400 matrixTimer->start();
401 if (numProcs == 1) {
402 AMGX_matrix_upload_all(A_, N_, nnz, 1, 1, &ia[0], &ja[0], &a[0], NULL);
403
404 } else {
405 // Transform the matrix
406 std::vector<int> ia_new(ia.size());
407 std::vector<int> ja_new(ja.size());
408 std::vector<double> a_new (a.size());
409
410 ia_new[0] = 0;
411 for (int i = 0; i < N_; i++) {
412 int oldRow = amgx2muelu[i];
413
414 ia_new[i+1] = ia_new[i] + (ia[oldRow+1] - ia[oldRow]);
415
416 for (int j = ia[oldRow]; j < ia[oldRow+1]; j++) {
417 int offset = j - ia[oldRow];
418 ja_new[ia_new[i] + offset] = muelu2amgx_[ja[j]];
419 a_new [ia_new[i] + offset] = a[j];
420 }
421 // Do bubble sort on two arrays
422 // NOTE: There are multiple possible optimizations here (even of bubble sort)
423 bool swapped;
424 do {
425 swapped = false;
426
427 for (int j = ia_new[i]; j < ia_new[i+1]-1; j++)
428 if (ja_new[j] > ja_new[j+1]) {
429 std::swap(ja_new[j], ja_new[j+1]);
430 std::swap(a_new [j], a_new [j+1]);
431 swapped = true;
432 }
433 } while (swapped == true);
434 }
435
436 AMGX_matrix_upload_all(A_, N_, nnz, 1, 1, &ia_new[0], &ja_new[0], &a_new[0], NULL);
437 }
438 matrixTimer->stop();
439 matrixTimer->incrementNumCalls();
440
441 domainMap_ = inA->getDomainMap();
442 rangeMap_ = inA->getRangeMap();
443
444 RCP<Teuchos::Time> realSetupTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: setup (total)");
445 realSetupTimer->start();
446 AMGX_solver_setup(Solver_, A_);
447 realSetupTimer->stop();
448 realSetupTimer->incrementNumCalls();
449
450 vectorTimer1_ = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer vectors CPU->GPU");
451 vectorTimer2_ = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer vector GPU->CPU");
452 solverTimer_ = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: Solve (total)");
453 }
454
457 {
458 // Comment this out if you need rebuild to work. This causes AMGX_solver_destroy memory issues.
459 AMGX_SAFE_CALL(AMGX_solver_destroy(Solver_));
460 AMGX_SAFE_CALL(AMGX_vector_destroy(X_));
461 AMGX_SAFE_CALL(AMGX_vector_destroy(Y_));
462 AMGX_SAFE_CALL(AMGX_matrix_destroy(A_));
463 AMGX_SAFE_CALL(AMGX_resources_destroy(Resources_));
464 AMGX_SAFE_CALL(AMGX_config_destroy(Config_));
465 }
466
467
469 Teuchos::RCP<const Map> getDomainMap() const;
470
472 Teuchos::RCP<const Map> getRangeMap() const;
473
475
479 void apply(const MultiVector& X, MultiVector& Y, Teuchos::ETransp mode = Teuchos::NO_TRANS,
480 SC alpha = Teuchos::ScalarTraits<SC>::one(), SC beta = Teuchos::ScalarTraits<SC>::zero()) const;
481
483 bool hasTransposeApply() const;
484
485 RCP<MueLu::Hierarchy<SC,LO,GO,NO> > GetHierarchy() const {
486 throw Exceptions::RuntimeError("AMGXOperator does not hold a MueLu::Hierarchy object \n");
487 }
488
489 std::string filterValueToString(const Teuchos::ParameterEntry& entry ) {
490 return ( entry.isList() ? std::string("...") : toString(entry.getAny()) );
491 }
492
493 int sizeA() {
494 int sizeX, sizeY, n;
495 AMGX_matrix_get_size(A_, &n, &sizeX, &sizeY);
496 return n;
497 }
498
499 int iters() {
500 int it;
501 AMGX_solver_get_iterations_number(Solver_, &it);
502 return it;
503 }
504
505 AMGX_SOLVE_STATUS getStatus() {
506 AMGX_SOLVE_STATUS status;
507 AMGX_solver_get_status(Solver_, &status);
508 return status;
509 }
510
511
512 private:
513 AMGX_solver_handle Solver_;
514 AMGX_resources_handle Resources_;
515 AMGX_config_handle Config_;
516 AMGX_matrix_handle A_;
517 AMGX_vector_handle X_;
518 AMGX_vector_handle Y_;
519 int N_;
520
521 RCP<const Map> domainMap_;
522 RCP<const Map> rangeMap_;
523
524 std::vector<int> muelu2amgx_;
525
526 RCP<Teuchos::Time> vectorTimer1_;
527 RCP<Teuchos::Time> vectorTimer2_;
528 RCP<Teuchos::Time> solverTimer_;
529 };
530
531} // namespace
532
533#endif //HAVE_MUELU_AMGX
534#endif // MUELU_AMGXOPERATOR_DECL_HPP
MueLu::DefaultLocalOrdinal LocalOrdinal
MueLu::DefaultScalar Scalar
MueLu::DefaultGlobalOrdinal GlobalOrdinal
MueLu::DefaultNode Node
void printMaps(Teuchos::RCP< const Teuchos::Comm< int > > &comm, const std::vector< std::vector< int > > &vec, const std::vector< int > &perm, const int *nbrs, const Map &map, const std::string &label)
AMGXOperator(const Teuchos::RCP< Tpetra::CrsMatrix< SC, LO, GO, NO > > &inA, Teuchos::ParameterList &paramListIn)
std::string filterValueToString(const Teuchos::ParameterEntry &entry)
RCP< MueLu::Hierarchy< SC, LO, GO, NO > > GetHierarchy() const
Tpetra::MultiVector< SC, LO, GO, NO > MultiVector
Adapter for AmgX library from Nvidia.
virtual ~AMGXOperator()
Destructor.
void apply(const MultiVector &X, MultiVector &Y, Teuchos::ETransp mode=Teuchos::NO_TRANS, Scalar alpha=Teuchos::ScalarTraits< Scalar >::one(), Scalar beta=Teuchos::ScalarTraits< Scalar >::zero()) const
Returns a solution for the linear system AX=Y in the Tpetra::MultiVector X.
Teuchos::RCP< const Map > getRangeMap() const
Returns the Tpetra::Map object associated with the range of this operator.
AMGXOperator(const Teuchos::RCP< Tpetra::CrsMatrix< SC, LO, GO, NO > > &InA, Teuchos::ParameterList &paramListIn)
Constructor.
Tpetra::MultiVector< SC, LO, GO, NO > MultiVector
RCP< MueLu::Hierarchy< SC, LO, GO, NO > > GetHierarchy() const
Tpetra::Map< LO, GO, NO > Map
bool hasTransposeApply() const
Indicates whether this operator supports applying the adjoint operator.
Teuchos::RCP< const Map > getDomainMap() const
Returns the Tpetra::Map object associated with the domain of this operator.
Base class for MueLu classes.
Exception throws to report errors in the internal logical of the program.
Wraps an existing MueLu::Hierarchy as a Tpetra::Operator.
Namespace for MueLu classes and methods.
std::string toString(const T &what)
Little helper function to convert non-string types to strings.