Teuchos - Trilinos Tools Package Version of the Day
Loading...
Searching...
No Matches
Teuchos_DefaultComm.hpp
1// @HEADER
2// ***********************************************************************
3//
4// Teuchos: Common Tools Package
5// Copyright (2004) Sandia Corporation
6//
7// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8// license for use of this work by or on behalf of the U.S. Government.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38//
39// ***********************************************************************
40// @HEADER
41
42#ifndef TEUCHOS_DEFAULT_COMM_HPP
43#define TEUCHOS_DEFAULT_COMM_HPP
44
45#include "Teuchos_RCP.hpp"
46#include "Teuchos_DefaultSerialComm.hpp"
47#ifdef HAVE_MPI
49#endif
50
51namespace Teuchos {
52
53#ifdef HAVE_MPI
54namespace Details {
55
56template<class OrdinalType>
57int
58mpiFreeDefaultComm (MPI_Comm, int, void*, void*);
59
60template<class OrdinalType>
61int
62mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*);
63
64} // namespace Details
65#endif // HAVE_MPI
66
93template<typename OrdinalType>
95public:
109
118
119private:
125 static const Comm<OrdinalType>* comm_;
126
128 static const Comm<OrdinalType>* defaultSerialComm_;
129
130#ifdef HAVE_MPI
132 template<class OT>
133 friend int
134 Details::mpiFreeDefaultComm (MPI_Comm, int, void*, void*);
135
137 template<class OT>
138 friend int
139 Details::mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*);
140#endif // HAVE_MPI
141
143 static void freeDefaultComm () {
144 if (comm_ != NULL) {
145 delete comm_;
146 comm_ = NULL;
147 }
148 }
149
151 static void freeDefaultSerialComm () {
152 if (defaultSerialComm_ != NULL) {
153 delete defaultSerialComm_;
154 defaultSerialComm_ = NULL;
155 }
156 }
157};
158
159#ifdef HAVE_MPI
160namespace Details {
161
162template<class OrdinalType>
163int
164mpiFreeDefaultComm (MPI_Comm, int, void*, void*)
165{
166 try {
168 } catch (...) {
169 // Destructors must not throw exceptions, so we must accept the
170 // possible memory leak and move on.
171 std::cerr << "Teuchos::DefaultComm: Failed to free default Comm! We can't "
172 "throw an exception here because this is a singleton destructor that "
173 "should only be called at MPI_Finalize or (if not building with MPI) at "
174 "exit from main()." << std::endl;
175 // FIXME (mfh 16 Nov 2014) There might be some way to create a
176 // custom return code with MPI error reporting. For now, we just
177 // pick some error code not equal to MPI_SUCCESS. It could
178 // perhaps overlap with some existing error code.
179 return (MPI_SUCCESS == 0) ? -1 : 0;
180 }
181 return MPI_SUCCESS;
182}
183
184template<class OrdinalType>
185int
186mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*)
187{
188 try {
190 } catch (...) {
191 // Destructors must not throw exceptions, so we must accept the
192 // possible memory leak and move on.
193 std::cerr << "Teuchos::DefaultComm: Failed to free default serial Comm! "
194 "We can't throw an exception here because this is a singleton destructor "
195 "that should only be called at MPI_Finalize or (if not building with MPI)"
196 " at exit from main()." << std::endl;
197 // FIXME (mfh 16 Nov 2014) There might be some way to create a
198 // custom return code with MPI error reporting. For now, we just
199 // pick some error code not equal to MPI_SUCCESS. It could
200 // perhaps overlap with some existing error code.
201 return (MPI_SUCCESS == 0) ? -1 : 0;
202 }
203 return MPI_SUCCESS;
204}
205
206} // namespace Details
207#endif // HAVE_MPI
208
209
210template<typename OrdinalType>
213{
214 if (comm_ == NULL) {
215#ifdef HAVE_MPI
216# if MPI_VERSION >= 2
217
218 comm_ = new MpiComm<OrdinalType> (MPI_COMM_WORLD);
219
220 // We want comm_ to be deallocated when MPI_Finalize is called.
221 // The standard idiom for this (look in the MPI standard) is to
222 // register an attribute ((key,value) pair) with MPI_COMM_SELF,
223 // with a custom "destructor" to be called at MPI_Finalize.
224
225 // 'key' is an output argument of MPI_Comm_create_keyval.
226 int key = MPI_KEYVAL_INVALID;
227 int err =
228 MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN,
229 Details::mpiFreeDefaultComm<OrdinalType>,
230 &key,
231 NULL); // no extra state
232 if (err != MPI_SUCCESS) {
233 if (comm_ != NULL) { // clean up if MPI call fails
234 delete comm_;
235 comm_ = NULL;
236 }
237 TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
238 "Teuchos::DefaultComm::getComm: MPI_Comm_create_keyval failed!");
239 }
240 int val = key; // doesn't matter
241
242 // Attach the attribute to MPI_COMM_SELF.
243 err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
244 if (err != MPI_SUCCESS) {
245 // MPI (versions up to and including 3.0) doesn't promise
246 // correct behavior after any function returns something other
247 // than MPI_SUCCESS. Thus, it's not required to try to free the
248 // new key via MPI_Comm_free_keyval. Furthermore, if something
249 // went wrong with MPI_Comm_set_attr, it's likely that the
250 // attribute mechanism is broken. Thus, it would be unwise to
251 // call MPI_Comm_free_keyval. However, we can still clean up
252 // other data.
253 if (comm_ != NULL) { // clean up if MPI call fails
254 delete comm_;
255 comm_ = NULL;
256 }
257 TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
258 "Teuchos::DefaultComm::getComm: MPI_Comm_set_attr failed!");
259 }
260
261 // It looks weird to "free" the key right away. However, this
262 // does not actually cause the "destructor" to be called. It only
263 // gets called at MPI_FINALIZE. See MPI 3.0 standard, Section
264 // 6.7.2, MPI_COMM_FREE_KEYVAL:
265 //
266 // "Note that it is not erroneous to free an attribute key that is
267 // in use, because the actual free does not transpire until after
268 // all references (in other communicators on the process) to the
269 // key have been freed. These references need to be explicitly
270 // freed by the program, either via calls to MPI_COMM_DELETE_ATTR
271 // that free one attribute instance, or by calls to MPI_COMM_FREE
272 // that free all attribute instances associated with the freed
273 // communicator."
274 //
275 // We rely here on the latter mechanism. MPI_FINALIZE calls
276 // MPI_COMM_FREE on MPI_COMM_SELF, so we do not need to call it
277 // explicitly.
278 //
279 // It's not clear what to do if the MPI_* calls above succeeded,
280 // but this call fails (i.e., returns != MPI_SUCCESS). We could
281 // throw; this would make sense to do, because MPI (versions up to
282 // and including 3.0) doesn't promise correct behavior after any
283 // MPI function returns something other than MPI_SUCCESS. We
284 // could also be optimistic and just ignore the return value,
285 // hoping that if the above calls succeeded, then the communicator
286 // will get freed at MPI_FINALIZE, even though the unfreed key may
287 // leak memory (see Bug 6338). I've chosen the latter.
288 (void) MPI_Comm_free_keyval (&key);
289
290# else // MPI_VERSION < 2
291# error "Sorry, you need an MPI implementation that supports at least MPI 2.0 in order to build this code. MPI 2.0 came out in 1997. I wrote this comment in 2017. If you really _really_ want MPI 1.x support, please file a GitHub issue for this feature request at github.com/trilinos/trilinos/issues with an expression of its priority and we will get to it as soon as we can."
292# endif // MPI_VERSION >= 2
293
294#else // NOT HAVE_MPI
295 comm_ = new SerialComm<OrdinalType> ();
296 // We want comm_ to be deallocated when main exits, so register
297 // its deallocation function as an atexit handler.
298 //
299 // The POSIX standard allows atexit to fail, in particular if it
300 // lacks space for registering more functions. "[T]he application
301 // should call sysconf() to obtain the value of {ATEXIT_MAX}, the
302 // [maximum] number of functions that can be registered. There is
303 // no way for an application to tell how many functions have
304 // already been registered with atexit()."
305 //
306 // We don't do this here. Instead, we just check atexit's return
307 // code. If it fails, we throw.
308 int err = atexit (freeDefaultComm);
309 if (err != 0) {
310 if (comm_ != NULL) { // clean up if atexit fails
311 delete comm_;
312 comm_ = NULL;
313 }
314 TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
315 "Teuchos::DefaultComm::getComm: atexit failed!");
316 }
317#endif // HAVE_MPI
318 }
319
321 (comm_ == NULL, std::logic_error, "Teuchos::DefaultComm::getComm: "
322 "comm_ == NULL before return. This should never happen. "
323 "Please report this bug to the Teuchos developers.");
324
325 // Return a nonowning RCP, because we need to ensure that
326 // destruction happens at MPI_Finalize (or at exit of main(), if not
327 // building with MPI).
328 return rcp (comm_, false);
329}
330
331template<typename OrdinalType>
335{
336 if (! comm.is_null ()) {
337 return comm;
338 } else {
339 if (defaultSerialComm_ == NULL) {
340#ifdef HAVE_MPI
341# if MPI_VERSION >= 2
342 //defaultSerialComm_ = new MpiComm<OrdinalType> (MPI_COMM_SELF);
343 defaultSerialComm_ = new SerialComm<OrdinalType> ();
344
345 // Register an MPI_Finalize hook to free defaultSerialComm_.
346 // (See getComm implementation above in this file for details.)
347
348 int key = MPI_KEYVAL_INVALID;
349 int err =
350 MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN,
351 Details::mpiFreeDefaultSerialComm<OrdinalType>,
352 &key,
353 NULL); // no extra state
354 if (err != MPI_SUCCESS) {
355 if (defaultSerialComm_ != NULL) { // clean up if MPI call fails
356 delete defaultSerialComm_;
357 defaultSerialComm_ = NULL;
358 }
360 true, std::runtime_error, "Teuchos::DefaultComm::getDefaultSerialComm"
361 ": MPI_Comm_create_keyval failed!");
362 }
363 int val = key; // doesn't matter
364
365 // Attach the attribute to MPI_COMM_SELF.
366 err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
367 if (err != MPI_SUCCESS) {
368 // See comments in getComm implementation above to see why we
369 // don't call MPI_Comm_free_keyval here.
370 if (defaultSerialComm_ != NULL) { // clean up if MPI call fails
371 delete defaultSerialComm_;
372 defaultSerialComm_ = NULL;
373 }
375 true, std::runtime_error, "Teuchos::DefaultComm::getDefaultSerialComm"
376 ": MPI_Comm_set_attr failed!");
377 }
378
379 // See comments in getComm implementation above to see why we
380 // _do_ call MPI_Comm_free_keyval here, and why we don't check
381 // the return code.
382 (void) MPI_Comm_free_keyval (&key);
383
384# else // MPI_VERSION < 2
385# error "Sorry, you need an MPI implementation that supports at least MPI 2.0 in order to build this code. MPI 2.0 came out in 1997. I wrote this comment in 2017. If you really _really_ want MPI 1.x support, please file a GitHub issue for this feature request at github.com/trilinos/trilinos/issues with an expression of its priority and we will get to it as soon as we can."
386# endif // MPI_VERSION >= 2
387
388#else // NOT HAVE_MPI
389 defaultSerialComm_ = new SerialComm<OrdinalType> ();
390 // We want defaultSerialComm_ to be deallocated when main exits,
391 // so register its deallocation function as an atexit handler.
392 //
393 // The POSIX standard allows atexit to fail, in particular if it
394 // lacks space for registering more functions. "[T]he
395 // application should call sysconf() to obtain the value of
396 // {ATEXIT_MAX}, the [maximum] number of functions that can be
397 // registered. There is no way for an application to tell how
398 // many functions have already been registered with atexit()."
399 //
400 // We don't do this here. Instead, we just check atexit's
401 // return code. If it fails, we throw.
402 int err = atexit (freeDefaultComm);
403 if (err != 0) {
404 if (defaultSerialComm_ != NULL) { // clean up if atexit fails
405 delete defaultSerialComm_;
406 defaultSerialComm_ = NULL;
407 }
408 TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
409 "Teuchos::DefaultComm::getDefaultSerialComm: atexit failed!");
410 }
411#endif // HAVE_MPI
412 }
413
415 (defaultSerialComm_ == NULL, std::logic_error, "Teuchos::DefaultComm::"
416 "getDefaultSerialComm: defaultSerialComm_ == NULL before return. This sh"
417 "ould never happen. Please report this bug to the Teuchos developers.");
418
419 // Return a nonowning RCP, because we need to ensure that
420 // destruction happens at MPI_Finalize (or at exit of main(), if not
421 // building with MPI).
422 return rcp (defaultSerialComm_, false);
423 }
424}
425
426template<typename OrdinalType>
429
430template<typename OrdinalType>
433
434} // namespace Teuchos
435
436#endif // TEUCHOS_DEFAULT_COMM_HPP
Implementation of Teuchos wrappers for MPI.
Reference-counted pointer class and non-member templated function implementations.
Abstract interface for distributed-memory communication.
Return a default global communicator appropriate for the build.
static Teuchos::RCP< const Comm< OrdinalType > > getDefaultSerialComm(const Teuchos::RCP< const Comm< OrdinalType > > &comm)
Return a serial Comm if the input Comm is null.
static Teuchos::RCP< const Comm< OrdinalType > > getComm()
Return the default global communicator.
Smart reference counting pointer class for automatic garbage collection.
Concrete serial communicator subclass.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Namespace of implementation details.
The Teuchos namespace contains all of the classes, structs and enums used by Teuchos,...
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.