Kokkos Core Kernels Package Version of the Day
Loading...
Searching...
No Matches
Kokkos_Atomic.hpp
Go to the documentation of this file.
1//@HEADER
2// ************************************************************************
3//
4// Kokkos v. 4.0
5// Copyright (2022) National Technology & Engineering
6// Solutions of Sandia, LLC (NTESS).
7//
8// Under the terms of Contract DE-NA0003525 with NTESS,
9// the U.S. Government retains certain rights in this software.
10//
11// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12// See https://kokkos.org/LICENSE for license information.
13// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14//
15//@HEADER
16
39
40#ifndef KOKKOS_ATOMIC_HPP
41#define KOKKOS_ATOMIC_HPP
42#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
43#define KOKKOS_IMPL_PUBLIC_INCLUDE
44#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC
45#endif
46
47#include <Kokkos_Macros.hpp>
48
49#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS
50#include <Kokkos_Atomics_Desul_Wrapper.hpp>
51#include <Kokkos_Atomics_Desul_Volatile_Wrapper.hpp>
52#include <impl/Kokkos_Utilities.hpp>
53
54// Helper functions for places where we really should have called SeqCst atomics
55// anyway These can go away when we call desul unconditionally Non-Desul
56// versions are below
57namespace Kokkos {
58namespace Impl {
59using desul::MemoryOrderSeqCst;
60using desul::MemoryScopeDevice;
61
62template <class T>
63KOKKOS_INLINE_FUNCTION void desul_atomic_dec(T* dest, MemoryOrderSeqCst,
64 MemoryScopeDevice) {
65 return desul::atomic_dec(const_cast<T*>(dest), desul::MemoryOrderSeqCst(),
66 desul::MemoryScopeDevice());
67}
68
69template <class T>
70KOKKOS_INLINE_FUNCTION void desul_atomic_inc(T* dest, MemoryOrderSeqCst,
71 MemoryScopeDevice) {
72 return desul::atomic_inc(const_cast<T*>(dest), desul::MemoryOrderSeqCst(),
73 desul::MemoryScopeDevice());
74}
75
76template <class T>
77KOKKOS_INLINE_FUNCTION T
78desul_atomic_exchange(T* dest, const Kokkos::Impl::type_identity_t<T> val,
79 MemoryOrderSeqCst, MemoryScopeDevice) {
80 return desul::atomic_exchange(const_cast<T*>(dest), val,
81 desul::MemoryOrderSeqCst(),
82 desul::MemoryScopeDevice());
83}
84
85template <class T>
86KOKKOS_INLINE_FUNCTION T desul_atomic_compare_exchange(
87 T* dest, Kokkos::Impl::type_identity_t<const T> compare,
88 Kokkos::Impl::type_identity_t<const T> val, MemoryOrderSeqCst,
89 MemoryScopeDevice) {
90 return desul::atomic_compare_exchange(dest, compare, val,
91 desul::MemoryOrderSeqCst(),
92 desul::MemoryScopeDevice());
93}
94
95} // namespace Impl
96} // namespace Kokkos
97#else
98
99#include <Kokkos_HostSpace.hpp>
100#include <impl/Kokkos_Traits.hpp>
101
102//----------------------------------------------------------------------------
103
104// Need to fix this for pure clang on windows
105#if defined(_WIN32)
106#define KOKKOS_ENABLE_WINDOWS_ATOMICS
107
108#if defined(KOKKOS_ENABLE_CUDA)
109#define KOKKOS_ENABLE_CUDA_ATOMICS
110#if defined(KOKKOS_COMPILER_CLANG)
111#define KOKKOS_ENABLE_GNU_ATOMICS
112#endif
113#endif
114
115#else // _WIN32
116#if defined(KOKKOS_ENABLE_CUDA)
117
118// Compiling NVIDIA device code, must use Cuda atomics:
119
120#define KOKKOS_ENABLE_CUDA_ATOMICS
121
122#elif defined(KOKKOS_ENABLE_HIP)
123
124#define KOKKOS_ENABLE_HIP_ATOMICS
125
126#endif
127
128#if !defined(KOKKOS_ENABLE_GNU_ATOMICS) && \
129 !defined(KOKKOS_ENABLE_INTEL_ATOMICS) && \
130 !defined(KOKKOS_ENABLE_OPENMP_ATOMICS) && \
131 !defined(KOKKOS_ENABLE_STD_ATOMICS) && \
132 !defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
133
134// Compiling for non-Cuda atomic implementation has not been pre-selected.
135// Choose the best implementation for the detected compiler.
136// Preference: GCC, INTEL, OMP31
137
138#if defined(KOKKOS_INTERNAL_NOT_PARALLEL)
139
140#define KOKKOS_ENABLE_SERIAL_ATOMICS
141
142#elif defined(KOKKOS_COMPILER_GNU) || defined(KOKKOS_COMPILER_CLANG) || \
143 defined(KOKKOS_COMPILER_NVCC)
144
145#define KOKKOS_ENABLE_GNU_ATOMICS
146
147#elif defined(KOKKOS_COMPILER_INTEL) || defined(KOKKOS_COMPILER_CRAYC)
148
149#define KOKKOS_ENABLE_INTEL_ATOMICS
150
151#elif defined(_OPENMP) && (201107 <= _OPENMP)
152
153#define KOKKOS_ENABLE_OPENMP_ATOMICS
154
155#else
156
157#error "KOKKOS_ATOMICS_USE : Unsupported compiler"
158
159#endif
160
161#endif /* Not pre-selected atomic implementation */
162#endif
163
164#ifdef KOKKOS_ENABLE_CUDA
165#include <Cuda/Kokkos_Cuda_Locks.hpp>
166#endif
167
168namespace Kokkos {
169template <typename T>
170KOKKOS_INLINE_FUNCTION void atomic_add(volatile T* const dest, const T src);
171
172// Atomic increment
173template <typename T>
174KOKKOS_INLINE_FUNCTION void atomic_increment(volatile T* a);
175
176template <typename T>
177KOKKOS_INLINE_FUNCTION void atomic_decrement(volatile T* a);
178} // namespace Kokkos
179
180namespace Kokkos {
181
182inline const char* atomic_query_version() {
183#if defined(KOKKOS_ENABLE_CUDA_ATOMICS)
184 return "KOKKOS_ENABLE_CUDA_ATOMICS";
185#elif defined(KOKKOS_ENABLE_GNU_ATOMICS)
186 return "KOKKOS_ENABLE_GNU_ATOMICS";
187#elif defined(KOKKOS_ENABLE_INTEL_ATOMICS)
188 return "KOKKOS_ENABLE_INTEL_ATOMICS";
189#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS)
190 return "KOKKOS_ENABLE_OPENMP_ATOMICS";
191#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS)
192 return "KOKKOS_ENABLE_WINDOWS_ATOMICS";
193#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
194 return "KOKKOS_ENABLE_SERIAL_ATOMICS";
195#else
196#error "No valid response for atomic_query_version!"
197#endif
198}
199
200} // namespace Kokkos
201
202//----------------------------------------------------------------------------
203// Atomic Memory Orders
204//
205// Implements Strongly-typed analogs of C++ standard memory orders
206#include "impl/Kokkos_Atomic_Memory_Order.hpp"
207
208#if defined(KOKKOS_ENABLE_HIP)
209#include <HIP/Kokkos_HIP_Atomic.hpp>
210#endif
211
212#if defined(KOKKOS_ENABLE_WINDOWS_ATOMICS)
213#include "impl/Kokkos_Atomic_Windows.hpp"
214#endif
215//----------------------------------------------------------------------------
216// Atomic Assembly
217//
218// Implements CAS128-bit in assembly
219
220#include "impl/Kokkos_Atomic_Assembly.hpp"
221
222//----------------------------------------------------------------------------
223// Memory fence
224//
225// All loads and stores from this thread will be globally consistent before
226// continuing
227//
228// void memory_fence() {...};
229#include "impl/Kokkos_Memory_Fence.hpp"
230
231//----------------------------------------------------------------------------
232// Atomic exchange
233//
234// template< typename T >
235// T atomic_exchange( volatile T* const dest , const T val )
236// { T tmp = *dest ; *dest = val ; return tmp ; }
237
238#include "impl/Kokkos_Atomic_Exchange.hpp"
239
240//----------------------------------------------------------------------------
241// Atomic compare-and-exchange
242//
243// template<class T>
244// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare,
245// const T val) { bool equal = compare == *dest ; if ( equal ) { *dest = val ; }
246// return equal ; }
247
248#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp"
249
250#include "impl/Kokkos_Atomic_Generic.hpp"
251
252//----------------------------------------------------------------------------
253// Atomic fetch and add
254//
255// template<class T>
256// T atomic_fetch_add(volatile T* const dest, const T val)
257// { T tmp = *dest ; *dest += val ; return tmp ; }
258
259#include "impl/Kokkos_Atomic_Fetch_Add.hpp"
260
261//----------------------------------------------------------------------------
262// Atomic increment
263//
264// template<class T>
265// T atomic_increment(volatile T* const dest)
266// { dest++; }
267
268#include "impl/Kokkos_Atomic_Increment.hpp"
269
270//----------------------------------------------------------------------------
271// Atomic Decrement
272//
273// template<class T>
274// T atomic_decrement(volatile T* const dest)
275// { dest--; }
276
277#include "impl/Kokkos_Atomic_Decrement.hpp"
278
279//----------------------------------------------------------------------------
280// Atomic fetch and sub
281//
282// template<class T>
283// T atomic_fetch_sub(volatile T* const dest, const T val)
284// { T tmp = *dest ; *dest -= val ; return tmp ; }
285
286#include "impl/Kokkos_Atomic_Fetch_Sub.hpp"
287
288//----------------------------------------------------------------------------
289// Atomic fetch and or
290//
291// template<class T>
292// T atomic_fetch_or(volatile T* const dest, const T val)
293// { T tmp = *dest ; *dest = tmp | val ; return tmp ; }
294
295#include "impl/Kokkos_Atomic_Fetch_Or.hpp"
296
297//----------------------------------------------------------------------------
298// Atomic fetch and and
299//
300// template<class T>
301// T atomic_fetch_and(volatile T* const dest, const T val)
302// { T tmp = *dest ; *dest = tmp & val ; return tmp ; }
303
304#include "impl/Kokkos_Atomic_Fetch_And.hpp"
305
306//----------------------------------------------------------------------------
307// Atomic MinMax
308//
309// template<class T>
310// T atomic_min(volatile T* const dest, const T val)
311// { T tmp = *dest ; *dest = min(*dest, val); return tmp ; }
312// template<class T>
313// T atomic_max(volatile T* const dest, const T val)
314// { T tmp = *dest ; *dest = max(*dest, val); return tmp ; }
315
316#include "impl/Kokkos_Atomic_MinMax.hpp"
317
318//----------------------------------------------------------------------------
319// Provide volatile_load and safe_load
320//
321// T volatile_load(T const volatile * const ptr);
322//
323// T const& safe_load(T const * const ptr);
324// XEON PHI
325// T safe_load(T const * const ptr
326
327#include "impl/Kokkos_Volatile_Load.hpp"
328
329//----------------------------------------------------------------------------
330// Provide atomic loads and stores with memory order semantics
331
332#include "impl/Kokkos_Atomic_Load.hpp"
333#include "impl/Kokkos_Atomic_Store.hpp"
334
335// Generic functions using the above defined functions
336#include "impl/Kokkos_Atomic_Generic_Secondary.hpp"
337//----------------------------------------------------------------------------
338// This atomic-style macro should be an inlined function, not a macro
339
340#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
341 !defined(__CUDA_ARCH__)
342
343#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
344#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
345
346#else
347
348#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
349#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
350
351#endif
352
353//----------------------------------------------------------------------------
354
355// Helper functions for places where we really should have called SeqCst atomics
356// anyway These can go away when we call desul unconditionally
357namespace Kokkos {
358namespace Impl {
359struct MemoryOrderSeqCst {};
360struct MemoryScopeDevice {};
361
362template <class T>
363KOKKOS_INLINE_FUNCTION void desul_atomic_dec(T* dest, MemoryOrderSeqCst,
364 MemoryScopeDevice) {
365 return Kokkos::atomic_decrement(dest);
366}
367
368template <class T>
369KOKKOS_INLINE_FUNCTION void desul_atomic_inc(T* dest, MemoryOrderSeqCst,
370 MemoryScopeDevice) {
371 return Kokkos::atomic_increment(dest);
372}
373
374template <class T>
375KOKKOS_INLINE_FUNCTION T
376desul_atomic_exchange(T* dest, Kokkos::Impl::type_identity_t<const T> val,
377 MemoryOrderSeqCst, MemoryScopeDevice) {
378 return Kokkos::atomic_exchange(dest, val);
379}
380
381template <class T>
382KOKKOS_INLINE_FUNCTION T desul_atomic_compare_exchange(
383 T* dest, Kokkos::Impl::type_identity_t<const T> compare,
384 Kokkos::Impl::type_identity_t<const T> val, MemoryOrderSeqCst,
385 MemoryScopeDevice) {
386 return Kokkos::atomic_compare_exchange(dest, compare, val);
387}
388
389} // namespace Impl
390} // namespace Kokkos
391
392#endif /* !KOKKOS_ENABLE_IMPL_DESUL_ATOMICS */
393#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC
394#undef KOKKOS_IMPL_PUBLIC_INCLUDE
395#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC
396#endif
397#endif /* KOKKOS_ATOMIC_HPP */