Kokkos Core Kernels Package Version of the Day
Loading...
Searching...
No Matches
Kokkos_DynamicView.hpp
1//@HEADER
2// ************************************************************************
3//
4// Kokkos v. 4.0
5// Copyright (2022) National Technology & Engineering
6// Solutions of Sandia, LLC (NTESS).
7//
8// Under the terms of Contract DE-NA0003525 with NTESS,
9// the U.S. Government retains certain rights in this software.
10//
11// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12// See https://kokkos.org/LICENSE for license information.
13// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14//
15//@HEADER
16
17#ifndef KOKKOS_DYNAMIC_VIEW_HPP
18#define KOKKOS_DYNAMIC_VIEW_HPP
19#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
20#define KOKKOS_IMPL_PUBLIC_INCLUDE
21#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DYNAMICVIEW
22#endif
23
24#include <cstdio>
25
26#include <Kokkos_Core.hpp>
27#include <impl/Kokkos_Error.hpp>
28
29namespace Kokkos {
30namespace Experimental {
31
32namespace Impl {
33
37template <typename MemorySpace, typename ValueType>
38struct ChunkedArrayManager {
39 using value_type = ValueType;
40 using pointer_type = ValueType*;
41 using track_type = Kokkos::Impl::SharedAllocationTracker;
42
43 ChunkedArrayManager() = default;
44 ChunkedArrayManager(ChunkedArrayManager const&) = default;
45 ChunkedArrayManager(ChunkedArrayManager&&) = default;
46 ChunkedArrayManager& operator=(ChunkedArrayManager&&) = default;
47 ChunkedArrayManager& operator=(const ChunkedArrayManager&) = default;
48
49 template <typename Space, typename Value>
50 friend struct ChunkedArrayManager;
51
52 template <typename Space, typename Value>
53 inline ChunkedArrayManager(const ChunkedArrayManager<Space, Value>& rhs)
54 : m_valid(rhs.m_valid),
55 m_chunk_max(rhs.m_chunk_max),
56 m_chunks((ValueType**)(rhs.m_chunks)),
57 m_track(rhs.m_track),
58 m_chunk_size(rhs.m_chunk_size) {
59 static_assert(
60 Kokkos::Impl::MemorySpaceAccess<MemorySpace, Space>::assignable,
61 "Incompatible ChunkedArrayManager copy construction");
62 }
63
64 ChunkedArrayManager(const unsigned arg_chunk_max,
65 const unsigned arg_chunk_size)
66 : m_chunk_max(arg_chunk_max), m_chunk_size(arg_chunk_size) {}
67
68 private:
69 struct ACCESSIBLE_TAG {};
70 struct INACCESSIBLE_TAG {};
71
72 ChunkedArrayManager(ACCESSIBLE_TAG, pointer_type* arg_chunks,
73 const unsigned arg_chunk_max)
74 : m_valid(true), m_chunk_max(arg_chunk_max), m_chunks(arg_chunks) {}
75
76 ChunkedArrayManager(INACCESSIBLE_TAG, const unsigned arg_chunk_max,
77 const unsigned arg_chunk_size)
78 : m_chunk_max(arg_chunk_max), m_chunk_size(arg_chunk_size) {}
79
80 public:
81 template <typename Space, typename Enable_ = void>
82 struct IsAccessibleFrom;
83
84 template <typename Space>
85 struct IsAccessibleFrom<
86 Space, typename std::enable_if_t<Kokkos::Impl::MemorySpaceAccess<
87 MemorySpace, Space>::accessible>> : std::true_type {};
88
89 template <typename Space>
90 struct IsAccessibleFrom<
91 Space, typename std::enable_if_t<!Kokkos::Impl::MemorySpaceAccess<
92 MemorySpace, Space>::accessible>> : std::false_type {};
93
94 template <typename Space>
95 static ChunkedArrayManager<Space, ValueType> create_mirror(
96 ChunkedArrayManager<MemorySpace, ValueType> const& other,
97 std::enable_if_t<IsAccessibleFrom<Space>::value>* = nullptr) {
98 return ChunkedArrayManager<Space, ValueType>{
99 ACCESSIBLE_TAG{}, other.m_chunks, other.m_chunk_max};
100 }
101
102 template <typename Space>
103 static ChunkedArrayManager<Space, ValueType> create_mirror(
104 ChunkedArrayManager<MemorySpace, ValueType> const& other,
105 std::enable_if_t<!IsAccessibleFrom<Space>::value>* = nullptr) {
106 using tag_type =
107 typename ChunkedArrayManager<Space, ValueType>::INACCESSIBLE_TAG;
108 return ChunkedArrayManager<Space, ValueType>{tag_type{}, other.m_chunk_max,
109 other.m_chunk_size};
110 }
111
112 public:
113 void allocate_device(const std::string& label) {
114 if (m_chunks == nullptr) {
115 m_chunks = reinterpret_cast<pointer_type*>(MemorySpace().allocate(
116 label.c_str(), (sizeof(pointer_type) * (m_chunk_max + 2))));
117 }
118 }
119
120 void initialize() {
121 for (unsigned i = 0; i < m_chunk_max + 2; i++) {
122 m_chunks[i] = nullptr;
123 }
124 m_valid = true;
125 }
126
127 private:
130 template <typename Space>
131 struct Destroy {
132 Destroy() = default;
133 Destroy(Destroy&&) = default;
134 Destroy(const Destroy&) = default;
135 Destroy& operator=(Destroy&&) = default;
136 Destroy& operator=(const Destroy&) = default;
137
138 Destroy(std::string label, value_type** arg_chunk,
139 const unsigned arg_chunk_max, const unsigned arg_chunk_size,
140 value_type** arg_linked)
141 : m_label(label),
142 m_chunks(arg_chunk),
143 m_linked(arg_linked),
144 m_chunk_max(arg_chunk_max),
145 m_chunk_size(arg_chunk_size) {}
146
147 void execute() {
148 // Destroy the array of chunk pointers.
149 // Two entries beyond the max chunks are allocation counters.
150 uintptr_t const len =
151 *reinterpret_cast<uintptr_t*>(m_chunks + m_chunk_max);
152 for (unsigned i = 0; i < len; i++) {
153 Space().deallocate(m_label.c_str(), m_chunks[i],
154 sizeof(value_type) * m_chunk_size);
155 }
156 // Destroy the linked allocation if we have one.
157 if (m_linked != nullptr) {
158 // NOLINTNEXTLINE(bugprone-multi-level-implicit-pointer-conversion)
159 Space().deallocate(m_label.c_str(), m_linked,
160 (sizeof(value_type*) * (m_chunk_max + 2)));
161 }
162 }
163
164 void destroy_shared_allocation() { execute(); }
165
166 std::string m_label;
167 value_type** m_chunks = nullptr;
168 value_type** m_linked = nullptr;
169 unsigned m_chunk_max;
170 unsigned m_chunk_size;
171 };
172
173 public:
174 template <typename Space>
175 void allocate_with_destroy(const std::string& label,
176 pointer_type* linked_allocation = nullptr) {
177 using destroy_type = Destroy<Space>;
178 using record_type =
179 Kokkos::Impl::SharedAllocationRecord<MemorySpace, destroy_type>;
180
181 // Allocate + 2 extra slots so that *m_chunk[m_chunk_max] ==
182 // num_chunks_alloc and *m_chunk[m_chunk_max+1] == extent This must match in
183 // Destroy's execute(...) method
184 record_type* const record = record_type::allocate(
185 MemorySpace(), label, (sizeof(pointer_type) * (m_chunk_max + 2)));
186 m_chunks = static_cast<pointer_type*>(record->data());
187 m_track.assign_allocated_record_to_uninitialized(record);
188
189 record->m_destroy = destroy_type(label, m_chunks, m_chunk_max, m_chunk_size,
190 linked_allocation);
191 }
192
193 pointer_type* get_ptr() const { return m_chunks; }
194
195 template <typename OtherMemorySpace, typename ExecutionSpace>
196 void deep_copy_to(
197 const ExecutionSpace& exec_space,
198 ChunkedArrayManager<OtherMemorySpace, ValueType> const& other) const {
199 // use of ad-hoc unmanaged views
200 Kokkos::deep_copy(
201 exec_space,
202 Kokkos::View<uintptr_t*, OtherMemorySpace>(
203 reinterpret_cast<uintptr_t*>(other.m_chunks), m_chunk_max + 2),
204 Kokkos::View<uintptr_t*, MemorySpace>(
205 reinterpret_cast<uintptr_t*>(m_chunks), m_chunk_max + 2));
206 }
207
208 KOKKOS_INLINE_FUNCTION
209 pointer_type* operator+(int i) const { return m_chunks + i; }
210
211 KOKKOS_INLINE_FUNCTION
212 pointer_type& operator[](int i) const { return m_chunks[i]; }
213
214 track_type const& track() const { return m_track; }
215
216 KOKKOS_INLINE_FUNCTION
217 bool valid() const { return m_valid; }
218
219 private:
220 bool m_valid = false;
221 unsigned m_chunk_max = 0;
222 pointer_type* m_chunks = nullptr;
223 track_type m_track;
224 unsigned m_chunk_size = 0;
225};
226
227} /* end namespace Impl */
228
233template <typename DataType, typename... P>
234class DynamicView : public Kokkos::ViewTraits<DataType, P...> {
235 public:
236 using traits = Kokkos::ViewTraits<DataType, P...>;
237
238 using value_type = typename traits::value_type;
239 using device_space = typename traits::memory_space;
240 using host_space =
241 typename Kokkos::Impl::HostMirror<device_space>::Space::memory_space;
242 using device_accessor = Impl::ChunkedArrayManager<device_space, value_type>;
243 using host_accessor = Impl::ChunkedArrayManager<host_space, value_type>;
244
245 private:
246 template <class, class...>
247 friend class DynamicView;
248
249 using track_type = Kokkos::Impl::SharedAllocationTracker;
250
251 static_assert(traits::rank == 1 && traits::rank_dynamic == 1,
252 "DynamicView must be rank-one");
253
254 // It is assumed that the value_type is trivially copyable;
255 // when this is not the case, potential problems can occur.
256 static_assert(std::is_void_v<typename traits::specialize>,
257 "DynamicView only implemented for non-specialized View type");
258
259 private:
260 device_accessor m_chunks;
261 host_accessor m_chunks_host;
262 unsigned m_chunk_shift; // ceil(log2(m_chunk_size))
263 unsigned m_chunk_mask; // m_chunk_size - 1
264 unsigned m_chunk_max; // number of entries in the chunk array - each pointing
265 // to a chunk of extent == m_chunk_size entries
266 unsigned m_chunk_size; // 2 << (m_chunk_shift - 1)
267
268 public:
269 //----------------------------------------------------------------------
270
272 using array_type =
273 DynamicView<typename traits::data_type, typename traits::device_type>;
274
276 using const_type = DynamicView<typename traits::const_data_type,
277 typename traits::device_type>;
278
280 using non_const_type = DynamicView<typename traits::non_const_data_type,
281 typename traits::device_type>;
282
284 using HostMirror = DynamicView;
285
287 using uniform_device =
288 Kokkos::Device<typename traits::device_type::execution_space,
289 Kokkos::AnonymousSpace>;
290 using uniform_type = array_type;
291 using uniform_const_type = const_type;
292 using uniform_runtime_type = array_type;
293 using uniform_runtime_const_type = const_type;
294 using uniform_nomemspace_type =
295 DynamicView<typename traits::data_type, uniform_device>;
296 using uniform_const_nomemspace_type =
297 DynamicView<typename traits::const_data_type, uniform_device>;
298 using uniform_runtime_nomemspace_type =
299 DynamicView<typename traits::data_type, uniform_device>;
300 using uniform_runtime_const_nomemspace_type =
301 DynamicView<typename traits::const_data_type, uniform_device>;
302
303 //----------------------------------------------------------------------
304
305 enum { Rank = 1 };
306
307 KOKKOS_INLINE_FUNCTION
308 size_t allocation_extent() const noexcept {
309 uintptr_t n =
310 *reinterpret_cast<const uintptr_t*>(m_chunks_host + m_chunk_max);
311 return (n << m_chunk_shift);
312 }
313
314 KOKKOS_INLINE_FUNCTION
315 size_t chunk_size() const noexcept { return m_chunk_size; }
316
317 KOKKOS_INLINE_FUNCTION
318 size_t chunk_max() const noexcept { return m_chunk_max; }
319
320 KOKKOS_INLINE_FUNCTION
321 size_t size() const noexcept {
322 size_t extent_0 =
323 *reinterpret_cast<const size_t*>(m_chunks_host + m_chunk_max + 1);
324 return extent_0;
325 }
326
327 template <typename iType>
328 KOKKOS_INLINE_FUNCTION size_t extent(const iType& r) const {
329 return r == 0 ? size() : 1;
330 }
331
332 template <typename iType>
333 KOKKOS_INLINE_FUNCTION size_t extent_int(const iType& r) const {
334 return r == 0 ? size() : 1;
335 }
336
337 KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { return 0; }
338 KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { return 0; }
339 KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { return 0; }
340 KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { return 0; }
341 KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { return 0; }
342 KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { return 0; }
343 KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { return 0; }
344 KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { return 0; }
345
346 template <typename iType>
347 KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
348 *s = 0;
349 }
350
351 //----------------------------------------
352 // Allocation tracking properties
353
354 KOKKOS_INLINE_FUNCTION
355 int use_count() const { return m_chunks_host.track().use_count(); }
356
357 inline const std::string label() const {
358 return m_chunks_host.track().template get_label<host_space>();
359 }
360
361 //----------------------------------------------------------------------
362 // Range span is the span which contains all members.
363
364 using reference_type = typename traits::value_type&;
365 using pointer_type = typename traits::value_type*;
366
367 enum {
368 reference_type_is_lvalue_reference =
369 std::is_lvalue_reference_v<reference_type>
370 };
371
372 KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
373 return false;
374 }
375 KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return 0; }
376 KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { return 0; }
377
378 //----------------------------------------
379
380 template <typename I0, class... Args>
381 KOKKOS_INLINE_FUNCTION reference_type
382 operator()(const I0& i0, const Args&... /*args*/) const {
383 static_assert(Kokkos::Impl::are_integral<I0, Args...>::value,
384 "Indices must be integral type");
385
386 Kokkos::Impl::runtime_check_memory_access_violation<
387 typename traits::memory_space>(
388 "Kokkos::DynamicView ERROR: attempt to access inaccessible memory "
389 "space");
390
391 // Which chunk is being indexed.
392 const uintptr_t ic = uintptr_t(i0) >> m_chunk_shift;
393
394#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
395 const uintptr_t n = *reinterpret_cast<uintptr_t*>(m_chunks + m_chunk_max);
396 if (n <= ic) Kokkos::abort("Kokkos::DynamicView array bounds error");
397#endif
398
399 typename traits::value_type** const ch = m_chunks + ic;
400 return (*ch)[i0 & m_chunk_mask];
401 }
402
403 //----------------------------------------
407 template <typename IntType>
408 inline void resize_serial(IntType const& n) {
409 using local_value_type = typename traits::value_type;
410 using value_pointer_type = local_value_type*;
411
412 const uintptr_t NC =
413 (n + m_chunk_mask) >>
414 m_chunk_shift; // New total number of chunks needed for resize
415
416 if (m_chunk_max < NC) {
417 Kokkos::abort("DynamicView::resize_serial exceeded maximum size");
418 }
419
420 // *m_chunks[m_chunk_max] stores the current number of chunks being used
421 uintptr_t* const pc =
422 reinterpret_cast<uintptr_t*>(m_chunks_host + m_chunk_max);
423 std::string _label = m_chunks_host.track().template get_label<host_space>();
424
425 if (*pc < NC) {
426 while (*pc < NC) {
427 m_chunks_host[*pc] =
428 reinterpret_cast<value_pointer_type>(device_space().allocate(
429 _label.c_str(), sizeof(local_value_type) << m_chunk_shift));
430 ++*pc;
431 }
432 } else {
433 while (NC + 1 <= *pc) {
434 --*pc;
435 device_space().deallocate(_label.c_str(), m_chunks_host[*pc],
436 sizeof(local_value_type) << m_chunk_shift);
437 m_chunks_host[*pc] = nullptr;
438 }
439 }
440 // *m_chunks_host[m_chunk_max+1] stores the 'extent' requested by resize
441 *(pc + 1) = n;
442
443 typename device_space::execution_space exec{};
444 m_chunks_host.deep_copy_to(exec, m_chunks);
445 exec.fence(
446 "DynamicView::resize_serial: Fence after copying chunks to the device");
447 }
448
449 KOKKOS_INLINE_FUNCTION bool is_allocated() const {
450 if (m_chunks_host.valid()) {
451 // *m_chunks_host[m_chunk_max] stores the current number of chunks being
452 // used
453 uintptr_t* const pc =
454 reinterpret_cast<uintptr_t*>(m_chunks_host + m_chunk_max);
455 return (*(pc + 1) > 0);
456 } else {
457 return false;
458 }
459 }
460
461 KOKKOS_FUNCTION const device_accessor& impl_get_chunks() const {
462 return m_chunks;
463 }
464
465 KOKKOS_FUNCTION device_accessor& impl_get_chunks() { return m_chunks; }
466
467 //----------------------------------------------------------------------
468
469 ~DynamicView() = default;
470 DynamicView() = default;
471 DynamicView(DynamicView&&) = default;
472 DynamicView(const DynamicView&) = default;
473 DynamicView& operator=(DynamicView&&) = default;
474 DynamicView& operator=(const DynamicView&) = default;
475
476 template <class RT, class... RP>
477 DynamicView(const DynamicView<RT, RP...>& rhs)
478 : m_chunks(rhs.m_chunks),
479 m_chunks_host(rhs.m_chunks_host),
480 m_chunk_shift(rhs.m_chunk_shift),
481 m_chunk_mask(rhs.m_chunk_mask),
482 m_chunk_max(rhs.m_chunk_max),
483 m_chunk_size(rhs.m_chunk_size) {
484 using SrcTraits = typename DynamicView<RT, RP...>::traits;
485 using Mapping = Kokkos::Impl::ViewMapping<traits, SrcTraits, void>;
486 static_assert(Mapping::is_assignable,
487 "Incompatible DynamicView copy construction");
488 }
489
496 template <class... Prop>
497 DynamicView(const Kokkos::Impl::ViewCtorProp<Prop...>& arg_prop,
498 const unsigned min_chunk_size,
499 const unsigned max_extent)
500 : // The chunk size is guaranteed to be a power of two
501 m_chunk_shift(min_chunk_size ? Kokkos::bit_width(min_chunk_size - 1)
502 : 0) // div ceil(log2(min_chunk_size))
503 ,
504 m_chunk_mask((1 << m_chunk_shift) - 1) // mod
505 ,
506 m_chunk_max((max_extent + m_chunk_mask) >>
507 m_chunk_shift) // max num pointers-to-chunks in array
508 ,
509 m_chunk_size(2 << (m_chunk_shift - 1)) {
510 m_chunks = device_accessor(m_chunk_max, m_chunk_size);
511
512 const std::string& label =
513 Kokkos::Impl::get_property<Kokkos::Impl::LabelTag>(arg_prop);
514
515 if (device_accessor::template IsAccessibleFrom<host_space>::value) {
516 m_chunks.template allocate_with_destroy<device_space>(label);
517 m_chunks.initialize();
518 m_chunks_host =
519 device_accessor::template create_mirror<host_space>(m_chunks);
520 } else {
521 m_chunks.allocate_device(label);
522 m_chunks_host =
523 device_accessor::template create_mirror<host_space>(m_chunks);
524 m_chunks_host.template allocate_with_destroy<device_space>(
525 label, m_chunks.get_ptr());
526 m_chunks_host.initialize();
527
528 using alloc_prop_input = Kokkos::Impl::ViewCtorProp<Prop...>;
529
530 auto arg_prop_copy = ::Kokkos::Impl::with_properties_if_unset(
531 arg_prop, typename device_space::execution_space{});
532
533 const auto& exec =
534 Kokkos::Impl::get_property<Kokkos::Impl::ExecutionSpaceTag>(
535 arg_prop_copy);
536 m_chunks_host.deep_copy_to(exec, m_chunks);
537 if (!alloc_prop_input::has_execution_space)
538 exec.fence(
539 "DynamicView::DynamicView(): Fence after copying chunks to the "
540 "device");
541 }
542 }
543
544 DynamicView(const std::string& arg_label, const unsigned min_chunk_size,
545 const unsigned max_extent)
546 : DynamicView(Kokkos::view_alloc(arg_label), min_chunk_size, max_extent) {
547 }
548};
549
550} // namespace Experimental
551
552template <class>
553struct is_dynamic_view : public std::false_type {};
554
555template <class D, class... P>
556struct is_dynamic_view<Kokkos::Experimental::DynamicView<D, P...>>
557 : public std::true_type {};
558
559template <class T>
560inline constexpr bool is_dynamic_view_v = is_dynamic_view<T>::value;
561
562} // namespace Kokkos
563
564namespace Kokkos {
565
566namespace Impl {
567
568// Deduce Mirror Types
569template <class Space, class T, class... P>
570struct MirrorDynamicViewType {
571 // The incoming view_type
572 using src_view_type = typename Kokkos::Experimental::DynamicView<T, P...>;
573 // The memory space for the mirror view
574 using memory_space = typename Space::memory_space;
575 // Check whether it is the same memory space
576 enum {
577 is_same_memspace =
578 std::is_same_v<memory_space, typename src_view_type::memory_space>
579 };
580 // The array_layout
581 using array_layout = typename src_view_type::array_layout;
582 // The data type (we probably want it non-const since otherwise we can't even
583 // deep_copy to it.)
584 using data_type = typename src_view_type::non_const_data_type;
585 // The destination view type if it is not the same memory space
586 using dest_view_type =
587 Kokkos::Experimental::DynamicView<data_type, array_layout, Space>;
588 // If it is the same memory_space return the existing view_type
589 // This will also keep the unmanaged trait if necessary
590 using view_type =
591 std::conditional_t<is_same_memspace, src_view_type, dest_view_type>;
592};
593} // namespace Impl
594
595namespace Impl {
596
597// create a mirror
598// private interface that accepts arbitrary view constructor args passed by a
599// view_alloc
600template <class T, class... P, class... ViewCtorArgs>
601inline auto create_mirror(const Kokkos::Experimental::DynamicView<T, P...>& src,
602 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
603 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
604 check_view_ctor_args_create_mirror<ViewCtorArgs...>();
605
606 auto prop_copy = Impl::with_properties_if_unset(
607 arg_prop, std::string(src.label()).append("_mirror"));
608
609 if constexpr (Impl::ViewCtorProp<ViewCtorArgs...>::has_memory_space) {
610 using MemorySpace = typename alloc_prop_input::memory_space;
611
612 auto ret = typename Kokkos::Impl::MirrorDynamicViewType<
613 MemorySpace, T, P...>::view_type(prop_copy, src.chunk_size(),
614 src.chunk_max() * src.chunk_size());
615
616 ret.resize_serial(src.extent(0));
617
618 return ret;
619 } else {
620 auto ret = typename Kokkos::Experimental::DynamicView<T, P...>::HostMirror(
621 prop_copy, src.chunk_size(), src.chunk_max() * src.chunk_size());
622
623 ret.resize_serial(src.extent(0));
624
625 return ret;
626 }
627#if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
628 !defined(KOKKOS_COMPILER_MSVC)
629 __builtin_unreachable();
630#endif
631}
632
633} // namespace Impl
634
635// public interface
636template <class T, class... P,
637 typename Enable = std::enable_if_t<
638 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
639inline auto create_mirror(
640 const Kokkos::Experimental::DynamicView<T, P...>& src) {
641 return Impl::create_mirror(src, Impl::ViewCtorProp<>{});
642}
643
644// public interface that accepts a without initializing flag
645template <class T, class... P,
646 typename Enable = std::enable_if_t<
647 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
648inline auto create_mirror(
649 Kokkos::Impl::WithoutInitializing_t wi,
650 const Kokkos::Experimental::DynamicView<T, P...>& src) {
651 return Impl::create_mirror(src, Kokkos::view_alloc(wi));
652}
653
654// public interface that accepts a space
655template <class Space, class T, class... P,
656 typename Enable = std::enable_if_t<
657 Kokkos::is_space<Space>::value &&
658 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
659inline auto create_mirror(
660 const Space&, const Kokkos::Experimental::DynamicView<T, P...>& src) {
661 return Impl::create_mirror(
662 src, Kokkos::view_alloc(typename Space::memory_space{}));
663}
664
665// public interface that accepts a space and a without initializing flag
666template <class Space, class T, class... P,
667 typename Enable = std::enable_if_t<
668 Kokkos::is_space<Space>::value &&
669 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
670inline auto create_mirror(
671 Kokkos::Impl::WithoutInitializing_t wi, const Space&,
672 const Kokkos::Experimental::DynamicView<T, P...>& src) {
673 return Impl::create_mirror(
674 src, Kokkos::view_alloc(wi, typename Space::memory_space{}));
675}
676
677// public interface that accepts arbitrary view constructor args passed by a
678// view_alloc
679template <class T, class... P, class... ViewCtorArgs,
680 typename Enable = std::enable_if_t<
681 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
682inline auto create_mirror(
683 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
684 const Kokkos::Experimental::DynamicView<T, P...>& src) {
685 return Impl::create_mirror(src, arg_prop);
686}
687
688namespace Impl {
689
690// create a mirror view
691// private interface that accepts arbitrary view constructor args passed by a
692// view_alloc
693template <class T, class... P, class... ViewCtorArgs>
694inline auto create_mirror_view(
695 const Kokkos::Experimental::DynamicView<T, P...>& src,
696 [[maybe_unused]] const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
697 if constexpr (!Impl::ViewCtorProp<ViewCtorArgs...>::has_memory_space) {
698 if constexpr (std::is_same_v<typename Kokkos::Experimental::DynamicView<
699 T, P...>::memory_space,
700 typename Kokkos::Experimental::DynamicView<
701 T, P...>::HostMirror::memory_space> &&
702 std::is_same_v<typename Kokkos::Experimental::DynamicView<
703 T, P...>::data_type,
704 typename Kokkos::Experimental::DynamicView<
705 T, P...>::HostMirror::data_type>) {
706 return
707 typename Kokkos::Experimental::DynamicView<T, P...>::HostMirror(src);
708 } else {
709 return Kokkos::Impl::choose_create_mirror(src, arg_prop);
710 }
711 } else {
712 if constexpr (Impl::MirrorDynamicViewType<
713 typename Impl::ViewCtorProp<
714 ViewCtorArgs...>::memory_space,
715 T, P...>::is_same_memspace) {
716 return typename Impl::MirrorDynamicViewType<
717 typename Impl::ViewCtorProp<ViewCtorArgs...>::memory_space, T,
718 P...>::view_type(src);
719 } else {
720 return Kokkos::Impl::choose_create_mirror(src, arg_prop);
721 }
722 }
723#if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
724 !defined(KOKKOS_COMPILER_MSVC)
725 __builtin_unreachable();
726#endif
727}
728
729} // namespace Impl
730
731// public interface
732template <class T, class... P>
733inline auto create_mirror_view(
734 const typename Kokkos::Experimental::DynamicView<T, P...>& src) {
735 return Impl::create_mirror_view(src, Impl::ViewCtorProp<>{});
736}
737
738// public interface that accepts a without initializing flag
739template <class T, class... P>
740inline auto create_mirror_view(
741 Kokkos::Impl::WithoutInitializing_t wi,
742 const typename Kokkos::Experimental::DynamicView<T, P...>& src) {
743 return Impl::create_mirror_view(src, Kokkos::view_alloc(wi));
744}
745
746// public interface that accepts a space
747template <class Space, class T, class... P,
748 class Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
749inline auto create_mirror_view(
750 const Space&, const Kokkos::Experimental::DynamicView<T, P...>& src) {
751 return Impl::create_mirror_view(src,
752 view_alloc(typename Space::memory_space{}));
753}
754
755// public interface that accepts a space and a without initializing flag
756template <class Space, class T, class... P,
757 class Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
758inline auto create_mirror_view(
759 Kokkos::Impl::WithoutInitializing_t wi, const Space&,
760 const Kokkos::Experimental::DynamicView<T, P...>& src) {
761 return Impl::create_mirror_view(
762 src, Kokkos::view_alloc(wi, typename Space::memory_space{}));
763}
764
765// public interface that accepts arbitrary view constructor args passed by a
766// view_alloc
767template <class T, class... P, class... ViewCtorArgs>
768inline auto create_mirror_view(
769 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
770 const Kokkos::Experimental::DynamicView<T, P...>& src) {
771 return Impl::create_mirror_view(src, arg_prop);
772}
773
774template <class T, class... DP, class... SP>
775inline void deep_copy(const Kokkos::Experimental::DynamicView<T, DP...>& dst,
776 const Kokkos::Experimental::DynamicView<T, SP...>& src) {
777 using dst_type = Kokkos::Experimental::DynamicView<T, DP...>;
778 using src_type = Kokkos::Experimental::DynamicView<T, SP...>;
779
780 using dst_execution_space = typename ViewTraits<T, DP...>::execution_space;
781 using src_execution_space = typename ViewTraits<T, SP...>::execution_space;
782 using dst_memory_space = typename ViewTraits<T, DP...>::memory_space;
783 using src_memory_space = typename ViewTraits<T, SP...>::memory_space;
784
785 constexpr bool DstExecCanAccessSrc =
786 Kokkos::SpaceAccessibility<dst_execution_space,
787 src_memory_space>::accessible;
788 constexpr bool SrcExecCanAccessDst =
789 Kokkos::SpaceAccessibility<src_execution_space,
790 dst_memory_space>::accessible;
791
792 if (DstExecCanAccessSrc || SrcExecCanAccessDst)
793 Kokkos::Impl::ViewRemap<dst_type, src_type>(dst, src);
794 else
795 src.impl_get_chunks().deep_copy_to(dst_execution_space{},
796 dst.impl_get_chunks());
797 Kokkos::fence("Kokkos::deep_copy(DynamicView)");
798}
799
800template <class ExecutionSpace, class T, class... DP, class... SP>
801inline void deep_copy(const ExecutionSpace& exec,
802 const Kokkos::Experimental::DynamicView<T, DP...>& dst,
803 const Kokkos::Experimental::DynamicView<T, SP...>& src) {
804 using dst_type = Kokkos::Experimental::DynamicView<T, DP...>;
805 using src_type = Kokkos::Experimental::DynamicView<T, SP...>;
806
807 using dst_execution_space = typename ViewTraits<T, DP...>::execution_space;
808 using src_execution_space = typename ViewTraits<T, SP...>::execution_space;
809 using dst_memory_space = typename ViewTraits<T, DP...>::memory_space;
810 using src_memory_space = typename ViewTraits<T, SP...>::memory_space;
811
812 constexpr bool DstExecCanAccessSrc =
813 Kokkos::SpaceAccessibility<dst_execution_space,
814 src_memory_space>::accessible;
815 constexpr bool SrcExecCanAccessDst =
816 Kokkos::SpaceAccessibility<src_execution_space,
817 dst_memory_space>::accessible;
818
819 // FIXME use execution space
820 if (DstExecCanAccessSrc || SrcExecCanAccessDst)
821 Kokkos::Impl::ViewRemap<dst_type, src_type>(dst, src);
822 else
823 src.impl_get_chunks().deep_copy_to(exec, dst.impl_get_chunks());
824}
825
826template <class T, class... DP, class... SP>
827inline void deep_copy(const View<T, DP...>& dst,
828 const Kokkos::Experimental::DynamicView<T, SP...>& src) {
829 using dst_type = View<T, DP...>;
830 using src_type = Kokkos::Experimental::DynamicView<T, SP...>;
831
832 using dst_execution_space = typename ViewTraits<T, DP...>::execution_space;
833 using src_memory_space = typename ViewTraits<T, SP...>::memory_space;
834
835 constexpr bool DstExecCanAccessSrc =
836 Kokkos::SpaceAccessibility<dst_execution_space,
837 src_memory_space>::accessible;
838 static_assert(
839 DstExecCanAccessSrc,
840 "deep_copy given views that would require a temporary allocation");
841
842 // Copying data between views in accessible memory spaces and either
843 // non-contiguous or incompatible shape.
844 Kokkos::Impl::ViewRemap<dst_type, src_type>(dst, src);
845 Kokkos::fence("Kokkos::deep_copy(DynamicView)");
846}
847
848template <class T, class... DP, class... SP>
849inline void deep_copy(const Kokkos::Experimental::DynamicView<T, DP...>& dst,
850 const View<T, SP...>& src) {
851 using dst_type = Kokkos::Experimental::DynamicView<T, DP...>;
852 using src_type = View<T, SP...>;
853
854 using dst_execution_space = typename ViewTraits<T, DP...>::execution_space;
855 using src_memory_space = typename ViewTraits<T, SP...>::memory_space;
856
857 constexpr bool DstExecCanAccessSrc =
858 Kokkos::SpaceAccessibility<dst_execution_space,
859 src_memory_space>::accessible;
860 static_assert(
861 DstExecCanAccessSrc,
862 "deep_copy given views that would require a temporary allocation");
863
864 // Copying data between views in accessible memory spaces and either
865 // non-contiguous or incompatible shape.
866 Kokkos::Impl::ViewRemap<dst_type, src_type>(dst, src);
867 Kokkos::fence("Kokkos::deep_copy(DynamicView)");
868}
869
870namespace Impl {
871template <class Arg0, class... DP, class... SP>
872struct CommonSubview<Kokkos::Experimental::DynamicView<DP...>,
873 Kokkos::Experimental::DynamicView<SP...>, Arg0> {
874 using DstType = Kokkos::Experimental::DynamicView<DP...>;
875 using SrcType = Kokkos::Experimental::DynamicView<SP...>;
876 using dst_subview_type = DstType;
877 using src_subview_type = SrcType;
878 dst_subview_type dst_sub;
879 src_subview_type src_sub;
880 CommonSubview(const DstType& dst, const SrcType& src, const Arg0& /*arg0*/)
881 : dst_sub(dst), src_sub(src) {}
882};
883
884template <class... DP, class SrcType, class Arg0>
885struct CommonSubview<Kokkos::Experimental::DynamicView<DP...>, SrcType, Arg0> {
886 using DstType = Kokkos::Experimental::DynamicView<DP...>;
887 using dst_subview_type = DstType;
888 using src_subview_type = typename Kokkos::Subview<SrcType, Arg0>;
889 dst_subview_type dst_sub;
890 src_subview_type src_sub;
891 CommonSubview(const DstType& dst, const SrcType& src, const Arg0& arg0)
892 : dst_sub(dst), src_sub(src, arg0) {}
893};
894
895template <class DstType, class... SP, class Arg0>
896struct CommonSubview<DstType, Kokkos::Experimental::DynamicView<SP...>, Arg0> {
897 using SrcType = Kokkos::Experimental::DynamicView<SP...>;
898 using dst_subview_type = typename Kokkos::Subview<DstType, Arg0>;
899 using src_subview_type = SrcType;
900 dst_subview_type dst_sub;
901 src_subview_type src_sub;
902 CommonSubview(const DstType& dst, const SrcType& src, const Arg0& arg0)
903 : dst_sub(dst, arg0), src_sub(src) {}
904};
905
906template <class... DP, class ViewTypeB, class Layout, class ExecSpace,
907 typename iType>
908struct ViewCopy<Kokkos::Experimental::DynamicView<DP...>, ViewTypeB, Layout,
909 ExecSpace, 1, iType> {
910 Kokkos::Experimental::DynamicView<DP...> a;
911 ViewTypeB b;
912
913 using policy_type = Kokkos::RangePolicy<ExecSpace, Kokkos::IndexType<iType>>;
914
915 ViewCopy(const Kokkos::Experimental::DynamicView<DP...>& a_,
916 const ViewTypeB& b_)
917 : a(a_), b(b_) {
918 Kokkos::parallel_for("Kokkos::ViewCopy-1D", policy_type(0, b.extent(0)),
919 *this);
920 }
921
922 KOKKOS_INLINE_FUNCTION
923 void operator()(const iType& i0) const { a(i0) = b(i0); }
924};
925
926template <class... DP, class... SP, class Layout, class ExecSpace,
927 typename iType>
928struct ViewCopy<Kokkos::Experimental::DynamicView<DP...>,
929 Kokkos::Experimental::DynamicView<SP...>, Layout, ExecSpace, 1,
930 iType> {
931 Kokkos::Experimental::DynamicView<DP...> a;
932 Kokkos::Experimental::DynamicView<SP...> b;
933
934 using policy_type = Kokkos::RangePolicy<ExecSpace, Kokkos::IndexType<iType>>;
935
936 ViewCopy(const Kokkos::Experimental::DynamicView<DP...>& a_,
937 const Kokkos::Experimental::DynamicView<SP...>& b_)
938 : a(a_), b(b_) {
939 const iType n = std::min(a.extent(0), b.extent(0));
940 Kokkos::parallel_for("Kokkos::ViewCopy-1D", policy_type(0, n), *this);
941 }
942
943 KOKKOS_INLINE_FUNCTION
944 void operator()(const iType& i0) const { a(i0) = b(i0); }
945};
946
947} // namespace Impl
948
949// create a mirror view and deep copy it
950// public interface that accepts arbitrary view constructor args passed by a
951// view_alloc
952template <class... ViewCtorArgs, class T, class... P,
953 class Enable = std::enable_if_t<
954 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
955auto create_mirror_view_and_copy(
956 [[maybe_unused]] const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
957 const Kokkos::Experimental::DynamicView<T, P...>& src) {
958 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
959
960 Impl::check_view_ctor_args_create_mirror_view_and_copy<ViewCtorArgs...>();
961
962 if constexpr (Impl::MirrorDynamicViewType<
963 typename Impl::ViewCtorProp<ViewCtorArgs...>::memory_space,
964 T, P...>::is_same_memspace) {
965 // same behavior as deep_copy(src, src)
966 if constexpr (!alloc_prop_input::has_execution_space)
967 fence(
968 "Kokkos::create_mirror_view_and_copy: fence before returning src "
969 "view");
970 return src;
971 } else {
972 using Space = typename alloc_prop_input::memory_space;
973 using Mirror =
974 typename Impl::MirrorDynamicViewType<Space, T, P...>::view_type;
975
976 auto arg_prop_copy = Impl::with_properties_if_unset(
977 arg_prop, std::string{}, WithoutInitializing,
978 typename Space::execution_space{});
979
980 std::string& label = Impl::get_property<Impl::LabelTag>(arg_prop_copy);
981 if (label.empty()) label = src.label();
982 auto mirror = typename Mirror::non_const_type(
983 arg_prop_copy, src.chunk_size(), src.chunk_max() * src.chunk_size());
984 mirror.resize_serial(src.extent(0));
985 if constexpr (alloc_prop_input::has_execution_space) {
986 deep_copy(Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop_copy),
987 mirror, src);
988 } else
989 deep_copy(mirror, src);
990 return mirror;
991 }
992#if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
993 !defined(KOKKOS_COMPILER_MSVC)
994 __builtin_unreachable();
995#endif
996}
997
998template <class Space, class T, class... P,
999 typename Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
1000auto create_mirror_view_and_copy(
1001 const Space&, const Kokkos::Experimental::DynamicView<T, P...>& src,
1002 std::string const& name = "") {
1003 return create_mirror_view_and_copy(
1004 Kokkos::view_alloc(typename Space::memory_space{}, name), src);
1005}
1006
1007} // namespace Kokkos
1008
1009#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DYNAMICVIEW
1010#undef KOKKOS_IMPL_PUBLIC_INCLUDE
1011#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DYNAMICVIEW
1012#endif
1013#endif /* #ifndef KOKKOS_DYNAMIC_VIEW_HPP */
View
View
Definition Kokkos_View.hpp:157
Kokkos::Bitset
A thread safe view to a bitset.
Definition Kokkos_Bitset.hpp:49
Kokkos::Experimental::DynamicView
Dynamic views are restricted to rank-one and no layout. Resize only occurs on host outside of paralle...
Definition Kokkos_DynamicView.hpp:234
Kokkos::Experimental::DynamicView::HostMirror
DynamicView HostMirror
Must be accessible everywhere.
Definition Kokkos_DynamicView.hpp:284
Kokkos::Experimental::DynamicView::DynamicView
DynamicView(const Kokkos::Impl::ViewCtorProp< Prop... > &arg_prop, const unsigned min_chunk_size, const unsigned max_extent)
Allocation constructor.
Definition Kokkos_DynamicView.hpp:497
Kokkos::Experimental::DynamicView::array_type
DynamicView< typename traits::data_type, typename traits::device_type > array_type
Compatible view of array of scalar types.
Definition Kokkos_DynamicView.hpp:273
Kokkos::Experimental::DynamicView::resize_serial
void resize_serial(IntType const &n)
Resizing in serial can grow or shrink the array size up to the maximum number of chunks.
Definition Kokkos_DynamicView.hpp:408
Kokkos::Experimental::DynamicView::const_type
DynamicView< typename traits::const_data_type, typename traits::device_type > const_type
Compatible view of const data type.
Definition Kokkos_DynamicView.hpp:277
Kokkos::Impl::MemorySpaceAccess
Access relationship between DstMemorySpace and SrcMemorySpace.
Definition Kokkos_Concepts.hpp:309
Kokkos::SpaceAccessibility
Can AccessSpace access MemorySpace ?
Definition Kokkos_Concepts.hpp:359
Generated by doxygen 1.9.8