Kokkos Core Kernels Package Version of the Day
Loading...
Searching...
No Matches
Kokkos_CopyViews.hpp
1//@HEADER
2// ************************************************************************
3//
4// Kokkos v. 4.0
5// Copyright (2022) National Technology & Engineering
6// Solutions of Sandia, LLC (NTESS).
7//
8// Under the terms of Contract DE-NA0003525 with NTESS,
9// the U.S. Government retains certain rights in this software.
10//
11// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12// See https://kokkos.org/LICENSE for license information.
13// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14//
15//@HEADER
16
17#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18#include <Kokkos_Macros.hpp>
19static_assert(false,
20 "Including non-public Kokkos header files is not allowed.");
21#endif
22#ifndef KOKKOS_COPYVIEWS_HPP_
23#define KOKKOS_COPYVIEWS_HPP_
24#include <string>
25#include <sstream>
26#include <Kokkos_Parallel.hpp>
27#include <KokkosExp_MDRangePolicy.hpp>
28#include <Kokkos_Layout.hpp>
29#include <impl/Kokkos_HostSpace_ZeroMemset.hpp>
30
31//----------------------------------------------------------------------------
32//----------------------------------------------------------------------------
33
34namespace Kokkos {
35
36namespace Impl {
37
38template <class Layout>
39struct ViewFillLayoutSelector {};
40
41template <>
42struct ViewFillLayoutSelector<Kokkos::LayoutLeft> {
43 static const Kokkos::Iterate iterate = Kokkos::Iterate::Left;
44};
45
46template <>
47struct ViewFillLayoutSelector<Kokkos::LayoutRight> {
48 static const Kokkos::Iterate iterate = Kokkos::Iterate::Right;
49};
50
51} // namespace Impl
52} // namespace Kokkos
53
54namespace Kokkos {
55namespace Impl {
56
57template <class ViewType, class Layout, class ExecSpace, typename iType>
58struct ViewFill<ViewType, Layout, ExecSpace, 0, iType> {
59 using ST = typename ViewType::non_const_value_type;
60 ViewFill(const ViewType& a, const ST& val, const ExecSpace& space) {
61 Kokkos::Impl::DeepCopy<typename ViewType::memory_space, Kokkos::HostSpace,
62 ExecSpace>(space, a.data(), &val, sizeof(ST));
63 }
64};
65
66template <class ViewType, class Layout, class ExecSpace, typename iType>
67struct ViewFill<ViewType, Layout, ExecSpace, 1, iType> {
68 ViewType a;
69 typename ViewType::const_value_type val;
70 using policy_type = Kokkos::RangePolicy<ExecSpace, Kokkos::IndexType<iType>>;
71
72 ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
73 const ExecSpace& space)
74 : a(a_), val(val_) {
75 Kokkos::parallel_for("Kokkos::ViewFill-1D",
76 policy_type(space, 0, a.extent(0)), *this);
77 }
78
79 KOKKOS_INLINE_FUNCTION
80 void operator()(const iType& i) const { a(i) = val; }
81};
82
83template <class ViewType, class Layout, class ExecSpace, typename iType>
84struct ViewFill<ViewType, Layout, ExecSpace, 2, iType> {
85 ViewType a;
86 typename ViewType::const_value_type val;
87
88 using iterate_type = Kokkos::Rank<2, ViewFillLayoutSelector<Layout>::iterate,
89 ViewFillLayoutSelector<Layout>::iterate>;
90 using policy_type =
91 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
92
93 ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
94 const ExecSpace& space)
95 : a(a_), val(val_) {
96 Kokkos::parallel_for("Kokkos::ViewFill-2D",
97 policy_type(space, {0, 0}, {a.extent(0), a.extent(1)}),
98 *this);
99 }
100
101 KOKKOS_INLINE_FUNCTION
102 void operator()(const iType& i0, const iType& i1) const { a(i0, i1) = val; }
103};
104
105template <class ViewType, class Layout, class ExecSpace, typename iType>
106struct ViewFill<ViewType, Layout, ExecSpace, 3, iType> {
107 ViewType a;
108 typename ViewType::const_value_type val;
109
110 using iterate_type = Kokkos::Rank<3, ViewFillLayoutSelector<Layout>::iterate,
111 ViewFillLayoutSelector<Layout>::iterate>;
112 using policy_type =
113 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
114
115 ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
116 const ExecSpace& space)
117 : a(a_), val(val_) {
118 Kokkos::parallel_for(
119 "Kokkos::ViewFill-3D",
120 policy_type(space, {0, 0, 0}, {a.extent(0), a.extent(1), a.extent(2)}),
121 *this);
122 }
123
124 KOKKOS_INLINE_FUNCTION
125 void operator()(const iType& i0, const iType& i1, const iType& i2) const {
126 a(i0, i1, i2) = val;
127 }
128};
129
130template <class ViewType, class Layout, class ExecSpace, typename iType>
131struct ViewFill<ViewType, Layout, ExecSpace, 4, iType> {
132 ViewType a;
133 typename ViewType::const_value_type val;
134
135 using iterate_type = Kokkos::Rank<4, ViewFillLayoutSelector<Layout>::iterate,
136 ViewFillLayoutSelector<Layout>::iterate>;
137 using policy_type =
138 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
139
140 ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
141 const ExecSpace& space)
142 : a(a_), val(val_) {
143 Kokkos::parallel_for(
144 "Kokkos::ViewFill-4D",
145 policy_type(space, {0, 0, 0, 0},
146 {a.extent(0), a.extent(1), a.extent(2), a.extent(3)}),
147 *this);
148 }
149
150 KOKKOS_INLINE_FUNCTION
151 void operator()(const iType& i0, const iType& i1, const iType& i2,
152 const iType& i3) const {
153 a(i0, i1, i2, i3) = val;
154 }
155};
156
157template <class ViewType, class Layout, class ExecSpace, typename iType>
158struct ViewFill<ViewType, Layout, ExecSpace, 5, iType> {
159 ViewType a;
160 typename ViewType::const_value_type val;
161
162 using iterate_type = Kokkos::Rank<5, ViewFillLayoutSelector<Layout>::iterate,
163 ViewFillLayoutSelector<Layout>::iterate>;
164 using policy_type =
165 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
166
167 ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
168 const ExecSpace& space)
169 : a(a_), val(val_) {
170 Kokkos::parallel_for("Kokkos::ViewFill-5D",
171 policy_type(space, {0, 0, 0, 0, 0},
172 {a.extent(0), a.extent(1), a.extent(2),
173 a.extent(3), a.extent(4)}),
174 *this);
175 }
176
177 KOKKOS_INLINE_FUNCTION
178 void operator()(const iType& i0, const iType& i1, const iType& i2,
179 const iType& i3, const iType& i4) const {
180 a(i0, i1, i2, i3, i4) = val;
181 }
182};
183
184template <class ViewType, class Layout, class ExecSpace, typename iType>
185struct ViewFill<ViewType, Layout, ExecSpace, 6, iType> {
186 ViewType a;
187 typename ViewType::const_value_type val;
188
189 using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
190 ViewFillLayoutSelector<Layout>::iterate>;
191 using policy_type =
192 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
193
194 ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
195 const ExecSpace& space)
196 : a(a_), val(val_) {
197 Kokkos::parallel_for("Kokkos::ViewFill-6D",
198 policy_type(space, {0, 0, 0, 0, 0, 0},
199 {a.extent(0), a.extent(1), a.extent(2),
200 a.extent(3), a.extent(4), a.extent(5)}),
201 *this);
202 }
203
204 KOKKOS_INLINE_FUNCTION
205 void operator()(const iType& i0, const iType& i1, const iType& i2,
206 const iType& i3, const iType& i4, const iType& i5) const {
207 a(i0, i1, i2, i3, i4, i5) = val;
208 }
209};
210
211template <class ViewType, class Layout, class ExecSpace, typename iType>
212struct ViewFill<ViewType, Layout, ExecSpace, 7, iType> {
213 ViewType a;
214 typename ViewType::const_value_type val;
215
216 using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
217 ViewFillLayoutSelector<Layout>::iterate>;
218 using policy_type =
219 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
220
221 ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
222 const ExecSpace& space)
223 : a(a_), val(val_) {
224 // MDRangePolicy is not supported for 7D views
225 // Iterate separately over extent(2)
226 Kokkos::parallel_for("Kokkos::ViewFill-7D",
227 policy_type(space, {0, 0, 0, 0, 0, 0},
228 {a.extent(0), a.extent(1), a.extent(3),
229 a.extent(4), a.extent(5), a.extent(6)}),
230 *this);
231 }
232
233 KOKKOS_INLINE_FUNCTION
234 void operator()(const iType& i0, const iType& i1, const iType& i3,
235 const iType& i4, const iType& i5, const iType& i6) const {
236 for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
237 a(i0, i1, i2, i3, i4, i5, i6) = val;
238 }
239};
240
241template <class ViewType, class Layout, class ExecSpace, typename iType>
242struct ViewFill<ViewType, Layout, ExecSpace, 8, iType> {
243 ViewType a;
244 typename ViewType::const_value_type val;
245
246 using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
247 ViewFillLayoutSelector<Layout>::iterate>;
248 using policy_type =
249 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
250
251 ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
252 const ExecSpace& space)
253 : a(a_), val(val_) {
254 // MDRangePolicy is not supported for 8D views
255 // Iterate separately over extent(2) and extent(4)
256 Kokkos::parallel_for("Kokkos::ViewFill-8D",
257 policy_type(space, {0, 0, 0, 0, 0, 0},
258 {a.extent(0), a.extent(1), a.extent(3),
259 a.extent(5), a.extent(6), a.extent(7)}),
260 *this);
261 }
262
263 KOKKOS_INLINE_FUNCTION
264 void operator()(const iType& i0, const iType& i1, const iType& i3,
265 const iType& i5, const iType& i6, const iType& i7) const {
266 for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
267 for (iType i4 = 0; i4 < iType(a.extent(4)); i4++)
268 a(i0, i1, i2, i3, i4, i5, i6, i7) = val;
269 }
270};
271
272template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
273 typename iType>
274struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 1, iType> {
275 ViewTypeA a;
276 ViewTypeB b;
277
278 using policy_type = Kokkos::RangePolicy<ExecSpace, Kokkos::IndexType<iType>>;
279 using value_type = typename ViewTypeA::value_type;
280
281 ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
282 const ExecSpace space = ExecSpace())
283 : a(a_), b(b_) {
284 Kokkos::parallel_for("Kokkos::ViewCopy-1D",
285 policy_type(space, 0, a.extent(0)), *this);
286 }
287
288 KOKKOS_INLINE_FUNCTION
289 void operator()(const iType& i0) const {
290 a(i0) = static_cast<value_type>(b(i0));
291 }
292};
293
294template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
295 typename iType>
296struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 2, iType> {
297 ViewTypeA a;
298 ViewTypeB b;
299 static const Kokkos::Iterate outer_iteration_pattern =
300 Kokkos::Impl::layout_iterate_type_selector<
301 Layout>::outer_iteration_pattern;
302 static const Kokkos::Iterate inner_iteration_pattern =
303 Kokkos::Impl::layout_iterate_type_selector<
304 Layout>::inner_iteration_pattern;
305 using iterate_type =
306 Kokkos::Rank<2, outer_iteration_pattern, inner_iteration_pattern>;
307 using policy_type =
308 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
309 using value_type = typename ViewTypeA::value_type;
310
311 ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
312 const ExecSpace space = ExecSpace())
313 : a(a_), b(b_) {
314 Kokkos::parallel_for("Kokkos::ViewCopy-2D",
315 policy_type(space, {0, 0}, {a.extent(0), a.extent(1)}),
316 *this);
317 }
318
319 KOKKOS_INLINE_FUNCTION
320 void operator()(const iType& i0, const iType& i1) const {
321 a(i0, i1) = static_cast<value_type>(b(i0, i1));
322 }
323};
324
325template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
326 typename iType>
327struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 3, iType> {
328 ViewTypeA a;
329 ViewTypeB b;
330
331 static const Kokkos::Iterate outer_iteration_pattern =
332 Kokkos::Impl::layout_iterate_type_selector<
333 Layout>::outer_iteration_pattern;
334 static const Kokkos::Iterate inner_iteration_pattern =
335 Kokkos::Impl::layout_iterate_type_selector<
336 Layout>::inner_iteration_pattern;
337 using iterate_type =
338 Kokkos::Rank<3, outer_iteration_pattern, inner_iteration_pattern>;
339 using policy_type =
340 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
341 using value_type = typename ViewTypeA::value_type;
342
343 ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
344 const ExecSpace space = ExecSpace())
345 : a(a_), b(b_) {
346 Kokkos::parallel_for(
347 "Kokkos::ViewCopy-3D",
348 policy_type(space, {0, 0, 0}, {a.extent(0), a.extent(1), a.extent(2)}),
349 *this);
350 }
351
352 KOKKOS_INLINE_FUNCTION
353 void operator()(const iType& i0, const iType& i1, const iType& i2) const {
354 a(i0, i1, i2) = static_cast<value_type>(b(i0, i1, i2));
355 }
356};
357
358template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
359 typename iType>
360struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 4, iType> {
361 ViewTypeA a;
362 ViewTypeB b;
363
364 static const Kokkos::Iterate outer_iteration_pattern =
365 Kokkos::Impl::layout_iterate_type_selector<
366 Layout>::outer_iteration_pattern;
367 static const Kokkos::Iterate inner_iteration_pattern =
368 Kokkos::Impl::layout_iterate_type_selector<
369 Layout>::inner_iteration_pattern;
370 using iterate_type =
371 Kokkos::Rank<4, outer_iteration_pattern, inner_iteration_pattern>;
372 using policy_type =
373 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
374
375 ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
376 const ExecSpace space = ExecSpace())
377 : a(a_), b(b_) {
378 Kokkos::parallel_for(
379 "Kokkos::ViewCopy-4D",
380 policy_type(space, {0, 0, 0, 0},
381 {a.extent(0), a.extent(1), a.extent(2), a.extent(3)}),
382 *this);
383 }
384
385 KOKKOS_INLINE_FUNCTION
386 void operator()(const iType& i0, const iType& i1, const iType& i2,
387 const iType& i3) const {
388 a(i0, i1, i2, i3) = b(i0, i1, i2, i3);
389 }
390};
391
392template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
393 typename iType>
394struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 5, iType> {
395 ViewTypeA a;
396 ViewTypeB b;
397
398 static const Kokkos::Iterate outer_iteration_pattern =
399 Kokkos::Impl::layout_iterate_type_selector<
400 Layout>::outer_iteration_pattern;
401 static const Kokkos::Iterate inner_iteration_pattern =
402 Kokkos::Impl::layout_iterate_type_selector<
403 Layout>::inner_iteration_pattern;
404 using iterate_type =
405 Kokkos::Rank<5, outer_iteration_pattern, inner_iteration_pattern>;
406 using policy_type =
407 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
408
409 ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
410 const ExecSpace space = ExecSpace())
411 : a(a_), b(b_) {
412 Kokkos::parallel_for("Kokkos::ViewCopy-5D",
413 policy_type(space, {0, 0, 0, 0, 0},
414 {a.extent(0), a.extent(1), a.extent(2),
415 a.extent(3), a.extent(4)}),
416 *this);
417 }
418
419 KOKKOS_INLINE_FUNCTION
420 void operator()(const iType& i0, const iType& i1, const iType& i2,
421 const iType& i3, const iType& i4) const {
422 a(i0, i1, i2, i3, i4) = b(i0, i1, i2, i3, i4);
423 }
424};
425
426template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
427 typename iType>
428struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 6, iType> {
429 ViewTypeA a;
430 ViewTypeB b;
431
432 static const Kokkos::Iterate outer_iteration_pattern =
433 Kokkos::Impl::layout_iterate_type_selector<
434 Layout>::outer_iteration_pattern;
435 static const Kokkos::Iterate inner_iteration_pattern =
436 Kokkos::Impl::layout_iterate_type_selector<
437 Layout>::inner_iteration_pattern;
438 using iterate_type =
439 Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
440 using policy_type =
441 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
442
443 ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
444 const ExecSpace space = ExecSpace())
445 : a(a_), b(b_) {
446 Kokkos::parallel_for("Kokkos::ViewCopy-6D",
447 policy_type(space, {0, 0, 0, 0, 0, 0},
448 {a.extent(0), a.extent(1), a.extent(2),
449 a.extent(3), a.extent(4), a.extent(5)}),
450 *this);
451 }
452
453 KOKKOS_INLINE_FUNCTION
454 void operator()(const iType& i0, const iType& i1, const iType& i2,
455 const iType& i3, const iType& i4, const iType& i5) const {
456 a(i0, i1, i2, i3, i4, i5) = b(i0, i1, i2, i3, i4, i5);
457 }
458};
459
460template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
461 typename iType>
462struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 7, iType> {
463 ViewTypeA a;
464 ViewTypeB b;
465
466 static const Kokkos::Iterate outer_iteration_pattern =
467 Kokkos::Impl::layout_iterate_type_selector<
468 Layout>::outer_iteration_pattern;
469 static const Kokkos::Iterate inner_iteration_pattern =
470 Kokkos::Impl::layout_iterate_type_selector<
471 Layout>::inner_iteration_pattern;
472 using iterate_type =
473 Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
474 using policy_type =
475 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
476
477 ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
478 const ExecSpace space = ExecSpace())
479 : a(a_), b(b_) {
480 // MDRangePolicy is not supported for 7D views
481 // Iterate separately over extent(2)
482 Kokkos::parallel_for("Kokkos::ViewCopy-7D",
483 policy_type(space, {0, 0, 0, 0, 0, 0},
484 {a.extent(0), a.extent(1), a.extent(3),
485 a.extent(4), a.extent(5), a.extent(6)}),
486 *this);
487 }
488
489 KOKKOS_INLINE_FUNCTION
490 void operator()(const iType& i0, const iType& i1, const iType& i3,
491 const iType& i4, const iType& i5, const iType& i6) const {
492 for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
493 a(i0, i1, i2, i3, i4, i5, i6) = b(i0, i1, i2, i3, i4, i5, i6);
494 }
495};
496
497template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
498 typename iType>
499struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 8, iType> {
500 ViewTypeA a;
501 ViewTypeB b;
502
503 static const Kokkos::Iterate outer_iteration_pattern =
504 Kokkos::Impl::layout_iterate_type_selector<
505 Layout>::outer_iteration_pattern;
506 static const Kokkos::Iterate inner_iteration_pattern =
507 Kokkos::Impl::layout_iterate_type_selector<
508 Layout>::inner_iteration_pattern;
509 using iterate_type =
510 Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
511 using policy_type =
512 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
513
514 ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
515 const ExecSpace space = ExecSpace())
516 : a(a_), b(b_) {
517 // MDRangePolicy is not supported for 8D views
518 // Iterate separately over extent(2) and extent(4)
519 Kokkos::parallel_for("Kokkos::ViewCopy-8D",
520 policy_type(space, {0, 0, 0, 0, 0, 0},
521 {a.extent(0), a.extent(1), a.extent(3),
522 a.extent(5), a.extent(6), a.extent(7)}),
523 *this);
524 }
525
526 KOKKOS_INLINE_FUNCTION
527 void operator()(const iType& i0, const iType& i1, const iType& i3,
528 const iType& i5, const iType& i6, const iType& i7) const {
529 for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
530 for (iType i4 = 0; i4 < iType(a.extent(4)); i4++)
531 a(i0, i1, i2, i3, i4, i5, i6, i7) = b(i0, i1, i2, i3, i4, i5, i6, i7);
532 }
533};
534
535} // namespace Impl
536} // namespace Kokkos
537
538namespace Kokkos {
539namespace Impl {
540
541template <class DstType>
542Kokkos::Iterate get_iteration_order(const DstType& dst) {
543 int64_t strides[DstType::rank + 1];
544 dst.stride(strides);
545 Kokkos::Iterate iterate;
546 if (std::is_same_v<typename DstType::array_layout, Kokkos::LayoutRight>) {
547 iterate = Kokkos::Iterate::Right;
548 } else if (std::is_same_v<typename DstType::array_layout,
549 Kokkos::LayoutLeft>) {
550 iterate = Kokkos::Iterate::Left;
551 } else if (std::is_same_v<typename DstType::array_layout,
552 Kokkos::LayoutStride>) {
553 if (strides[0] > strides[DstType::rank - 1])
554 iterate = Kokkos::Iterate::Right;
555 else
556 iterate = Kokkos::Iterate::Left;
557 } else {
558 if (std::is_same_v<typename DstType::execution_space::array_layout,
559 Kokkos::LayoutRight>)
560 iterate = Kokkos::Iterate::Right;
561 else
562 iterate = Kokkos::Iterate::Left;
563 }
564 return iterate;
565}
566
567template <class ExecutionSpace, class DstType, class SrcType>
568void view_copy(const ExecutionSpace& space, const DstType& dst,
569 const SrcType& src) {
570 using dst_memory_space = typename DstType::memory_space;
571 using src_memory_space = typename SrcType::memory_space;
572
573 constexpr bool ExecCanAccessSrc =
574 Kokkos::SpaceAccessibility<ExecutionSpace, src_memory_space>::accessible;
575 constexpr bool ExecCanAccessDst =
576 Kokkos::SpaceAccessibility<ExecutionSpace, dst_memory_space>::accessible;
577
578 if (!(ExecCanAccessSrc && ExecCanAccessDst)) {
579 Kokkos::Impl::throw_runtime_exception(
580 "Kokkos::Impl::view_copy called with invalid execution space");
581 } else {
582 // Figure out iteration order in case we need it
583 Kokkos::Iterate iterate = get_iteration_order(dst);
584
585 if ((dst.span() >= size_t(std::numeric_limits<int>::max())) ||
586 (src.span() >= size_t(std::numeric_limits<int>::max()))) {
587 if (iterate == Kokkos::Iterate::Right)
588 Kokkos::Impl::ViewCopy<
589 typename DstType::uniform_runtime_nomemspace_type,
590 typename SrcType::uniform_runtime_const_nomemspace_type,
591 Kokkos::LayoutRight, ExecutionSpace, DstType::rank, int64_t>(
592 dst, src, space);
593 else
594 Kokkos::Impl::ViewCopy<
595 typename DstType::uniform_runtime_nomemspace_type,
596 typename SrcType::uniform_runtime_const_nomemspace_type,
597 Kokkos::LayoutLeft, ExecutionSpace, DstType::rank, int64_t>(
598 dst, src, space);
599 } else {
600 if (iterate == Kokkos::Iterate::Right)
601 Kokkos::Impl::ViewCopy<
602 typename DstType::uniform_runtime_nomemspace_type,
603 typename SrcType::uniform_runtime_const_nomemspace_type,
604 Kokkos::LayoutRight, ExecutionSpace, DstType::rank, int>(dst, src,
605 space);
606 else
607 Kokkos::Impl::ViewCopy<
608 typename DstType::uniform_runtime_nomemspace_type,
609 typename SrcType::uniform_runtime_const_nomemspace_type,
610 Kokkos::LayoutLeft, ExecutionSpace, DstType::rank, int>(dst, src,
611 space);
612 }
613 }
614}
615
616template <class DstType, class SrcType>
617void view_copy(const DstType& dst, const SrcType& src) {
618 using dst_execution_space = typename DstType::execution_space;
619 using src_execution_space = typename SrcType::execution_space;
620 using dst_memory_space = typename DstType::memory_space;
621 using src_memory_space = typename SrcType::memory_space;
622
623 constexpr bool DstExecCanAccessSrc =
624 Kokkos::SpaceAccessibility<dst_execution_space,
625 src_memory_space>::accessible;
626
627 constexpr bool SrcExecCanAccessDst =
628 Kokkos::SpaceAccessibility<src_execution_space,
629 dst_memory_space>::accessible;
630
631 if (!DstExecCanAccessSrc && !SrcExecCanAccessDst) {
632 std::ostringstream ss;
633 ss << "Error: Kokkos::deep_copy with no available copy mechanism: "
634 << "from source view (\"" << src.label() << "\") to destination view (\""
635 << dst.label() << "\").\n"
636 << "There is no common execution space that can access both source's "
637 "space\n"
638 << "(" << src_memory_space().name() << ") and destination's space ("
639 << dst_memory_space().name() << "), "
640 << "so source and destination\n"
641 << "must be contiguous and have the same layout.\n";
642 Kokkos::Impl::throw_runtime_exception(ss.str());
643 }
644
645 using ExecutionSpace =
646 std::conditional_t<DstExecCanAccessSrc, dst_execution_space,
647 src_execution_space>;
648
649 // Figure out iteration order in case we need it
650 Kokkos::Iterate iterate = get_iteration_order(dst);
651
652 if ((dst.span() >= size_t(std::numeric_limits<int>::max())) ||
653 (src.span() >= size_t(std::numeric_limits<int>::max()))) {
654 if (iterate == Kokkos::Iterate::Right)
655 Kokkos::Impl::ViewCopy<
656 typename DstType::uniform_runtime_nomemspace_type,
657 typename SrcType::uniform_runtime_const_nomemspace_type,
658 Kokkos::LayoutRight, ExecutionSpace, DstType::rank, int64_t>(dst,
659 src);
660 else
661 Kokkos::Impl::ViewCopy<
662 typename DstType::uniform_runtime_nomemspace_type,
663 typename SrcType::uniform_runtime_const_nomemspace_type,
664 Kokkos::LayoutLeft, ExecutionSpace, DstType::rank, int64_t>(dst, src);
665 } else {
666 if (iterate == Kokkos::Iterate::Right)
667 Kokkos::Impl::ViewCopy<
668 typename DstType::uniform_runtime_nomemspace_type,
669 typename SrcType::uniform_runtime_const_nomemspace_type,
670 Kokkos::LayoutRight, ExecutionSpace, DstType::rank, int>(dst, src);
671 else
672 Kokkos::Impl::ViewCopy<
673 typename DstType::uniform_runtime_nomemspace_type,
674 typename SrcType::uniform_runtime_const_nomemspace_type,
675 Kokkos::LayoutLeft, ExecutionSpace, DstType::rank, int>(dst, src);
676 }
677}
678
679template <class DstType, class SrcType, class... Args>
680struct CommonSubview {
681 using dst_subview_type = typename Kokkos::Subview<DstType, Args...>;
682 using src_subview_type = typename Kokkos::Subview<SrcType, Args...>;
683 dst_subview_type dst_sub;
684 src_subview_type src_sub;
685 CommonSubview(const DstType& dst, const SrcType& src, const Args&... args)
686 : dst_sub(dst, args...), src_sub(src, args...) {}
687};
688
689template <class DstType, class SrcType, int Rank = DstType::rank>
690struct ViewRemap;
691
692template <class DstType, class SrcType>
693struct ViewRemap<DstType, SrcType, 1> {
694 using p_type = Kokkos::pair<int64_t, int64_t>;
695
696 template <typename... OptExecSpace>
697 ViewRemap(const DstType& dst, const SrcType& src,
698 const OptExecSpace&... exec_space) {
699 static_assert(
700 sizeof...(OptExecSpace) <= 1,
701 "OptExecSpace must be either empty or be an execution space!");
702
703 if (dst.extent(0) == src.extent(0)) {
704 view_copy(exec_space..., dst, src);
705 } else {
706 p_type ext0(0, std::min(dst.extent(0), src.extent(0)));
707 CommonSubview common_subview(dst, src, ext0);
708 view_copy(exec_space..., common_subview.dst_sub, common_subview.src_sub);
709 }
710 }
711};
712
713template <class DstType, class SrcType, std::size_t... I>
714auto create_common_subview_first_and_last_match(const DstType& dst,
715 const SrcType& src,
716 std::index_sequence<I...>) {
717 using p_type = Kokkos::pair<int64_t, int64_t>;
718 CommonSubview common_subview(
719 dst, src, Kokkos::ALL,
720 (p_type(0, std::min(dst.extent(I + 1), src.extent(I + 1))))...,
721 Kokkos::ALL);
722 return common_subview;
723}
724
725template <class DstType, class SrcType, std::size_t... I>
726auto create_common_subview_first_match(const DstType& dst, const SrcType& src,
727 std::index_sequence<I...>) {
728 using p_type = Kokkos::pair<int64_t, int64_t>;
729 CommonSubview common_subview(
730 dst, src, Kokkos::ALL,
731 (p_type(0, std::min(dst.extent(I + 1), src.extent(I + 1))))...);
732 return common_subview;
733}
734
735template <class DstType, class SrcType, std::size_t... I>
736auto create_common_subview_last_match(const DstType& dst, const SrcType& src,
737 std::index_sequence<I...>) {
738 using p_type = Kokkos::pair<int64_t, int64_t>;
739 CommonSubview common_subview(
740 dst, src, (p_type(0, std::min(dst.extent(I), src.extent(I))))...,
741 Kokkos::ALL);
742 return common_subview;
743}
744
745template <class DstType, class SrcType, std::size_t... I>
746auto create_common_subview_no_match(const DstType& dst, const SrcType& src,
747 std::index_sequence<I...>) {
748 using p_type = Kokkos::pair<int64_t, int64_t>;
749 CommonSubview common_subview(
750 dst, src, (p_type(0, std::min(dst.extent(I), src.extent(I))))...);
751 return common_subview;
752}
753
754template <class DstType, class SrcType, int Rank>
755struct ViewRemap {
756 using p_type = Kokkos::pair<int64_t, int64_t>;
757
758 template <typename... OptExecSpace>
759 ViewRemap(const DstType& dst, const SrcType& src,
760 const OptExecSpace&... exec_space) {
761 static_assert(
762 sizeof...(OptExecSpace) <= 1,
763 "OptExecSpace must be either empty or be an execution space!");
764
765 if (dst.extent(0) == src.extent(0)) {
766 if (dst.extent(Rank - 1) == src.extent(Rank - 1)) {
767 if constexpr (Rank < 3)
768 view_copy(exec_space..., dst, src);
769 else {
770 auto common_subview = create_common_subview_first_and_last_match(
771 dst, src, std::make_index_sequence<Rank - 2>{});
772 view_copy(exec_space..., common_subview.dst_sub,
773 common_subview.src_sub);
774 }
775 } else {
776 auto common_subview = create_common_subview_first_match(
777 dst, src, std::make_index_sequence<Rank - 1>{});
778 view_copy(exec_space..., common_subview.dst_sub,
779 common_subview.src_sub);
780 }
781 } else {
782 if (dst.extent(Rank - 1) == src.extent(Rank - 1)) {
783 auto common_subview = create_common_subview_last_match(
784 dst, src, std::make_index_sequence<Rank - 1>{});
785 view_copy(exec_space..., common_subview.dst_sub,
786 common_subview.src_sub);
787 } else {
788 auto common_subview = create_common_subview_no_match(
789 dst, src, std::make_index_sequence<Rank>{});
790 view_copy(exec_space..., common_subview.dst_sub,
791 common_subview.src_sub);
792 }
793 }
794 }
795};
796
797template <typename ExecutionSpace, class DT, class... DP>
798inline void contiguous_fill(
799 const ExecutionSpace& exec_space, const View<DT, DP...>& dst,
800 typename ViewTraits<DT, DP...>::const_value_type& value) {
801 using ViewType = View<DT, DP...>;
802 using ViewTypeFlat = Kokkos::View<
803 typename ViewType::value_type*, Kokkos::LayoutRight,
804 Kokkos::Device<typename ViewType::execution_space,
805 std::conditional_t<ViewType::rank == 0,
806 typename ViewType::memory_space,
807 Kokkos::AnonymousSpace>>,
808 Kokkos::MemoryTraits<>>;
809
810 ViewTypeFlat dst_flat(dst.data(), dst.size());
811 if (dst.span() < static_cast<size_t>(std::numeric_limits<int>::max())) {
812 Kokkos::Impl::ViewFill<ViewTypeFlat, Kokkos::LayoutRight, ExecutionSpace,
813 ViewTypeFlat::rank, int>(dst_flat, value,
814 exec_space);
815 } else
816 Kokkos::Impl::ViewFill<ViewTypeFlat, Kokkos::LayoutRight, ExecutionSpace,
817 ViewTypeFlat::rank, int64_t>(dst_flat, value,
818 exec_space);
819}
820
821// Default implementation for execution spaces that don't provide a definition
822template <typename ExecutionSpace>
823struct ZeroMemset {
824 ZeroMemset(const ExecutionSpace& exec_space, void* dst, size_t cnt) {
825 contiguous_fill(
826 exec_space,
827 Kokkos::View<std::byte*, ExecutionSpace, Kokkos::MemoryUnmanaged>(
828 static_cast<std::byte*>(dst), cnt),
829 std::byte{});
830 }
831};
832
833// Returns true when we can safely determine that the object has all 0 bits,
834// false otherwise. It is intended to determine whether to perform zero memset
835// as an optimization.
836template <typename T>
837bool has_all_zero_bits(const T& value) {
838 static_assert(std::is_trivially_copyable_v<T>);
839
840 if constexpr (std::is_scalar_v<T>) {
841 return value == T();
842 }
843
844 KOKKOS_IMPL_DISABLE_UNREACHABLE_WARNINGS_PUSH()
845 if constexpr (std::is_standard_layout_v<T> &&
846 std::has_unique_object_representations_v<T>) {
847 constexpr std::byte all_zeroes[sizeof(T)] = {};
848 return std::memcmp(&value, all_zeroes, sizeof(T)) == 0;
849 }
850
851 return false;
852 KOKKOS_IMPL_DISABLE_UNREACHABLE_WARNINGS_POP()
853}
854
855template <typename ExecutionSpace, class DT, class... DP>
856inline std::enable_if_t<
857 std::is_trivially_copyable_v<typename ViewTraits<DT, DP...>::value_type> &&
858 !ViewTraits<DT, DP...>::impl_is_customized>
859contiguous_fill_or_memset(
860 const ExecutionSpace& exec_space, const View<DT, DP...>& dst,
861 typename ViewTraits<DT, DP...>::const_value_type& value) {
862 // With OpenMP, using memset has significant performance issues.
863 if (has_all_zero_bits(value)
864#ifdef KOKKOS_ENABLE_OPENMP
865 && !std::is_same_v<ExecutionSpace, Kokkos::OpenMP>
866#endif
867 )
868 ZeroMemset(exec_space, dst.data(),
869 dst.size() * sizeof(typename ViewTraits<DT, DP...>::value_type));
870 else
871 contiguous_fill(exec_space, dst, value);
872}
873
874template <typename ExecutionSpace, class DT, class... DP>
875inline std::enable_if_t<
876 !std::is_trivially_copyable_v<typename ViewTraits<DT, DP...>::value_type> ||
877 ViewTraits<DT, DP...>::impl_is_customized>
878contiguous_fill_or_memset(
879 const ExecutionSpace& exec_space, const View<DT, DP...>& dst,
880 typename ViewTraits<DT, DP...>::const_value_type& value) {
881 contiguous_fill(exec_space, dst, value);
882}
883
884template <class DT, class... DP>
885void contiguous_fill_or_memset(
886 const View<DT, DP...>& dst,
887 typename ViewTraits<DT, DP...>::const_value_type& value) {
888 using ViewType = View<DT, DP...>;
889 using exec_space_type = typename ViewType::execution_space;
890
891 contiguous_fill_or_memset(exec_space_type(), dst, value);
892}
893} // namespace Impl
894
896template <class DT, class... DP>
897inline void deep_copy(
898 const View<DT, DP...>& dst,
899 typename ViewTraits<DT, DP...>::const_value_type& value,
900 std::enable_if_t<std::is_same_v<typename ViewTraits<DT, DP...>::specialize,
901 void>>* = nullptr) {
902 using ViewType = View<DT, DP...>;
903 using exec_space_type = typename ViewType::execution_space;
904
905 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
906 Kokkos::Profiling::beginDeepCopy(
907 Kokkos::Profiling::make_space_handle(ViewType::memory_space::name()),
908 dst.label(), dst.data(),
909 Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
910 "Scalar", &value, dst.span() * sizeof(typename ViewType::value_type));
911 }
912
913 if (dst.data() == nullptr) {
914 Kokkos::fence(
915 "Kokkos::deep_copy: scalar copy, fence because destination is null");
916 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
917 Kokkos::Profiling::endDeepCopy();
918 }
919 return;
920 }
921
922 Kokkos::fence("Kokkos::deep_copy: scalar copy, pre copy fence");
923 static_assert(std::is_same_v<typename ViewType::non_const_value_type,
924 typename ViewType::value_type>,
925 "deep_copy requires non-const type");
926
927 // If contiguous we can simply do a 1D flat loop or use memset
928 // Do not use shortcut if there is a custom accessor
929 if (dst.span_is_contiguous() && !ViewType::traits::impl_is_customized) {
930 Impl::contiguous_fill_or_memset(dst, value);
931 Kokkos::fence("Kokkos::deep_copy: scalar copy, post copy fence");
932 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
933 Kokkos::Profiling::endDeepCopy();
934 }
935 return;
936 }
937
938 // Figure out iteration order to do the ViewFill
939 int64_t strides[ViewType::rank + 1];
940 dst.stride(strides);
941 Kokkos::Iterate iterate;
942 if (std::is_same_v<typename ViewType::array_layout, Kokkos::LayoutRight>) {
943 iterate = Kokkos::Iterate::Right;
944 } else if (std::is_same_v<typename ViewType::array_layout,
945 Kokkos::LayoutLeft>) {
946 iterate = Kokkos::Iterate::Left;
947 } else if (std::is_same_v<typename ViewType::array_layout,
948 Kokkos::LayoutStride>) {
949 if (strides[0] > strides[ViewType::rank > 0 ? ViewType::rank - 1 : 0])
950 iterate = Kokkos::Iterate::Right;
951 else
952 iterate = Kokkos::Iterate::Left;
953 } else {
954 if (std::is_same_v<typename ViewType::execution_space::array_layout,
955 Kokkos::LayoutRight>)
956 iterate = Kokkos::Iterate::Right;
957 else
958 iterate = Kokkos::Iterate::Left;
959 }
960
961 // Lets call the right ViewFill functor based on integer space needed and
962 // iteration type
963 using ViewTypeUniform =
964 std::conditional_t<ViewType::rank == 0,
965 typename ViewType::uniform_runtime_type,
966 typename ViewType::uniform_runtime_nomemspace_type>;
967 if (dst.span() > static_cast<size_t>(std::numeric_limits<int>::max())) {
968 if (iterate == Kokkos::Iterate::Right)
969 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight,
970 exec_space_type, ViewType::rank, int64_t>(
971 dst, value, exec_space_type());
972 else
973 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutLeft,
974 exec_space_type, ViewType::rank, int64_t>(
975 dst, value, exec_space_type());
976 } else {
977 if (iterate == Kokkos::Iterate::Right)
978 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight,
979 exec_space_type, ViewType::rank, int>(
980 dst, value, exec_space_type());
981 else
982 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutLeft,
983 exec_space_type, ViewType::rank, int>(
984 dst, value, exec_space_type());
985 }
986 Kokkos::fence("Kokkos::deep_copy: scalar copy, post copy fence");
987
988 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
989 Kokkos::Profiling::endDeepCopy();
990 }
991}
992
994template <class ST, class... SP>
995inline void deep_copy(
996 typename ViewTraits<ST, SP...>::non_const_value_type& dst,
997 const View<ST, SP...>& src,
998 std::enable_if_t<std::is_same_v<typename ViewTraits<ST, SP...>::specialize,
999 void>>* = nullptr) {
1000 using src_traits = ViewTraits<ST, SP...>;
1001 using src_memory_space = typename src_traits::memory_space;
1002
1003 static_assert(src_traits::rank == 0,
1004 "ERROR: Non-rank-zero view in deep_copy( value , View )");
1005
1006 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
1007 Kokkos::Profiling::beginDeepCopy(
1008 Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
1009 "Scalar", &dst,
1010 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1011 src.label(), src.data(),
1012 src.span() * sizeof(typename src_traits::value_type));
1013 }
1014
1015 if (src.data() == nullptr) {
1016 Kokkos::fence("Kokkos::deep_copy: copy into scalar, src is null");
1017 } else {
1018 Kokkos::fence("Kokkos::deep_copy: copy into scalar, pre copy fence");
1019 Kokkos::Impl::DeepCopy<HostSpace, src_memory_space>(&dst, src.data(),
1020 sizeof(ST));
1021 Kokkos::fence("Kokkos::deep_copy: copy into scalar, post copy fence");
1022 }
1023
1024 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1025 Kokkos::Profiling::endDeepCopy();
1026 }
1027}
1028
1029//----------------------------------------------------------------------------
1031template <class DT, class... DP, class ST, class... SP>
1032inline void deep_copy(
1033 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1034 std::enable_if_t<
1035 (std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
1036 std::is_void_v<typename ViewTraits<ST, SP...>::specialize> &&
1037 (unsigned(ViewTraits<DT, DP...>::rank) == unsigned(0) &&
1038 unsigned(ViewTraits<ST, SP...>::rank) == unsigned(0)))>* = nullptr) {
1039 using dst_type = View<DT, DP...>;
1040 using src_type = View<ST, SP...>;
1041
1042 using value_type = typename dst_type::value_type;
1043 using dst_memory_space = typename dst_type::memory_space;
1044 using src_memory_space = typename src_type::memory_space;
1045
1046 static_assert(std::is_same_v<typename dst_type::value_type,
1047 typename src_type::non_const_value_type>,
1048 "deep_copy requires matching non-const destination type");
1049
1050 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
1051 Kokkos::Profiling::beginDeepCopy(
1052 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
1053 dst.label(), dst.data(),
1054 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1055 src.label(), src.data(),
1056 src.span() * sizeof(typename dst_type::value_type));
1057 }
1058
1059 if (dst.data() == nullptr && src.data() == nullptr) {
1060 Kokkos::fence(
1061 "Kokkos::deep_copy: scalar to scalar copy, both pointers null");
1062 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1063 Kokkos::Profiling::endDeepCopy();
1064 }
1065 return;
1066 }
1067
1068 Kokkos::fence("Kokkos::deep_copy: scalar to scalar copy, pre copy fence");
1069 if (dst.data() != src.data()) {
1070 Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space>(
1071 dst.data(), src.data(), sizeof(value_type));
1072 Kokkos::fence("Kokkos::deep_copy: scalar to scalar copy, post copy fence");
1073 }
1074 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1075 Kokkos::Profiling::endDeepCopy();
1076 }
1077}
1078
1079//----------------------------------------------------------------------------
1083template <class DT, class... DP, class ST, class... SP>
1084inline void deep_copy(
1085 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1086 std::enable_if_t<
1087 (std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
1088 std::is_void_v<typename ViewTraits<ST, SP...>::specialize> &&
1089 (unsigned(ViewTraits<DT, DP...>::rank) != 0 ||
1090 unsigned(ViewTraits<ST, SP...>::rank) != 0))>* = nullptr) {
1091 using dst_type = View<DT, DP...>;
1092 using src_type = View<ST, SP...>;
1093 using dst_memory_space = typename dst_type::memory_space;
1094 using src_memory_space = typename src_type::memory_space;
1095 using dst_ptr_type = decltype(dst.data());
1096 using src_ptr_type = decltype(src.data());
1097
1098 static_assert(std::is_same_v<typename dst_type::value_type,
1099 typename dst_type::non_const_value_type>,
1100 "deep_copy requires non-const destination type");
1101
1102 static_assert((unsigned(dst_type::rank) == unsigned(src_type::rank)),
1103 "deep_copy requires Views of equal rank");
1104
1105 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
1106 Kokkos::Profiling::beginDeepCopy(
1107 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
1108 dst.label(), dst.data(),
1109 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1110 src.label(), src.data(),
1111 src.span() * sizeof(typename dst_type::value_type));
1112 }
1113
1114 if (dst.data() == nullptr || src.data() == nullptr) {
1115 // throw if dimension mismatch
1116 if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
1117 (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
1118 (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
1119 (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
1120 std::string message(
1121 "Deprecation Error: Kokkos::deep_copy extents of views don't "
1122 "match: ");
1123 message += dst.label();
1124 message += "(";
1125 message += std::to_string(dst.extent(0));
1126 for (size_t r = 1; r < dst_type::rank; r++) {
1127 message += ",";
1128 message += std::to_string(dst.extent(r));
1129 }
1130 message += ") ";
1131 message += src.label();
1132 message += "(";
1133 message += std::to_string(src.extent(0));
1134 for (size_t r = 1; r < src_type::rank; r++) {
1135 message += ",";
1136 message += std::to_string(src.extent(r));
1137 }
1138 message += ") ";
1139
1140 Kokkos::Impl::throw_runtime_exception(message);
1141 }
1142 Kokkos::fence(
1143 "Kokkos::deep_copy: copy between contiguous views, fence due to null "
1144 "argument");
1145 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1146 Kokkos::Profiling::endDeepCopy();
1147 }
1148 return;
1149 }
1150
1151 // Checking for Overlapping Views.
1152 dst_ptr_type dst_start = dst.data();
1153 src_ptr_type src_start = src.data();
1154#ifndef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
1155 dst_ptr_type dst_end = dst.data() + allocation_size_from_mapping_and_accessor(
1156 dst.mapping(), dst.accessor());
1157 src_ptr_type src_end = src.data() + allocation_size_from_mapping_and_accessor(
1158 src.mapping(), src.accessor());
1159#else
1160 dst_ptr_type dst_end = dst.data() + dst.span();
1161 src_ptr_type src_end = src.data() + src.span();
1162#endif
1163 if (((std::ptrdiff_t)dst_start == (std::ptrdiff_t)src_start) &&
1164 ((std::ptrdiff_t)dst_end == (std::ptrdiff_t)src_end) &&
1165 (dst.span_is_contiguous() && src.span_is_contiguous())) {
1166 Kokkos::fence(
1167 "Kokkos::deep_copy: copy between contiguous views, fence due to same "
1168 "spans");
1169 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1170 Kokkos::Profiling::endDeepCopy();
1171 }
1172 return;
1173 }
1174
1175 if ((((std::ptrdiff_t)dst_start < (std::ptrdiff_t)src_end) &&
1176 ((std::ptrdiff_t)dst_end > (std::ptrdiff_t)src_start)) &&
1177 ((dst.span_is_contiguous() && src.span_is_contiguous()))) {
1178 std::string message("Error: Kokkos::deep_copy of overlapping views: ");
1179 message += dst.label();
1180 message += "(";
1181 message += std::to_string((std::ptrdiff_t)dst_start);
1182 message += ",";
1183 message += std::to_string((std::ptrdiff_t)dst_end);
1184 message += ") ";
1185 message += src.label();
1186 message += "(";
1187 message += std::to_string((std::ptrdiff_t)src_start);
1188 message += ",";
1189 message += std::to_string((std::ptrdiff_t)src_end);
1190 message += ") ";
1191 Kokkos::Impl::throw_runtime_exception(message);
1192 }
1193
1194 // Check for same extents
1195 if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
1196 (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
1197 (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
1198 (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
1199 std::string message(
1200 "Deprecation Error: Kokkos::deep_copy extents of views don't match: ");
1201 message += dst.label();
1202 message += "(";
1203 message += std::to_string(dst.extent(0));
1204 for (size_t r = 1; r < dst_type::rank; r++) {
1205 message += ",";
1206 message += std::to_string(dst.extent(r));
1207 }
1208 message += ") ";
1209 message += src.label();
1210 message += "(";
1211 message += std::to_string(src.extent(0));
1212 for (size_t r = 1; r < src_type::rank; r++) {
1213 message += ",";
1214 message += std::to_string(src.extent(r));
1215 }
1216 message += ") ";
1217
1218 Kokkos::Impl::throw_runtime_exception(message);
1219 }
1220
1221 // If same type, equal layout, equal dimensions, equal span, and contiguous
1222 // memory then can byte-wise copy
1223
1224 if (std::is_same_v<typename dst_type::value_type,
1225 typename src_type::non_const_value_type> &&
1226 (std::is_same_v<typename dst_type::array_layout,
1227 typename src_type::array_layout> ||
1228 (dst_type::rank == 1 && src_type::rank == 1)) &&
1229 dst.span_is_contiguous() && src.span_is_contiguous() &&
1230 ((dst_type::rank < 1) || (dst.stride_0() == src.stride_0())) &&
1231 ((dst_type::rank < 2) || (dst.stride_1() == src.stride_1())) &&
1232 ((dst_type::rank < 3) || (dst.stride_2() == src.stride_2())) &&
1233 ((dst_type::rank < 4) || (dst.stride_3() == src.stride_3())) &&
1234 ((dst_type::rank < 5) || (dst.stride_4() == src.stride_4())) &&
1235 ((dst_type::rank < 6) || (dst.stride_5() == src.stride_5())) &&
1236 ((dst_type::rank < 7) || (dst.stride_6() == src.stride_6())) &&
1237 ((dst_type::rank < 8) || (dst.stride_7() == src.stride_7()))) {
1238#ifndef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
1239 const size_t nbytes = allocation_size_from_mapping_and_accessor(
1240 src.mapping(), src.accessor()) *
1241 sizeof(std::remove_pointer_t<dst_ptr_type>);
1242#else
1243 const size_t nbytes = sizeof(typename dst_type::value_type) * dst.span();
1244#endif
1245 Kokkos::fence(
1246 "Kokkos::deep_copy: copy between contiguous views, pre view equality "
1247 "check");
1248 if ((void*)dst.data() != (void*)src.data() && 0 < nbytes) {
1249 Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space>(
1250 dst.data(), src.data(), nbytes);
1251 Kokkos::fence(
1252 "Kokkos::deep_copy: copy between contiguous views, post deep copy "
1253 "fence");
1254 }
1255 } else {
1256 Kokkos::fence(
1257 "Kokkos::deep_copy: copy between contiguous views, pre copy fence");
1258 Impl::view_copy(dst, src);
1259 Kokkos::fence(
1260 "Kokkos::deep_copy: copy between contiguous views, post copy fence");
1261 }
1262 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1263 Kokkos::Profiling::endDeepCopy();
1264 }
1265}
1266
1267//----------------------------------------------------------------------------
1268//----------------------------------------------------------------------------
1269namespace Experimental {
1273template <class TeamType, class DT, class... DP, class ST, class... SP>
1274void KOKKOS_INLINE_FUNCTION
1275local_deep_copy_contiguous(const TeamType& team, const View<DT, DP...>& dst,
1276 const View<ST, SP...>& src) {
1277 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, src.span()),
1278 [&](const int& i) { dst.data()[i] = src.data()[i]; });
1279}
1280//----------------------------------------------------------------------------
1281template <class DT, class... DP, class ST, class... SP>
1282void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1283 const View<DT, DP...>& dst, const View<ST, SP...>& src) {
1284 for (size_t i = 0; i < src.span(); ++i) {
1285 dst.data()[i] = src.data()[i];
1286 }
1287}
1288//----------------------------------------------------------------------------
1289template <class TeamType, class DT, class... DP, class ST, class... SP>
1290void KOKKOS_INLINE_FUNCTION local_deep_copy(
1291 const TeamType& team, const View<DT, DP...>& dst,
1292 const View<ST, SP...>& src,
1293 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 1 &&
1294 unsigned(ViewTraits<ST, SP...>::rank) == 1)>* = nullptr) {
1295 if (dst.data() == nullptr) {
1296 return;
1297 }
1298
1299 const size_t N = dst.extent(0);
1300
1301 team.team_barrier();
1302 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N),
1303 [&](const int& i) { dst(i) = src(i); });
1304 team.team_barrier();
1305}
1306//----------------------------------------------------------------------------
1307template <class TeamType, class DT, class... DP, class ST, class... SP>
1308void KOKKOS_INLINE_FUNCTION local_deep_copy(
1309 const TeamType& team, const View<DT, DP...>& dst,
1310 const View<ST, SP...>& src,
1311 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 2 &&
1312 unsigned(ViewTraits<ST, SP...>::rank) == 2)>* = nullptr) {
1313 if (dst.data() == nullptr) {
1314 return;
1315 }
1316
1317 const size_t N = dst.extent(0) * dst.extent(1);
1318
1319 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1320 team.team_barrier();
1321 local_deep_copy_contiguous(team, dst, src);
1322 team.team_barrier();
1323 } else {
1324 team.team_barrier();
1325 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1326 int i0 = i % dst.extent(0);
1327 int i1 = i / dst.extent(0);
1328 dst(i0, i1) = src(i0, i1);
1329 });
1330 team.team_barrier();
1331 }
1332}
1333//----------------------------------------------------------------------------
1334template <class TeamType, class DT, class... DP, class ST, class... SP>
1335void KOKKOS_INLINE_FUNCTION local_deep_copy(
1336 const TeamType& team, const View<DT, DP...>& dst,
1337 const View<ST, SP...>& src,
1338 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 3 &&
1339 unsigned(ViewTraits<ST, SP...>::rank) == 3)>* = nullptr) {
1340 if (dst.data() == nullptr) {
1341 return;
1342 }
1343
1344 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2);
1345
1346 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1347 team.team_barrier();
1348 local_deep_copy_contiguous(team, dst, src);
1349 team.team_barrier();
1350 } else {
1351 team.team_barrier();
1352 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1353 int i0 = i % dst.extent(0);
1354 int itmp = i / dst.extent(0);
1355 int i1 = itmp % dst.extent(1);
1356 int i2 = itmp / dst.extent(1);
1357 dst(i0, i1, i2) = src(i0, i1, i2);
1358 });
1359 team.team_barrier();
1360 }
1361}
1362//----------------------------------------------------------------------------
1363template <class TeamType, class DT, class... DP, class ST, class... SP>
1364void KOKKOS_INLINE_FUNCTION local_deep_copy(
1365 const TeamType& team, const View<DT, DP...>& dst,
1366 const View<ST, SP...>& src,
1367 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 4 &&
1368 unsigned(ViewTraits<ST, SP...>::rank) == 4)>* = nullptr) {
1369 if (dst.data() == nullptr) {
1370 return;
1371 }
1372
1373 const size_t N =
1374 dst.extent(0) * dst.extent(1) * dst.extent(2) * dst.extent(3);
1375
1376 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1377 team.team_barrier();
1378 local_deep_copy_contiguous(team, dst, src);
1379 team.team_barrier();
1380 } else {
1381 team.team_barrier();
1382 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1383 int i0 = i % dst.extent(0);
1384 int itmp = i / dst.extent(0);
1385 int i1 = itmp % dst.extent(1);
1386 itmp = itmp / dst.extent(1);
1387 int i2 = itmp % dst.extent(2);
1388 int i3 = itmp / dst.extent(2);
1389 dst(i0, i1, i2, i3) = src(i0, i1, i2, i3);
1390 });
1391 team.team_barrier();
1392 }
1393}
1394//----------------------------------------------------------------------------
1395template <class TeamType, class DT, class... DP, class ST, class... SP>
1396void KOKKOS_INLINE_FUNCTION local_deep_copy(
1397 const TeamType& team, const View<DT, DP...>& dst,
1398 const View<ST, SP...>& src,
1399 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 5 &&
1400 unsigned(ViewTraits<ST, SP...>::rank) == 5)>* = nullptr) {
1401 if (dst.data() == nullptr) {
1402 return;
1403 }
1404
1405 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1406 dst.extent(3) * dst.extent(4);
1407
1408 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1409 team.team_barrier();
1410 local_deep_copy_contiguous(team, dst, src);
1411 team.team_barrier();
1412 } else {
1413 team.team_barrier();
1414 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1415 int i0 = i % dst.extent(0);
1416 int itmp = i / dst.extent(0);
1417 int i1 = itmp % dst.extent(1);
1418 itmp = itmp / dst.extent(1);
1419 int i2 = itmp % dst.extent(2);
1420 itmp = itmp / dst.extent(2);
1421 int i3 = itmp % dst.extent(3);
1422 int i4 = itmp / dst.extent(3);
1423 dst(i0, i1, i2, i3, i4) = src(i0, i1, i2, i3, i4);
1424 });
1425 team.team_barrier();
1426 }
1427}
1428//----------------------------------------------------------------------------
1429template <class TeamType, class DT, class... DP, class ST, class... SP>
1430void KOKKOS_INLINE_FUNCTION local_deep_copy(
1431 const TeamType& team, const View<DT, DP...>& dst,
1432 const View<ST, SP...>& src,
1433 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 6 &&
1434 unsigned(ViewTraits<ST, SP...>::rank) == 6)>* = nullptr) {
1435 if (dst.data() == nullptr) {
1436 return;
1437 }
1438
1439 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1440 dst.extent(3) * dst.extent(4) * dst.extent(5);
1441
1442 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1443 team.team_barrier();
1444 local_deep_copy_contiguous(team, dst, src);
1445 team.team_barrier();
1446 } else {
1447 team.team_barrier();
1448 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1449 int i0 = i % dst.extent(0);
1450 int itmp = i / dst.extent(0);
1451 int i1 = itmp % dst.extent(1);
1452 itmp = itmp / dst.extent(1);
1453 int i2 = itmp % dst.extent(2);
1454 itmp = itmp / dst.extent(2);
1455 int i3 = itmp % dst.extent(3);
1456 itmp = itmp / dst.extent(3);
1457 int i4 = itmp % dst.extent(4);
1458 int i5 = itmp / dst.extent(4);
1459 dst(i0, i1, i2, i3, i4, i5) = src(i0, i1, i2, i3, i4, i5);
1460 });
1461 team.team_barrier();
1462 }
1463}
1464//----------------------------------------------------------------------------
1465template <class TeamType, class DT, class... DP, class ST, class... SP>
1466void KOKKOS_INLINE_FUNCTION local_deep_copy(
1467 const TeamType& team, const View<DT, DP...>& dst,
1468 const View<ST, SP...>& src,
1469 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 7 &&
1470 unsigned(ViewTraits<ST, SP...>::rank) == 7)>* = nullptr) {
1471 if (dst.data() == nullptr) {
1472 return;
1473 }
1474
1475 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1476 dst.extent(3) * dst.extent(4) * dst.extent(5) *
1477 dst.extent(6);
1478
1479 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1480 team.team_barrier();
1481 local_deep_copy_contiguous(team, dst, src);
1482 team.team_barrier();
1483 } else {
1484 team.team_barrier();
1485 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1486 int i0 = i % dst.extent(0);
1487 int itmp = i / dst.extent(0);
1488 int i1 = itmp % dst.extent(1);
1489 itmp = itmp / dst.extent(1);
1490 int i2 = itmp % dst.extent(2);
1491 itmp = itmp / dst.extent(2);
1492 int i3 = itmp % dst.extent(3);
1493 itmp = itmp / dst.extent(3);
1494 int i4 = itmp % dst.extent(4);
1495 itmp = itmp / dst.extent(4);
1496 int i5 = itmp % dst.extent(5);
1497 int i6 = itmp / dst.extent(5);
1498 dst(i0, i1, i2, i3, i4, i5, i6) = src(i0, i1, i2, i3, i4, i5, i6);
1499 });
1500 team.team_barrier();
1501 }
1502}
1503//----------------------------------------------------------------------------
1504template <class DT, class... DP, class ST, class... SP>
1505void KOKKOS_INLINE_FUNCTION local_deep_copy(
1506 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1507 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 1 &&
1508 unsigned(ViewTraits<ST, SP...>::rank) == 1)>* = nullptr) {
1509 if (dst.data() == nullptr) {
1510 return;
1511 }
1512
1513 const size_t N = dst.extent(0);
1514
1515 for (size_t i = 0; i < N; ++i) {
1516 dst(i) = src(i);
1517 }
1518}
1519//----------------------------------------------------------------------------
1520template <class DT, class... DP, class ST, class... SP>
1521void KOKKOS_INLINE_FUNCTION local_deep_copy(
1522 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1523 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 2 &&
1524 unsigned(ViewTraits<ST, SP...>::rank) == 2)>* = nullptr) {
1525 if (dst.data() == nullptr) {
1526 return;
1527 }
1528
1529 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1530 local_deep_copy_contiguous(dst, src);
1531 } else {
1532 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1533 for (size_t i1 = 0; i1 < dst.extent(1); ++i1) dst(i0, i1) = src(i0, i1);
1534 }
1535}
1536//----------------------------------------------------------------------------
1537template <class DT, class... DP, class ST, class... SP>
1538void KOKKOS_INLINE_FUNCTION local_deep_copy(
1539 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1540 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 3 &&
1541 unsigned(ViewTraits<ST, SP...>::rank) == 3)>* = nullptr) {
1542 if (dst.data() == nullptr) {
1543 return;
1544 }
1545
1546 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1547 local_deep_copy_contiguous(dst, src);
1548 } else {
1549 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1550 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1551 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1552 dst(i0, i1, i2) = src(i0, i1, i2);
1553 }
1554}
1555//----------------------------------------------------------------------------
1556template <class DT, class... DP, class ST, class... SP>
1557void KOKKOS_INLINE_FUNCTION local_deep_copy(
1558 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1559 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 4 &&
1560 unsigned(ViewTraits<ST, SP...>::rank) == 4)>* = nullptr) {
1561 if (dst.data() == nullptr) {
1562 return;
1563 }
1564
1565 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1566 local_deep_copy_contiguous(dst, src);
1567 } else {
1568 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1569 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1570 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1571 for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1572 dst(i0, i1, i2, i3) = src(i0, i1, i2, i3);
1573 }
1574}
1575//----------------------------------------------------------------------------
1576template <class DT, class... DP, class ST, class... SP>
1577void KOKKOS_INLINE_FUNCTION local_deep_copy(
1578 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1579 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 5 &&
1580 unsigned(ViewTraits<ST, SP...>::rank) == 5)>* = nullptr) {
1581 if (dst.data() == nullptr) {
1582 return;
1583 }
1584
1585 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1586 local_deep_copy_contiguous(dst, src);
1587 } else {
1588 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1589 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1590 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1591 for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1592 for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1593 dst(i0, i1, i2, i3, i4) = src(i0, i1, i2, i3, i4);
1594 }
1595}
1596//----------------------------------------------------------------------------
1597template <class DT, class... DP, class ST, class... SP>
1598void KOKKOS_INLINE_FUNCTION local_deep_copy(
1599 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1600 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 6 &&
1601 unsigned(ViewTraits<ST, SP...>::rank) == 6)>* = nullptr) {
1602 if (dst.data() == nullptr) {
1603 return;
1604 }
1605
1606 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1607 local_deep_copy_contiguous(dst, src);
1608 } else {
1609 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1610 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1611 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1612 for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1613 for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1614 for (size_t i5 = 0; i5 < dst.extent(5); ++i5)
1615 dst(i0, i1, i2, i3, i4, i5) = src(i0, i1, i2, i3, i4, i5);
1616 }
1617}
1618//----------------------------------------------------------------------------
1619template <class DT, class... DP, class ST, class... SP>
1620void KOKKOS_INLINE_FUNCTION local_deep_copy(
1621 const View<DT, DP...>& dst, const View<ST, SP...>& src,
1622 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 7 &&
1623 unsigned(ViewTraits<ST, SP...>::rank) == 7)>* = nullptr) {
1624 if (dst.data() == nullptr) {
1625 return;
1626 }
1627
1628 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1629 local_deep_copy_contiguous(dst, src);
1630 } else {
1631 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1632 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1633 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1634 for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1635 for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1636 for (size_t i5 = 0; i5 < dst.extent(5); ++i5)
1637 for (size_t i6 = 0; i6 < dst.extent(6); ++i6)
1638 dst(i0, i1, i2, i3, i4, i5, i6) =
1639 src(i0, i1, i2, i3, i4, i5, i6);
1640 }
1641}
1642//----------------------------------------------------------------------------
1643//----------------------------------------------------------------------------
1645template <class TeamType, class DT, class... DP>
1646void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1647 const TeamType& team, const View<DT, DP...>& dst,
1648 typename ViewTraits<DT, DP...>::const_value_type& value,
1649 std::enable_if_t<std::is_same_v<typename ViewTraits<DT, DP...>::specialize,
1650 void>>* = nullptr) {
1651 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, dst.span()),
1652 [&](const int& i) { dst.data()[i] = value; });
1653}
1654//----------------------------------------------------------------------------
1655template <class DT, class... DP>
1656void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1657 const View<DT, DP...>& dst,
1658 typename ViewTraits<DT, DP...>::const_value_type& value,
1659 std::enable_if_t<std::is_same_v<typename ViewTraits<DT, DP...>::specialize,
1660 void>>* = nullptr) {
1661 for (size_t i = 0; i < dst.span(); ++i) {
1662 dst.data()[i] = value;
1663 }
1664}
1665//----------------------------------------------------------------------------
1666template <class TeamType, class DT, class... DP>
1667void KOKKOS_INLINE_FUNCTION local_deep_copy(
1668 const TeamType& team, const View<DT, DP...>& dst,
1669 typename ViewTraits<DT, DP...>::const_value_type& value,
1670 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 1)>* = nullptr) {
1671 if (dst.data() == nullptr) {
1672 return;
1673 }
1674
1675 const size_t N = dst.extent(0);
1676
1677 team.team_barrier();
1678 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N),
1679 [&](const int& i) { dst(i) = value; });
1680 team.team_barrier();
1681}
1682//----------------------------------------------------------------------------
1683template <class TeamType, class DT, class... DP>
1684void KOKKOS_INLINE_FUNCTION local_deep_copy(
1685 const TeamType& team, const View<DT, DP...>& dst,
1686 typename ViewTraits<DT, DP...>::const_value_type& value,
1687 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 2)>* = nullptr) {
1688 if (dst.data() == nullptr) {
1689 return;
1690 }
1691
1692 const size_t N = dst.extent(0) * dst.extent(1);
1693
1694 if (dst.span_is_contiguous()) {
1695 // FIXME We might want to check the traits for customization here but we
1696 // aren't aware of a use case where that is necessary.
1697 if constexpr (std::is_same_v<decltype(dst.data()),
1698 typename View<DT, DP...>::element_type*>) {
1699 team.team_barrier();
1700 local_deep_copy_contiguous(team, dst, value);
1701 team.team_barrier();
1702 return;
1703 }
1704 }
1705 team.team_barrier();
1706 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1707 int i0 = i % dst.extent(0);
1708 int i1 = i / dst.extent(0);
1709 dst(i0, i1) = value;
1710 });
1711 team.team_barrier();
1712}
1713//----------------------------------------------------------------------------
1714template <class TeamType, class DT, class... DP>
1715void KOKKOS_INLINE_FUNCTION local_deep_copy(
1716 const TeamType& team, const View<DT, DP...>& dst,
1717 typename ViewTraits<DT, DP...>::const_value_type& value,
1718 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 3)>* = nullptr) {
1719 if (dst.data() == nullptr) {
1720 return;
1721 }
1722
1723 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2);
1724
1725 if (dst.span_is_contiguous()) {
1726 // FIXME We might want to check the traits for customization here but we
1727 // aren't aware of a use case where that is necessary.
1728 if constexpr (std::is_same_v<decltype(dst.data()),
1729 typename View<DT, DP...>::element_type*>) {
1730 team.team_barrier();
1731 local_deep_copy_contiguous(team, dst, value);
1732 team.team_barrier();
1733 return;
1734 }
1735 }
1736 team.team_barrier();
1737 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1738 int i0 = i % dst.extent(0);
1739 int itmp = i / dst.extent(0);
1740 int i1 = itmp % dst.extent(1);
1741 int i2 = itmp / dst.extent(1);
1742 dst(i0, i1, i2) = value;
1743 });
1744 team.team_barrier();
1745}
1746//----------------------------------------------------------------------------
1747template <class TeamType, class DT, class... DP>
1748void KOKKOS_INLINE_FUNCTION local_deep_copy(
1749 const TeamType& team, const View<DT, DP...>& dst,
1750 typename ViewTraits<DT, DP...>::const_value_type& value,
1751 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 4)>* = nullptr) {
1752 if (dst.data() == nullptr) {
1753 return;
1754 }
1755
1756 const size_t N =
1757 dst.extent(0) * dst.extent(1) * dst.extent(2) * dst.extent(3);
1758
1759 if (dst.span_is_contiguous()) {
1760 // FIXME We might want to check the traits for customization here but we
1761 // aren't aware of a use case where that is necessary.
1762 if constexpr (std::is_same_v<decltype(dst.data()),
1763 typename View<DT, DP...>::element_type*>) {
1764 team.team_barrier();
1765 local_deep_copy_contiguous(team, dst, value);
1766 team.team_barrier();
1767 return;
1768 }
1769 }
1770 team.team_barrier();
1771 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1772 int i0 = i % dst.extent(0);
1773 int itmp = i / dst.extent(0);
1774 int i1 = itmp % dst.extent(1);
1775 itmp = itmp / dst.extent(1);
1776 int i2 = itmp % dst.extent(2);
1777 int i3 = itmp / dst.extent(2);
1778 dst(i0, i1, i2, i3) = value;
1779 });
1780 team.team_barrier();
1781}
1782//----------------------------------------------------------------------------
1783template <class TeamType, class DT, class... DP>
1784void KOKKOS_INLINE_FUNCTION local_deep_copy(
1785 const TeamType& team, const View<DT, DP...>& dst,
1786 typename ViewTraits<DT, DP...>::const_value_type& value,
1787 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 5)>* = nullptr) {
1788 if (dst.data() == nullptr) {
1789 return;
1790 }
1791
1792 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1793 dst.extent(3) * dst.extent(4);
1794
1795 if (dst.span_is_contiguous()) {
1796 // FIXME We might want to check the traits for customization here but we
1797 // aren't aware of a use case where that is necessary.
1798 if constexpr (std::is_same_v<decltype(dst.data()),
1799 typename View<DT, DP...>::element_type*>) {
1800 team.team_barrier();
1801 local_deep_copy_contiguous(team, dst, value);
1802 team.team_barrier();
1803 return;
1804 }
1805 }
1806 team.team_barrier();
1807 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1808 int i0 = i % dst.extent(0);
1809 int itmp = i / dst.extent(0);
1810 int i1 = itmp % dst.extent(1);
1811 itmp = itmp / dst.extent(1);
1812 int i2 = itmp % dst.extent(2);
1813 itmp = itmp / dst.extent(2);
1814 int i3 = itmp % dst.extent(3);
1815 int i4 = itmp / dst.extent(3);
1816 dst(i0, i1, i2, i3, i4) = value;
1817 });
1818 team.team_barrier();
1819}
1820//----------------------------------------------------------------------------
1821template <class TeamType, class DT, class... DP>
1822void KOKKOS_INLINE_FUNCTION local_deep_copy(
1823 const TeamType& team, const View<DT, DP...>& dst,
1824 typename ViewTraits<DT, DP...>::const_value_type& value,
1825 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 6)>* = nullptr) {
1826 if (dst.data() == nullptr) {
1827 return;
1828 }
1829
1830 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1831 dst.extent(3) * dst.extent(4) * dst.extent(5);
1832
1833 if (dst.span_is_contiguous()) {
1834 // FIXME We might want to check the traits for customization here but we
1835 // aren't aware of a use case where that is necessary.
1836 if constexpr (std::is_same_v<decltype(dst.data()),
1837 typename View<DT, DP...>::element_type*>) {
1838 team.team_barrier();
1839 local_deep_copy_contiguous(team, dst, value);
1840 team.team_barrier();
1841 return;
1842 }
1843 }
1844 team.team_barrier();
1845 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1846 int i0 = i % dst.extent(0);
1847 int itmp = i / dst.extent(0);
1848 int i1 = itmp % dst.extent(1);
1849 itmp = itmp / dst.extent(1);
1850 int i2 = itmp % dst.extent(2);
1851 itmp = itmp / dst.extent(2);
1852 int i3 = itmp % dst.extent(3);
1853 itmp = itmp / dst.extent(3);
1854 int i4 = itmp % dst.extent(4);
1855 int i5 = itmp / dst.extent(4);
1856 dst(i0, i1, i2, i3, i4, i5) = value;
1857 });
1858 team.team_barrier();
1859}
1860//----------------------------------------------------------------------------
1861template <class TeamType, class DT, class... DP>
1862void KOKKOS_INLINE_FUNCTION local_deep_copy(
1863 const TeamType& team, const View<DT, DP...>& dst,
1864 typename ViewTraits<DT, DP...>::const_value_type& value,
1865 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 7)>* = nullptr) {
1866 if (dst.data() == nullptr) {
1867 return;
1868 }
1869
1870 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1871 dst.extent(3) * dst.extent(4) * dst.extent(5) *
1872 dst.extent(6);
1873
1874 if (dst.span_is_contiguous()) {
1875 // FIXME We might want to check the traits for customization here but we
1876 // aren't aware of a use case where that is necessary.
1877 if constexpr (std::is_same_v<decltype(dst.data()),
1878 typename View<DT, DP...>::element_type*>) {
1879 team.team_barrier();
1880 local_deep_copy_contiguous(team, dst, value);
1881 team.team_barrier();
1882 return;
1883 }
1884 }
1885 team.team_barrier();
1886 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1887 int i0 = i % dst.extent(0);
1888 int itmp = i / dst.extent(0);
1889 int i1 = itmp % dst.extent(1);
1890 itmp = itmp / dst.extent(1);
1891 int i2 = itmp % dst.extent(2);
1892 itmp = itmp / dst.extent(2);
1893 int i3 = itmp % dst.extent(3);
1894 itmp = itmp / dst.extent(3);
1895 int i4 = itmp % dst.extent(4);
1896 itmp = itmp / dst.extent(4);
1897 int i5 = itmp % dst.extent(5);
1898 int i6 = itmp / dst.extent(5);
1899 dst(i0, i1, i2, i3, i4, i5, i6) = value;
1900 });
1901 team.team_barrier();
1902}
1903//----------------------------------------------------------------------------
1904template <class DT, class... DP>
1905void KOKKOS_INLINE_FUNCTION local_deep_copy(
1906 const View<DT, DP...>& dst,
1907 typename ViewTraits<DT, DP...>::const_value_type& value,
1908 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 1)>* = nullptr) {
1909 if (dst.data() == nullptr) {
1910 return;
1911 }
1912
1913 const size_t N = dst.extent(0);
1914
1915 for (size_t i = 0; i < N; ++i) {
1916 dst(i) = value;
1917 }
1918}
1919//----------------------------------------------------------------------------
1920template <class DT, class... DP>
1921void KOKKOS_INLINE_FUNCTION local_deep_copy(
1922 const View<DT, DP...>& dst,
1923 typename ViewTraits<DT, DP...>::const_value_type& value,
1924 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 2)>* = nullptr) {
1925 if (dst.data() == nullptr) {
1926 return;
1927 }
1928
1929 if (dst.span_is_contiguous()) {
1930 // FIXME We might want to check the traits for customization here but we
1931 // aren't aware of a use case where that is necessary.
1932 if constexpr (std::is_same_v<decltype(dst.data()),
1933 typename View<DT, DP...>::element_type*>) {
1934 local_deep_copy_contiguous(dst, value);
1935 return;
1936 }
1937 }
1938 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1939 for (size_t i1 = 0; i1 < dst.extent(1); ++i1) dst(i0, i1) = value;
1940}
1941//----------------------------------------------------------------------------
1942template <class DT, class... DP>
1943void KOKKOS_INLINE_FUNCTION local_deep_copy(
1944 const View<DT, DP...>& dst,
1945 typename ViewTraits<DT, DP...>::const_value_type& value,
1946 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 3)>* = nullptr) {
1947 if (dst.data() == nullptr) {
1948 return;
1949 }
1950
1951 if (dst.span_is_contiguous()) {
1952 // FIXME We might want to check the traits for customization here but we
1953 // aren't aware of a use case where that is necessary.
1954 if constexpr (std::is_same_v<decltype(dst.data()),
1955 typename View<DT, DP...>::element_type*>) {
1956 local_deep_copy_contiguous(dst, value);
1957 return;
1958 }
1959 }
1960 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1961 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1962 for (size_t i2 = 0; i2 < dst.extent(2); ++i2) dst(i0, i1, i2) = value;
1963}
1964//----------------------------------------------------------------------------
1965template <class DT, class... DP>
1966void KOKKOS_INLINE_FUNCTION local_deep_copy(
1967 const View<DT, DP...>& dst,
1968 typename ViewTraits<DT, DP...>::const_value_type& value,
1969 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 4)>* = nullptr) {
1970 if (dst.data() == nullptr) {
1971 return;
1972 }
1973
1974 if (dst.span_is_contiguous()) {
1975 // FIXME We might want to check the traits for customization here but we
1976 // aren't aware of a use case where that is necessary.
1977 if constexpr (std::is_same_v<decltype(dst.data()),
1978 typename View<DT, DP...>::element_type*>) {
1979 local_deep_copy_contiguous(dst, value);
1980 return;
1981 }
1982 }
1983 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1984 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1985 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1986 for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1987 dst(i0, i1, i2, i3) = value;
1988}
1989//----------------------------------------------------------------------------
1990template <class DT, class... DP>
1991void KOKKOS_INLINE_FUNCTION local_deep_copy(
1992 const View<DT, DP...>& dst,
1993 typename ViewTraits<DT, DP...>::const_value_type& value,
1994 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 5)>* = nullptr) {
1995 if (dst.data() == nullptr) {
1996 return;
1997 }
1998
1999 if (dst.span_is_contiguous()) {
2000 // FIXME We might want to check the traits for customization here but we
2001 // aren't aware of a use case where that is necessary.
2002 if constexpr (std::is_same_v<decltype(dst.data()),
2003 typename View<DT, DP...>::element_type*>) {
2004 local_deep_copy_contiguous(dst, value);
2005 return;
2006 }
2007 }
2008 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
2009 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
2010 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
2011 for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
2012 for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
2013 dst(i0, i1, i2, i3, i4) = value;
2014}
2015//----------------------------------------------------------------------------
2016template <class DT, class... DP>
2017void KOKKOS_INLINE_FUNCTION local_deep_copy(
2018 const View<DT, DP...>& dst,
2019 typename ViewTraits<DT, DP...>::const_value_type& value,
2020 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 6)>* = nullptr) {
2021 if (dst.data() == nullptr) {
2022 return;
2023 }
2024
2025 if (dst.span_is_contiguous()) {
2026 // FIXME We might want to check the traits for customization here but we
2027 // aren't aware of a use case where that is necessary.
2028 if constexpr (std::is_same_v<decltype(dst.data()),
2029 typename View<DT, DP...>::element_type*>) {
2030 local_deep_copy_contiguous(dst, value);
2031 return;
2032 }
2033 }
2034 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
2035 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
2036 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
2037 for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
2038 for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
2039 for (size_t i5 = 0; i5 < dst.extent(5); ++i5)
2040 dst(i0, i1, i2, i3, i4, i5) = value;
2041}
2042//----------------------------------------------------------------------------
2043template <class DT, class... DP>
2044void KOKKOS_INLINE_FUNCTION local_deep_copy(
2045 const View<DT, DP...>& dst,
2046 typename ViewTraits<DT, DP...>::const_value_type& value,
2047 std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 7)>* = nullptr) {
2048 if (dst.data() == nullptr) {
2049 return;
2050 }
2051
2052 if (dst.span_is_contiguous()) {
2053 // FIXME We might want to check the traits for customization here but we
2054 // aren't aware of a use case where that is necessary.
2055 if constexpr (std::is_same_v<decltype(dst.data()),
2056 typename View<DT, DP...>::element_type*>) {
2057 local_deep_copy_contiguous(dst, value);
2058 return;
2059 }
2060 }
2061 for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
2062 for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
2063 for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
2064 for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
2065 for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
2066 for (size_t i5 = 0; i5 < dst.extent(5); ++i5)
2067 for (size_t i6 = 0; i6 < dst.extent(6); ++i6)
2068 dst(i0, i1, i2, i3, i4, i5, i6) = value;
2069}
2070} /* namespace Experimental */
2071} /* namespace Kokkos */
2072
2073//----------------------------------------------------------------------------
2074//----------------------------------------------------------------------------
2075
2076namespace Kokkos {
2077
2080template <class ExecSpace, class DT, class... DP>
2081inline void deep_copy(
2082 const ExecSpace& space, const View<DT, DP...>& dst,
2083 typename ViewTraits<DT, DP...>::const_value_type& value,
2084 std::enable_if_t<
2085 Kokkos::is_execution_space<ExecSpace>::value &&
2086 std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
2087 Kokkos::SpaceAccessibility<ExecSpace, typename ViewTraits<DT, DP...>::
2088 memory_space>::accessible>* =
2089 nullptr) {
2090 using dst_traits = ViewTraits<DT, DP...>;
2091 static_assert(std::is_same_v<typename dst_traits::non_const_value_type,
2092 typename dst_traits::value_type>,
2093 "deep_copy requires non-const type");
2094 using dst_memory_space = typename dst_traits::memory_space;
2095 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2096 Kokkos::Profiling::beginDeepCopy(
2097 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2098 dst.label(), dst.data(),
2099 Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
2100 "(none)", &value, dst.span() * sizeof(typename dst_traits::value_type));
2101 }
2102 if (dst.data() == nullptr) {
2103 space.fence("Kokkos::deep_copy: scalar copy on space, dst data is null");
2104 } else if (dst.span_is_contiguous() &&
2105 !ViewTraits<DT, DP...>::impl_is_customized) {
2106 Impl::contiguous_fill_or_memset(space, dst, value);
2107 } else {
2108 using ViewType = View<DT, DP...>;
2109 // Figure out iteration order to do the ViewFill
2110 int64_t strides[ViewType::rank + 1];
2111 dst.stride(strides);
2112 Kokkos::Iterate iterate;
2113 if (std::is_same_v<typename ViewType::array_layout, Kokkos::LayoutRight>) {
2114 iterate = Kokkos::Iterate::Right;
2115 } else if (std::is_same_v<typename ViewType::array_layout,
2116 Kokkos::LayoutLeft>) {
2117 iterate = Kokkos::Iterate::Left;
2118 } else if (std::is_same_v<typename ViewType::array_layout,
2119 Kokkos::LayoutStride>) {
2120 if (strides[0] > strides[ViewType::rank > 0 ? ViewType::rank - 1 : 0])
2121 iterate = Kokkos::Iterate::Right;
2122 else
2123 iterate = Kokkos::Iterate::Left;
2124 } else {
2125 if (std::is_same_v<typename ViewType::execution_space::array_layout,
2126 Kokkos::LayoutRight>)
2127 iterate = Kokkos::Iterate::Right;
2128 else
2129 iterate = Kokkos::Iterate::Left;
2130 }
2131
2132 // Lets call the right ViewFill functor based on integer space needed and
2133 // iteration type
2134 using ViewTypeUniform =
2135 std::conditional_t<ViewType::rank == 0,
2136 typename ViewType::uniform_runtime_type,
2137 typename ViewType::uniform_runtime_nomemspace_type>;
2138 if (dst.span() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
2139 if (iterate == Kokkos::Iterate::Right)
2140 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight, ExecSpace,
2141 ViewType::rank, int64_t>(dst, value, space);
2142 else
2143 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutLeft, ExecSpace,
2144 ViewType::rank, int64_t>(dst, value, space);
2145 } else {
2146 if (iterate == Kokkos::Iterate::Right)
2147 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight, ExecSpace,
2148 ViewType::rank, int32_t>(dst, value, space);
2149 else
2150 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutLeft, ExecSpace,
2151 ViewType::rank, int32_t>(dst, value, space);
2152 }
2153 }
2154 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2155 Kokkos::Profiling::endDeepCopy();
2156 }
2157}
2158
2161template <class ExecSpace, class DT, class... DP>
2162inline void deep_copy(
2163 const ExecSpace& space, const View<DT, DP...>& dst,
2164 typename ViewTraits<DT, DP...>::const_value_type& value,
2165 std::enable_if_t<
2166 Kokkos::is_execution_space<ExecSpace>::value &&
2167 std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
2168 !Kokkos::SpaceAccessibility<ExecSpace, typename ViewTraits<DT, DP...>::
2169 memory_space>::accessible>* =
2170 nullptr) {
2171 using dst_traits = ViewTraits<DT, DP...>;
2172 static_assert(std::is_same_v<typename dst_traits::non_const_value_type,
2173 typename dst_traits::value_type>,
2174 "deep_copy requires non-const type");
2175 using dst_memory_space = typename dst_traits::memory_space;
2176 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2177 Kokkos::Profiling::beginDeepCopy(
2178 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2179 dst.label(), dst.data(),
2180 Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
2181 "(none)", &value, dst.span() * sizeof(typename dst_traits::value_type));
2182 }
2183 if (dst.data() == nullptr) {
2184 space.fence(
2185 "Kokkos::deep_copy: scalar-to-view copy on space, dst data is null");
2186 } else {
2187 space.fence("Kokkos::deep_copy: scalar-to-view copy on space, pre copy");
2188 using fill_exec_space = typename dst_traits::memory_space::execution_space;
2189 if (dst.span_is_contiguous() &&
2190 !ViewTraits<DT, DP...>::impl_is_customized) {
2191 Impl::contiguous_fill_or_memset(fill_exec_space(), dst, value);
2192 } else {
2193 using ViewTypeUniform = std::conditional_t<
2194 View<DT, DP...>::rank == 0,
2195 typename View<DT, DP...>::uniform_runtime_type,
2196 typename View<DT, DP...>::uniform_runtime_nomemspace_type>;
2197 Kokkos::Impl::ViewFill<ViewTypeUniform, typename dst_traits::array_layout,
2198 fill_exec_space>(dst, value, fill_exec_space());
2199 }
2200 fill_exec_space().fence(
2201 "Kokkos::deep_copy: scalar-to-view copy on space, fence after fill");
2202 }
2203 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2204 Kokkos::Profiling::endDeepCopy();
2205 }
2206}
2207
2209template <class ExecSpace, class ST, class... SP>
2210inline void deep_copy(
2211 const ExecSpace& exec_space,
2212 typename ViewTraits<ST, SP...>::non_const_value_type& dst,
2213 const View<ST, SP...>& src,
2214 std::enable_if_t<Kokkos::is_execution_space<ExecSpace>::value &&
2215 std::is_same_v<typename ViewTraits<ST, SP...>::specialize,
2216 void>>* = nullptr) {
2217 using src_traits = ViewTraits<ST, SP...>;
2218 using src_memory_space = typename src_traits::memory_space;
2219 static_assert(src_traits::rank == 0,
2220 "ERROR: Non-rank-zero view in deep_copy( value , View )");
2221 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2222 Kokkos::Profiling::beginDeepCopy(
2223 Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
2224 "(none)", &dst,
2225 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2226 src.label(), src.data(), sizeof(ST));
2227 }
2228
2229 if (src.data() == nullptr) {
2230 exec_space.fence(
2231 "Kokkos::deep_copy: view-to-scalar copy on space, src data is null");
2232 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2233 Kokkos::Profiling::endDeepCopy();
2234 }
2235 return;
2236 }
2237
2238 Kokkos::Impl::DeepCopy<HostSpace, src_memory_space, ExecSpace>(
2239 exec_space, &dst, src.data(), sizeof(ST));
2240 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2241 Kokkos::Profiling::endDeepCopy();
2242 }
2243}
2244
2245//----------------------------------------------------------------------------
2247template <class ExecSpace, class DT, class... DP, class ST, class... SP>
2248inline void deep_copy(
2249 const ExecSpace& exec_space, const View<DT, DP...>& dst,
2250 const View<ST, SP...>& src,
2251 std::enable_if_t<
2252 (Kokkos::is_execution_space<ExecSpace>::value &&
2253 std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
2254 std::is_void_v<typename ViewTraits<ST, SP...>::specialize> &&
2255 (unsigned(ViewTraits<DT, DP...>::rank) == unsigned(0) &&
2256 unsigned(ViewTraits<ST, SP...>::rank) == unsigned(0)))>* = nullptr) {
2257 using src_traits = ViewTraits<ST, SP...>;
2258 using dst_traits = ViewTraits<DT, DP...>;
2259
2260 using src_memory_space = typename src_traits::memory_space;
2261 using dst_memory_space = typename dst_traits::memory_space;
2262 static_assert(std::is_same_v<typename dst_traits::value_type,
2263 typename src_traits::non_const_value_type>,
2264 "deep_copy requires matching non-const destination type");
2265
2266 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2267 Kokkos::Profiling::beginDeepCopy(
2268 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2269 dst.label(), dst.data(),
2270 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2271 src.label(), src.data(), sizeof(DT));
2272 }
2273
2274 if (dst.data() == nullptr && src.data() == nullptr) {
2275 exec_space.fence(
2276 "Kokkos::deep_copy: view-to-view copy on space, data is null");
2277 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2278 Kokkos::Profiling::endDeepCopy();
2279 }
2280 return;
2281 }
2282
2283 if (dst.data() != src.data()) {
2284 Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space, ExecSpace>(
2285 exec_space, dst.data(), src.data(),
2286 sizeof(typename dst_traits::value_type));
2287 }
2288 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2289 Kokkos::Profiling::endDeepCopy();
2290 }
2291}
2292
2293//----------------------------------------------------------------------------
2297template <class ExecSpace, class DT, class... DP, class ST, class... SP>
2298inline void deep_copy(
2299 const ExecSpace& exec_space, const View<DT, DP...>& dst,
2300 const View<ST, SP...>& src,
2301 std::enable_if_t<
2302 (Kokkos::is_execution_space<ExecSpace>::value &&
2303 std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
2304 std::is_void_v<typename ViewTraits<ST, SP...>::specialize> &&
2305 (unsigned(ViewTraits<DT, DP...>::rank) != 0 ||
2306 unsigned(ViewTraits<ST, SP...>::rank) != 0))>* = nullptr) {
2307 using dst_type = View<DT, DP...>;
2308 using src_type = View<ST, SP...>;
2309
2310 static_assert(std::is_same_v<typename dst_type::value_type,
2311 typename dst_type::non_const_value_type>,
2312 "deep_copy requires non-const destination type");
2313
2314 static_assert((unsigned(dst_type::rank) == unsigned(src_type::rank)),
2315 "deep_copy requires Views of equal rank");
2316
2317 using dst_execution_space = typename dst_type::execution_space;
2318 using src_execution_space = typename src_type::execution_space;
2319 using dst_memory_space = typename dst_type::memory_space;
2320 using src_memory_space = typename src_type::memory_space;
2321 using dst_value_type = typename dst_type::value_type;
2322 using src_value_type = typename src_type::value_type;
2323
2324 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2325 Kokkos::Profiling::beginDeepCopy(
2326 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2327 dst.label(), dst.data(),
2328 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2329 src.label(), src.data(), dst.span() * sizeof(dst_value_type));
2330 }
2331
2332 dst_value_type* dst_start = dst.data();
2333 dst_value_type* dst_end = dst.data() + dst.span();
2334 src_value_type* src_start = src.data();
2335 src_value_type* src_end = src.data() + src.span();
2336
2337 // Early dropout if identical range
2338 if ((dst_start == nullptr || src_start == nullptr) ||
2339 ((std::ptrdiff_t(dst_start) == std::ptrdiff_t(src_start)) &&
2340 (std::ptrdiff_t(dst_end) == std::ptrdiff_t(src_end)))) {
2341 // throw if dimension mismatch
2342 if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
2343 (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
2344 (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
2345 (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
2346 std::string message(
2347 "Deprecation Error: Kokkos::deep_copy extents of views don't "
2348 "match: ");
2349 message += dst.label();
2350 message += "(";
2351 message += std::to_string(dst.extent(0));
2352 for (size_t r = 1; r < dst_type::rank; r++) {
2353 message += ",";
2354 message += std::to_string(dst.extent(r));
2355 }
2356 message += ") ";
2357 message += src.label();
2358 message += "(";
2359 message += std::to_string(src.extent(0));
2360 for (size_t r = 1; r < src_type::rank; r++) {
2361 message += ",";
2362 message += std::to_string(src.extent(r));
2363 }
2364 message += ") ";
2365
2366 Kokkos::Impl::throw_runtime_exception(message);
2367 }
2368 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2369 Kokkos::Profiling::endDeepCopy();
2370 }
2371 return;
2372 }
2373
2374 // Error out for non-identical overlapping views.
2375 if ((((std::ptrdiff_t)dst_start < (std::ptrdiff_t)src_end) &&
2376 ((std::ptrdiff_t)dst_end > (std::ptrdiff_t)src_start)) &&
2377 ((dst.span_is_contiguous() && src.span_is_contiguous()))) {
2378 std::string message("Error: Kokkos::deep_copy of overlapping views: ");
2379 message += dst.label();
2380 message += "(";
2381 message += std::to_string((std::ptrdiff_t)dst_start);
2382 message += ",";
2383 message += std::to_string((std::ptrdiff_t)dst_end);
2384 message += ") ";
2385 message += src.label();
2386 message += "(";
2387 message += std::to_string((std::ptrdiff_t)src_start);
2388 message += ",";
2389 message += std::to_string((std::ptrdiff_t)src_end);
2390 message += ") ";
2391 Kokkos::Impl::throw_runtime_exception(message);
2392 }
2393
2394 // Check for same extents
2395 if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
2396 (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
2397 (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
2398 (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
2399 std::string message(
2400 "Deprecation Error: Kokkos::deep_copy extents of views don't match: ");
2401 message += dst.label();
2402 message += "(";
2403 message += std::to_string(dst.extent(0));
2404 for (size_t r = 1; r < dst_type::rank; r++) {
2405 message += ",";
2406 message += std::to_string(dst.extent(r));
2407 }
2408 message += ") ";
2409 message += src.label();
2410 message += "(";
2411 message += std::to_string(src.extent(0));
2412 for (size_t r = 1; r < src_type::rank; r++) {
2413 message += ",";
2414 message += std::to_string(src.extent(r));
2415 }
2416 message += ") ";
2417
2418 Kokkos::Impl::throw_runtime_exception(message);
2419 }
2420
2421 // If same type, equal layout, equal dimensions, equal span, and contiguous
2422 // memory then can byte-wise copy
2423
2424 if (std::is_same_v<typename dst_type::value_type,
2425 typename src_type::non_const_value_type> &&
2426 (std::is_same_v<typename dst_type::array_layout,
2427 typename src_type::array_layout> ||
2428 (dst_type::rank == 1 && src_type::rank == 1)) &&
2429 dst.span_is_contiguous() && src.span_is_contiguous() &&
2430 ((dst_type::rank < 1) || (dst.stride_0() == src.stride_0())) &&
2431 ((dst_type::rank < 2) || (dst.stride_1() == src.stride_1())) &&
2432 ((dst_type::rank < 3) || (dst.stride_2() == src.stride_2())) &&
2433 ((dst_type::rank < 4) || (dst.stride_3() == src.stride_3())) &&
2434 ((dst_type::rank < 5) || (dst.stride_4() == src.stride_4())) &&
2435 ((dst_type::rank < 6) || (dst.stride_5() == src.stride_5())) &&
2436 ((dst_type::rank < 7) || (dst.stride_6() == src.stride_6())) &&
2437 ((dst_type::rank < 8) || (dst.stride_7() == src.stride_7()))) {
2438 const size_t nbytes = sizeof(typename dst_type::value_type) * dst.span();
2439 if ((void*)dst.data() != (void*)src.data() && 0 < nbytes) {
2440 Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space, ExecSpace>(
2441 exec_space, dst.data(), src.data(), nbytes);
2442 }
2443 } else {
2444 // Copying data between views in accessible memory spaces and either
2445 // non-contiguous or incompatible shape.
2446
2447 constexpr bool ExecCanAccessSrcDst =
2448 Kokkos::SpaceAccessibility<ExecSpace, dst_memory_space>::accessible &&
2449 Kokkos::SpaceAccessibility<ExecSpace, src_memory_space>::accessible;
2450 constexpr bool DstExecCanAccessSrc =
2451 Kokkos::SpaceAccessibility<dst_execution_space,
2452 src_memory_space>::accessible;
2453 constexpr bool SrcExecCanAccessDst =
2454 Kokkos::SpaceAccessibility<src_execution_space,
2455 dst_memory_space>::accessible;
2456
2457 if constexpr (ExecCanAccessSrcDst) {
2458 Impl::view_copy(exec_space, dst, src);
2459 } else if constexpr (DstExecCanAccessSrc || SrcExecCanAccessDst) {
2460 using cpy_exec_space =
2461 std::conditional_t<DstExecCanAccessSrc, dst_execution_space,
2462 src_execution_space>;
2463 exec_space.fence(
2464 "Kokkos::deep_copy: view-to-view noncontiguous copy on space, pre "
2465 "copy");
2466 Impl::view_copy(cpy_exec_space(), dst, src);
2467 cpy_exec_space().fence(
2468 "Kokkos::deep_copy: view-to-view noncontiguous copy on space, post "
2469 "copy");
2470 } else {
2471 Kokkos::Impl::throw_runtime_exception(
2472 "deep_copy given views that would require a temporary allocation");
2473 }
2474 }
2475 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2476 Kokkos::Profiling::endDeepCopy();
2477 }
2478}
2479
2480} /* namespace Kokkos */
2481
2482//----------------------------------------------------------------------------
2483//----------------------------------------------------------------------------
2484
2485namespace Kokkos {
2486
2487namespace Impl {
2488template <typename ViewType>
2489bool size_mismatch(const ViewType& view, unsigned int max_extent,
2490 const size_t new_extents[8]) {
2491 for (unsigned int dim = 0; dim < max_extent; ++dim)
2492 if (new_extents[dim] != view.extent(dim)) {
2493 return true;
2494 }
2495 for (unsigned int dim = max_extent; dim < 8; ++dim)
2496 if (new_extents[dim] != KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2497 return true;
2498 }
2499 return false;
2500}
2501
2502} // namespace Impl
2503
2506template <class T, class... P, class... ViewCtorArgs>
2507inline std::enable_if_t<
2508 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2509 Kokkos::LayoutLeft> ||
2510 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2511 Kokkos::LayoutRight>>
2512impl_resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2513 Kokkos::View<T, P...>& v, const size_t n0, const size_t n1,
2514 const size_t n2, const size_t n3, const size_t n4, const size_t n5,
2515 const size_t n6, const size_t n7) {
2516 using view_type = Kokkos::View<T, P...>;
2517 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2518
2519 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2520 "Can only resize managed views");
2521 static_assert(!alloc_prop_input::has_label,
2522 "The view constructor arguments passed to Kokkos::resize "
2523 "must not include a label!");
2524 static_assert(!alloc_prop_input::has_pointer,
2525 "The view constructor arguments passed to Kokkos::resize must "
2526 "not include a pointer!");
2527 static_assert(!alloc_prop_input::has_memory_space,
2528 "The view constructor arguments passed to Kokkos::resize must "
2529 "not include a memory space instance!");
2530
2531 // TODO (mfh 27 Jun 2017) If the old View has enough space but just
2532 // different dimensions (e.g., if the product of the dimensions,
2533 // including extra space for alignment, will not change), then
2534 // consider just reusing storage. For now, Kokkos always
2535 // reallocates if any of the dimensions change, even if the old View
2536 // has enough space.
2537
2538 const size_t new_extents[8] = {n0, n1, n2, n3, n4, n5, n6, n7};
2539 const bool sizeMismatch = Impl::size_mismatch(v, v.rank_dynamic, new_extents);
2540
2541 if (sizeMismatch) {
2542#ifdef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
2543 auto prop_copy = Impl::with_properties_if_unset(
2544 arg_prop, typename view_type::execution_space{}, v.label());
2545
2546 view_type v_resized(prop_copy, n0, n1, n2, n3, n4, n5, n6, n7);
2547#else
2548 auto prop_copy = [&]() {
2549 if constexpr (view_type::traits::impl_is_customized) {
2550 // FIXME SACADO: this is specializing for sacado, might need a better
2551 // thing
2552 Kokkos::Impl::AccessorArg_t acc_arg{new_extents[view_type::rank()]};
2553 return Impl::with_properties_if_unset(
2554 arg_prop, acc_arg, typename view_type::execution_space{},
2555 v.label());
2556 } else
2557 return Impl::with_properties_if_unset(
2558 arg_prop, typename view_type::execution_space{}, v.label());
2559 ;
2560 }();
2561
2562 view_type v_resized;
2563 if constexpr (view_type::rank() == 0) {
2564 v_resized = view_type(prop_copy);
2565 } else if constexpr (view_type::rank() == 1) {
2566 v_resized = view_type(prop_copy, n0);
2567 } else if constexpr (view_type::rank() == 2) {
2568 v_resized = view_type(prop_copy, n0, n1);
2569 } else if constexpr (view_type::rank() == 3) {
2570 v_resized = view_type(prop_copy, n0, n1, n2);
2571 } else if constexpr (view_type::rank() == 4) {
2572 v_resized = view_type(prop_copy, n0, n1, n2, n3);
2573 } else if constexpr (view_type::rank() == 5) {
2574 v_resized = view_type(prop_copy, n0, n1, n2, n3, n4);
2575 } else if constexpr (view_type::rank() == 6) {
2576 v_resized = view_type(prop_copy, n0, n1, n2, n3, n4, n5);
2577 } else if constexpr (view_type::rank() == 7) {
2578 v_resized = view_type(prop_copy, n0, n1, n2, n3, n4, n5, n6);
2579 } else {
2580 v_resized = view_type(prop_copy, n0, n1, n2, n3, n4, n5, n6, n7);
2581 }
2582#endif
2583
2584 if constexpr (alloc_prop_input::has_execution_space)
2585 Kokkos::Impl::ViewRemap<view_type, view_type>(
2586 v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(prop_copy));
2587 else {
2588 Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2589 Kokkos::fence("Kokkos::resize(View)");
2590 }
2591
2592 v = v_resized;
2593 }
2594}
2595
2596template <class T, class... P, class... ViewCtorArgs>
2597inline std::enable_if_t<
2598 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2599 Kokkos::LayoutLeft> ||
2600 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2601 Kokkos::LayoutRight>>
2602resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2603 Kokkos::View<T, P...>& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2604 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2605 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2606 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2607 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2608 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2609 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2610 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2611 impl_resize(arg_prop, v, n0, n1, n2, n3, n4, n5, n6, n7);
2612}
2613
2614template <class T, class... P>
2615inline std::enable_if_t<
2616 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2617 Kokkos::LayoutLeft> ||
2618 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2619 Kokkos::LayoutRight>>
2620resize(Kokkos::View<T, P...>& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2621 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2622 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2623 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2624 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2625 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2626 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2627 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2628 impl_resize(Impl::ViewCtorProp<>{}, v, n0, n1, n2, n3, n4, n5, n6, n7);
2629}
2630
2631template <class I, class T, class... P>
2632inline std::enable_if_t<
2633 (Impl::is_view_ctor_property<I>::value ||
2634 Kokkos::is_execution_space<I>::value) &&
2635 (std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2636 Kokkos::LayoutLeft> ||
2637 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2638 Kokkos::LayoutRight>)>
2639resize(const I& arg_prop, Kokkos::View<T, P...>& v,
2640 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2641 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2642 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2643 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2644 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2645 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2646 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2647 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2648 impl_resize(Kokkos::view_alloc(arg_prop), v, n0, n1, n2, n3, n4, n5, n6, n7);
2649}
2650
2651template <class T, class... P, class... ViewCtorArgs>
2652inline std::enable_if_t<
2653 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2654 Kokkos::LayoutLeft> ||
2655 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2656 Kokkos::LayoutRight> ||
2657 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2658 Kokkos::LayoutStride>>
2659impl_resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2660 Kokkos::View<T, P...>& v,
2661 const typename Kokkos::View<T, P...>::array_layout& layout) {
2662 using view_type = Kokkos::View<T, P...>;
2663 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2664
2665 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2666 "Can only resize managed views");
2667 static_assert(!alloc_prop_input::has_label,
2668 "The view constructor arguments passed to Kokkos::resize "
2669 "must not include a label!");
2670 static_assert(!alloc_prop_input::has_pointer,
2671 "The view constructor arguments passed to Kokkos::resize must "
2672 "not include a pointer!");
2673 static_assert(!alloc_prop_input::has_memory_space,
2674 "The view constructor arguments passed to Kokkos::resize must "
2675 "not include a memory space instance!");
2676
2677 if (v.layout() != layout) {
2678 auto prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2679
2680 view_type v_resized(prop_copy, layout);
2681
2682 if constexpr (alloc_prop_input::has_execution_space)
2683 Kokkos::Impl::ViewRemap<view_type, view_type>(
2684 v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop));
2685 else {
2686 Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2687 Kokkos::fence("Kokkos::resize(View)");
2688 }
2689
2690 v = v_resized;
2691 }
2692}
2693
2694// FIXME User-provided (custom) layouts are not required to have a comparison
2695// operator. Hence, there is no way to check if the requested layout is actually
2696// the same as the existing one.
2697template <class T, class... P, class... ViewCtorArgs>
2698inline std::enable_if_t<
2699 !(std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2700 Kokkos::LayoutLeft> ||
2701 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2702 Kokkos::LayoutRight> ||
2703 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2704 Kokkos::LayoutStride>)>
2705impl_resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2706 Kokkos::View<T, P...>& v,
2707 const typename Kokkos::View<T, P...>::array_layout& layout) {
2708 using view_type = Kokkos::View<T, P...>;
2709 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2710
2711 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2712 "Can only resize managed views");
2713 static_assert(!alloc_prop_input::has_label,
2714 "The view constructor arguments passed to Kokkos::resize "
2715 "must not include a label!");
2716 static_assert(!alloc_prop_input::has_pointer,
2717 "The view constructor arguments passed to Kokkos::resize must "
2718 "not include a pointer!");
2719 static_assert(!alloc_prop_input::has_memory_space,
2720 "The view constructor arguments passed to Kokkos::resize must "
2721 "not include a memory space instance!");
2722
2723 auto prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2724
2725 view_type v_resized(prop_copy, layout);
2726
2727 if constexpr (alloc_prop_input::has_execution_space)
2728 Kokkos::Impl::ViewRemap<view_type, view_type>(
2729 v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop));
2730 else {
2731 Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2732 Kokkos::fence("Kokkos::resize(View)");
2733 }
2734
2735 v = v_resized;
2736}
2737
2738template <class T, class... P, class... ViewCtorArgs>
2739inline void resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2740 Kokkos::View<T, P...>& v,
2741 const typename Kokkos::View<T, P...>::array_layout& layout) {
2742 impl_resize(arg_prop, v, layout);
2743}
2744
2745template <class I, class T, class... P>
2746inline std::enable_if_t<Impl::is_view_ctor_property<I>::value ||
2747 Kokkos::is_execution_space<I>::value>
2748resize(const I& arg_prop, Kokkos::View<T, P...>& v,
2749 const typename Kokkos::View<T, P...>::array_layout& layout) {
2750 impl_resize(arg_prop, v, layout);
2751}
2752
2753template <class ExecutionSpace, class T, class... P>
2754inline void resize(const ExecutionSpace& exec_space, Kokkos::View<T, P...>& v,
2755 const typename Kokkos::View<T, P...>::array_layout& layout) {
2756 impl_resize(Impl::ViewCtorProp<>(), exec_space, v, layout);
2757}
2758
2759template <class T, class... P>
2760inline void resize(Kokkos::View<T, P...>& v,
2761 const typename Kokkos::View<T, P...>::array_layout& layout) {
2762 impl_resize(Impl::ViewCtorProp<>{}, v, layout);
2763}
2764
2766template <class T, class... P, class... ViewCtorArgs>
2767inline std::enable_if_t<
2768 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2769 Kokkos::LayoutLeft> ||
2770 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2771 Kokkos::LayoutRight>>
2772impl_realloc(Kokkos::View<T, P...>& v, const size_t n0, const size_t n1,
2773 const size_t n2, const size_t n3, const size_t n4, const size_t n5,
2774 const size_t n6, const size_t n7,
2775 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2776 using view_type = Kokkos::View<T, P...>;
2777 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2778
2779 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2780 "Can only realloc managed views");
2781 static_assert(!alloc_prop_input::has_label,
2782 "The view constructor arguments passed to Kokkos::realloc must "
2783 "not include a label!");
2784 static_assert(!alloc_prop_input::has_pointer,
2785 "The view constructor arguments passed to Kokkos::realloc must "
2786 "not include a pointer!");
2787 static_assert(!alloc_prop_input::has_memory_space,
2788 "The view constructor arguments passed to Kokkos::realloc must "
2789 "not include a memory space instance!");
2790
2791 const size_t new_extents[8] = {n0, n1, n2, n3, n4, n5, n6, n7};
2792 const bool sizeMismatch = Impl::size_mismatch(v, v.rank_dynamic, new_extents);
2793
2794 if (sizeMismatch) {
2795 auto arg_prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2796 v = view_type(); // Best effort to deallocate in case no other view refers
2797 // to the shared allocation
2798 v = view_type(arg_prop_copy, n0, n1, n2, n3, n4, n5, n6, n7);
2799 return;
2800 }
2801
2802 if constexpr (alloc_prop_input::initialize) {
2803 if constexpr (alloc_prop_input::has_execution_space) {
2804 const auto& exec_space =
2805 Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
2806 Kokkos::deep_copy(exec_space, v, typename view_type::value_type{});
2807 } else
2808 Kokkos::deep_copy(v, typename view_type::value_type{});
2809 }
2810}
2811
2812template <class T, class... P, class... ViewCtorArgs>
2813inline std::enable_if_t<
2814 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2815 Kokkos::LayoutLeft> ||
2816 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2817 Kokkos::LayoutRight>>
2818realloc(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2819 Kokkos::View<T, P...>& v,
2820 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2821 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2822 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2823 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2824 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2825 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2826 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2827 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2828 impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, arg_prop);
2829}
2830
2831template <class T, class... P>
2832inline std::enable_if_t<
2833 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2834 Kokkos::LayoutLeft> ||
2835 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2836 Kokkos::LayoutRight>>
2837realloc(Kokkos::View<T, P...>& v,
2838 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2839 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2840 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2841 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2842 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2843 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2844 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2845 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2846 impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, Impl::ViewCtorProp<>{});
2847}
2848
2849template <class I, class T, class... P>
2850inline std::enable_if_t<
2851 Impl::is_view_ctor_property<I>::value &&
2852 (std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2853 Kokkos::LayoutLeft> ||
2854 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2855 Kokkos::LayoutRight>)>
2856realloc(const I& arg_prop, Kokkos::View<T, P...>& v,
2857 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2858 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2859 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2860 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2861 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2862 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2863 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2864 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2865 impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, Kokkos::view_alloc(arg_prop));
2866}
2867
2868template <class T, class... P, class... ViewCtorArgs>
2869inline std::enable_if_t<
2870 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2871 Kokkos::LayoutLeft> ||
2872 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2873 Kokkos::LayoutRight> ||
2874 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2875 Kokkos::LayoutStride>>
2876impl_realloc(Kokkos::View<T, P...>& v,
2877 const typename Kokkos::View<T, P...>::array_layout& layout,
2878 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2879 using view_type = Kokkos::View<T, P...>;
2880 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2881
2882 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2883 "Can only realloc managed views");
2884 static_assert(!alloc_prop_input::has_label,
2885 "The view constructor arguments passed to Kokkos::realloc must "
2886 "not include a label!");
2887 static_assert(!alloc_prop_input::has_pointer,
2888 "The view constructor arguments passed to Kokkos::realloc must "
2889 "not include a pointer!");
2890 static_assert(!alloc_prop_input::has_memory_space,
2891 "The view constructor arguments passed to Kokkos::realloc must "
2892 "not include a memory space instance!");
2893
2894 if (v.layout() != layout) {
2895 v = view_type(); // Deallocate first, if the only view to allocation
2896 v = view_type(arg_prop, layout);
2897 return;
2898 }
2899
2900 if constexpr (alloc_prop_input::initialize) {
2901 if constexpr (alloc_prop_input::has_execution_space) {
2902 const auto& exec_space =
2903 Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
2904 Kokkos::deep_copy(exec_space, v, typename view_type::value_type{});
2905 } else
2906 Kokkos::deep_copy(v, typename view_type::value_type{});
2907 }
2908}
2909
2910// FIXME User-provided (custom) layouts are not required to have a comparison
2911// operator. Hence, there is no way to check if the requested layout is actually
2912// the same as the existing one.
2913template <class T, class... P, class... ViewCtorArgs>
2914inline std::enable_if_t<
2915 !(std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2916 Kokkos::LayoutLeft> ||
2917 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2918 Kokkos::LayoutRight> ||
2919 std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2920 Kokkos::LayoutStride>)>
2921impl_realloc(Kokkos::View<T, P...>& v,
2922 const typename Kokkos::View<T, P...>::array_layout& layout,
2923 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2924 using view_type = Kokkos::View<T, P...>;
2925 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2926
2927 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2928 "Can only realloc managed views");
2929 static_assert(!alloc_prop_input::has_label,
2930 "The view constructor arguments passed to Kokkos::realloc must "
2931 "not include a label!");
2932 static_assert(!alloc_prop_input::has_pointer,
2933 "The view constructor arguments passed to Kokkos::realloc must "
2934 "not include a pointer!");
2935 static_assert(!alloc_prop_input::has_memory_space,
2936 "The view constructor arguments passed to Kokkos::realloc must "
2937 "not include a memory space instance!");
2938
2939 auto arg_prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2940
2941 v = view_type(); // Deallocate first, if the only view to allocation
2942 v = view_type(arg_prop_copy, layout);
2943}
2944
2945template <class T, class... P, class... ViewCtorArgs>
2946inline void realloc(
2947 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2948 Kokkos::View<T, P...>& v,
2949 const typename Kokkos::View<T, P...>::array_layout& layout) {
2950 impl_realloc(v, layout, arg_prop);
2951}
2952
2953template <class I, class T, class... P>
2954inline std::enable_if_t<Impl::is_view_ctor_property<I>::value> realloc(
2955 const I& arg_prop, Kokkos::View<T, P...>& v,
2956 const typename Kokkos::View<T, P...>::array_layout& layout) {
2957 impl_realloc(v, layout, Kokkos::view_alloc(arg_prop));
2958}
2959
2960template <class T, class... P>
2961inline void realloc(
2962 Kokkos::View<T, P...>& v,
2963 const typename Kokkos::View<T, P...>::array_layout& layout) {
2964 impl_realloc(v, layout, Impl::ViewCtorProp<>{});
2965}
2966
2967} /* namespace Kokkos */
2968
2969//----------------------------------------------------------------------------
2970//----------------------------------------------------------------------------
2971
2972namespace Kokkos {
2973namespace Impl {
2974
2975// Deduce Mirror Types
2976template <class Space, class T, class... P>
2977struct MirrorViewType {
2978 // The incoming view_type
2979 using src_view_type = typename Kokkos::View<T, P...>;
2980 // The memory space for the mirror view
2981 using memory_space = typename Space::memory_space;
2982 // Check whether it is the same memory space
2983 static constexpr bool is_same_memspace =
2984 std::is_same_v<memory_space, typename src_view_type::memory_space>;
2985 // The array_layout
2986 using array_layout = typename src_view_type::array_layout;
2987 // The data type (we probably want it non-const since otherwise we can't even
2988 // deep_copy to it.
2989 using data_type = typename src_view_type::non_const_data_type;
2990 // The destination view type if it is not the same memory space
2991 using dest_view_type = Kokkos::View<data_type, array_layout, Space>;
2992 // If it is the same memory_space return the existsing view_type
2993 // This will also keep the unmanaged trait if necessary
2994 using view_type =
2995 std::conditional_t<is_same_memspace, src_view_type, dest_view_type>;
2996};
2997
2998// collection of static asserts for create_mirror and create_mirror_view
2999template <class... ViewCtorArgs>
3000void check_view_ctor_args_create_mirror() {
3001 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
3002
3003 static_assert(
3004 !alloc_prop_input::has_label,
3005 "The view constructor arguments passed to Kokkos::create_mirror[_view] "
3006 "must not include a label!");
3007 static_assert(!alloc_prop_input::has_pointer,
3008 "The view constructor arguments passed to "
3009 "Kokkos::create_mirror[_view] must "
3010 "not include a pointer!");
3011 static_assert(!alloc_prop_input::allow_padding,
3012 "The view constructor arguments passed to "
3013 "Kokkos::create_mirror[_view] must "
3014 "not explicitly allow padding!");
3015}
3016
3017// create a mirror
3018// private interface that accepts arbitrary view constructor args passed by a
3019// view_alloc
3020template <class T, class... P, class... ViewCtorArgs>
3021inline auto create_mirror(const Kokkos::View<T, P...>& src,
3022 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
3023 check_view_ctor_args_create_mirror<ViewCtorArgs...>();
3024
3025 auto prop_copy = Impl::with_properties_if_unset(
3026 arg_prop, std::string(src.label()).append("_mirror"));
3027
3028 if constexpr (Impl::ViewCtorProp<ViewCtorArgs...>::has_memory_space) {
3029 using memory_space = typename decltype(prop_copy)::memory_space;
3030 using dst_type =
3031 typename Impl::MirrorViewType<memory_space, T, P...>::dest_view_type;
3032#ifndef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
3033 // This is necessary because constructing non-const element type from
3034 // const element type accessors is not generally supported
3035 // We don't construct from the src accessor generally because our accessors
3036 // aren't generally constructible from each other.
3037 // We could change that, but all our internal accessors are stateless anyway
3038 // right now. So for now if you have custom accessors that need to carry
3039 // forward information you just have to make the conversion constructors
3040 // work.
3041 if constexpr (std::is_constructible_v<
3042 typename dst_type::accessor_type,
3043 typename Kokkos::View<T, P...>::accessor_type>)
3044 return dst_type(prop_copy, src.mapping(), src.accessor());
3045 else
3046 return dst_type(prop_copy, src.layout());
3047#else
3048 return dst_type(prop_copy, src.layout());
3049#endif
3050 } else {
3051 using dst_type = typename View<T, P...>::HostMirror;
3052#ifndef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
3053 // This is necessary because constructing non-const element type from
3054 // const element type accessors is not generally supported
3055 if constexpr (std::is_constructible_v<
3056 typename dst_type::accessor_type,
3057 typename Kokkos::View<T, P...>::accessor_type>)
3058 return dst_type(prop_copy, src.mapping(), src.accessor());
3059 else
3060 return dst_type(prop_copy, src.layout());
3061#else
3062 return dst_type(prop_copy, src.layout());
3063#endif
3064 }
3065#if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3066 !defined(KOKKOS_COMPILER_MSVC)
3067 __builtin_unreachable();
3068#endif
3069}
3070} // namespace Impl
3071
3072// public interface
3073template <class T, class... P,
3074 typename = std::enable_if_t<
3075 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3076auto create_mirror(Kokkos::View<T, P...> const& src) {
3077 return Impl::create_mirror(src, Impl::ViewCtorProp<>{});
3078}
3079
3080// public interface that accepts a without initializing flag
3081template <class T, class... P,
3082 typename = std::enable_if_t<
3083 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3084auto create_mirror(Kokkos::Impl::WithoutInitializing_t wi,
3085 Kokkos::View<T, P...> const& src) {
3086 return Impl::create_mirror(src, view_alloc(wi));
3087}
3088
3089// public interface that accepts a space
3090template <class Space, class T, class... P,
3091 typename Enable = std::enable_if_t<
3092 Kokkos::is_space<Space>::value &&
3093 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3094auto create_mirror(Space const&, Kokkos::View<T, P...> const& src) {
3095 return Impl::create_mirror(src, view_alloc(typename Space::memory_space{}));
3096}
3097
3098// public interface that accepts arbitrary view constructor args passed by a
3099// view_alloc
3100template <class T, class... P, class... ViewCtorArgs,
3101 typename = std::enable_if_t<
3102 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3103auto create_mirror(Impl::ViewCtorProp<ViewCtorArgs...> const& arg_prop,
3104 Kokkos::View<T, P...> const& src) {
3105 return Impl::create_mirror(src, arg_prop);
3106}
3107
3108// public interface that accepts a space and a without initializing flag
3109template <class Space, class T, class... P,
3110 typename Enable = std::enable_if_t<
3111 Kokkos::is_space<Space>::value &&
3112 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3113auto create_mirror(Kokkos::Impl::WithoutInitializing_t wi, Space const&,
3114 Kokkos::View<T, P...> const& src) {
3115 return Impl::create_mirror(src,
3116 view_alloc(typename Space::memory_space{}, wi));
3117}
3118
3119namespace Impl {
3120
3121// choose a `Kokkos::create_mirror` adapted for the provided view and the
3122// provided arguments
3123template <class View, class... ViewCtorArgs>
3124inline auto choose_create_mirror(
3125 const View& src, const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
3126 // Due to the fact that users can overload `Kokkos::create_mirror`, but also
3127 // that they may not have implemented all of its different possible
3128 // variations, this function chooses the correct private or public version of
3129 // it to call.
3130 // This helper should be used by any overload of
3131 // `Kokkos::Impl::create_mirror_view`.
3132
3133 if constexpr (std::is_void_v<typename View::traits::specialize>) {
3134 // if the view is not specialized, just call the Impl function
3135
3136 // using ADL to find the later defined overload of the function
3137 using namespace Kokkos::Impl;
3138
3139 return create_mirror(src, arg_prop);
3140 } else {
3141 // otherwise, recreate the public call
3142 using ViewProp = Impl::ViewCtorProp<ViewCtorArgs...>;
3143
3144 // using ADL to find the later defined overload of the function
3145 using namespace Kokkos;
3146
3147 if constexpr (sizeof...(ViewCtorArgs) == 0) {
3148 // if there are no view constructor args, call the specific public
3149 // function
3150 return create_mirror(src);
3151 } else if constexpr (sizeof...(ViewCtorArgs) == 1 &&
3152 ViewProp::has_memory_space) {
3153 // if there is one view constructor arg and it has a memory space, call
3154 // the specific public function
3155 return create_mirror(typename ViewProp::memory_space{}, src);
3156 } else if constexpr (sizeof...(ViewCtorArgs) == 1 &&
3157 !ViewProp::initialize) {
3158 // if there is one view constructor arg and it has a without initializing
3159 // mark, call the specific public function
3160 return create_mirror(typename Kokkos::Impl::WithoutInitializing_t{}, src);
3161 } else if constexpr (sizeof...(ViewCtorArgs) == 2 &&
3162 ViewProp::has_memory_space && !ViewProp::initialize) {
3163 // if there is two view constructor args and they have a memory space and
3164 // a without initializing mark, call the specific public function
3165 return create_mirror(typename Kokkos::Impl::WithoutInitializing_t{},
3166 typename ViewProp::memory_space{}, src);
3167 } else {
3168 // if there are other constructor args, call the generic public function
3169
3170 // Beware, there are some libraries using Kokkos that don't implement
3171 // this overload (hence the reason for this present function to exist).
3172 return create_mirror(arg_prop, src);
3173 }
3174 }
3175
3176#if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3177 !defined(KOKKOS_COMPILER_MSVC)
3178 __builtin_unreachable();
3179#endif
3180}
3181
3182// create a mirror view
3183// private interface that accepts arbitrary view constructor args passed by a
3184// view_alloc
3185template <class T, class... P, class... ViewCtorArgs>
3186inline auto create_mirror_view(
3187 const Kokkos::View<T, P...>& src,
3188 [[maybe_unused]] const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
3189 if constexpr (!Impl::ViewCtorProp<ViewCtorArgs...>::has_memory_space) {
3190 if constexpr (std::is_same_v<typename Kokkos::View<T, P...>::memory_space,
3191 typename Kokkos::View<
3192 T, P...>::HostMirror::memory_space> &&
3193 std::is_same_v<
3194 typename Kokkos::View<T, P...>::data_type,
3195 typename Kokkos::View<T, P...>::HostMirror::data_type>) {
3196 check_view_ctor_args_create_mirror<ViewCtorArgs...>();
3197 return typename Kokkos::View<T, P...>::HostMirror(src);
3198 } else {
3199 return Kokkos::Impl::choose_create_mirror(src, arg_prop);
3200 }
3201 } else {
3202 if constexpr (Impl::MirrorViewType<typename Impl::ViewCtorProp<
3203 ViewCtorArgs...>::memory_space,
3204 T, P...>::is_same_memspace) {
3205 check_view_ctor_args_create_mirror<ViewCtorArgs...>();
3206 return typename Impl::MirrorViewType<
3207 typename Impl::ViewCtorProp<ViewCtorArgs...>::memory_space, T,
3208 P...>::view_type(src);
3209 } else {
3210 return Kokkos::Impl::choose_create_mirror(src, arg_prop);
3211 }
3212 }
3213#if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3214 !defined(KOKKOS_COMPILER_MSVC)
3215 __builtin_unreachable();
3216#endif
3217}
3218} // namespace Impl
3219
3220// public interface
3221template <class T, class... P>
3222auto create_mirror_view(const Kokkos::View<T, P...>& src) {
3223 return Impl::create_mirror_view(src, view_alloc());
3224}
3225
3226// public interface that accepts a without initializing flag
3227template <class T, class... P>
3228auto create_mirror_view(Kokkos::Impl::WithoutInitializing_t wi,
3229 Kokkos::View<T, P...> const& src) {
3230 return Impl::create_mirror_view(src, view_alloc(wi));
3231}
3232
3233// public interface that accepts a space
3234template <class Space, class T, class... P,
3235 class Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3236auto create_mirror_view(const Space&, const Kokkos::View<T, P...>& src) {
3237 return Impl::create_mirror_view(src,
3238 view_alloc(typename Space::memory_space()));
3239}
3240
3241// public interface that accepts a space and a without initializing flag
3242template <class Space, class T, class... P,
3243 typename Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3244auto create_mirror_view(Kokkos::Impl::WithoutInitializing_t wi, Space const&,
3245 Kokkos::View<T, P...> const& src) {
3246 return Impl::create_mirror_view(
3247 src, view_alloc(typename Space::memory_space{}, wi));
3248}
3249
3250// public interface that accepts arbitrary view constructor args passed by a
3251// view_alloc
3252template <class T, class... P, class... ViewCtorArgs,
3253 typename = std::enable_if_t<
3254 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3255auto create_mirror_view(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
3256 const Kokkos::View<T, P...>& src) {
3257 return Impl::create_mirror_view(src, arg_prop);
3258}
3259
3260namespace Impl {
3261
3262// collection of static asserts for create_mirror_view_and_copy
3263template <class... ViewCtorArgs>
3264void check_view_ctor_args_create_mirror_view_and_copy() {
3265 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
3266
3267 static_assert(
3268 alloc_prop_input::has_memory_space,
3269 "The view constructor arguments passed to "
3270 "Kokkos::create_mirror_view_and_copy must include a memory space!");
3271 static_assert(!alloc_prop_input::has_pointer,
3272 "The view constructor arguments passed to "
3273 "Kokkos::create_mirror_view_and_copy must "
3274 "not include a pointer!");
3275 static_assert(!alloc_prop_input::allow_padding,
3276 "The view constructor arguments passed to "
3277 "Kokkos::create_mirror_view_and_copy must "
3278 "not explicitly allow padding!");
3279}
3280
3281} // namespace Impl
3282
3283// create a mirror view and deep copy it
3284// public interface that accepts arbitrary view constructor args passed by a
3285// view_alloc
3286template <class... ViewCtorArgs, class T, class... P,
3287 class Enable = std::enable_if_t<
3288 std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3289auto create_mirror_view_and_copy(
3290 [[maybe_unused]] const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
3291 const Kokkos::View<T, P...>& src) {
3292 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
3293
3294 Impl::check_view_ctor_args_create_mirror_view_and_copy<ViewCtorArgs...>();
3295
3296 if constexpr (Impl::MirrorViewType<typename alloc_prop_input::memory_space, T,
3297 P...>::is_same_memspace) {
3298 // same behavior as deep_copy(src, src)
3299 if constexpr (!alloc_prop_input::has_execution_space)
3300 fence(
3301 "Kokkos::create_mirror_view_and_copy: fence before returning src "
3302 "view");
3303 return src;
3304 } else {
3305 using Space = typename alloc_prop_input::memory_space;
3306 using Mirror = typename Impl::MirrorViewType<Space, T, P...>::view_type;
3307
3308 auto arg_prop_copy = Impl::with_properties_if_unset(
3309 arg_prop, std::string{}, WithoutInitializing,
3310 typename Space::execution_space{});
3311
3312 std::string& label = Impl::get_property<Impl::LabelTag>(arg_prop_copy);
3313 if (label.empty()) label = src.label();
3314 auto mirror = typename Mirror::non_const_type{arg_prop_copy, src.layout()};
3315 if constexpr (alloc_prop_input::has_execution_space) {
3316 deep_copy(Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop_copy),
3317 mirror, src);
3318 } else
3319 deep_copy(mirror, src);
3320 return mirror;
3321 }
3322#if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3323 !defined(KOKKOS_COMPILER_MSVC)
3324 __builtin_unreachable();
3325#endif
3326}
3327
3328// Previously when using auto here, the intel compiler 19.3 would
3329// sometimes not create a symbol, guessing that it somehow is a combination
3330// of auto and just forwarding arguments (see issue #5196)
3331template <class Space, class T, class... P,
3332 typename Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3333typename Impl::MirrorViewType<Space, T, P...>::view_type
3334create_mirror_view_and_copy(
3335 const Space&, const Kokkos::View<T, P...>& src,
3336 std::string const& name = "",
3337 std::enable_if_t<
3338 std::is_void_v<typename ViewTraits<T, P...>::specialize>>* = nullptr) {
3339 return create_mirror_view_and_copy(
3340 Kokkos::view_alloc(typename Space::memory_space{}, name), src);
3341}
3342
3343} /* namespace Kokkos */
3344
3345//----------------------------------------------------------------------------
3346//----------------------------------------------------------------------------
3347
3348#endif
View
View
Definition Kokkos_View.hpp:157
Kokkos_Layout.hpp
Declaration of various MemoryLayout options.
Kokkos_Parallel.hpp
Declaration of parallel operators.
Kokkos::Bitset
A thread safe view to a bitset.
Definition Kokkos_Bitset.hpp:49
Kokkos::HostSpace
Memory management for host memory.
Definition Kokkos_HostSpace.hpp:49
Kokkos::HostSpace::name
static constexpr const char * name()
Return Name of the MemorySpace.
Definition Kokkos_HostSpace.hpp:127
Kokkos::Impl
ScopeGuard Some user scope issues have been identified with some Kokkos::finalize calls; ScopeGuard a...
Definition Kokkos_Abort.hpp:33
Kokkos::LayoutLeft
Memory layout tag indicating left-to-right (Fortran scheme) striding of multi-indices.
Definition Kokkos_Layout.hpp:50
Kokkos::LayoutRight
Memory layout tag indicating right-to-left (C or lexigraphical scheme) striding of multi-indices.
Definition Kokkos_Layout.hpp:104
Kokkos::LayoutStride
Memory layout tag indicated arbitrarily strided multi-index mapping into contiguous memory.
Definition Kokkos_Layout.hpp:148
Kokkos::SpaceAccessibility
Can AccessSpace access MemorySpace ?
Definition Kokkos_Concepts.hpp:359
Generated by doxygen 1.9.8