Intrepid2
Intrepid2_Data.hpp
Go to the documentation of this file.
1// @HEADER
2// *****************************************************************************
3// Intrepid2 Package
4//
5// Copyright 2007 NTESS and the Intrepid2 contributors.
6// SPDX-License-Identifier: BSD-3-Clause
7// *****************************************************************************
8// @HEADER
9//
10// Intrepid2_Data.hpp
11// QuadraturePerformance
12//
13// Created by Roberts, Nathan V on 8/24/20.
14//
15
16#ifndef Intrepid2_Data_h
17#define Intrepid2_Data_h
18
23#include "Intrepid2_ScalarView.hpp"
24#include "Intrepid2_Utils.hpp"
25
26#include <variant>
27
33namespace Intrepid2 {
34
43template<class DataScalar,typename DeviceType>
44class ZeroView {
45public:
46 static ScalarView<DataScalar,DeviceType> zeroView()
47 {
48 static ScalarView<DataScalar,DeviceType> zeroView = ScalarView<DataScalar,DeviceType>("zero",1);
49 static bool havePushedFinalizeHook = false;
50 if (!havePushedFinalizeHook)
51 {
52 Kokkos::push_finalize_hook( [=] {
53 zeroView = ScalarView<DataScalar,DeviceType>();
54 });
55 havePushedFinalizeHook = true;
56 }
57 return zeroView;
58 }
59};
60
78 template<class DataScalar,typename DeviceType>
79 class Data {
80 public:
81 using value_type = DataScalar;
82 using execution_space = typename DeviceType::execution_space;
83
84 using reference_type = typename ScalarView<DataScalar,DeviceType>::reference_type;
85 using const_reference_type = typename ScalarView<const DataScalar,DeviceType>::reference_type;
86 private:
87 ordinal_type dataRank_;
88 using View1 = Kokkos::View<DataScalar*, DeviceType>;
89 using View2 = Kokkos::View<DataScalar**, DeviceType>;
90 using View3 = Kokkos::View<DataScalar***, DeviceType>;
91 using View4 = Kokkos::View<DataScalar****, DeviceType>;
92 using View5 = Kokkos::View<DataScalar*****, DeviceType>;
93 using View6 = Kokkos::View<DataScalar******, DeviceType>;
94 using View7 = Kokkos::View<DataScalar*******, DeviceType>;
95 std::variant<View1,View2,View3,View4,View5,View6,View7> underlyingView_;
96 Kokkos::Array<int,7> extents_; // logical extents in each dimension
97 Kokkos::Array<DataVariationType,7> variationType_; // for each dimension, whether the data varies in that dimension
98 Kokkos::Array<int,7> variationModulus_; // for each dimension, a value by which indices should be modulused (only used when variationType_ is MODULAR)
99 int blockPlusDiagonalLastNonDiagonal_ = -1; // last row/column that is part of the non-diagonal part of the matrix indicated by BLOCK_PLUS_DIAGONAL (if any dimensions are thus marked)
100
101 bool underlyingMatchesLogical_; // if true, this Data object has the same rank and extent as the underlying view
102 Kokkos::Array<ordinal_type,7> activeDims_;
103 int numActiveDims_; // how many of the 7 entries are actually filled in
104
105 ordinal_type rank_;
106
107 // we use (const_)reference_type as the return for operator() for performance reasons, especially significant when using Sacado types
108 using return_type = const_reference_type;
109
110 ScalarView<DataScalar,DeviceType> zeroView_; // one-entry (zero); used to allow getEntry() to return 0 for off-diagonal entries in BLOCK_PLUS_DIAGONAL
111
113 KOKKOS_INLINE_FUNCTION
114 static int blockPlusDiagonalNumNondiagonalEntries(const int &lastNondiagonal)
115 {
116 return (lastNondiagonal + 1) * (lastNondiagonal + 1);
117 }
118
120 KOKKOS_INLINE_FUNCTION
121 static int blockPlusDiagonalBlockEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i, const int &j)
122 {
123 return i * (lastNondiagonal + 1) + j;
124 }
125
127 KOKKOS_INLINE_FUNCTION
128 static int blockPlusDiagonalDiagonalEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i)
129 {
130 return i - (lastNondiagonal + 1) + numNondiagonalEntries;
131 }
132
133 template <class T>
134 KOKKOS_INLINE_FUNCTION const T& get_fixed_view(const std::variant<View1,View2,View3,View4,View5,View6,View7> & v) const {
135 return *reinterpret_cast<const T*>(&v);
136 }
137
139 KOKKOS_INLINE_FUNCTION
140 int getUnderlyingViewExtent(const int &dim) const
141 {
142 switch (dataRank_)
143 {
144 case 1: return get_fixed_view<View1>(underlyingView_).extent_int(dim);
145 case 2: return get_fixed_view<View2>(underlyingView_).extent_int(dim);
146 case 3: return get_fixed_view<View3>(underlyingView_).extent_int(dim);
147 case 4: return get_fixed_view<View4>(underlyingView_).extent_int(dim);
148 case 5: return get_fixed_view<View5>(underlyingView_).extent_int(dim);
149 case 6: return get_fixed_view<View6>(underlyingView_).extent_int(dim);
150 case 7: return get_fixed_view<View7>(underlyingView_).extent_int(dim);
151 default: return -1;
152 }
153 }
154
157 {
158 INTREPID2_TEST_FOR_EXCEPTION(underlyingView_.valueless_by_exception(), std::invalid_argument, "underlyingView_ not correctly set!");
159 zeroView_ = ZeroView<DataScalar,DeviceType>::zeroView(); // one-entry (zero); used to allow getEntry() to return 0 for off-diagonal entries in BLOCK_PLUS_DIAGONAL
160 // check that rank is compatible with the claimed extents:
161 for (int d=rank_; d<7; d++)
162 {
163 INTREPID2_TEST_FOR_EXCEPTION(extents_[d] > 1, std::invalid_argument, "Nominal extents may not be > 1 in dimensions beyond the rank of the container");
164 }
165
166 numActiveDims_ = 0;
167 int blockPlusDiagonalCount = 0;
168 underlyingMatchesLogical_ = true;
169 for (ordinal_type i=0; i<7; i++)
170 {
171 if (variationType_[i] == GENERAL)
172 {
173 if (extents_[i] != 0)
174 {
175 variationModulus_[i] = extents_[i];
176 }
177 else
178 {
179 variationModulus_[i] = 1;
180 }
181 activeDims_[numActiveDims_] = i;
182 numActiveDims_++;
183 }
184 else if (variationType_[i] == MODULAR)
185 {
186 underlyingMatchesLogical_ = false;
187 if (extents_[i] != getUnderlyingViewExtent(numActiveDims_))
188 {
189 const int dataExtent = getUnderlyingViewExtent(numActiveDims_);
190 const int logicalExtent = extents_[i];
191 const int modulus = dataExtent;
192
193 INTREPID2_TEST_FOR_EXCEPTION( dataExtent * (logicalExtent / dataExtent) != logicalExtent, std::invalid_argument, "data extent must evenly divide logical extent");
194
195 variationModulus_[i] = modulus;
196 }
197 else
198 {
199 variationModulus_[i] = extents_[i];
200 }
201 activeDims_[numActiveDims_] = i;
202 numActiveDims_++;
203 }
204 else if (variationType_[i] == BLOCK_PLUS_DIAGONAL)
205 {
206 underlyingMatchesLogical_ = false;
207 blockPlusDiagonalCount++;
208 if (blockPlusDiagonalCount == 1) // first dimension thus marked --> active
209 {
210
211#ifdef HAVE_INTREPID2_DEBUG
212 const int numNondiagonalEntries = blockPlusDiagonalNumNondiagonalEntries(blockPlusDiagonalLastNonDiagonal_);
213 const int dataExtent = getUnderlyingViewExtent(numActiveDims_); // flat storage of all matrix entries
214 const int logicalExtent = extents_[i];
215 const int numDiagonalEntries = logicalExtent - (blockPlusDiagonalLastNonDiagonal_ + 1);
216 const int expectedDataExtent = numNondiagonalEntries + numDiagonalEntries;
217 INTREPID2_TEST_FOR_EXCEPTION(dataExtent != expectedDataExtent, std::invalid_argument, ("BLOCK_PLUS_DIAGONAL data extent of " + std::to_string(dataExtent) + " does not match expected based on blockPlusDiagonalLastNonDiagonal setting of " + std::to_string(blockPlusDiagonalLastNonDiagonal_)).c_str());
218#endif
219
220 activeDims_[numActiveDims_] = i;
221 numActiveDims_++;
222 }
223 variationModulus_[i] = getUnderlyingViewExtent(numActiveDims_);
224 INTREPID2_TEST_FOR_EXCEPTION(variationType_[i+1] != BLOCK_PLUS_DIAGONAL, std::invalid_argument, "BLOCK_PLUS_DIAGONAL ranks must be contiguous");
225 i++; // skip over the next BLOCK_PLUS_DIAGONAL
226 variationModulus_[i] = 1; // trivial modulus (should not ever be used)
227 INTREPID2_TEST_FOR_EXCEPTION(blockPlusDiagonalCount > 1, std::invalid_argument, "BLOCK_PLUS_DIAGONAL can only apply to two ranks");
228 }
229 else // CONSTANT
230 {
231 if (i < rank_)
232 {
233 underlyingMatchesLogical_ = false;
234 }
235 variationModulus_[i] = 1; // trivial modulus
236 }
237 }
238
239 if (rank_ != dataRank_)
240 {
241 underlyingMatchesLogical_ = false;
242 }
243
244 for (int d=numActiveDims_; d<7; d++)
245 {
246 // for *inactive* dims, the activeDims_ map just is the identity
247 // (this allows getEntry() to work even when the logical rank of the Data object is lower than that of the underlying View. This can happen for gradients in 1D.)
248 activeDims_[d] = d;
249 }
250 for (int d=0; d<7; d++)
251 {
252 INTREPID2_TEST_FOR_EXCEPTION(variationModulus_[d] == 0, std::logic_error, "variationModulus should not ever be 0");
253 }
254 }
255
256 public:
258 template<class UnaryOperator>
259 void applyOperator(UnaryOperator unaryOperator)
260 {
261 using ExecutionSpace = typename DeviceType::execution_space;
262
263 switch (dataRank_)
264 {
265 case 1:
266 {
267 const int dataRank = 1;
268 auto view = getUnderlyingView<dataRank>();
269
270 const int dataExtent = this->getDataExtent(0);
271 Kokkos::RangePolicy<ExecutionSpace> policy(ExecutionSpace(),0,dataExtent);
272 Kokkos::parallel_for("apply operator in-place", policy,
273 KOKKOS_LAMBDA (const int &i0) {
274 view(i0) = unaryOperator(view(i0));
275 });
276
277 }
278 break;
279 case 2:
280 {
281 const int dataRank = 2;
282 auto policy = dataExtentRangePolicy<dataRank>();
283 auto view = getUnderlyingView<dataRank>();
284
285 Kokkos::parallel_for("apply operator in-place", policy,
286 KOKKOS_LAMBDA (const int &i0, const int &i1) {
287 view(i0,i1) = unaryOperator(view(i0,i1));
288 });
289 }
290 break;
291 case 3:
292 {
293 const int dataRank = 3;
294 auto policy = dataExtentRangePolicy<dataRank>();
295 auto view = getUnderlyingView<dataRank>();
296
297 Kokkos::parallel_for("apply operator in-place", policy,
298 KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2) {
299 view(i0,i1,i2) = unaryOperator(view(i0,i1,i2));
300 });
301 }
302 break;
303 case 4:
304 {
305 const int dataRank = 4;
306 auto policy = dataExtentRangePolicy<dataRank>();
307 auto view = getUnderlyingView<dataRank>();
308
309 Kokkos::parallel_for("apply operator in-place", policy,
310 KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3) {
311 view(i0,i1,i2,i3) = unaryOperator(view(i0,i1,i2,i3));
312 });
313 }
314 break;
315 case 5:
316 {
317 const int dataRank = 5;
318 auto policy = dataExtentRangePolicy<dataRank>();
319 auto view = getUnderlyingView<dataRank>();
320
321 Kokkos::parallel_for("apply operator in-place", policy,
322 KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3, const int &i4) {
323 view(i0,i1,i2,i3,i4) = unaryOperator(view(i0,i1,i2,i3,i4));
324 });
325 }
326 break;
327 case 6:
328 {
329 const int dataRank = 6;
330 auto policy = dataExtentRangePolicy<dataRank>();
331 auto view = getUnderlyingView<dataRank>();
332
333 Kokkos::parallel_for("apply operator in-place", policy,
334 KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3, const int &i4, const int &i5) {
335 view(i0,i1,i2,i3,i4,i5) = unaryOperator(view(i0,i1,i2,i3,i4,i5));
336 });
337 }
338 break;
339 case 7:
340 {
341 const int dataRank = 7;
342 auto policy6 = dataExtentRangePolicy<6>();
343 auto view = getUnderlyingView<dataRank>();
344
345 const int dim_i6 = view.extent_int(6);
346
347 Kokkos::parallel_for("apply operator in-place", policy6,
348 KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3, const int &i4, const int &i5) {
349 for (int i6=0; i6<dim_i6; i6++)
350 {
351 view(i0,i1,i2,i3,i4,i5,i6) = unaryOperator(view(i0,i1,i2,i3,i4,i5,i6));
352 }
353 });
354 }
355 break;
356 default:
357 INTREPID2_TEST_FOR_EXCEPTION(true,std::invalid_argument,"Unsupported data rank");
358 }
359 }
360
362 template<class ...IntArgs>
363 KOKKOS_INLINE_FUNCTION
364 reference_type getWritableEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs... intArgs) const
365 {
366 #ifdef INTREPID2_HAVE_DEBUG
367 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(numArgs != rank_, std::invalid_argument, "getWritableEntry() should have the same number of arguments as the logical rank.");
368 #endif
369 constexpr int numArgs = sizeof...(intArgs);
370 if (underlyingMatchesLogical_)
371 {
372 // in this case, we require that numArgs == dataRank_
373 return getUnderlyingView<numArgs>()(intArgs...);
374 }
375
376 // extract the type of the first argument; use that for the arrays below
377 using int_type = std::tuple_element_t<0, std::tuple<IntArgs...>>;
378
379 const Kokkos::Array<int_type, numArgs+1> args {intArgs...,0}; // we pad with one extra entry (0) to avoid gcc compiler warnings about references beyond the bounds of the array (the [d+1]'s below)
380 Kokkos::Array<int_type, 7> refEntry;
381 for (int d=0; d<numArgs; d++)
382 {
383 switch (variationType_[d])
384 {
385 case CONSTANT: refEntry[d] = 0; break;
386 case GENERAL: refEntry[d] = args[d]; break;
387 case MODULAR: refEntry[d] = args[d] % variationModulus_[d]; break;
389 {
390 if (passThroughBlockDiagonalArgs)
391 {
392 refEntry[d] = args[d];
393 refEntry[d+1] = args[d+1]; // this had better be == 0
394 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(args[d+1] != 0, std::invalid_argument, "getWritableEntry() called with passThroughBlockDiagonalArgs = true, but nonzero second matrix argument.");
395 }
396 else
397 {
398 const int numNondiagonalEntries = blockPlusDiagonalNumNondiagonalEntries(blockPlusDiagonalLastNonDiagonal_);
399
400 const int_type &i = args[d];
401 if (d+1 >= numArgs)
402 {
403 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::invalid_argument, "BLOCK_PLUS_DIAGONAL must be present for two dimensions; here, encountered only one");
404 }
405 else
406 {
407 const int_type &j = args[d+1];
408
409 if ((i > static_cast<int_type>(blockPlusDiagonalLastNonDiagonal_)) || (j > static_cast<int_type>(blockPlusDiagonalLastNonDiagonal_)))
410 {
411 if (i != j)
412 {
413 // off diagonal: zero
414 return zeroView_(0); // NOTE: this branches in an argument-dependent way; this is not great for CUDA performance. When using BLOCK_PLUS_DIAGONAL, should generally avoid calls to this getEntry() method. (Use methods that directly take advantage of the data packing instead.)
415 }
416 else
417 {
418 refEntry[d] = blockPlusDiagonalDiagonalEntryIndex(blockPlusDiagonalLastNonDiagonal_, numNondiagonalEntries, i);
419 }
420 }
421 else
422 {
423 refEntry[d] = blockPlusDiagonalBlockEntryIndex(blockPlusDiagonalLastNonDiagonal_, numNondiagonalEntries, i, j);
424 }
425
426 // skip next d (this is required also to be BLOCK_PLUS_DIAGONAL, and we've consumed its arg as j above)
427 refEntry[d+1] = 0;
428 }
429 }
430 d++;
431 }
432 }
433 }
434 // refEntry should be zero-filled beyond numArgs, for cases when rank_ < dataRank_ (this only is allowed if the extra dimensions each has extent 1).
435 for (int d=numArgs; d<7; d++)
436 {
437 refEntry[d] = 0;
438 }
439
440 if (dataRank_ == 1)
441 {
442 return get_fixed_view<View1>(underlyingView_)(refEntry[activeDims_[0]]);
443 }
444 else if (dataRank_ == 2)
445 {
446 return get_fixed_view<View2>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]]);
447 }
448 else if (dataRank_ == 3)
449 {
450 return get_fixed_view<View3>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]]);
451 }
452 else if (dataRank_ == 4)
453 {
454 return get_fixed_view<View4>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]]);
455 }
456 else if (dataRank_ == 5)
457 {
458 return get_fixed_view<View5>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
459 refEntry[activeDims_[4]]);
460 }
461 else if (dataRank_ == 6)
462 {
463 return get_fixed_view<View6>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
464 refEntry[activeDims_[4]],refEntry[activeDims_[5]]);
465 }
466 else // dataRank_ == 7
467 {
468 return get_fixed_view<View7>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
469 refEntry[activeDims_[4]],refEntry[activeDims_[5]],refEntry[activeDims_[6]]);
470 }
471
472 }
473
475 template<class ...IntArgs>
476 KOKKOS_INLINE_FUNCTION
477 reference_type getWritableEntry(const IntArgs... intArgs) const
478 {
479 return getWritableEntryWithPassThroughOption(false, intArgs...);
480 }
481 public:
483 template<class ToContainer, class FromContainer>
484 static void copyContainer(ToContainer to, FromContainer from)
485 {
486// std::cout << "Entered copyContainer().\n";
487 auto policy = Kokkos::MDRangePolicy<execution_space,Kokkos::Rank<6>>({0,0,0,0,0,0},{from.extent_int(0),from.extent_int(1),from.extent_int(2), from.extent_int(3), from.extent_int(4), from.extent_int(5)});
488
489 Kokkos::parallel_for("copyContainer", policy,
490 KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3, const int &i4, const int &i5) {
491 for (int i6=0; i6<from.extent_int(6); i6++)
492 {
493 to.access(i0,i1,i2,i3,i4,i5,i6) = from.access(i0,i1,i2,i3,i4,i5,i6);
494 }
495 });
496 }
497
499 void allocateAndCopyFromDynRankView(ScalarView<DataScalar,DeviceType> data)
500 {
501// std::cout << "Entered allocateAndCopyFromDynRankView().\n";
502 switch (dataRank_)
503 {
504 case 1: underlyingView_ = Kokkos::View<DataScalar*, DeviceType>("Intrepid2 Data", data.extent_int(0)); break;
505 case 2: underlyingView_ = Kokkos::View<DataScalar**, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1)); break;
506 case 3: underlyingView_ = Kokkos::View<DataScalar***, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2)); break;
507 case 4: underlyingView_ = Kokkos::View<DataScalar****, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3)); break;
508 case 5: underlyingView_ = Kokkos::View<DataScalar*****, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4)); break;
509 case 6: underlyingView_ = Kokkos::View<DataScalar******, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4), data.extent_int(5)); break;
510 case 7: underlyingView_ = Kokkos::View<DataScalar*******, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4), data.extent_int(5), data.extent_int(6)); break;
511 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
512 }
513
514 switch (dataRank_)
515 {
516 case 1: copyContainer(get_fixed_view<View1>(underlyingView_),data); break;
517 case 2: copyContainer(get_fixed_view<View2>(underlyingView_),data); break;
518 case 3: copyContainer(get_fixed_view<View3>(underlyingView_),data); break;
519 case 4: copyContainer(get_fixed_view<View4>(underlyingView_),data); break;
520 case 5: copyContainer(get_fixed_view<View5>(underlyingView_),data); break;
521 case 6: copyContainer(get_fixed_view<View6>(underlyingView_),data); break;
522 case 7: copyContainer(get_fixed_view<View7>(underlyingView_),data); break;
523 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
524 }
525 }
526
528 Data(std::vector<DimensionInfo> dimInfoVector)
529 :
530 // initialize member variables as if default constructor; if dimInfoVector is empty, we want default constructor behavior.
531 dataRank_(0), extents_({0,0,0,0,0,0,0}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(dimInfoVector.size())
532 {
533 // If dimInfoVector is empty, the member initialization above is correct; otherwise, we set as below.
534 // Either way, once members are initialized, we must call setActiveDims().
535 if (dimInfoVector.size() != 0)
536 {
537 std::vector<int> dataExtents;
538
539 bool blockPlusDiagonalEncountered = false;
540 for (int d=0; d<rank_; d++)
541 {
542 const DimensionInfo & dimInfo = dimInfoVector[d];
543 extents_[d] = dimInfo.logicalExtent;
544 variationType_[d] = dimInfo.variationType;
545 const bool isBlockPlusDiagonal = (variationType_[d] == BLOCK_PLUS_DIAGONAL);
546 const bool isSecondBlockPlusDiagonal = isBlockPlusDiagonal && blockPlusDiagonalEncountered;
547 if (isBlockPlusDiagonal)
548 {
549 blockPlusDiagonalEncountered = true;
550 blockPlusDiagonalLastNonDiagonal_ = dimInfo.blockPlusDiagonalLastNonDiagonal;
551 }
552 if ((variationType_[d] != CONSTANT) && (!isSecondBlockPlusDiagonal))
553 {
554 dataExtents.push_back(dimInfo.dataExtent);
555 }
556 }
557 if (dataExtents.size() == 0)
558 {
559 // constant data
560 dataExtents.push_back(1);
561 }
562 dataRank_ = dataExtents.size();
563 switch (dataRank_)
564 {
565 case 1: underlyingView_ = Kokkos::View<DataScalar*, DeviceType>("Intrepid2 Data", dataExtents[0]); break;
566 case 2: underlyingView_ = Kokkos::View<DataScalar**, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1]); break;
567 case 3: underlyingView_ = Kokkos::View<DataScalar***, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2]); break;
568 case 4: underlyingView_ = Kokkos::View<DataScalar****, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3]); break;
569 case 5: underlyingView_ = Kokkos::View<DataScalar*****, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4]); break;
570 case 6: underlyingView_ = Kokkos::View<DataScalar******, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4], dataExtents[5]); break;
571 case 7: underlyingView_ = Kokkos::View<DataScalar*******, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4], dataExtents[5], dataExtents[6]); break;
572 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
573 }
574 }
576 }
577
579 Data(const ScalarView<DataScalar,DeviceType> &data, int rank, Kokkos::Array<int,7> extents, Kokkos::Array<DataVariationType,7> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
580 :
581 dataRank_(data.rank()), extents_(extents), variationType_(variationType), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
582 {
585 }
586
588 template<typename OtherDeviceType, class = typename std::enable_if< std::is_same<typename DeviceType::memory_space, typename OtherDeviceType::memory_space>::value>::type,
589 class = typename std::enable_if<!std::is_same<DeviceType,OtherDeviceType>::value>::type>
590 Data(const Data<DataScalar,OtherDeviceType> &data)
591 :
592 dataRank_(data.getUnderlyingViewRank()), extents_(data.getExtents()), variationType_(data.getVariationTypes()), blockPlusDiagonalLastNonDiagonal_(data.blockPlusDiagonalLastNonDiagonal()), rank_(data.rank())
593 {
594// std::cout << "Entered copy-like Data constructor.\n";
595 if (dataRank_ != 0) // dataRank_ == 0 indicates an invalid Data object (a placeholder, can indicate zero value)
596 {
597 const auto view = data.getUnderlyingView();
598 switch (dataRank_)
599 {
600 case 1: underlyingView_ = data.getUnderlyingView1(); break;
601 case 2: underlyingView_ = data.getUnderlyingView2(); break;
602 case 3: underlyingView_ = data.getUnderlyingView3(); break;
603 case 4: underlyingView_ = data.getUnderlyingView4(); break;
604 case 5: underlyingView_ = data.getUnderlyingView5(); break;
605 case 6: underlyingView_ = data.getUnderlyingView6(); break;
606 case 7: underlyingView_ = data.getUnderlyingView7(); break;
607 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
608 }
609 }
610 setActiveDims();
611 }
612
614 template<typename OtherDeviceType, class = typename std::enable_if<!std::is_same<typename DeviceType::memory_space, typename OtherDeviceType::memory_space>::value>::type>
616 :
617 dataRank_(data.getUnderlyingViewRank()), extents_(data.getExtents()), variationType_(data.getVariationTypes()), blockPlusDiagonalLastNonDiagonal_(data.blockPlusDiagonalLastNonDiagonal()), rank_(data.rank())
618 {
619// std::cout << "Entered copy-like Data constructor.\n";
620 if (dataRank_ != 0) // dataRank_ == 0 indicates an invalid Data object (a placeholder, can indicate zero value)
621 {
622 const auto view = data.getUnderlyingView();
623 switch (dataRank_)
624 {
625 case 1: underlyingView_ = Kokkos::View<DataScalar*, DeviceType>("Intrepid2 Data", view.extent_int(0)); break;
626 case 2: underlyingView_ = Kokkos::View<DataScalar**, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1)); break;
627 case 3: underlyingView_ = Kokkos::View<DataScalar***, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2)); break;
628 case 4: underlyingView_ = Kokkos::View<DataScalar****, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3)); break;
629 case 5: underlyingView_ = Kokkos::View<DataScalar*****, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4)); break;
630 case 6: underlyingView_ = Kokkos::View<DataScalar******, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5)); break;
631 case 7: underlyingView_ = Kokkos::View<DataScalar*******, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5), view.extent_int(6)); break;
632 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
633 }
634
635 // copy
636 // (Note: Kokkos::deep_copy() will not generally work if the layouts are different; that's why we do a manual copy here once we have the data on the host):
637 // first, mirror and copy dataView; then copy to the appropriate data_ member
638 using MemorySpace = typename DeviceType::memory_space;
639 switch (dataRank_)
640 {
641 case 1: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView1()); copyContainer(get_fixed_view<View1>(underlyingView_), dataViewMirror);} break;
642 case 2: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView2()); copyContainer(get_fixed_view<View2>(underlyingView_), dataViewMirror);} break;
643 case 3: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView3()); copyContainer(get_fixed_view<View3>(underlyingView_), dataViewMirror);} break;
644 case 4: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView4()); copyContainer(get_fixed_view<View4>(underlyingView_), dataViewMirror);} break;
645 case 5: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView5()); copyContainer(get_fixed_view<View5>(underlyingView_), dataViewMirror);} break;
646 case 6: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView6()); copyContainer(get_fixed_view<View6>(underlyingView_), dataViewMirror);} break;
647 case 7: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView7()); copyContainer(get_fixed_view<View7>(underlyingView_), dataViewMirror);} break;
648 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
649 }
650 }
652 }
653
655// Data(const Data<DataScalar,ExecSpaceType> &data)
656// :
657// dataRank_(data.getUnderlyingViewRank()), extents_(data.getExtents()), variationType_(data.getVariationTypes()), blockPlusDiagonalLastNonDiagonal_(data.blockPlusDiagonalLastNonDiagonal()), rank_(data.rank())
658// {
659// std::cout << "Entered Data copy constructor.\n";
660// if (dataRank_ != 0) // dataRank_ == 0 indicates an invalid Data object (a placeholder, can indicate zero value)
661// {
662// const auto view = data.getUnderlyingView();
663// switch (dataRank_)
664// {
665// case 1: underlyingView_ = Kokkos::View<DataScalar*, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0)); break;
666// case 2: underlyingView_ = Kokkos::View<DataScalar**, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1)); break;
667// case 3: underlyingView_ = Kokkos::View<DataScalar***, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2)); break;
668// case 4: underlyingView_ = Kokkos::View<DataScalar****, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3)); break;
669// case 5: underlyingView_ = Kokkos::View<DataScalar*****, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4)); break;
670// case 6: underlyingView_ = Kokkos::View<DataScalar******, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5)); break;
671// case 7: underlyingView_ = Kokkos::View<DataScalar*******, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5), view.extent_int(6)); break;
672// default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
673// }
674//
675// // copy
676// // (Note: Kokkos::deep_copy() will not generally work if the layouts are different; that's why we do a manual copy here once we have the data on the host):
677// // first, mirror and copy dataView; then copy to the appropriate data_ member
678// using MemorySpace = typename DeviceType::memory_space;
679// switch (dataRank_)
680// {
681// case 1: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView1()); copyContainer(get_fixed_view<View1>(underlyingView_), dataViewMirror);} break;
682// case 2: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView2()); copyContainer(get_fixed_view<View2>(underlyingView_), dataViewMirror);} break;
683// case 3: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView3()); copyContainer(get_fixed_view<View3>(underlyingView_), dataViewMirror);} break;
684// case 4: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView4()); copyContainer(get_fixed_view<View4>(underlyingView_), dataViewMirror);} break;
685// case 5: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView5()); copyContainer(get_fixed_view<View5>(underlyingView_), dataViewMirror);} break;
686// case 6: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView6()); copyContainer(get_fixed_view<View6>(underlyingView_), dataViewMirror);} break;
687// case 7: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView7()); copyContainer(get_fixed_view<View7>(underlyingView_), dataViewMirror);} break;
688// default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
689// }
690// }
691//
692// setActiveDims();
693// }
694
696 Data(ScalarView<DataScalar,DeviceType> data)
697 :
698 Data(data,
699 data.rank(),
700 Kokkos::Array<int,7> {data.extent_int(0),data.extent_int(1),data.extent_int(2),data.extent_int(3),data.extent_int(4),data.extent_int(5),data.extent_int(6)},
701 Kokkos::Array<DataVariationType,7> {GENERAL,GENERAL,GENERAL,GENERAL,GENERAL,GENERAL,GENERAL}, -1)
702 {}
703
705 template<size_t rank, class ...DynRankViewProperties>
706 Data(const Kokkos::DynRankView<DataScalar,DeviceType, DynRankViewProperties...> &data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
707 :
708 dataRank_(data.rank()), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
709 {
710// std::cout << "Entered a DynRankView Data() constructor.\n";
712 for (unsigned d=0; d<rank; d++)
713 {
714 extents_[d] = extents[d];
715 variationType_[d] = variationType[d];
716 }
718 }
719
720 template<size_t rank, class ...ViewProperties>
721 Data(Kokkos::View<DataScalar*,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
722 :
723 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
724 {
725 underlyingView_ = data;
726 for (unsigned d=0; d<rank; d++)
727 {
728 extents_[d] = extents[d];
729 variationType_[d] = variationType[d];
730 }
732 }
733
734 template<size_t rank, class ...ViewProperties>
735 Data(Kokkos::View<DataScalar**,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
736 :
737 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
738 {
739 underlyingView_ = data;
740 for (unsigned d=0; d<rank; d++)
741 {
742 extents_[d] = extents[d];
743 variationType_[d] = variationType[d];
744 }
746 }
747
748 template<size_t rank, class ...ViewProperties>
749 Data(Kokkos::View<DataScalar***,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
750 :
751 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
752 {
753 underlyingView_ = data;
754 for (unsigned d=0; d<rank; d++)
755 {
756 extents_[d] = extents[d];
757 variationType_[d] = variationType[d];
758 }
760 }
761
762 template<size_t rank, class ...ViewProperties>
763 Data(Kokkos::View<DataScalar****,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
764 :
765 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
766 {
767 underlyingView_ = data;
768 for (unsigned d=0; d<rank; d++)
769 {
770 extents_[d] = extents[d];
771 variationType_[d] = variationType[d];
772 }
774 }
775
776 template<size_t rank, class ...ViewProperties>
777 Data(Kokkos::View<DataScalar*****,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
778 :
779 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
780 {
781 underlyingView_ = data;
782 for (unsigned d=0; d<rank; d++)
783 {
784 extents_[d] = extents[d];
785 variationType_[d] = variationType[d];
786 }
788 }
789
790 template<size_t rank, class ...ViewProperties>
791 Data(Kokkos::View<DataScalar******,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
792 :
793 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
794 {
795 underlyingView_ = data;
796 for (unsigned d=0; d<rank; d++)
797 {
798 extents_[d] = extents[d];
799 variationType_[d] = variationType[d];
800 }
802 }
803
804 template<size_t rank, class ...ViewProperties>
805 Data(Kokkos::View<DataScalar*******,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
806 :
807 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
808 {
809 underlyingView_ = data;
810 for (unsigned d=0; d<rank; d++)
811 {
812 extents_[d] = extents[d];
813 variationType_[d] = variationType[d];
814 }
816 }
817
819 template<class ViewScalar, class ...ViewProperties>
820 Data(const unsigned rank, Kokkos::View<ViewScalar,DeviceType, ViewProperties...> data, Kokkos::Array<int,7> extents, Kokkos::Array<DataVariationType,7> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
821 :
822 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
823 {
824 underlyingView_ = data;
825 for (unsigned d=0; d<rank; d++)
826 {
827 extents_[d] = extents[d];
828 variationType_[d] = variationType[d];
829 }
831 }
832
834 template<size_t rank>
835 Data(DataScalar constantValue, Kokkos::Array<int,rank> extents)
836 :
837 dataRank_(1), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(rank)
838 {
839 underlyingView_ = Kokkos::View<DataScalar*,DeviceType>("Constant Data",1);
840 Kokkos::deep_copy(get_fixed_view<View1>(underlyingView_), constantValue);
841 for (unsigned d=0; d<rank; d++)
842 {
843 extents_[d] = extents[d];
844 }
846 }
847
850 :
851 dataRank_(0), extents_({0,0,0,0,0,0,0}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(0)
852 {
854 }
855
857 KOKKOS_INLINE_FUNCTION
859 {
860 return blockPlusDiagonalLastNonDiagonal_;
861 }
862
864 KOKKOS_INLINE_FUNCTION
865 Kokkos::Array<int,7> getExtents() const
866 {
867 return extents_;
868 }
869
871 KOKKOS_INLINE_FUNCTION
872 DimensionInfo getDimensionInfo(const int &dim) const
873 {
874 DimensionInfo dimInfo;
875
876 dimInfo.logicalExtent = extent_int(dim);
877 dimInfo.variationType = variationType_[dim];
878 dimInfo.dataExtent = getDataExtent(dim);
879 dimInfo.variationModulus = variationModulus_[dim];
880
881 if (dimInfo.variationType == BLOCK_PLUS_DIAGONAL)
882 {
883 dimInfo.blockPlusDiagonalLastNonDiagonal = blockPlusDiagonalLastNonDiagonal_;
884 }
885 return dimInfo;
886 }
887
889 KOKKOS_INLINE_FUNCTION
890 DimensionInfo combinedDataDimensionInfo(const Data &otherData, const int &dim) const
891 {
892 const DimensionInfo myDimInfo = getDimensionInfo(dim);
893 const DimensionInfo otherDimInfo = otherData.getDimensionInfo(dim);
894
895 return combinedDimensionInfo(myDimInfo, otherDimInfo);
896 }
897
899 template<int rank>
900 KOKKOS_INLINE_FUNCTION
901 enable_if_t<rank==1, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
903 {
904 #ifdef HAVE_INTREPID2_DEBUG
905 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
906 #endif
907 return get_fixed_view<View1>(underlyingView_);
908 }
909
911 template<int rank>
912 KOKKOS_INLINE_FUNCTION
913 enable_if_t<rank==2, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
915 {
916 #ifdef HAVE_INTREPID2_DEBUG
917 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
918 #endif
919 return get_fixed_view<View2>(underlyingView_);
920 }
921
923 template<int rank>
924 KOKKOS_INLINE_FUNCTION
925 enable_if_t<rank==3, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
927 {
928 #ifdef HAVE_INTREPID2_DEBUG
929 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
930 #endif
931 return get_fixed_view<View3>(underlyingView_);
932 }
933
935 template<int rank>
936 KOKKOS_INLINE_FUNCTION
937 enable_if_t<rank==4, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
939 {
940 #ifdef HAVE_INTREPID2_DEBUG
941 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
942 #endif
943 return get_fixed_view<View4>(underlyingView_);
944 }
945
947 template<int rank>
948 KOKKOS_INLINE_FUNCTION
949 enable_if_t<rank==5, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
951 {
952 #ifdef HAVE_INTREPID2_DEBUG
953 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
954 #endif
955 return get_fixed_view<View5>(underlyingView_);
956 }
957
959 template<int rank>
960 KOKKOS_INLINE_FUNCTION
961 enable_if_t<rank==6, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
963 {
964 #ifdef HAVE_INTREPID2_DEBUG
965 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
966 #endif
967 return get_fixed_view<View6>(underlyingView_);
968 }
969
971 template<int rank>
972 KOKKOS_INLINE_FUNCTION
973 enable_if_t<rank==7, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
975 {
976 #ifdef HAVE_INTREPID2_DEBUG
977 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
978 #endif
979 return get_fixed_view<View7>(underlyingView_);
980 }
981
983 KOKKOS_INLINE_FUNCTION
984 const Kokkos::View<DataScalar*, DeviceType> & getUnderlyingView1() const
985 {
986 return getUnderlyingView<1>();
987 }
988
990 KOKKOS_INLINE_FUNCTION
991 const Kokkos::View<DataScalar**, DeviceType> & getUnderlyingView2() const
992 {
993 return getUnderlyingView<2>();
994 }
995
997 KOKKOS_INLINE_FUNCTION
998 const Kokkos::View<DataScalar***, DeviceType> & getUnderlyingView3() const
999 {
1000 return getUnderlyingView<3>();
1001 }
1002
1004 KOKKOS_INLINE_FUNCTION
1005 const Kokkos::View<DataScalar****, DeviceType> & getUnderlyingView4() const
1006 {
1007 return getUnderlyingView<4>();
1008 }
1009
1011 KOKKOS_INLINE_FUNCTION
1012 const Kokkos::View<DataScalar*****, DeviceType> & getUnderlyingView5() const
1013 {
1014 return getUnderlyingView<5>();
1015 }
1016
1018 KOKKOS_INLINE_FUNCTION
1019 const Kokkos::View<DataScalar******, DeviceType> & getUnderlyingView6() const
1020 {
1021 return getUnderlyingView<6>();
1022 }
1023
1025 KOKKOS_INLINE_FUNCTION
1026 const Kokkos::View<DataScalar*******, DeviceType> & getUnderlyingView7() const
1027 {
1028 return getUnderlyingView<7>();
1029 }
1030
1032 ScalarView<DataScalar,DeviceType> getUnderlyingView() const
1033 {
1034 switch (dataRank_)
1035 {
1036 case 1: return get_fixed_view<View1>(underlyingView_);
1037 case 2: return get_fixed_view<View2>(underlyingView_);
1038 case 3: return get_fixed_view<View3>(underlyingView_);
1039 case 4: return get_fixed_view<View4>(underlyingView_);
1040 case 5: return get_fixed_view<View5>(underlyingView_);
1041 case 6: return get_fixed_view<View6>(underlyingView_);
1042 case 7: return get_fixed_view<View7>(underlyingView_);
1043 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1044 }
1045 }
1046
1048 KOKKOS_INLINE_FUNCTION
1049 ordinal_type getUnderlyingViewRank() const
1050 {
1051 return dataRank_;
1052 }
1053
1055 KOKKOS_INLINE_FUNCTION
1056 ordinal_type getUnderlyingViewSize() const
1057 {
1058 ordinal_type size = 1;
1059 for (ordinal_type r=0; r<dataRank_; r++)
1060 {
1061 size *= getUnderlyingViewExtent(r);
1062 }
1063 return size;
1064 }
1065
1067 ScalarView<DataScalar,DeviceType> allocateDynRankViewMatchingUnderlying() const
1068 {
1069 switch (dataRank_)
1070 {
1071 case 1: return getMatchingViewWithLabel(get_fixed_view<View1>(underlyingView_), "Intrepid2 Data", get_fixed_view<View1>(underlyingView_).extent_int(0));
1072 case 2: return getMatchingViewWithLabel(get_fixed_view<View2>(underlyingView_), "Intrepid2 Data", get_fixed_view<View2>(underlyingView_).extent_int(0), get_fixed_view<View2>(underlyingView_).extent_int(1));
1073 case 3: return getMatchingViewWithLabel(get_fixed_view<View3>(underlyingView_), "Intrepid2 Data", get_fixed_view<View3>(underlyingView_).extent_int(0), get_fixed_view<View3>(underlyingView_).extent_int(1), get_fixed_view<View3>(underlyingView_).extent_int(2));
1074 case 4: return getMatchingViewWithLabel(get_fixed_view<View4>(underlyingView_), "Intrepid2 Data", get_fixed_view<View4>(underlyingView_).extent_int(0), get_fixed_view<View4>(underlyingView_).extent_int(1), get_fixed_view<View4>(underlyingView_).extent_int(2), get_fixed_view<View4>(underlyingView_).extent_int(3));
1075 case 5: return getMatchingViewWithLabel(get_fixed_view<View5>(underlyingView_), "Intrepid2 Data", get_fixed_view<View5>(underlyingView_).extent_int(0), get_fixed_view<View5>(underlyingView_).extent_int(1), get_fixed_view<View5>(underlyingView_).extent_int(2), get_fixed_view<View5>(underlyingView_).extent_int(3), get_fixed_view<View5>(underlyingView_).extent_int(4));
1076 case 6: return getMatchingViewWithLabel(get_fixed_view<View6>(underlyingView_), "Intrepid2 Data", get_fixed_view<View6>(underlyingView_).extent_int(0), get_fixed_view<View6>(underlyingView_).extent_int(1), get_fixed_view<View6>(underlyingView_).extent_int(2), get_fixed_view<View6>(underlyingView_).extent_int(3), get_fixed_view<View6>(underlyingView_).extent_int(4), get_fixed_view<View6>(underlyingView_).extent_int(5));
1077 case 7: return getMatchingViewWithLabel(get_fixed_view<View7>(underlyingView_), "Intrepid2 Data", get_fixed_view<View7>(underlyingView_).extent_int(0), get_fixed_view<View7>(underlyingView_).extent_int(1), get_fixed_view<View7>(underlyingView_).extent_int(2), get_fixed_view<View7>(underlyingView_).extent_int(3), get_fixed_view<View7>(underlyingView_).extent_int(4), get_fixed_view<View7>(underlyingView_).extent_int(5), get_fixed_view<View7>(underlyingView_).extent_int(6));
1078 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1079 }
1080 }
1081
1083 template<class ... DimArgs>
1084 ScalarView<DataScalar,DeviceType> allocateDynRankViewMatchingUnderlying(DimArgs... dims) const
1085 {
1086 switch (dataRank_)
1087 {
1088 case 1: return getMatchingViewWithLabel(get_fixed_view<View1>(underlyingView_), "Intrepid2 Data", dims...);
1089 case 2: return getMatchingViewWithLabel(get_fixed_view<View2>(underlyingView_), "Intrepid2 Data", dims...);
1090 case 3: return getMatchingViewWithLabel(get_fixed_view<View3>(underlyingView_), "Intrepid2 Data", dims...);
1091 case 4: return getMatchingViewWithLabel(get_fixed_view<View4>(underlyingView_), "Intrepid2 Data", dims...);
1092 case 5: return getMatchingViewWithLabel(get_fixed_view<View5>(underlyingView_), "Intrepid2 Data", dims...);
1093 case 6: return getMatchingViewWithLabel(get_fixed_view<View6>(underlyingView_), "Intrepid2 Data", dims...);
1094 case 7: return getMatchingViewWithLabel(get_fixed_view<View7>(underlyingView_), "Intrepid2 Data", dims...);
1095 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1096 }
1097 }
1098
1100 void clear() const
1101 {
1102 switch (dataRank_)
1103 {
1104 case 1: Kokkos::deep_copy(get_fixed_view<View1>(underlyingView_), 0.0); break;
1105 case 2: Kokkos::deep_copy(get_fixed_view<View2>(underlyingView_), 0.0); break;
1106 case 3: Kokkos::deep_copy(get_fixed_view<View3>(underlyingView_), 0.0); break;
1107 case 4: Kokkos::deep_copy(get_fixed_view<View4>(underlyingView_), 0.0); break;
1108 case 5: Kokkos::deep_copy(get_fixed_view<View5>(underlyingView_), 0.0); break;
1109 case 6: Kokkos::deep_copy(get_fixed_view<View6>(underlyingView_), 0.0); break;
1110 case 7: Kokkos::deep_copy(get_fixed_view<View7>(underlyingView_), 0.0); break;
1111 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1112 }
1113 }
1114
1116 void copyDataFromDynRankViewMatchingUnderlying(const ScalarView<DataScalar,DeviceType> &dynRankView) const
1117 {
1118// std::cout << "Entered copyDataFromDynRankViewMatchingUnderlying().\n";
1119 switch (dataRank_)
1120 {
1121 case 1: copyContainer(get_fixed_view<View1>(underlyingView_),dynRankView); break;
1122 case 2: copyContainer(get_fixed_view<View2>(underlyingView_),dynRankView); break;
1123 case 3: copyContainer(get_fixed_view<View3>(underlyingView_),dynRankView); break;
1124 case 4: copyContainer(get_fixed_view<View4>(underlyingView_),dynRankView); break;
1125 case 5: copyContainer(get_fixed_view<View5>(underlyingView_),dynRankView); break;
1126 case 6: copyContainer(get_fixed_view<View6>(underlyingView_),dynRankView); break;
1127 case 7: copyContainer(get_fixed_view<View7>(underlyingView_),dynRankView); break;
1128 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1129 }
1130 }
1131
1133 KOKKOS_INLINE_FUNCTION int getDataExtent(const ordinal_type &d) const
1134 {
1135 for (int i=0; i<numActiveDims_; i++)
1136 {
1137 if (activeDims_[i] == d)
1138 {
1139 return getUnderlyingViewExtent(i);
1140 }
1141 else if (activeDims_[i] > d)
1142 {
1143 return 1; // data does not vary in the specified dimension
1144 }
1145 }
1146 return 1; // data does not vary in the specified dimension
1147 }
1148
1160 KOKKOS_INLINE_FUNCTION
1161 int getVariationModulus(const int &d) const
1162 {
1163 return variationModulus_[d];
1164 }
1165
1167 KOKKOS_INLINE_FUNCTION
1168 const Kokkos::Array<DataVariationType,7> & getVariationTypes() const
1169 {
1170 return variationType_;
1171 }
1172
1174 template<class ...IntArgs>
1175 KOKKOS_INLINE_FUNCTION
1176 return_type getEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs&... intArgs) const
1177 {
1178 return getWritableEntryWithPassThroughOption(passThroughBlockDiagonalArgs, intArgs...);
1179 }
1180
1182 template<class ...IntArgs>
1183 KOKKOS_INLINE_FUNCTION
1184 return_type getEntry(const IntArgs&... intArgs) const
1185 {
1186 return getEntryWithPassThroughOption(false, intArgs...);
1187 }
1188
1189 template <bool...> struct bool_pack;
1190
1191 template <bool... v>
1192 using all_true = std::is_same<bool_pack<true, v...>, bool_pack<v..., true>>;
1193
1194 template <class ...IntArgs>
1195 using valid_args = all_true<std::is_integral<IntArgs>{}...>;
1196
1197 static_assert(valid_args<int,long,unsigned>::value, "valid args works");
1198
1200 template <class ...IntArgs>
1201 KOKKOS_INLINE_FUNCTION
1202#ifndef __INTEL_COMPILER
1203 // icc has a bug that prevents compilation with this enable_if_t
1204 // (possibly the same as https://community.intel.com/t5/Intel-C-Compiler/Intel-Compiler-bug-while-deducing-template-arguments-inside/m-p/1164358)
1205 // so with icc we'll just skip the argument type/count check
1206 enable_if_t<valid_args<IntArgs...>::value && (sizeof...(IntArgs) <= 7),return_type>
1207#else
1208 return_type
1209#endif
1210 operator()(const IntArgs&... intArgs) const {
1211 return getEntry(intArgs...);
1212 }
1213
1215 KOKKOS_INLINE_FUNCTION
1216 int extent_int(const int& r) const
1217 {
1218 return extents_[r];
1219 }
1220
1221 template <typename iType>
1222 KOKKOS_INLINE_FUNCTION constexpr
1223 typename std::enable_if<std::is_integral<iType>::value, size_t>::type
1224 extent(const iType& r) const {
1225 return extents_[r];
1226 }
1227
1229 KOKKOS_INLINE_FUNCTION bool isDiagonal() const
1230 {
1231 if (blockPlusDiagonalLastNonDiagonal_ >= 1) return false;
1232 int numBlockPlusDiagonalTypes = 0;
1233 for (unsigned r = 0; r<variationType_.size(); r++)
1234 {
1235 const auto &entryType = variationType_[r];
1236 if (entryType == BLOCK_PLUS_DIAGONAL) numBlockPlusDiagonalTypes++;
1237 }
1238 // 2 BLOCK_PLUS_DIAGONAL entries, combined with blockPlusDiagonalLastNonDiagonal being -1 or 0 indicates diagonal
1239 if (numBlockPlusDiagonalTypes == 2) return true;
1240 else if (numBlockPlusDiagonalTypes == 0) return false; // no BLOCK_PLUS_DIAGONAL --> not a diagonal matrix
1241 else INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::invalid_argument, "Unexpected number of ranks marked as BLOCK_PLUS_DIAGONAL (should be 0 or 2)");
1242 return false; // statement should be unreachable; included because compilers don't necessarily recognize that fact...
1243 }
1244
1249 {
1250 return Data<DataScalar,DeviceType>(value, this->getExtents());
1251 }
1252
1259 {
1260 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.rank() != B.rank(), std::invalid_argument, "A and B must have the same logical shape");
1261 const int rank = A.rank();
1262 std::vector<DimensionInfo> dimInfo(rank);
1263 for (int d=0; d<rank; d++)
1264 {
1265 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.extent_int(d) != B.extent_int(d), std::invalid_argument, "A and B must have the same logical shape");
1266 dimInfo[d] = A.combinedDataDimensionInfo(B, d);
1267 }
1268 Data<DataScalar,DeviceType> result(dimInfo);
1269 return result;
1270 }
1271
1278 static Data<DataScalar,DeviceType> allocateMatMatResult( const bool transposeA, const Data<DataScalar,DeviceType> &A_MatData, const bool transposeB, const Data<DataScalar,DeviceType> &B_MatData )
1279 {
1280 // we treat last two logical dimensions of matData as the matrix; last dimension of vecData as the vector
1281 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_MatData.rank() != B_MatData.rank(), std::invalid_argument, "AmatData and BmatData have incompatible ranks");
1282
1283 const int D1_DIM = A_MatData.rank() - 2;
1284 const int D2_DIM = A_MatData.rank() - 1;
1285
1286 const int A_rows = A_MatData.extent_int(D1_DIM);
1287 const int A_cols = A_MatData.extent_int(D2_DIM);
1288 const int B_rows = B_MatData.extent_int(D1_DIM);
1289 const int B_cols = B_MatData.extent_int(D2_DIM);
1290
1291 const int leftRows = transposeA ? A_cols : A_rows;
1292 const int leftCols = transposeA ? A_rows : A_cols;
1293 const int rightRows = transposeB ? B_cols : B_rows;
1294 const int rightCols = transposeB ? B_rows : B_cols;
1295
1296 INTREPID2_TEST_FOR_EXCEPTION(leftCols != rightRows, std::invalid_argument, "incompatible matrix dimensions");
1297
1298 Kokkos::Array<int,7> resultExtents; // logical extents
1299 Kokkos::Array<DataVariationType,7> resultVariationTypes; // for each dimension, whether the data varies in that dimension
1300
1301 resultExtents[D1_DIM] = leftRows;
1302 resultExtents[D2_DIM] = rightCols;
1303 int resultBlockPlusDiagonalLastNonDiagonal = -1;
1304 if ( (A_MatData.getVariationTypes()[D1_DIM] == BLOCK_PLUS_DIAGONAL) && (B_MatData.getVariationTypes()[D1_DIM] == BLOCK_PLUS_DIAGONAL) )
1305 {
1306 // diagonal times diagonal is diagonal; the result will have the maximum of A and B's non-diagonal block size
1307 resultVariationTypes[D1_DIM] = BLOCK_PLUS_DIAGONAL;
1308 resultVariationTypes[D2_DIM] = BLOCK_PLUS_DIAGONAL;
1309 resultBlockPlusDiagonalLastNonDiagonal = std::max(A_MatData.blockPlusDiagonalLastNonDiagonal(), B_MatData.blockPlusDiagonalLastNonDiagonal());
1310 }
1311
1312 const int resultRank = A_MatData.rank();
1313
1314 auto A_VariationTypes = A_MatData.getVariationTypes();
1315 auto B_VariationTypes = B_MatData.getVariationTypes();
1316
1317 Kokkos::Array<int,7> resultActiveDims;
1318 Kokkos::Array<int,7> resultDataDims;
1319 int resultNumActiveDims = 0; // how many of the 7 entries are actually filled in
1320 // the following loop is over the dimensions *prior* to matrix dimensions
1321 for (int i=0; i<resultRank-2; i++)
1322 {
1323 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_MatData.extent_int(i) != B_MatData.extent_int(i), std::invalid_argument, "A and B extents must match in each non-matrix dimension");
1324
1325 resultExtents[i] = A_MatData.extent_int(i);
1326
1327 const DataVariationType &A_VariationType = A_VariationTypes[i];
1328 const DataVariationType &B_VariationType = B_VariationTypes[i];
1329
1330 // BLOCK_PLUS_DIAGONAL should only occur in matData, and only in the matrix (final) dimensions
1331 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_VariationType == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "unsupported variationType");
1332 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(B_VariationType == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "unsupported variationType");
1333
1334 int dataSize = 0;
1335 DataVariationType resultVariationType;
1336 if ((A_VariationType == GENERAL) || (B_VariationType == GENERAL))
1337 {
1338 resultVariationType = GENERAL;
1339 dataSize = resultExtents[i];
1340 }
1341 else if ((B_VariationType == CONSTANT) && (A_VariationType == CONSTANT))
1342 {
1343 resultVariationType = CONSTANT;
1344 dataSize = 1;
1345 }
1346 else if ((B_VariationType == MODULAR) && (A_VariationType == CONSTANT))
1347 {
1348 resultVariationType = MODULAR;
1349 dataSize = B_MatData.getVariationModulus(i);
1350 }
1351 else if ((B_VariationType == CONSTANT) && (A_VariationType == MODULAR))
1352 {
1353 resultVariationType = MODULAR;
1354 dataSize = A_MatData.getVariationModulus(i);
1355 }
1356 else
1357 {
1358 // both are modular. We allow this if they agree on the modulus
1359 auto A_Modulus = A_MatData.getVariationModulus(i);
1360 auto B_Modulus = B_MatData.getVariationModulus(i);
1361 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_Modulus != B_Modulus, std::invalid_argument, "If both matrices have variation type MODULAR, they must agree on the modulus");
1362 resultVariationType = MODULAR;
1363 dataSize = A_Modulus;
1364 }
1365 resultVariationTypes[i] = resultVariationType;
1366
1367 if (resultVariationType != CONSTANT)
1368 {
1369 resultActiveDims[resultNumActiveDims] = i;
1370 resultDataDims[resultNumActiveDims] = dataSize;
1371 resultNumActiveDims++;
1372 }
1373 }
1374
1375 // set things for final dimensions:
1376 resultExtents[D1_DIM] = leftRows;
1377 resultExtents[D2_DIM] = rightCols;
1378
1379 if ( (A_MatData.getVariationTypes()[D1_DIM] == BLOCK_PLUS_DIAGONAL) && (B_MatData.getVariationTypes()[D1_DIM] == BLOCK_PLUS_DIAGONAL) )
1380 {
1381 // diagonal times diagonal is diagonal; the result will have the maximum of A and B's non-diagonal block size
1382 resultVariationTypes[D1_DIM] = BLOCK_PLUS_DIAGONAL;
1383 resultVariationTypes[D2_DIM] = BLOCK_PLUS_DIAGONAL;
1384 resultBlockPlusDiagonalLastNonDiagonal = std::max(A_MatData.blockPlusDiagonalLastNonDiagonal(), B_MatData.blockPlusDiagonalLastNonDiagonal());
1385
1386 resultActiveDims[resultNumActiveDims] = resultRank - 2;
1387
1388 const int numDiagonalEntries = leftRows - (resultBlockPlusDiagonalLastNonDiagonal + 1);
1389 const int numNondiagonalEntries = (resultBlockPlusDiagonalLastNonDiagonal + 1) * (resultBlockPlusDiagonalLastNonDiagonal + 1);
1390
1391 resultDataDims[resultNumActiveDims] = numDiagonalEntries + numNondiagonalEntries;
1392 resultNumActiveDims++;
1393 }
1394 else
1395 {
1396 // pretty much the only variation types that make sense for matrix dims are GENERAL and BLOCK_PLUS_DIAGONAL
1397 resultVariationTypes[D1_DIM] = GENERAL;
1398 resultVariationTypes[D2_DIM] = GENERAL;
1399
1400 resultActiveDims[resultNumActiveDims] = resultRank - 2;
1401 resultActiveDims[resultNumActiveDims+1] = resultRank - 1;
1402
1403 resultDataDims[resultNumActiveDims] = leftRows;
1404 resultDataDims[resultNumActiveDims+1] = rightCols;
1405 resultNumActiveDims += 2;
1406 }
1407
1408 for (int i=resultRank; i<7; i++)
1409 {
1410 resultVariationTypes[i] = CONSTANT;
1411 resultExtents[i] = 1;
1412 }
1413
1414 ScalarView<DataScalar,DeviceType> data; // new view will match this one in layout and fad dimension, if any
1415 auto viewToMatch = A_MatData.getUnderlyingView();
1416 if (resultNumActiveDims == 1)
1417 {
1418 data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0]);
1419 }
1420 else if (resultNumActiveDims == 2)
1421 {
1422 data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1]);
1423 }
1424 else if (resultNumActiveDims == 3)
1425 {
1426 data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2]);
1427 }
1428 else if (resultNumActiveDims == 4)
1429 {
1430 data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1431 resultDataDims[3]);
1432 }
1433 else if (resultNumActiveDims == 5)
1434 {
1435 data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1436 resultDataDims[3], resultDataDims[4]);
1437 }
1438 else if (resultNumActiveDims == 6)
1439 {
1440 data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1441 resultDataDims[3], resultDataDims[4], resultDataDims[5]);
1442 }
1443 else // resultNumActiveDims == 7
1444 {
1445 data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1446 resultDataDims[3], resultDataDims[4], resultDataDims[5], resultDataDims[6]);
1447 }
1448
1449 return Data<DataScalar,DeviceType>(data,resultRank,resultExtents,resultVariationTypes,resultBlockPlusDiagonalLastNonDiagonal);
1450 }
1451
1459 {
1460 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.rank() != B.rank(), std::invalid_argument, "A and B must have the same logical shape");
1461 const int rank = A.rank();
1462 const int resultRank = rank - numContractionDims;
1463 std::vector<DimensionInfo> dimInfo(resultRank);
1464 for (int d=0; d<resultRank; d++)
1465 {
1466 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.extent_int(d) != B.extent_int(d), std::invalid_argument, "A and B must have the same logical shape");
1467 dimInfo[d] = A.combinedDataDimensionInfo(B, d);
1468 }
1469 Data<DataScalar,DeviceType> result(dimInfo);
1470 return result;
1471 }
1472
1482
1485 static Data<DataScalar,DeviceType> allocateMatVecResult( const Data<DataScalar,DeviceType> &matData, const Data<DataScalar,DeviceType> &vecData, const bool transposeMatrix = false )
1486 {
1487 // we treat last two logical dimensions of matData as the matrix; last dimension of vecData as the vector
1488 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(matData.rank() != vecData.rank() + 1, std::invalid_argument, "matData and vecData have incompatible ranks");
1489 const int vecDim = vecData.extent_int(vecData.rank() - 1);
1490
1491 const int D1_DIM = matData.rank() - 2;
1492 const int D2_DIM = matData.rank() - 1;
1493
1494 const int matRows = matData.extent_int(D1_DIM);
1495 const int matCols = matData.extent_int(D2_DIM);
1496
1497 const int rows = transposeMatrix ? matCols : matRows;
1498 const int cols = transposeMatrix ? matRows : matCols;
1499
1500 const int resultRank = vecData.rank();
1501
1502 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(cols != vecDim, std::invalid_argument, "matData column count != vecData dimension");
1503
1504 Kokkos::Array<int,7> resultExtents; // logical extents
1505 Kokkos::Array<DataVariationType,7> resultVariationTypes; // for each dimension, whether the data varies in that dimension
1506 auto vecVariationTypes = vecData.getVariationTypes();
1507 auto matVariationTypes = matData.getVariationTypes();
1508
1509 Kokkos::Array<int,7> resultActiveDims;
1510 Kokkos::Array<int,7> resultDataDims;
1511 int resultNumActiveDims = 0; // how many of the 7 entries are actually filled in
1512 // the following loop is over the dimensions *prior* to matrix/vector dimensions
1513 for (int i=0; i<resultRank-1; i++)
1514 {
1515 resultExtents[i] = vecData.extent_int(i);
1516 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(vecData.extent_int(i) != matData.extent_int(i), std::invalid_argument, "matData and vecData extents must match in each non-matrix/vector dimension");
1517
1518 const DataVariationType &vecVariationType = vecVariationTypes[i];
1519 const DataVariationType &matVariationType = matVariationTypes[i];
1520
1521 // BLOCK_PLUS_DIAGONAL should only occur in matData, and only in the matrix (final) dimensions
1522 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(vecVariationType == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "unsupported variationType");
1523 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(matVariationType == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "unsupported variationType");
1524
1525 int dataSize = 0;
1526 DataVariationType resultVariationType;
1527 if ((vecVariationType == GENERAL) || (matVariationType == GENERAL))
1528 {
1529 resultVariationType = GENERAL;
1530 dataSize = resultExtents[i];
1531 }
1532 else if ((matVariationType == CONSTANT) && (vecVariationType == CONSTANT))
1533 {
1534 resultVariationType = CONSTANT;
1535 dataSize = 1;
1536 }
1537 else if ((matVariationType == MODULAR) && (vecVariationType == CONSTANT))
1538 {
1539 resultVariationType = MODULAR;
1540 dataSize = matData.getVariationModulus(i);
1541 }
1542 else if ((matVariationType == CONSTANT) && (vecVariationType == MODULAR))
1543 {
1544 resultVariationType = MODULAR;
1545 dataSize = matData.getVariationModulus(i);
1546 }
1547 else
1548 {
1549 // both are modular. We allow this if they agree on the modulus
1550 auto matModulus = matData.getVariationModulus(i);
1551 auto vecModulus = vecData.getVariationModulus(i);
1552 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(matModulus != vecModulus, std::invalid_argument, "If matrix and vector both have variation type MODULAR, they must agree on the modulus");
1553 resultVariationType = MODULAR;
1554 dataSize = matModulus;
1555 }
1556 resultVariationTypes[i] = resultVariationType;
1557
1558 if (resultVariationType != CONSTANT)
1559 {
1560 resultActiveDims[resultNumActiveDims] = i;
1561 resultDataDims[resultNumActiveDims] = dataSize;
1562 resultNumActiveDims++;
1563 }
1564 }
1565 // for the final dimension, the variation type is always GENERAL
1566 // (Some combinations, e.g. CONSTANT/CONSTANT *would* generate a CONSTANT result, but constant matrices don't make a lot of sense beyond 1x1 matrices…)
1567 resultActiveDims[resultNumActiveDims] = resultRank - 1;
1568 resultDataDims[resultNumActiveDims] = rows;
1569 resultNumActiveDims++;
1570
1571 for (int i=resultRank; i<7; i++)
1572 {
1573 resultVariationTypes[i] = CONSTANT;
1574 resultExtents[i] = 1;
1575 }
1576 resultVariationTypes[resultRank-1] = GENERAL;
1577 resultExtents[resultRank-1] = rows;
1578
1579 ScalarView<DataScalar,DeviceType> data;
1580 if (resultNumActiveDims == 1)
1581 {
1582 data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0]);
1583 }
1584 else if (resultNumActiveDims == 2)
1585 {
1586 data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1]);
1587 }
1588 else if (resultNumActiveDims == 3)
1589 {
1590 data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2]);
1591 }
1592 else if (resultNumActiveDims == 4)
1593 {
1594 data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2],
1595 resultDataDims[3]);
1596 }
1597 else if (resultNumActiveDims == 5)
1598 {
1599 data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2],
1600 resultDataDims[3], resultDataDims[4]);
1601 }
1602 else if (resultNumActiveDims == 6)
1603 {
1604 data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2],
1605 resultDataDims[3], resultDataDims[4], resultDataDims[5]);
1606 }
1607 else // resultNumActiveDims == 7
1608 {
1609 data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2],
1610 resultDataDims[3], resultDataDims[4], resultDataDims[5], resultDataDims[6]);
1611 }
1612
1613 return Data<DataScalar,DeviceType>(data,resultRank,resultExtents,resultVariationTypes);
1614 }
1615
1617 template<int rank>
1618 enable_if_t<(rank!=1) && (rank!=7), Kokkos::MDRangePolicy<typename DeviceType::execution_space,Kokkos::Rank<rank>> >
1620 {
1621 using ExecutionSpace = typename DeviceType::execution_space;
1622 Kokkos::Array<int,rank> startingOrdinals;
1623 Kokkos::Array<int,rank> extents;
1624
1625 for (int d=0; d<rank; d++)
1626 {
1627 startingOrdinals[d] = 0;
1628 extents[d] = getDataExtent(d);
1629 }
1630 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<rank>>(startingOrdinals,extents);
1631 return policy;
1632 }
1633
1635 template<int rank>
1636 enable_if_t<rank==7, Kokkos::MDRangePolicy<typename DeviceType::execution_space,Kokkos::Rank<6>> >
1638 {
1639 using ExecutionSpace = typename DeviceType::execution_space;
1640 Kokkos::Array<int,6> startingOrdinals;
1641 Kokkos::Array<int,6> extents;
1642
1643 for (int d=0; d<6; d++)
1644 {
1645 startingOrdinals[d] = 0;
1646 extents[d] = getDataExtent(d);
1647 }
1648 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<6>>(startingOrdinals,extents);
1649 return policy;
1650 }
1651
1652 template<int rank>
1653 inline
1654 enable_if_t<rank==1, Kokkos::RangePolicy<typename DeviceType::execution_space> >
1656 {
1657 using ExecutionSpace = typename DeviceType::execution_space;
1658 Kokkos::RangePolicy<ExecutionSpace> policy(ExecutionSpace(),0,getDataExtent(0));
1659 return policy;
1660 }
1661
1663 Data shallowCopy(const int rank, const Kokkos::Array<int,7> &extents, const Kokkos::Array<DataVariationType,7> &variationTypes) const
1664 {
1665 switch (dataRank_)
1666 {
1667 case 1: return Data(rank, get_fixed_view<View1>(underlyingView_), extents, variationTypes);
1668 case 2: return Data(rank, get_fixed_view<View2>(underlyingView_), extents, variationTypes);
1669 case 3: return Data(rank, get_fixed_view<View3>(underlyingView_), extents, variationTypes);
1670 case 4: return Data(rank, get_fixed_view<View4>(underlyingView_), extents, variationTypes);
1671 case 5: return Data(rank, get_fixed_view<View5>(underlyingView_), extents, variationTypes);
1672 case 6: return Data(rank, get_fixed_view<View6>(underlyingView_), extents, variationTypes);
1673 case 7: return Data(rank, get_fixed_view<View7>(underlyingView_), extents, variationTypes);
1674 default:
1675 INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Unhandled dataRank_");
1676 }
1677 }
1678
1681 {
1682 const int D_DIM = A.rank() - 1;
1683 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.extent_int(D_DIM) != B.extent_int(D_DIM), std::invalid_argument, "A and B have different extents");
1684 const int vectorComponents = A.extent_int(D_DIM);
1685
1686 // shallow copy of this to avoid implicit references to this in call to getWritableEntry() below
1687 Data<DataScalar,DeviceType> thisData = *this;
1688
1689 using ExecutionSpace = typename DeviceType::execution_space;
1690 // note the use of getDataExtent() below: we only range over the possibly-distinct entries
1691 if (rank_ == 1) // contraction result rank; e.g., (P)
1692 {
1693 Kokkos::parallel_for("compute dot product", getDataExtent(0),
1694 KOKKOS_LAMBDA (const int &pointOrdinal) {
1695 auto & val = thisData.getWritableEntry(pointOrdinal);
1696 val = 0;
1697 for (int i=0; i<vectorComponents; i++)
1698 {
1699 val += A(pointOrdinal,i) * B(pointOrdinal,i);
1700 }
1701 });
1702 }
1703 else if (rank_ == 2) // contraction result rank; e.g., (C,P)
1704 {
1705 // typical case for e.g. gradient data: (C,P,D)
1706 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<2>>({0,0},{getDataExtent(0),getDataExtent(1)});
1707 Kokkos::parallel_for("compute dot product", policy,
1708 KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal) {
1709 auto & val = thisData.getWritableEntry(cellOrdinal, pointOrdinal);
1710 val = 0;
1711 for (int i=0; i<vectorComponents; i++)
1712 {
1713 val += A(cellOrdinal,pointOrdinal,i) * B(cellOrdinal,pointOrdinal,i);
1714 }
1715 });
1716 }
1717 else if (rank_ == 3)
1718 {
1719 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<3>>({0,0,0},{getDataExtent(0),getDataExtent(1),getDataExtent(2)});
1720 Kokkos::parallel_for("compute dot product", policy,
1721 KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal, const int &d) {
1722 auto & val = thisData.getWritableEntry(cellOrdinal, pointOrdinal,d);
1723 val = 0;
1724 for (int i=0; i<vectorComponents; i++)
1725 {
1726 val += A(cellOrdinal,pointOrdinal,d,i) * B(cellOrdinal,pointOrdinal,d,i);
1727 }
1728 });
1729 }
1730 else
1731 {
1732 // TODO: handle other cases
1733 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::logic_error, "rank not yet supported");
1734 }
1735 }
1736
1738 template<class BinaryOperator>
1739 void storeInPlaceCombination(const Data<DataScalar,DeviceType> &A, const Data<DataScalar,DeviceType> &B, BinaryOperator binaryOperator);
1740
1747
1754
1761
1768
1770 void storeMatVec( const Data<DataScalar,DeviceType> &matData, const Data<DataScalar,DeviceType> &vecData, const bool transposeMatrix = false )
1771 {
1772 // TODO: add a compile-time (SFINAE-type) guard against DataScalar types that do not support arithmetic operations. (We support Orientation as a DataScalar type; it might suffice just to compare DataScalar to Orientation, and eliminate this method for that case.)
1773 // TODO: check for invalidly shaped containers.
1774
1775 const int D1_DIM = matData.rank() - 2;
1776 const int D2_DIM = matData.rank() - 1;
1777
1778 const int matRows = matData.extent_int(D1_DIM);
1779 const int matCols = matData.extent_int(D2_DIM);
1780
1781 const int rows = transposeMatrix ? matCols : matRows;
1782 const int cols = transposeMatrix ? matRows : matCols;
1783
1784 // shallow copy of this to avoid implicit references to this in call to getWritableEntry() below
1785 Data<DataScalar,DeviceType> thisData = *this;
1786
1787 using ExecutionSpace = typename DeviceType::execution_space;
1788 // note the use of getDataExtent() below: we only range over the possibly-distinct entries
1789 if (rank_ == 3)
1790 {
1791 // typical case for e.g. gradient data: (C,P,D)
1792 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<3>>({0,0,0},{getDataExtent(0),getDataExtent(1),rows});
1793 Kokkos::parallel_for("compute mat-vec", policy,
1794 KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal, const int &i) {
1795 auto & val_i = thisData.getWritableEntry(cellOrdinal, pointOrdinal, i);
1796 val_i = 0;
1797 for (int j=0; j<cols; j++)
1798 {
1799 const auto & mat_ij = transposeMatrix ? matData(cellOrdinal,pointOrdinal,j,i) : matData(cellOrdinal,pointOrdinal,i,j);
1800 val_i += mat_ij * vecData(cellOrdinal,pointOrdinal,j);
1801 }
1802 });
1803 }
1804 else if (rank_ == 2)
1805 {
1806 //
1807 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<2>>({0,0},{getDataExtent(0),rows});
1808 Kokkos::parallel_for("compute mat-vec", policy,
1809 KOKKOS_LAMBDA (const int &vectorOrdinal, const int &i) {
1810 auto & val_i = thisData.getWritableEntry(vectorOrdinal, i);
1811 val_i = 0;
1812 for (int j=0; j<cols; j++)
1813 {
1814 const auto & mat_ij = transposeMatrix ? matData(vectorOrdinal,j,i) : matData(vectorOrdinal,i,j);
1815 val_i += mat_ij * vecData(vectorOrdinal,j);
1816 }
1817 });
1818 }
1819 else if (rank_ == 1)
1820 {
1821 // single-vector case
1822 Kokkos::RangePolicy<ExecutionSpace> policy(0,rows);
1823 Kokkos::parallel_for("compute mat-vec", policy,
1824 KOKKOS_LAMBDA (const int &i) {
1825 auto & val_i = thisData.getWritableEntry(i);
1826 val_i = 0;
1827 for (int j=0; j<cols; j++)
1828 {
1829 const auto & mat_ij = transposeMatrix ? matData(j,i) : matData(i,j);
1830 val_i += mat_ij * vecData(j);
1831 }
1832 });
1833 }
1834 else
1835 {
1836 // TODO: handle other cases
1837 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::logic_error, "rank not yet supported");
1838 }
1839 }
1840
1847 void storeMatMat( const bool transposeA, const Data<DataScalar,DeviceType> &A_MatData, const bool transposeB, const Data<DataScalar,DeviceType> &B_MatData )
1848 {
1849 // TODO: add a compile-time (SFINAE-type) guard against DataScalar types that do not support arithmetic operations. (We support Orientation as a DataScalar type; it might suffice just to compare DataScalar to Orientation, and eliminate this method for that case.)
1850 // TODO: check for invalidly shaped containers.
1851
1852 // we treat last two logical dimensions of matData as the matrix; last dimension of vecData as the vector
1853 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_MatData.rank() != B_MatData.rank(), std::invalid_argument, "AmatData and BmatData have incompatible ranks");
1854
1855 const int D1_DIM = A_MatData.rank() - 2;
1856 const int D2_DIM = A_MatData.rank() - 1;
1857
1858 const int A_rows = A_MatData.extent_int(D1_DIM);
1859 const int A_cols = A_MatData.extent_int(D2_DIM);
1860 const int B_rows = B_MatData.extent_int(D1_DIM);
1861 const int B_cols = B_MatData.extent_int(D2_DIM);
1862
1863 const int leftRows = transposeA ? A_cols : A_rows;
1864 const int leftCols = transposeA ? A_rows : A_cols;
1865 const int rightCols = transposeB ? B_rows : B_cols;
1866
1867#ifdef INTREPID2_HAVE_DEBUG
1868 const int rightRows = transposeB ? B_cols : B_rows;
1869 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(leftCols != rightRows, std::invalid_argument, "inner dimensions do not match");
1870#endif
1871
1872 // shallow copy of this to avoid implicit references to this in call to getWritableEntry() below
1873 Data<DataScalar,DeviceType> thisData = *this;
1874
1875 using ExecutionSpace = typename DeviceType::execution_space;
1876
1877 const int diagonalStart = (variationType_[D1_DIM] == BLOCK_PLUS_DIAGONAL) ? blockPlusDiagonalLastNonDiagonal_ + 1 : leftRows;
1878 // note the use of getDataExtent() below: we only range over the possibly-distinct entries
1879 if (rank_ == 3)
1880 {
1881 // (C,D,D), say
1882 auto policy = Kokkos::RangePolicy<ExecutionSpace>(0,getDataExtent(0));
1883 Kokkos::parallel_for("compute mat-mat", policy,
1884 KOKKOS_LAMBDA (const int &matrixOrdinal) {
1885 for (int i=0; i<diagonalStart; i++)
1886 {
1887 for (int j=0; j<rightCols; j++)
1888 {
1889 auto & val_ij = thisData.getWritableEntry(matrixOrdinal, i, j);
1890 val_ij = 0;
1891 for (int k=0; k<leftCols; k++)
1892 {
1893 const auto & left = transposeA ? A_MatData(matrixOrdinal,k,i) : A_MatData(matrixOrdinal,i,k);
1894 const auto & right = transposeB ? B_MatData(matrixOrdinal,j,k) : B_MatData(matrixOrdinal,k,j);
1895 val_ij += left * right;
1896 }
1897 }
1898 }
1899 for (int i=diagonalStart; i<leftRows; i++)
1900 {
1901 auto & val_ii = thisData.getWritableEntry(matrixOrdinal, i, i);
1902 const auto & left = A_MatData(matrixOrdinal,i,i);
1903 const auto & right = B_MatData(matrixOrdinal,i,i);
1904 val_ii = left * right;
1905 }
1906 });
1907 }
1908 else if (rank_ == 4)
1909 {
1910 // (C,P,D,D), perhaps
1911 auto policy = Kokkos::MDRangePolicy<ExecutionSpace, Kokkos::Rank<2> >({0,0},{getDataExtent(0),getDataExtent(1)});
1912 if (underlyingMatchesLogical_) // receiving data object is completely expanded
1913 {
1914 Kokkos::parallel_for("compute mat-mat", policy,
1915 KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal) {
1916 for (int i=0; i<leftRows; i++)
1917 {
1918 for (int j=0; j<rightCols; j++)
1919 {
1920 auto & val_ij = thisData.getUnderlyingView4()(cellOrdinal,pointOrdinal, i, j);
1921 val_ij = 0;
1922 for (int k=0; k<leftCols; k++)
1923 {
1924 const auto & left = transposeA ? A_MatData(cellOrdinal,pointOrdinal,k,i) : A_MatData(cellOrdinal,pointOrdinal,i,k);
1925 const auto & right = transposeB ? B_MatData(cellOrdinal,pointOrdinal,j,k) : B_MatData(cellOrdinal,pointOrdinal,k,j);
1926 val_ij += left * right;
1927 }
1928 }
1929 }
1930 });
1931 }
1932 else
1933 {
1934 Kokkos::parallel_for("compute mat-mat", policy,
1935 KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal) {
1936 for (int i=0; i<diagonalStart; i++)
1937 {
1938 for (int j=0; j<rightCols; j++)
1939 {
1940 auto & val_ij = thisData.getWritableEntry(cellOrdinal,pointOrdinal, i, j);
1941 val_ij = 0;
1942 for (int k=0; k<leftCols; k++)
1943 {
1944 const auto & left = transposeA ? A_MatData(cellOrdinal,pointOrdinal,k,i) : A_MatData(cellOrdinal,pointOrdinal,i,k);
1945 const auto & right = transposeB ? B_MatData(cellOrdinal,pointOrdinal,j,k) : B_MatData(cellOrdinal,pointOrdinal,k,j);
1946 val_ij += left * right;
1947 }
1948 }
1949 }
1950 for (int i=diagonalStart; i<leftRows; i++)
1951 {
1952 auto & val_ii = thisData.getWritableEntry(cellOrdinal,pointOrdinal, i, i);
1953 const auto & left = A_MatData(cellOrdinal,pointOrdinal,i,i);
1954 const auto & right = B_MatData(cellOrdinal,pointOrdinal,i,i);
1955 val_ii = left * right;
1956 }
1957 });
1958 }
1959 }
1960 else
1961 {
1962 // TODO: handle other cases
1963 INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::logic_error, "rank not yet supported");
1964 }
1965 }
1966
1968 KOKKOS_INLINE_FUNCTION constexpr bool isValid() const
1969 {
1970 return extents_[0] > 0;
1971 }
1972
1974 KOKKOS_INLINE_FUNCTION
1975 unsigned rank() const
1976 {
1977 return rank_;
1978 }
1979
1986 void setExtent(const ordinal_type &d, const ordinal_type &newExtent)
1987 {
1988 INTREPID2_TEST_FOR_EXCEPTION(variationType_[d] == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "setExtent is not supported for BLOCK_PLUS_DIAGONAL dimensions");
1989
1990 if (variationType_[d] == MODULAR)
1991 {
1992 bool dividesEvenly = ((newExtent / variationModulus_[d]) * variationModulus_[d] == newExtent);
1993 INTREPID2_TEST_FOR_EXCEPTION(!dividesEvenly, std::invalid_argument, "when setExtent is called on dimenisions with MODULAR variation, the modulus must divide the new extent evenly");
1994 }
1995
1996 if ((newExtent != extents_[d]) && (variationType_[d] == GENERAL))
1997 {
1998 // then we need to resize; let's determine the full set of new extents
1999 std::vector<ordinal_type> newExtents(dataRank_,-1);
2000 for (int r=0; r<dataRank_; r++)
2001 {
2002 if (activeDims_[r] == d)
2003 {
2004 // this is the changed dimension
2005 newExtents[r] = newExtent;
2006 }
2007 else
2008 {
2009 // unchanged; copy from existing
2010 newExtents[r] = getUnderlyingViewExtent(r);
2011 }
2012 }
2013
2014 switch (dataRank_)
2015 {
2016 case 1: Kokkos::resize(std::get<View1>(underlyingView_),newExtents[0]);
2017 break;
2018 case 2: Kokkos::resize(std::get<View2>(underlyingView_),newExtents[0],newExtents[1]);
2019 break;
2020 case 3: Kokkos::resize(std::get<View3>(underlyingView_),newExtents[0],newExtents[1],newExtents[2]);
2021 break;
2022 case 4: Kokkos::resize(std::get<View4>(underlyingView_),newExtents[0],newExtents[1],newExtents[2],newExtents[3]);
2023 break;
2024 case 5: Kokkos::resize(std::get<View5>(underlyingView_),newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4]);
2025 break;
2026 case 6: Kokkos::resize(std::get<View6>(underlyingView_),newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4],newExtents[5]);
2027 break;
2028 case 7: Kokkos::resize(std::get<View7>(underlyingView_),newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4],newExtents[5],newExtents[6]);
2029 break;
2030 default: INTREPID2_TEST_FOR_EXCEPTION(true, std::logic_error, "Unexpected dataRank_ value");
2031 }
2032 }
2033
2034 extents_[d] = newExtent;
2035 }
2036
2038 KOKKOS_INLINE_FUNCTION
2040 {
2041 return underlyingMatchesLogical_;
2042 }
2043 };
2044
2045 template<class DataScalar, typename DeviceType>
2046 KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const Data<DataScalar, DeviceType>& D) {
2047 return D.rank();
2048 }
2049}
2050
2051// we do ETI for doubles and default ExecutionSpace's device_type
2053
2054#include "Intrepid2_DataDef.hpp"
2055
2056#endif /* Intrepid2_Data_h */
Header file with various static argument-extractor classes. These are useful for writing efficient,...
Defines implementations for the Data class that are not present in the declaration.
Defines DimensionInfo struct that allows specification of a dimension within a Data object.
KOKKOS_INLINE_FUNCTION DimensionInfo combinedDimensionInfo(const DimensionInfo &myData, const DimensionInfo &otherData)
Returns DimensionInfo for a Data container that combines (through multiplication, say,...
Defines functors for use with Data objects: so far, we include simple arithmetical functors for sum,...
Defines DataVariationType enum that specifies the types of variation possible within a Data object.
@ GENERAL
arbitrary variation
@ BLOCK_PLUS_DIAGONAL
one of two dimensions in a matrix; bottom-right part of matrix is diagonal
@ MODULAR
varies according to modulus of the index
Header function for Intrepid2::Util class and other utility functions.
#define INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(test, x, msg)
Kokkos::DynRankView< typename ViewType::value_type, typename DeduceLayout< ViewType >::result_layout, typename ViewType::device_type > getMatchingViewWithLabel(const ViewType &view, const std::string &label, DimArgs... dims)
Creates and returns a view that matches the provided view in Kokkos Layout.
Wrapper around a Kokkos::View that allows data that is constant or repeating in various logical dimen...
void storeInPlaceProduct(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) product, A .* B, into this container.
static Data< DataScalar, DeviceType > allocateMatMatResult(const bool transposeA, const Data< DataScalar, DeviceType > &A_MatData, const bool transposeB, const Data< DataScalar, DeviceType > &B_MatData)
KOKKOS_INLINE_FUNCTION int getVariationModulus(const int &d) const
Variation modulus accessor.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==6, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 6.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==1, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 1.
KOKKOS_INLINE_FUNCTION reference_type getWritableEntry(const IntArgs... intArgs) const
Returns an l-value reference to the specified logical entry in the underlying view....
void copyDataFromDynRankViewMatchingUnderlying(const ScalarView< DataScalar, DeviceType > &dynRankView) const
Copies from the provided DynRankView into the underlying Kokkos::View container storing the unique da...
enable_if_t<(rank!=1) &&(rank!=7), Kokkos::MDRangePolicy< typename DeviceType::execution_space, Kokkos::Rank< rank > > > dataExtentRangePolicy()
returns an MDRangePolicy over the underlying data extents (but with the logical shape).
KOKKOS_INLINE_FUNCTION int extent_int(const int &r) const
Returns the logical extent in the specified dimension.
KOKKOS_INLINE_FUNCTION reference_type getWritableEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs... intArgs) const
Returns an l-value reference to the specified logical entry in the underlying view....
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar **, DeviceType > & getUnderlyingView2() const
returns the View that stores the unique data. For rank-2 underlying containers.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==2, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 2.
Data(const unsigned rank, Kokkos::View< ViewScalar, DeviceType, ViewProperties... > data, Kokkos::Array< int, 7 > extents, Kokkos::Array< DataVariationType, 7 > variationType, const int blockPlusDiagonalLastNonDiagonal=-1)
constructor with run-time rank (requires full-length extents, variationType inputs; those beyond the ...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ****, DeviceType > & getUnderlyingView4() const
returns the View that stores the unique data. For rank-4 underlying containers.
ScalarView< DataScalar, DeviceType > getUnderlyingView() const
Returns a DynRankView constructed atop the same underlying data as the fixed-rank Kokkos::View used i...
KOKKOS_INLINE_FUNCTION constexpr bool isValid() const
returns true for containers that have data; false for those that don't (namely, those that have been ...
static Data< DataScalar, DeviceType > allocateDotProductResult(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
Data()
default constructor (empty data)
static Data< DataScalar, DeviceType > allocateMatVecResult(const Data< DataScalar, DeviceType > &matData, const Data< DataScalar, DeviceType > &vecData, const bool transposeMatrix=false)
void storeMatVec(const Data< DataScalar, DeviceType > &matData, const Data< DataScalar, DeviceType > &vecData, const bool transposeMatrix=false)
Places the result of a matrix-vector multiply corresponding to the two provided containers into this ...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ***, DeviceType > & getUnderlyingView3() const
returns the View that stores the unique data. For rank-3 underlying containers.
void clear() const
Copies 0.0 to the underlying View.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==7, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 7.
KOKKOS_INLINE_FUNCTION bool isDiagonal() const
returns true for containers that have two dimensions marked as BLOCK_PLUS_DIAGONAL for which the non-...
ScalarView< DataScalar, DeviceType > allocateDynRankViewMatchingUnderlying() const
Returns a DynRankView that matches the underlying Kokkos::View object in value_type,...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==3, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 3.
KOKKOS_INLINE_FUNCTION return_type getEntry(const IntArgs &... intArgs) const
Returns a (read-only) value corresponding to the specified logical data location.
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ******, DeviceType > & getUnderlyingView6() const
returns the View that stores the unique data. For rank-6 underlying containers.
Data(const ScalarView< DataScalar, DeviceType > &data, int rank, Kokkos::Array< int, 7 > extents, Kokkos::Array< DataVariationType, 7 > variationType, const int blockPlusDiagonalLastNonDiagonal=-1)
DynRankView constructor. Will copy to a View of appropriate rank.
static void copyContainer(ToContainer to, FromContainer from)
Generic data copying method to allow construction of Data object from DynRankViews for which deep_cop...
KOKKOS_INLINE_FUNCTION const int & blockPlusDiagonalLastNonDiagonal() const
For a Data object containing data with variation type BLOCK_PLUS_DIAGONAL, returns the row and column...
void storeInPlaceSum(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) sum, A .+ B, into this container.
void applyOperator(UnaryOperator unaryOperator)
applies the specified unary operator to each entry
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalDiagonalEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i)
Returns flattened index of the specified (i,i) matrix entry, assuming that i > lastNondiagonal....
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalNumNondiagonalEntries(const int &lastNondiagonal)
Returns the number of non-diagonal entries based on the last non-diagonal. Only applicable for BLOCK_...
Data(ScalarView< DataScalar, DeviceType > data)
copy constructor modeled after the copy-like constructor above. Not as efficient as the implicit copy...
KOKKOS_INLINE_FUNCTION ordinal_type getUnderlyingViewSize() const
returns the number of entries in the View that stores the unique data
KOKKOS_INLINE_FUNCTION enable_if_t< rank==5, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 5.
void storeInPlaceQuotient(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) quotient, A ./ B, into this container.
static Data< DataScalar, DeviceType > allocateInPlaceCombinationResult(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
KOKKOS_INLINE_FUNCTION const Kokkos::Array< DataVariationType, 7 > & getVariationTypes() const
Returns an array with the variation types in each logical dimension.
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *, DeviceType > & getUnderlyingView1() const
returns the View that stores the unique data. For rank-1 underlying containers.
KOKKOS_INLINE_FUNCTION int getDataExtent(const ordinal_type &d) const
returns the true extent of the data corresponding to the logical dimension provided; if the data does...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *******, DeviceType > & getUnderlyingView7() const
returns the View that stores the unique data. For rank-7 underlying containers.
Data< DataScalar, DeviceType > allocateConstantData(const DataScalar &value)
void storeMatMat(const bool transposeA, const Data< DataScalar, DeviceType > &A_MatData, const bool transposeB, const Data< DataScalar, DeviceType > &B_MatData)
KOKKOS_INLINE_FUNCTION Kokkos::Array< int, 7 > getExtents() const
Returns an array containing the logical extents in each dimension.
ScalarView< DataScalar, DeviceType > allocateDynRankViewMatchingUnderlying(DimArgs... dims) const
Returns a DynRankView that matches the underlying Kokkos::View object value_type and layout,...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==4, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 4.
KOKKOS_INLINE_FUNCTION bool underlyingMatchesLogical() const
Returns true if the underlying container has exactly the same rank and extents as the logical contain...
KOKKOS_INLINE_FUNCTION ordinal_type getUnderlyingViewRank() const
returns the rank of the View that stores the unique data
void allocateAndCopyFromDynRankView(ScalarView< DataScalar, DeviceType > data)
allocate an underlying View that matches the provided DynRankView in dimensions, and copy....
void storeInPlaceDifference(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) difference, A .- B, into this container.
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalBlockEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i, const int &j)
//! Returns flattened index of the specified (i,j) matrix entry, assuming that i,j ≤ lastNondiagonal....
KOKKOS_INLINE_FUNCTION int getUnderlyingViewExtent(const int &dim) const
Returns the extent of the underlying view in the specified dimension.
void setActiveDims()
class initialization method. Called by constructors.
KOKKOS_INLINE_FUNCTION return_type getEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs &... intArgs) const
Returns a (read-only) value corresponding to the specified logical data location. If passThroughBlock...
Data(const Data< DataScalar, OtherDeviceType > &data)
copy-like constructor for differing execution spaces. This does a deep_copy of the underlying view.
void setExtent(const ordinal_type &d, const ordinal_type &newExtent)
sets the logical extent in the specified dimension. If needed, the underlying data container is resiz...
void storeInPlaceCombination(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B, BinaryOperator binaryOperator)
Places the result of an in-place combination (e.g., entrywise sum) into this data container.
KOKKOS_INLINE_FUNCTION DimensionInfo combinedDataDimensionInfo(const Data &otherData, const int &dim) const
Returns (DataVariationType, data extent) in the specified dimension for a Data container that combine...
static Data< DataScalar, DeviceType > allocateContractionResult(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B, const int &numContractionDims)
KOKKOS_INLINE_FUNCTION DimensionInfo getDimensionInfo(const int &dim) const
Returns an object fully specifying the indicated dimension. This is used in determining appropriate s...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *****, DeviceType > & getUnderlyingView5() const
returns the View that stores the unique data. For rank-5 underlying containers.
Data(std::vector< DimensionInfo > dimInfoVector)
Constructor in terms of DimensionInfo for each logical dimension; does not require a View to be speci...
enable_if_t< rank==7, Kokkos::MDRangePolicy< typename DeviceType::execution_space, Kokkos::Rank< 6 > > > dataExtentRangePolicy()
returns an MDRangePolicy over the first six underlying data extents (but with the logical shape).
Data(DataScalar constantValue, Kokkos::Array< int, rank > extents)
constructor for everywhere-constant data
Data(const Kokkos::DynRankView< DataScalar, DeviceType, DynRankViewProperties... > &data, Kokkos::Array< int, rank > extents, Kokkos::Array< DataVariationType, rank > variationType, const int blockPlusDiagonalLastNonDiagonal=-1)
Constructor that accepts a DynRankView as an argument. The data belonging to the DynRankView will be ...
Data shallowCopy(const int rank, const Kokkos::Array< int, 7 > &extents, const Kokkos::Array< DataVariationType, 7 > &variationTypes) const
Creates a new Data object with the same underlying view, but with the specified logical rank,...
KOKKOS_INLINE_FUNCTION unsigned rank() const
Returns the logical rank of the Data container.
KOKKOS_INLINE_FUNCTION enable_if_t< valid_args< IntArgs... >::value &&(sizeof...(IntArgs)<=7), return_type > operator()(const IntArgs &... intArgs) const
Returns a value corresponding to the specified logical data location.
void storeDotProduct(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
Places the result of a contraction along the final dimension of A and B into this data container.
A singleton class for a DynRankView containing exactly one zero entry. (Technically,...
Struct expressing all variation information about a Data object in a single dimension,...