Tpetra parallel linear algebra Version of the Day
Loading...
Searching...
No Matches
Tpetra_CrsGraph_def.hpp
Go to the documentation of this file.
1// @HEADER
2// *****************************************************************************
3// Tpetra: Templated Linear Algebra Services Package
4//
5// Copyright 2008 NTESS and the Tpetra contributors.
6// SPDX-License-Identifier: BSD-3-Clause
7// *****************************************************************************
8// @HEADER
9
10#ifndef TPETRA_CRSGRAPH_DEF_HPP
11#define TPETRA_CRSGRAPH_DEF_HPP
12
15
16#include <memory>
17#include "Teuchos_Assert.hpp"
18#include "Tpetra_Details_iallreduce.hpp"
19#ifdef KOKKOS_ENABLE_SYCL
20#include <sycl/sycl.hpp>
21#endif
22
23#include "Tpetra_Details_Behavior.hpp"
24#include "Tpetra_Details_computeOffsets.hpp"
25#include "Tpetra_Details_copyOffsets.hpp"
26#include "Tpetra_Details_gathervPrint.hpp"
27#include "Tpetra_Details_getGraphDiagOffsets.hpp"
28#include "Tpetra_Details_getGraphOffRankOffsets.hpp"
29#include "Tpetra_Details_makeColMap.hpp"
30#include "Tpetra_Details_Profiling.hpp"
31#include "Tpetra_Details_getEntryOnHost.hpp"
32#include "Tpetra_Details_crsUtils.hpp"
33#include "Tpetra_Distributor.hpp"
34#include "Teuchos_SerialDenseMatrix.hpp"
35#include "Tpetra_Vector.hpp"
36#include "Tpetra_Import_Util.hpp"
37#include "Tpetra_Import_Util2.hpp"
38#include "Tpetra_Details_packCrsGraph.hpp"
39#include "Tpetra_Details_unpackCrsGraphAndCombine.hpp"
40#include "Tpetra_Details_CrsPadding.hpp"
41#include "Tpetra_Util.hpp"
42#include <algorithm>
43#include <limits>
44#include <map>
45#include <sstream>
46#include <string>
47#include <type_traits>
48#include <utility>
49#include <vector>
50
51namespace Tpetra {
52namespace Details {
53namespace Impl {
54
55template <class MapIter>
56void verbosePrintMap(std::ostream& out,
57 MapIter beg,
58 MapIter end,
59 const size_t numEnt,
60 const char mapName[]) {
61 using ::Tpetra::Details::Behavior;
62 using ::Tpetra::Details::verbosePrintArray;
63
64 out << mapName << ": {";
65 const size_t maxNumToPrint =
66 Behavior::verbosePrintCountThreshold();
67 if (maxNumToPrint == 0) {
68 if (numEnt != 0) {
69 out << "...";
70 }
71 } else {
72 const size_t numToPrint = numEnt > maxNumToPrint ? maxNumToPrint : numEnt;
73 size_t count = 0;
74 for (MapIter it = beg; it != end; ++it) {
75 out << "(" << (*it).first << ", ";
76 verbosePrintArray(out, (*it).second, "gblColInds",
77 maxNumToPrint);
78 out << ")";
79 if (count + size_t(1) < numToPrint) {
80 out << ", ";
81 }
82 ++count;
83 }
84 if (count < numEnt) {
85 out << ", ...";
86 }
87 }
88 out << "}";
89}
90
91template <class LO, class GO, class Node>
92Teuchos::ArrayView<GO>
93getRowGraphGlobalRow(
94 std::vector<GO>& gblColIndsStorage,
95 const RowGraph<LO, GO, Node>& graph,
96 const GO gblRowInd) {
97 size_t origNumEnt = graph.getNumEntriesInGlobalRow(gblRowInd);
98 if (gblColIndsStorage.size() < origNumEnt) {
99 gblColIndsStorage.resize(origNumEnt);
100 }
101 typename CrsGraph<LO, GO, Node>::nonconst_global_inds_host_view_type gblColInds(gblColIndsStorage.data(),
102 origNumEnt);
103 graph.getGlobalRowCopy(gblRowInd, gblColInds, origNumEnt);
104 Teuchos::ArrayView<GO> retval(gblColIndsStorage.data(), origNumEnt);
105 return retval;
106}
107
108template <class LO, class GO, class DT, class OffsetType, class NumEntType>
109class ConvertColumnIndicesFromGlobalToLocal {
110 public:
111 ConvertColumnIndicesFromGlobalToLocal(const ::Kokkos::View<LO*, DT>& lclColInds,
112 const ::Kokkos::View<const GO*, DT>& gblColInds,
113 const ::Kokkos::View<const OffsetType*, DT>& ptr,
114 const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
115 const ::Kokkos::View<const NumEntType*, DT>& numRowEnt)
116 : lclColInds_(lclColInds)
117 , gblColInds_(gblColInds)
118 , ptr_(ptr)
119 , lclColMap_(lclColMap)
120 , numRowEnt_(numRowEnt) {}
121
122 KOKKOS_FUNCTION void
123 operator()(const LO& lclRow, OffsetType& curNumBad) const {
124 const OffsetType offset = ptr_(lclRow);
125 // NOTE (mfh 26 Jun 2016) It's always legal to cast the number
126 // of entries in a row to LO, as long as the row doesn't have
127 // too many duplicate entries.
128 const LO numEnt = static_cast<LO>(numRowEnt_(lclRow));
129 for (LO j = 0; j < numEnt; ++j) {
130 const GO gid = gblColInds_(offset + j);
131 const LO lid = lclColMap_.getLocalElement(gid);
132 lclColInds_(offset + j) = lid;
133 if (lid == ::Tpetra::Details::OrdinalTraits<LO>::invalid()) {
134 ++curNumBad;
135 }
136 }
137 }
138
139 static OffsetType
140 run(const ::Kokkos::View<LO*, DT>& lclColInds,
141 const ::Kokkos::View<const GO*, DT>& gblColInds,
142 const ::Kokkos::View<const OffsetType*, DT>& ptr,
143 const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
144 const ::Kokkos::View<const NumEntType*, DT>& numRowEnt) {
145 typedef ::Kokkos::RangePolicy<typename DT::execution_space, LO> range_type;
146 typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> functor_type;
147
148 const LO lclNumRows = ptr.extent(0) == 0 ? static_cast<LO>(0) : static_cast<LO>(ptr.extent(0) - 1);
149 OffsetType numBad = 0;
150 // Count of "bad" column indices is a reduction over rows.
151 ::Kokkos::parallel_reduce(range_type(0, lclNumRows),
152 functor_type(lclColInds, gblColInds, ptr,
153 lclColMap, numRowEnt),
154 numBad);
155 return numBad;
156 }
157
158 private:
159 ::Kokkos::View<LO*, DT> lclColInds_;
160 ::Kokkos::View<const GO*, DT> gblColInds_;
161 ::Kokkos::View<const OffsetType*, DT> ptr_;
162 ::Tpetra::Details::LocalMap<LO, GO, DT> lclColMap_;
163 ::Kokkos::View<const NumEntType*, DT> numRowEnt_;
164};
165
166} // namespace Impl
167
182template <class LO, class GO, class DT, class OffsetType, class NumEntType>
183OffsetType
184convertColumnIndicesFromGlobalToLocal(const Kokkos::View<LO*, DT>& lclColInds,
185 const Kokkos::View<const GO*, DT>& gblColInds,
186 const Kokkos::View<const OffsetType*, DT>& ptr,
187 const LocalMap<LO, GO, DT>& lclColMap,
188 const Kokkos::View<const NumEntType*, DT>& numRowEnt) {
189 using Impl::ConvertColumnIndicesFromGlobalToLocal;
190 typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> impl_type;
191 return impl_type::run(lclColInds, gblColInds, ptr, lclColMap, numRowEnt);
192}
193
194template <class ViewType, class LO>
195class MaxDifference {
196 public:
197 MaxDifference(const ViewType& ptr)
198 : ptr_(ptr) {}
199
200 KOKKOS_INLINE_FUNCTION void init(LO& dst) const {
201 dst = 0;
202 }
203
204 KOKKOS_INLINE_FUNCTION void
205 join(LO& dst, const LO& src) const {
206 dst = (src > dst) ? src : dst;
207 }
208
209 KOKKOS_INLINE_FUNCTION void
210 operator()(const LO lclRow, LO& maxNumEnt) const {
211 const LO numEnt = static_cast<LO>(ptr_(lclRow + 1) - ptr_(lclRow));
212 maxNumEnt = (numEnt > maxNumEnt) ? numEnt : maxNumEnt;
213 }
214
215 private:
216 typename ViewType::const_type ptr_;
217};
218
219template <class ViewType, class LO>
220typename ViewType::non_const_value_type
221maxDifference(const char kernelLabel[],
222 const ViewType& ptr,
223 const LO lclNumRows) {
224 if (lclNumRows == 0) {
225 // mfh 07 May 2018: Weirdly, I need this special case,
226 // otherwise I get the wrong answer.
227 return static_cast<LO>(0);
228 } else {
229 using execution_space = typename ViewType::execution_space;
230 using range_type = Kokkos::RangePolicy<execution_space, LO>;
231 LO theMaxNumEnt{0};
232 Kokkos::parallel_reduce(kernelLabel,
233 range_type(0, lclNumRows),
234 MaxDifference<ViewType, LO>(ptr),
235 theMaxNumEnt);
236 return theMaxNumEnt;
237 }
238}
239
240} // namespace Details
241
242template <class LocalOrdinal, class GlobalOrdinal, class Node>
243bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
244 getDebug() {
245 return Details::Behavior::debug("CrsGraph");
246}
247
248template <class LocalOrdinal, class GlobalOrdinal, class Node>
249bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
250 getVerbose() {
251 return Details::Behavior::verbose("CrsGraph");
252}
253
254template <class LocalOrdinal, class GlobalOrdinal, class Node>
255CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
256 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
257 const size_t maxNumEntriesPerRow,
258 const Teuchos::RCP<Teuchos::ParameterList>& params)
259 : dist_object_type(rowMap)
260 , rowMap_(rowMap)
261 , numAllocForAllRows_(maxNumEntriesPerRow) {
262 const char tfecfFuncName[] =
263 "CrsGraph(rowMap,maxNumEntriesPerRow,params): ";
264 staticAssertions();
265 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid(),
266 std::invalid_argument,
267 "The allocation hint maxNumEntriesPerRow must be "
268 "a valid size_t value, which in this case means it must not be "
269 "Teuchos::OrdinalTraits<size_t>::invalid().");
270 resumeFill(params);
271 checkInternalState();
272}
273
274template <class LocalOrdinal, class GlobalOrdinal, class Node>
275CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
276 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
277 const Teuchos::RCP<const map_type>& colMap,
278 const size_t maxNumEntriesPerRow,
279 const Teuchos::RCP<Teuchos::ParameterList>& params)
280 : dist_object_type(rowMap)
281 , rowMap_(rowMap)
282 , colMap_(colMap)
283 , numAllocForAllRows_(maxNumEntriesPerRow) {
284 const char tfecfFuncName[] =
285 "CrsGraph(rowMap,colMap,maxNumEntriesPerRow,params): ";
286 staticAssertions();
287 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
288 maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid(),
289 std::invalid_argument,
290 "The allocation hint maxNumEntriesPerRow must be "
291 "a valid size_t value, which in this case means it must not be "
292 "Teuchos::OrdinalTraits<size_t>::invalid().");
293 resumeFill(params);
294 checkInternalState();
295}
296
297template <class LocalOrdinal, class GlobalOrdinal, class Node>
298CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
299 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
300 const Teuchos::ArrayView<const size_t>& numEntPerRow,
301 const Teuchos::RCP<Teuchos::ParameterList>& params)
302 : dist_object_type(rowMap)
303 , rowMap_(rowMap)
304 , numAllocForAllRows_(0) {
305 const char tfecfFuncName[] =
306 "CrsGraph(rowMap,numEntPerRow,params): ";
307 staticAssertions();
308
309 const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
310 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
311 static_cast<size_t>(numEntPerRow.size()) != lclNumRows,
312 std::invalid_argument, "numEntPerRow has length " << numEntPerRow.size() << " != the local number of rows " << lclNumRows << " as specified by "
313 "the input row Map.");
314
315 if (debug_) {
316 for (size_t r = 0; r < lclNumRows; ++r) {
317 const size_t curRowCount = numEntPerRow[r];
318 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
319 std::invalid_argument, "numEntPerRow(" << r << ") "
320 "specifies an invalid number of entries "
321 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
322 }
323 }
324
325 // Deep-copy the (host-accessible) input into k_numAllocPerRow_.
326 // The latter is a const View, so we have to copy into a nonconst
327 // View first, then assign.
328 typedef decltype(k_numAllocPerRow_) out_view_type;
329 typedef typename out_view_type::non_const_type nc_view_type;
330 typedef Kokkos::View<const size_t*,
331 typename nc_view_type::array_layout,
332 Kokkos::HostSpace,
333 Kokkos::MemoryUnmanaged>
334 in_view_type;
335 in_view_type numAllocPerRowIn(numEntPerRow.getRawPtr(), lclNumRows);
336 nc_view_type numAllocPerRowOut("Tpetra::CrsGraph::numAllocPerRow",
337 lclNumRows);
338 // DEEP_COPY REVIEW - HOST-TO-HOSTMIRROR
339 using exec_space = typename nc_view_type::execution_space;
340 Kokkos::deep_copy(exec_space(), numAllocPerRowOut, numAllocPerRowIn);
341 k_numAllocPerRow_ = numAllocPerRowOut;
342
343 resumeFill(params);
344 checkInternalState();
345}
346
347template <class LocalOrdinal, class GlobalOrdinal, class Node>
348CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
349 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
350 const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
351 const Teuchos::RCP<Teuchos::ParameterList>& params)
352 : dist_object_type(rowMap)
353 , rowMap_(rowMap)
354 , k_numAllocPerRow_(numEntPerRow.view_host())
355 , numAllocForAllRows_(0) {
356 const char tfecfFuncName[] =
357 "CrsGraph(rowMap,numEntPerRow,params): ";
358 staticAssertions();
359
360 const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
361 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
362 static_cast<size_t>(numEntPerRow.extent(0)) != lclNumRows,
363 std::invalid_argument, "numEntPerRow has length " << numEntPerRow.extent(0) << " != the local number of rows " << lclNumRows << " as specified by "
364 "the input row Map.");
365
366 if (debug_) {
367 for (size_t r = 0; r < lclNumRows; ++r) {
368 const size_t curRowCount = numEntPerRow.view_host()(r);
369 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
370 std::invalid_argument, "numEntPerRow(" << r << ") "
371 "specifies an invalid number of entries "
372 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
373 }
374 }
375
376 resumeFill(params);
377 checkInternalState();
378}
379
380template <class LocalOrdinal, class GlobalOrdinal, class Node>
381CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
382 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
383 const Teuchos::RCP<const map_type>& colMap,
384 const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
385 const Teuchos::RCP<Teuchos::ParameterList>& params)
386 : dist_object_type(rowMap)
387 , rowMap_(rowMap)
388 , colMap_(colMap)
389 , k_numAllocPerRow_(numEntPerRow.view_host())
390 , numAllocForAllRows_(0) {
391 const char tfecfFuncName[] =
392 "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
393 staticAssertions();
394
395 const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
396 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
397 static_cast<size_t>(numEntPerRow.extent(0)) != lclNumRows,
398 std::invalid_argument, "numEntPerRow has length " << numEntPerRow.extent(0) << " != the local number of rows " << lclNumRows << " as specified by "
399 "the input row Map.");
400
401 if (debug_) {
402 for (size_t r = 0; r < lclNumRows; ++r) {
403 const size_t curRowCount = numEntPerRow.view_host()(r);
404 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
405 std::invalid_argument, "numEntPerRow(" << r << ") "
406 "specifies an invalid number of entries "
407 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
408 }
409 }
410
411 resumeFill(params);
412 checkInternalState();
413}
414
415template <class LocalOrdinal, class GlobalOrdinal, class Node>
416CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
417 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
418 const Teuchos::RCP<const map_type>& colMap,
419 const Teuchos::ArrayView<const size_t>& numEntPerRow,
420 const Teuchos::RCP<Teuchos::ParameterList>& params)
421 : dist_object_type(rowMap)
422 , rowMap_(rowMap)
423 , colMap_(colMap)
424 , numAllocForAllRows_(0) {
425 const char tfecfFuncName[] =
426 "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
427 staticAssertions();
428
429 const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
430 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
431 static_cast<size_t>(numEntPerRow.size()) != lclNumRows,
432 std::invalid_argument, "numEntPerRow has length " << numEntPerRow.size() << " != the local number of rows " << lclNumRows << " as specified by "
433 "the input row Map.");
434
435 if (debug_) {
436 for (size_t r = 0; r < lclNumRows; ++r) {
437 const size_t curRowCount = numEntPerRow[r];
438 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
439 std::invalid_argument, "numEntPerRow(" << r << ") "
440 "specifies an invalid number of entries "
441 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
442 }
443 }
444
445 // Deep-copy the (host-accessible) input into k_numAllocPerRow_.
446 // The latter is a const View, so we have to copy into a nonconst
447 // View first, then assign.
448 typedef decltype(k_numAllocPerRow_) out_view_type;
449 typedef typename out_view_type::non_const_type nc_view_type;
450 typedef Kokkos::View<const size_t*,
451 typename nc_view_type::array_layout,
452 Kokkos::HostSpace,
453 Kokkos::MemoryUnmanaged>
454 in_view_type;
455 in_view_type numAllocPerRowIn(numEntPerRow.getRawPtr(), lclNumRows);
456 nc_view_type numAllocPerRowOut("Tpetra::CrsGraph::numAllocPerRow",
457 lclNumRows);
458 // DEEP_COPY REVIEW - HOST-TO-HOSTMIRROR
459 using exec_space = typename nc_view_type::execution_space;
460 Kokkos::deep_copy(exec_space(), numAllocPerRowOut, numAllocPerRowIn);
461 k_numAllocPerRow_ = numAllocPerRowOut;
462
463 resumeFill(params);
464 checkInternalState();
465}
466
467template <class LocalOrdinal, class GlobalOrdinal, class Node>
468CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
469 CrsGraph(CrsGraph<local_ordinal_type, global_ordinal_type, node_type>& originalGraph,
470 const Teuchos::RCP<const map_type>& rowMap,
471 const Teuchos::RCP<Teuchos::ParameterList>& params)
472 : dist_object_type(rowMap)
473 , rowMap_(rowMap)
474 , colMap_(originalGraph.colMap_)
475 , numAllocForAllRows_(originalGraph.numAllocForAllRows_)
476 , storageStatus_(originalGraph.storageStatus_)
477 , indicesAreAllocated_(originalGraph.indicesAreAllocated_)
478 , indicesAreLocal_(originalGraph.indicesAreLocal_)
479 , indicesAreSorted_(originalGraph.indicesAreSorted_) {
480 staticAssertions();
481
482 int numRows = rowMap->getLocalNumElements();
483 size_t numNonZeros = originalGraph.getRowPtrsPackedHost()(numRows);
484 auto rowsToUse = Kokkos::pair<size_t, size_t>(0, numRows + 1);
485
486 this->setRowPtrsUnpacked(Kokkos::subview(originalGraph.getRowPtrsUnpackedDevice(), rowsToUse));
487 this->setRowPtrsPacked(Kokkos::subview(originalGraph.getRowPtrsPackedDevice(), rowsToUse));
488
489 if (indicesAreLocal_) {
490 lclIndsUnpacked_wdv = local_inds_wdv_type(originalGraph.lclIndsUnpacked_wdv, 0, numNonZeros);
491 lclIndsPacked_wdv = local_inds_wdv_type(originalGraph.lclIndsPacked_wdv, 0, numNonZeros);
492 } else {
493 gblInds_wdv = global_inds_wdv_type(originalGraph.gblInds_wdv, 0, numNonZeros);
494 }
495
496 checkInternalState();
497}
498
499template <class LocalOrdinal, class GlobalOrdinal, class Node>
500CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
501 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
502 const Teuchos::RCP<const map_type>& colMap,
503 const CrsGraph<local_ordinal_type, global_ordinal_type, node_type>& originalGraph,
504 const Teuchos::RCP<Teuchos::ParameterList>& params)
505 : CrsGraph(rowMap, colMap, Teuchos::null, Teuchos::null, originalGraph, params) {}
506
507template <class LocalOrdinal, class GlobalOrdinal, class Node>
508CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
509 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
510 const Teuchos::RCP<const map_type>& colMap,
511 const Teuchos::RCP<const map_type>& domainMap,
512 const Teuchos::RCP<const map_type>& rangeMap,
513 const CrsGraph<local_ordinal_type, global_ordinal_type, node_type>& originalGraph,
514 const Teuchos::RCP<Teuchos::ParameterList>& params)
515 : dist_object_type(rowMap)
516 , rowMap_(rowMap)
517 , colMap_(colMap)
518 , numAllocForAllRows_(originalGraph.numAllocForAllRows_)
519 , storageStatus_(originalGraph.storageStatus_)
520 , indicesAreAllocated_(originalGraph.indicesAreAllocated_)
521 , indicesAreLocal_(originalGraph.indicesAreLocal_)
522 , indicesAreSorted_(originalGraph.indicesAreSorted_) {
523 staticAssertions();
524
525 const char tfecfFuncName[] = "CrsGraph(Map,Map,CrsGraph)";
526 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
527 originalGraph.getRowMap()->getLocalNumElements() != rowMap->getLocalNumElements(),
528 std::runtime_error,
529 ": The input row Map and the original graph need to have the same "
530 "number of elements.");
531 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
532 originalGraph.getColMap()->getLocalNumElements() != colMap->getLocalNumElements(),
533 std::runtime_error,
534 ": The input column Map and the original graph need to have the same "
535 "number of elements.");
536 if (!domainMap.is_null()) {
537 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
538 originalGraph.getDomainMap()->getLocalNumElements() != domainMap->getLocalNumElements(),
539 std::runtime_error,
540 ": The input domain Map and the original graph need to have the same "
541 "number of elements.");
542 }
543 if (!rangeMap.is_null()) {
544 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
545 originalGraph.getRangeMap()->getLocalNumElements() != rangeMap->getLocalNumElements(),
546 std::runtime_error,
547 ": The input range Map and the original graph need to have the same "
548 "number of elements.");
549 }
550
551 this->setRowPtrsUnpacked(originalGraph.getRowPtrsUnpackedDevice());
552 this->setRowPtrsPacked(originalGraph.getRowPtrsPackedDevice());
553
554 if (indicesAreLocal_) {
555 lclIndsUnpacked_wdv = originalGraph.lclIndsUnpacked_wdv;
556 lclIndsPacked_wdv = originalGraph.lclIndsPacked_wdv;
557 } else {
558 gblInds_wdv = originalGraph.gblInds_wdv;
559 }
560
561 setDomainRangeMaps(domainMap.is_null() ? originalGraph.getDomainMap() : domainMap,
562 rangeMap.is_null() ? originalGraph.getRangeMap() : rangeMap);
563
564 checkInternalState();
565}
566
567template <class LocalOrdinal, class GlobalOrdinal, class Node>
568CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
569 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
570 const Teuchos::RCP<const map_type>& colMap,
571 const typename local_graph_device_type::row_map_type& rowPointers,
572 const typename local_graph_device_type::entries_type::non_const_type& columnIndices,
573 const Teuchos::RCP<Teuchos::ParameterList>& params)
574 : dist_object_type(rowMap)
575 , rowMap_(rowMap)
576 , colMap_(colMap)
577 , numAllocForAllRows_(0)
578 , storageStatus_(Details::STORAGE_1D_PACKED)
579 , indicesAreAllocated_(true)
580 , indicesAreLocal_(true) {
581 staticAssertions();
582 if (!params.is_null() && params->isParameter("sorted") &&
583 !params->get<bool>("sorted")) {
584 indicesAreSorted_ = false;
585 } else {
586 indicesAreSorted_ = true;
587 }
588 setAllIndices(rowPointers, columnIndices);
589 checkInternalState();
590}
591
592template <class LocalOrdinal, class GlobalOrdinal, class Node>
593CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
594 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
595 const Teuchos::RCP<const map_type>& colMap,
596 const Teuchos::ArrayRCP<size_t>& rowPointers,
597 const Teuchos::ArrayRCP<LocalOrdinal>& columnIndices,
598 const Teuchos::RCP<Teuchos::ParameterList>& params)
599 : dist_object_type(rowMap)
600 , rowMap_(rowMap)
601 , colMap_(colMap)
602 , numAllocForAllRows_(0)
603 , storageStatus_(Details::STORAGE_1D_PACKED)
604 , indicesAreAllocated_(true)
605 , indicesAreLocal_(true) {
606 staticAssertions();
607 if (!params.is_null() && params->isParameter("sorted") &&
608 !params->get<bool>("sorted")) {
609 indicesAreSorted_ = false;
610 } else {
611 indicesAreSorted_ = true;
612 }
613 setAllIndices(rowPointers, columnIndices);
614 checkInternalState();
615}
616
617template <class LocalOrdinal, class GlobalOrdinal, class Node>
618CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
619 CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
620 const Teuchos::RCP<const map_type>& colMap,
621 const local_graph_device_type& k_local_graph_,
622 const Teuchos::RCP<Teuchos::ParameterList>& params)
623 : CrsGraph(k_local_graph_,
624 rowMap,
625 colMap,
626 Teuchos::null,
627 Teuchos::null,
628 params) {}
629
630template <class LocalOrdinal, class GlobalOrdinal, class Node>
631CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
632 CrsGraph(const local_graph_device_type& k_local_graph_,
633 const Teuchos::RCP<const map_type>& rowMap,
634 const Teuchos::RCP<const map_type>& colMap,
635 const Teuchos::RCP<const map_type>& domainMap,
636 const Teuchos::RCP<const map_type>& rangeMap,
637 const Teuchos::RCP<Teuchos::ParameterList>& params)
638 : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
639 , rowMap_(rowMap)
640 , colMap_(colMap)
641 , numAllocForAllRows_(0)
642 , storageStatus_(Details::STORAGE_1D_PACKED)
643 , indicesAreAllocated_(true)
644 , indicesAreLocal_(true) {
645 staticAssertions();
646 const char tfecfFuncName[] = "CrsGraph(Kokkos::LocalStaticCrsGraph,Map,Map,Map,Map)";
647
648 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
649 colMap.is_null(), std::runtime_error,
650 ": The input column Map must be nonnull.");
651 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
652 k_local_graph_.numRows() != rowMap->getLocalNumElements(),
653 std::runtime_error,
654 ": The input row Map and the input local graph need to have the same "
655 "number of rows. The row Map claims "
656 << rowMap->getLocalNumElements()
657 << " row(s), but the local graph claims " << k_local_graph_.numRows()
658 << " row(s).");
659
660 // NOTE (mfh 17 Mar 2014) getLocalNumRows() returns
661 // rowMap_->getLocalNumElements(), but it doesn't have to.
662 // TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
663 // k_local_graph_.numRows () != getLocalNumRows (), std::runtime_error,
664 // ": The input row Map and the input local graph need to have the same "
665 // "number of rows. The row Map claims " << getLocalNumRows () << " row(s), "
666 // "but the local graph claims " << k_local_graph_.numRows () << " row(s).");
667 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
668 lclIndsUnpacked_wdv.extent(0) != 0 || gblInds_wdv.extent(0) != 0, std::logic_error,
669 ": cannot have 1D data structures allocated.");
670
671 if (!params.is_null() && params->isParameter("sorted") &&
672 !params->get<bool>("sorted")) {
673 indicesAreSorted_ = false;
674 } else {
675 indicesAreSorted_ = true;
676 }
677
678 setDomainRangeMaps(domainMap.is_null() ? rowMap_ : domainMap,
679 rangeMap.is_null() ? rowMap_ : rangeMap);
680 Teuchos::Array<int> remotePIDs(0); // unused output argument
681 this->makeImportExport(remotePIDs, false);
682
683 lclIndsPacked_wdv = local_inds_wdv_type(k_local_graph_.entries);
684 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
685 this->setRowPtrs(k_local_graph_.row_map);
686
687 set_need_sync_host_uvm_access(); // lclGraph_ potentially still in a kernel
688
689 const bool callComputeGlobalConstants = params.get() == nullptr ||
690 params->get("compute global constants", true);
691
692 if (callComputeGlobalConstants) {
693 this->computeGlobalConstants();
694 }
695 this->fillComplete_ = true;
696 this->checkInternalState();
697}
698
699template <class LocalOrdinal, class GlobalOrdinal, class Node>
700CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
701 CrsGraph(const local_graph_device_type& lclGraph,
702 const Teuchos::RCP<const map_type>& rowMap,
703 const Teuchos::RCP<const map_type>& colMap,
704 const Teuchos::RCP<const map_type>& domainMap,
705 const Teuchos::RCP<const map_type>& rangeMap,
706 const Teuchos::RCP<const import_type>& importer,
707 const Teuchos::RCP<const export_type>& exporter,
708 const Teuchos::RCP<Teuchos::ParameterList>& params)
709 : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
710 , rowMap_(rowMap)
711 , colMap_(colMap)
712 , rangeMap_(rangeMap.is_null() ? rowMap : rangeMap)
713 , domainMap_(domainMap.is_null() ? rowMap : domainMap)
714 , importer_(importer)
715 , exporter_(exporter)
716 , numAllocForAllRows_(0)
717 , storageStatus_(Details::STORAGE_1D_PACKED)
718 , indicesAreAllocated_(true)
719 , indicesAreLocal_(true) {
720 staticAssertions();
721 const char tfecfFuncName[] =
722 "Tpetra::CrsGraph(local_graph_device_type,"
723 "Map,Map,Map,Map,Import,Export,params): ";
724
725 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(colMap.is_null(), std::runtime_error,
726 "The input column Map must be nonnull.");
727
728 lclIndsPacked_wdv = local_inds_wdv_type(lclGraph.entries);
729 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
730 setRowPtrs(lclGraph.row_map);
731
732 set_need_sync_host_uvm_access(); // lclGraph_ potentially still in a kernel
733
734 if (!params.is_null() && params->isParameter("sorted") &&
735 !params->get<bool>("sorted")) {
736 indicesAreSorted_ = false;
737 } else {
738 indicesAreSorted_ = true;
739 }
740
741 const bool callComputeGlobalConstants =
742 params.get() == nullptr ||
743 params->get("compute global constants", true);
744 if (callComputeGlobalConstants) {
745 this->computeGlobalConstants();
746 }
747 fillComplete_ = true;
748 checkInternalState();
749}
750
751template <class LocalOrdinal, class GlobalOrdinal, class Node>
752CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
753 CrsGraph(const row_ptrs_device_view_type& rowPointers,
754 const local_inds_wdv_type& columnIndices,
755 const Teuchos::RCP<const map_type>& rowMap,
756 const Teuchos::RCP<const map_type>& colMap,
757 const Teuchos::RCP<const map_type>& domainMap,
758 const Teuchos::RCP<const map_type>& rangeMap,
759 const Teuchos::RCP<const import_type>& importer,
760 const Teuchos::RCP<const export_type>& exporter,
761 const Teuchos::RCP<Teuchos::ParameterList>& params)
762 : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
763 , rowMap_(rowMap)
764 , colMap_(colMap)
765 , rangeMap_(rangeMap.is_null() ? rowMap : rangeMap)
766 , domainMap_(domainMap.is_null() ? rowMap : domainMap)
767 , importer_(importer)
768 , exporter_(exporter)
769 , numAllocForAllRows_(0)
770 , storageStatus_(Details::STORAGE_1D_PACKED)
771 , indicesAreAllocated_(true)
772 , indicesAreLocal_(true) {
773 staticAssertions();
774 const char tfecfFuncName[] =
775 "Tpetra::CrsGraph(row_ptrs_device_view_type,local_inds_wdv_type"
776 "Map,Map,Map,Map,Import,Export,params): ";
777
778 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(colMap.is_null(), std::runtime_error,
779 "The input column Map must be nonnull.");
780
781 lclIndsPacked_wdv = columnIndices;
782 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
783 setRowPtrs(rowPointers);
784
785 set_need_sync_host_uvm_access(); // lclGraph_ potentially still in a kernel
786
787 if (!params.is_null() && params->isParameter("sorted") &&
788 !params->get<bool>("sorted")) {
789 indicesAreSorted_ = false;
790 } else {
791 indicesAreSorted_ = true;
792 }
793
794 const bool callComputeGlobalConstants =
795 params.get() == nullptr ||
796 params->get("compute global constants", true);
797 if (callComputeGlobalConstants) {
798 this->computeGlobalConstants();
799 }
800 fillComplete_ = true;
801 checkInternalState();
802}
803
804template <class LocalOrdinal, class GlobalOrdinal, class Node>
805Teuchos::RCP<const Teuchos::ParameterList>
806CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
807 getValidParameters() const {
808 using Teuchos::ParameterList;
809 using Teuchos::parameterList;
810 using Teuchos::RCP;
811
812 RCP<ParameterList> params = parameterList("Tpetra::CrsGraph");
813
814 // Make a sublist for the Import.
815 RCP<ParameterList> importSublist = parameterList("Import");
816
817 // FIXME (mfh 02 Apr 2012) We should really have the Import and
818 // Export objects fill in these lists. However, we don't want to
819 // create an Import or Export unless we need them. For now, we
820 // know that the Import and Export just pass the list directly to
821 // their Distributor, so we can create a Distributor here
822 // (Distributor's constructor is a lightweight operation) and have
823 // it fill in the list.
824
825 // Fill in Distributor default parameters by creating a
826 // Distributor and asking it to do the work.
827 Distributor distributor(rowMap_->getComm(), importSublist);
828 params->set("Import", *importSublist, "How the Import performs communication.");
829
830 // Make a sublist for the Export. For now, it's a clone of the
831 // Import sublist. It's not a shallow copy, though, since we
832 // might like the Import to do communication differently than the
833 // Export.
834 params->set("Export", *importSublist, "How the Export performs communication.");
835
836 return params;
837}
838
839template <class LocalOrdinal, class GlobalOrdinal, class Node>
840void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
841 setParameterList(const Teuchos::RCP<Teuchos::ParameterList>& params) {
842 Teuchos::RCP<const Teuchos::ParameterList> validParams =
843 getValidParameters();
844 params->validateParametersAndSetDefaults(*validParams);
845 this->setMyParamList(params);
846}
847
848template <class LocalOrdinal, class GlobalOrdinal, class Node>
849global_size_t
850CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
851 getGlobalNumRows() const {
852 return rowMap_->getGlobalNumElements();
853}
854
855template <class LocalOrdinal, class GlobalOrdinal, class Node>
856global_size_t
857CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
858 getGlobalNumCols() const {
859 const char tfecfFuncName[] = "getGlobalNumCols: ";
860 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
861 !isFillComplete() || getDomainMap().is_null(), std::runtime_error,
862 "The graph does not have a domain Map. You may not call this method in "
863 "that case.");
864 return getDomainMap()->getGlobalNumElements();
865}
866
867template <class LocalOrdinal, class GlobalOrdinal, class Node>
868size_t
869CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
870 getLocalNumRows() const {
871 return this->rowMap_.is_null() ? static_cast<size_t>(0) : this->rowMap_->getLocalNumElements();
872}
873
874template <class LocalOrdinal, class GlobalOrdinal, class Node>
875size_t
876CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
877 getLocalNumCols() const {
878 const char tfecfFuncName[] = "getLocalNumCols: ";
879 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
880 !hasColMap(), std::runtime_error,
881 "The graph does not have a column Map. You may not call this method "
882 "unless the graph has a column Map. This requires either that a custom "
883 "column Map was given to the constructor, or that fillComplete() has "
884 "been called.");
885 return colMap_.is_null() ? static_cast<size_t>(0) : colMap_->getLocalNumElements();
886}
887
888template <class LocalOrdinal, class GlobalOrdinal, class Node>
889Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
894
895template <class LocalOrdinal, class GlobalOrdinal, class Node>
896Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
901
902template <class LocalOrdinal, class GlobalOrdinal, class Node>
903Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
908
909template <class LocalOrdinal, class GlobalOrdinal, class Node>
910Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
915
916template <class LocalOrdinal, class GlobalOrdinal, class Node>
917Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::import_type>
922
923template <class LocalOrdinal, class GlobalOrdinal, class Node>
924Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::export_type>
929
930template <class LocalOrdinal, class GlobalOrdinal, class Node>
931bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
932 hasColMap() const {
933 return !colMap_.is_null();
934}
935
936template <class LocalOrdinal, class GlobalOrdinal, class Node>
937bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
938 isStorageOptimized() const {
939 // FIXME (mfh 07 Aug 2014) Why wouldn't storage be optimized if
940 // getLocalNumRows() is zero?
941
942 const bool isOpt = indicesAreAllocated_ &&
943 k_numRowEntries_.extent(0) == 0 &&
944 getLocalNumRows() > 0;
945
946 return isOpt;
947}
948
949template <class LocalOrdinal, class GlobalOrdinal, class Node>
950global_size_t
951CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
952 getGlobalNumEntries() const {
953 const char tfecfFuncName[] = "getGlobalNumEntries: ";
954 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_, std::logic_error,
955 "The graph does not have global constants computed, "
956 "but the user has requested them.");
957
958 return globalNumEntries_;
959}
960
961template <class LocalOrdinal, class GlobalOrdinal, class Node>
962size_t
963CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
964 getLocalNumEntries() const {
965 const char tfecfFuncName[] = "getLocalNumEntries: ";
966 typedef LocalOrdinal LO;
967
968 Details::ProfilingRegion regionGLNE("Tpetra::CrsGraph::getLocalNumEntries");
969
970 if (this->indicesAreAllocated_) {
971 const LO lclNumRows = this->getLocalNumRows();
972 if (lclNumRows == 0) {
973 return static_cast<size_t>(0);
974 } else {
975 // Avoid the "*this capture" issue by creating a local Kokkos::View.
976 auto numEntPerRow = this->k_numRowEntries_;
977 const LO numNumEntPerRow = numEntPerRow.extent(0);
978 if (numNumEntPerRow == 0) {
979 if (static_cast<LO>(this->getRowPtrsPackedDevice().extent(0)) <
980 static_cast<LO>(lclNumRows + 1)) {
981 return static_cast<size_t>(0);
982 } else {
983 // indices are allocated and k_numRowEntries_ is not allocated,
984 // so we have packed storage and the length of lclIndsPacked_wdv
985 // must be the number of local entries.
986 if (debug_) {
987 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->getRowPtrsPackedHost()(lclNumRows) != lclIndsPacked_wdv.extent(0), std::logic_error,
988 "Final entry of packed host rowptrs doesn't match the length of lclIndsPacked");
989 }
990 return lclIndsPacked_wdv.extent(0);
991 }
992 } else { // k_numRowEntries_ is populated
993 // k_numRowEntries_ is actually be a host View, so we run
994 // the sum in its native execution space. This also means
995 // that we can use explicit capture (which could perhaps
996 // improve build time) instead of KOKKOS_LAMBDA, and avoid
997 // any CUDA build issues with trying to run a __device__ -
998 // only function on host.
999 typedef typename num_row_entries_type::execution_space
1000 host_exec_space;
1001 typedef Kokkos::RangePolicy<host_exec_space, LO> range_type;
1002
1003 const LO upperLoopBound = lclNumRows < numNumEntPerRow ? lclNumRows : numNumEntPerRow;
1004 size_t nodeNumEnt = 0;
1005 Kokkos::parallel_reduce(
1006 "Tpetra::CrsGraph::getNumNodeEntries",
1007 range_type(0, upperLoopBound),
1008 [=](const LO& k, size_t& lclSum) {
1009 lclSum += numEntPerRow(k);
1010 },
1011 nodeNumEnt);
1012 return nodeNumEnt;
1013 }
1014 }
1015 } else { // nothing allocated on this process, so no entries
1016 return static_cast<size_t>(0);
1017 }
1018}
1019
1020template <class LocalOrdinal, class GlobalOrdinal, class Node>
1021global_size_t
1022CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1023 getGlobalMaxNumRowEntries() const {
1024 const char tfecfFuncName[] = "getGlobalMaxNumRowEntries: ";
1025 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_, std::logic_error,
1026 "The graph does not have global constants computed, "
1027 "but the user has requested them.");
1028
1029 return globalMaxNumRowEntries_;
1030}
1031
1032template <class LocalOrdinal, class GlobalOrdinal, class Node>
1033size_t
1038
1039template <class LocalOrdinal, class GlobalOrdinal, class Node>
1040bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1041 isFillComplete() const {
1042 return fillComplete_;
1043}
1044
1045template <class LocalOrdinal, class GlobalOrdinal, class Node>
1046bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1047 isFillActive() const {
1048 return !fillComplete_;
1049}
1050
1051template <class LocalOrdinal, class GlobalOrdinal, class Node>
1052bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1053 isLocallyIndexed() const {
1054 return indicesAreLocal_;
1055}
1056
1057template <class LocalOrdinal, class GlobalOrdinal, class Node>
1058bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1059 isGloballyIndexed() const {
1060 return indicesAreGlobal_;
1061}
1062
1063template <class LocalOrdinal, class GlobalOrdinal, class Node>
1064size_t
1065CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1066 getLocalAllocationSize() const {
1067 typedef LocalOrdinal LO;
1068
1069 if (this->indicesAreAllocated_) {
1070 const LO lclNumRows = this->getLocalNumRows();
1071 if (lclNumRows == 0) {
1072 return static_cast<size_t>(0);
1073 } else if (storageStatus_ == Details::STORAGE_1D_PACKED) {
1074 if (static_cast<LO>(this->getRowPtrsPackedDevice().extent(0)) <
1075 static_cast<LO>(lclNumRows + 1)) {
1076 return static_cast<size_t>(0);
1077 } else {
1078 if (this->isLocallyIndexed())
1079 return lclIndsPacked_wdv.extent(0);
1080 else
1081 return gblInds_wdv.extent(0);
1082 }
1083 } else if (storageStatus_ == Details::STORAGE_1D_UNPACKED) {
1084 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1085 if (rowPtrsUnpacked_host.extent(0) == 0) {
1086 return static_cast<size_t>(0);
1087 } else {
1088 if (this->isLocallyIndexed())
1089 return lclIndsUnpacked_wdv.extent(0);
1090 else
1091 return gblInds_wdv.extent(0);
1092 }
1093 } else {
1094 return static_cast<size_t>(0);
1095 }
1096 } else {
1097 return Tpetra::Details::OrdinalTraits<size_t>::invalid();
1098 }
1099}
1100
1101template <class LocalOrdinal, class GlobalOrdinal, class Node>
1102Teuchos::RCP<const Teuchos::Comm<int>>
1103CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1104 getComm() const {
1105 return this->rowMap_.is_null() ? Teuchos::null : this->rowMap_->getComm();
1106}
1107
1108template <class LocalOrdinal, class GlobalOrdinal, class Node>
1109GlobalOrdinal
1110CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1111 getIndexBase() const {
1112 return rowMap_->getIndexBase();
1113}
1114
1115template <class LocalOrdinal, class GlobalOrdinal, class Node>
1116bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1117 indicesAreAllocated() const {
1118 return indicesAreAllocated_;
1119}
1120
1121template <class LocalOrdinal, class GlobalOrdinal, class Node>
1126
1127template <class LocalOrdinal, class GlobalOrdinal, class Node>
1132
1133template <class LocalOrdinal, class GlobalOrdinal, class Node>
1134void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1135 setLocallyModified() {
1136 // FIXME (mfh 07 May 2013) How do we know that the change
1137 // introduced a redundancy, or even that it invalidated the sorted
1138 // order of indices? CrsGraph has always made this conservative
1139 // guess. It could be a bit costly to check at insertion time,
1140 // though.
1141 indicesAreSorted_ = false;
1142 noRedundancies_ = false;
1143
1144 // We've modified the graph, so we'll have to recompute local
1145 // constants like the number of diagonal entries on this process.
1146 haveLocalConstants_ = false;
1147}
1148
1149template <class LocalOrdinal, class GlobalOrdinal, class Node>
1150void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1151 allocateIndices(const ELocalGlobal lg, const bool verbose) {
1152 using std::endl;
1153 using Teuchos::arcp;
1154 using Teuchos::Array;
1155 using Teuchos::ArrayRCP;
1156 typedef Teuchos::ArrayRCP<size_t>::size_type size_type;
1157 typedef typename local_graph_device_type::row_map_type::non_const_type
1158 non_const_row_map_type;
1159 const char tfecfFuncName[] = "allocateIndices: ";
1160 const char suffix[] =
1161 " Please report this bug to the Tpetra developers.";
1162
1163 Details::ProfilingRegion profRegion("Tpetra::CrsGraph::allocateIndices");
1164
1165 std::unique_ptr<std::string> prefix;
1166 if (verbose) {
1167 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
1168 std::ostringstream os;
1169 os << *prefix << "Start: lg="
1170 << (lg == GlobalIndices ? "GlobalIndices" : "LocalIndices")
1171 << ", numRows: " << this->getLocalNumRows() << endl;
1172 std::cerr << os.str();
1173 }
1174
1175 // This is a protected function, only callable by us. If it was
1176 // called incorrectly, it is our fault. That's why the tests
1177 // below throw std::logic_error instead of std::invalid_argument.
1178 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isLocallyIndexed() && lg == GlobalIndices, std::logic_error,
1179 ": The graph is locally indexed, but Tpetra code is calling "
1180 "this method with lg=GlobalIndices."
1181 << suffix);
1182 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed() && lg == LocalIndices, std::logic_error,
1183 ": The graph is globally indexed, but Tpetra code is calling "
1184 "this method with lg=LocalIndices."
1185 << suffix);
1186 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreAllocated(), std::logic_error,
1187 ": The graph's "
1188 "indices are already allocated, but Tpetra is calling "
1189 "allocateIndices again."
1190 << suffix);
1191 const size_t numRows = this->getLocalNumRows();
1192
1193 //
1194 // STATIC ALLOCATION PROFILE
1195 //
1196 size_type numInds = 0;
1197 {
1198 if (verbose) {
1199 std::ostringstream os;
1200 os << *prefix << "Allocate k_rowPtrs: " << (numRows + 1) << endl;
1201 std::cerr << os.str();
1202 }
1203 non_const_row_map_type k_rowPtrs("Tpetra::CrsGraph::ptr", numRows + 1);
1204
1205 if (this->k_numAllocPerRow_.extent(0) != 0) {
1206 // It's OK to throw std::invalid_argument here, because we
1207 // haven't incurred any side effects yet. Throwing that
1208 // exception (and not, say, std::logic_error) implies that the
1209 // instance can recover.
1210 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->k_numAllocPerRow_.extent(0) != numRows,
1211 std::invalid_argument,
1212 "k_numAllocPerRow_ is allocated, that is, "
1213 "has nonzero length "
1214 << this->k_numAllocPerRow_.extent(0)
1215 << ", but its length != numRows = " << numRows << ".");
1216
1217 // k_numAllocPerRow_ is a host View, but k_rowPtrs (the thing
1218 // we want to compute here) lives on device. That's OK;
1219 // computeOffsetsFromCounts can handle this case.
1220 using Details::computeOffsetsFromCounts;
1221
1222 // FIXME (mfh 27 Jun 2016) Currently, computeOffsetsFromCounts
1223 // doesn't attempt to check its input for "invalid" flag
1224 // values. For now, we omit that feature of the sequential
1225 // code disabled below.
1226 numInds = computeOffsetsFromCounts(k_rowPtrs, k_numAllocPerRow_);
1227 } else {
1228 // It's OK to throw std::invalid_argument here, because we
1229 // haven't incurred any side effects yet. Throwing that
1230 // exception (and not, say, std::logic_error) implies that the
1231 // instance can recover.
1232 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->numAllocForAllRows_ ==
1233 Tpetra::Details::OrdinalTraits<size_t>::invalid(),
1234 std::invalid_argument,
1235 "numAllocForAllRows_ has an invalid value, "
1236 "namely Tpetra::Details::OrdinalTraits<size_t>::invalid() = "
1237 << Tpetra::Details::OrdinalTraits<size_t>::invalid() << ".");
1238
1239 using Details::computeOffsetsFromConstantCount;
1240 numInds = computeOffsetsFromConstantCount(k_rowPtrs, this->numAllocForAllRows_);
1241 }
1242 // "Commit" the resulting row offsets.
1243 setRowPtrsUnpacked(k_rowPtrs);
1244 }
1245 if (debug_) {
1246 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numInds != size_type(this->getRowPtrsUnpackedHost()(numRows)), std::logic_error,
1247 ": Number of indices produced by computeOffsetsFrom[Constant]Counts "
1248 "does not match final entry of rowptrs unpacked");
1249 }
1250
1251 if (lg == LocalIndices) {
1252 if (verbose) {
1253 std::ostringstream os;
1254 os << *prefix << "Allocate local column indices "
1255 "lclIndsUnpacked_wdv: "
1256 << numInds << endl;
1257 std::cerr << os.str();
1258 }
1259 lclIndsUnpacked_wdv = local_inds_wdv_type(
1260 local_inds_dualv_type("Tpetra::CrsGraph::lclInd", numInds));
1261 } else {
1262 if (verbose) {
1263 std::ostringstream os;
1264 os << *prefix << "Allocate global column indices "
1265 "gblInds_wdv: "
1266 << numInds << endl;
1267 std::cerr << os.str();
1268 }
1269 gblInds_wdv = global_inds_wdv_type(
1270 global_inds_dualv_type("Tpetra::CrsGraph::gblInd", numInds));
1271 }
1272 storageStatus_ = Details::STORAGE_1D_UNPACKED;
1273
1274 this->indicesAreLocal_ = (lg == LocalIndices);
1275 this->indicesAreGlobal_ = (lg == GlobalIndices);
1276
1277 if (numRows > 0) { // reallocate k_numRowEntries_ & fill w/ 0s
1278 using Kokkos::ViewAllocateWithoutInitializing;
1279 const char label[] = "Tpetra::CrsGraph::numRowEntries";
1280 if (verbose) {
1281 std::ostringstream os;
1282 os << *prefix << "Allocate k_numRowEntries_: " << numRows
1283 << endl;
1284 std::cerr << os.str();
1285 }
1286 num_row_entries_type numRowEnt(ViewAllocateWithoutInitializing(label), numRows);
1287 // DEEP_COPY REVIEW - VALUE-TO-HOSTMIRROR
1288 Kokkos::deep_copy(execution_space(), numRowEnt, static_cast<size_t>(0)); // fill w/ 0s
1289 Kokkos::fence("CrsGraph::allocateIndices"); // TODO: Need to understand downstream failure points and move this fence.
1290 this->k_numRowEntries_ = numRowEnt; // "commit" our allocation
1291 }
1292
1293 // Once indices are allocated, CrsGraph needs to free this information.
1294 this->numAllocForAllRows_ = 0;
1295 this->k_numAllocPerRow_ = decltype(k_numAllocPerRow_)();
1296 this->indicesAreAllocated_ = true;
1297
1298 try {
1299 this->checkInternalState();
1300 } catch (std::logic_error& e) {
1301 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
1302 "At end of allocateIndices, "
1303 "checkInternalState threw std::logic_error: "
1304 << e.what());
1305 } catch (std::exception& e) {
1306 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1307 "At end of allocateIndices, "
1308 "checkInternalState threw std::exception: "
1309 << e.what());
1310 } catch (...) {
1311 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1312 "At end of allocateIndices, "
1313 "checkInternalState threw an exception "
1314 "not a subclass of std::exception.");
1315 }
1316
1317 if (verbose) {
1318 std::ostringstream os;
1319 os << *prefix << "Done" << endl;
1320 std::cerr << os.str();
1321 }
1322}
1323
1324template <class LocalOrdinal, class GlobalOrdinal, class Node>
1325typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1326 local_inds_dualv_type::t_host::const_type
1327 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1328 getLocalIndsViewHost(const RowInfo& rowinfo) const {
1329 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1330 return typename local_inds_dualv_type::t_host::const_type();
1331 else
1332 return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
1333 rowinfo.allocSize,
1334 Access::ReadOnly);
1335}
1336
1337template <class LocalOrdinal, class GlobalOrdinal, class Node>
1338typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1339 local_inds_dualv_type::t_host
1340 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1341 getLocalIndsViewHostNonConst(const RowInfo& rowinfo) {
1342 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1343 return typename local_inds_dualv_type::t_host();
1344 else
1345 return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
1346 rowinfo.allocSize,
1347 Access::ReadWrite);
1348}
1349
1350template <class LocalOrdinal, class GlobalOrdinal, class Node>
1351typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1352 global_inds_dualv_type::t_host::const_type
1353 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1354 getGlobalIndsViewHost(const RowInfo& rowinfo) const {
1355 if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
1356 return typename global_inds_dualv_type::t_host::const_type();
1357 else
1358 return gblInds_wdv.getHostSubview(rowinfo.offset1D,
1359 rowinfo.allocSize,
1360 Access::ReadOnly);
1361}
1362
1363template <class LocalOrdinal, class GlobalOrdinal, class Node>
1364typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1365 local_inds_dualv_type::t_dev::const_type
1366 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1367 getLocalIndsViewDevice(const RowInfo& rowinfo) const {
1368 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1369 return typename local_inds_dualv_type::t_dev::const_type();
1370 else
1371 return lclIndsUnpacked_wdv.getDeviceSubview(rowinfo.offset1D,
1372 rowinfo.allocSize,
1373 Access::ReadOnly);
1374}
1375
1376template <class LocalOrdinal, class GlobalOrdinal, class Node>
1377typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1378 global_inds_dualv_type::t_dev::const_type
1379 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1380 getGlobalIndsViewDevice(const RowInfo& rowinfo) const {
1381 if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
1382 return typename global_inds_dualv_type::t_dev::const_type();
1383 else
1384 return gblInds_wdv.getDeviceSubview(rowinfo.offset1D,
1385 rowinfo.allocSize,
1386 Access::ReadOnly);
1387}
1388
1389template <class LocalOrdinal, class GlobalOrdinal, class Node>
1390RowInfo
1391CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1392 getRowInfo(const LocalOrdinal myRow) const {
1393 const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid();
1394 RowInfo ret;
1395 if (this->rowMap_.is_null() || !this->rowMap_->isNodeLocalElement(myRow)) {
1396 ret.localRow = STINV;
1397 ret.allocSize = 0;
1398 ret.numEntries = 0;
1399 ret.offset1D = STINV;
1400 return ret;
1401 }
1402
1403 ret.localRow = static_cast<size_t>(myRow);
1404 if (this->indicesAreAllocated()) {
1405 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1406 // Offsets tell us the allocation size in this case.
1407 if (rowPtrsUnpacked_host.extent(0) == 0) {
1408 ret.offset1D = 0;
1409 ret.allocSize = 0;
1410 } else {
1411 ret.offset1D = rowPtrsUnpacked_host(myRow);
1412 ret.allocSize = rowPtrsUnpacked_host(myRow + 1) - rowPtrsUnpacked_host(myRow);
1413 }
1414
1415 ret.numEntries = (this->k_numRowEntries_.extent(0) == 0) ? ret.allocSize : this->k_numRowEntries_(myRow);
1416 } else { // haven't performed allocation yet; probably won't hit this code
1417 // FIXME (mfh 07 Aug 2014) We want graph's constructors to
1418 // allocate, rather than doing lazy allocation at first insert.
1419 // This will make k_numAllocPerRow_ obsolete.
1420 ret.allocSize = (this->k_numAllocPerRow_.extent(0) != 0) ? this->k_numAllocPerRow_(myRow) : // this is a host View
1421 this->numAllocForAllRows_;
1422 ret.numEntries = 0;
1423 ret.offset1D = STINV;
1424 }
1425
1426 return ret;
1427}
1428
1429template <class LocalOrdinal, class GlobalOrdinal, class Node>
1430RowInfo
1431CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1432 getRowInfoFromGlobalRowIndex(const GlobalOrdinal gblRow) const {
1433 const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid();
1434 RowInfo ret;
1435 if (this->rowMap_.is_null()) {
1436 ret.localRow = STINV;
1437 ret.allocSize = 0;
1438 ret.numEntries = 0;
1439 ret.offset1D = STINV;
1440 return ret;
1441 }
1442 const LocalOrdinal myRow = this->rowMap_->getLocalElement(gblRow);
1443 if (myRow == Teuchos::OrdinalTraits<LocalOrdinal>::invalid()) {
1444 ret.localRow = STINV;
1445 ret.allocSize = 0;
1446 ret.numEntries = 0;
1447 ret.offset1D = STINV;
1448 return ret;
1449 }
1450
1451 ret.localRow = static_cast<size_t>(myRow);
1452 if (this->indicesAreAllocated()) {
1453 // graph data structures have the info that we need
1454 //
1455 // if static graph, offsets tell us the allocation size
1456 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1457 if (rowPtrsUnpacked_host.extent(0) == 0) {
1458 ret.offset1D = 0;
1459 ret.allocSize = 0;
1460 } else {
1461 ret.offset1D = rowPtrsUnpacked_host(myRow);
1462 ret.allocSize = rowPtrsUnpacked_host(myRow + 1) - rowPtrsUnpacked_host(myRow);
1463 }
1464
1465 ret.numEntries = (this->k_numRowEntries_.extent(0) == 0) ? ret.allocSize : this->k_numRowEntries_(myRow);
1466 } else { // haven't performed allocation yet; probably won't hit this code
1467 // FIXME (mfh 07 Aug 2014) We want graph's constructors to
1468 // allocate, rather than doing lazy allocation at first insert.
1469 // This will make k_numAllocPerRow_ obsolete.
1470 ret.allocSize = (this->k_numAllocPerRow_.extent(0) != 0) ? this->k_numAllocPerRow_(myRow) : // this is a host View
1471 this->numAllocForAllRows_;
1472 ret.numEntries = 0;
1473 ret.offset1D = STINV;
1474 }
1475
1476 return ret;
1477}
1478
1479template <class LocalOrdinal, class GlobalOrdinal, class Node>
1480void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1481 staticAssertions() const {
1482 using Teuchos::OrdinalTraits;
1483 typedef LocalOrdinal LO;
1484 typedef GlobalOrdinal GO;
1485 typedef global_size_t GST;
1486
1487 // Assumption: sizeof(GlobalOrdinal) >= sizeof(LocalOrdinal):
1488 // This is so that we can store local indices in the memory
1489 // formerly occupied by global indices.
1490 static_assert(sizeof(GlobalOrdinal) >= sizeof(LocalOrdinal),
1491 "Tpetra::CrsGraph: sizeof(GlobalOrdinal) must be >= sizeof(LocalOrdinal).");
1492 // Assumption: max(size_t) >= max(LocalOrdinal)
1493 // This is so that we can represent any LocalOrdinal as a size_t.
1494 static_assert(sizeof(size_t) >= sizeof(LocalOrdinal),
1495 "Tpetra::CrsGraph: sizeof(size_t) must be >= sizeof(LocalOrdinal).");
1496 static_assert(sizeof(GST) >= sizeof(size_t),
1497 "Tpetra::CrsGraph: sizeof(Tpetra::global_size_t) must be >= sizeof(size_t).");
1498
1499 // FIXME (mfh 30 Sep 2015) We're not using
1500 // Teuchos::CompileTimeAssert any more. Can we do these checks
1501 // with static_assert?
1502
1503 // can't call max() with CompileTimeAssert, because it isn't a
1504 // constant expression; will need to make this a runtime check
1505 const char msg[] =
1506 "Tpetra::CrsGraph: Object cannot be created with the "
1507 "given template arguments: size assumptions are not valid.";
1508 TEUCHOS_TEST_FOR_EXCEPTION(
1509 static_cast<size_t>(Teuchos::OrdinalTraits<LO>::max()) > Teuchos::OrdinalTraits<size_t>::max(),
1510 std::runtime_error, msg);
1511 TEUCHOS_TEST_FOR_EXCEPTION(
1512 static_cast<GST>(Teuchos::OrdinalTraits<LO>::max()) > static_cast<GST>(Teuchos::OrdinalTraits<GO>::max()),
1513 std::runtime_error, msg);
1514 TEUCHOS_TEST_FOR_EXCEPTION(
1515 static_cast<size_t>(Teuchos::OrdinalTraits<GO>::max()) > Teuchos::OrdinalTraits<GST>::max(),
1516 std::runtime_error, msg);
1517 TEUCHOS_TEST_FOR_EXCEPTION(
1518 Teuchos::OrdinalTraits<size_t>::max() > Teuchos::OrdinalTraits<GST>::max(),
1519 std::runtime_error, msg);
1520}
1521
1522template <class LocalOrdinal, class GlobalOrdinal, class Node>
1523size_t
1524CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1525 insertIndices(RowInfo& rowinfo,
1526 const SLocalGlobalViews& newInds,
1527 const ELocalGlobal lg,
1528 const ELocalGlobal I) {
1529 using Teuchos::ArrayView;
1530 typedef LocalOrdinal LO;
1531 typedef GlobalOrdinal GO;
1532 const char tfecfFuncName[] = "insertIndices: ";
1533
1534 size_t oldNumEnt = 0;
1535 if (debug_) {
1536 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(lg != GlobalIndices && lg != LocalIndices, std::invalid_argument,
1537 "lg must be either GlobalIndices or LocalIndices.");
1538 oldNumEnt = this->getNumEntriesInLocalRow(rowinfo.localRow);
1539 }
1540
1541 size_t numNewInds = 0;
1542 if (lg == GlobalIndices) { // input indices are global
1543 ArrayView<const GO> new_ginds = newInds.ginds;
1544 numNewInds = new_ginds.size();
1545 if (I == GlobalIndices) { // store global indices
1546 auto gind_view = gblInds_wdv.getHostView(Access::ReadWrite);
1547 if (debug_) {
1548 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(gind_view.size()) <
1549 rowinfo.numEntries + numNewInds,
1550 std::logic_error,
1551 "gind_view.size() = " << gind_view.size()
1552 << " < rowinfo.numEntries (= " << rowinfo.numEntries
1553 << ") + numNewInds (= " << numNewInds << ").");
1554 }
1555 GO* const gblColInds_out = gind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
1556 for (size_t k = 0; k < numNewInds; ++k) {
1557 gblColInds_out[k] = new_ginds[k];
1558 }
1559 } else if (I == LocalIndices) { // store local indices
1560 auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1561 if (debug_) {
1562 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(lind_view.size()) <
1563 rowinfo.numEntries + numNewInds,
1564 std::logic_error,
1565 "lind_view.size() = " << lind_view.size()
1566 << " < rowinfo.numEntries (= " << rowinfo.numEntries
1567 << ") + numNewInds (= " << numNewInds << ").");
1568 }
1569 LO* const lclColInds_out = lind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
1570 for (size_t k = 0; k < numNewInds; ++k) {
1571 lclColInds_out[k] = colMap_->getLocalElement(new_ginds[k]);
1572 }
1573 }
1574 } else if (lg == LocalIndices) { // input indices are local
1575 ArrayView<const LO> new_linds = newInds.linds;
1576 numNewInds = new_linds.size();
1577 if (I == LocalIndices) { // store local indices
1578 auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1579 if (debug_) {
1580 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(lind_view.size()) <
1581 rowinfo.numEntries + numNewInds,
1582 std::logic_error,
1583 "lind_view.size() = " << lind_view.size()
1584 << " < rowinfo.numEntries (= " << rowinfo.numEntries
1585 << ") + numNewInds (= " << numNewInds << ").");
1586 }
1587 LO* const lclColInds_out = lind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
1588 for (size_t k = 0; k < numNewInds; ++k) {
1589 lclColInds_out[k] = new_linds[k];
1590 }
1591 } else if (I == GlobalIndices) {
1592 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
1593 "The case where the input indices are local "
1594 "and the indices to write are global (lg=LocalIndices, I="
1595 "GlobalIndices) is not implemented, because it does not make sense."
1596 << std::endl
1597 << "If you have correct local column indices, that "
1598 "means the graph has a column Map. In that case, you should be "
1599 "storing local indices.");
1600 }
1601 }
1602
1603 rowinfo.numEntries += numNewInds;
1604 this->k_numRowEntries_(rowinfo.localRow) += numNewInds;
1605 this->setLocallyModified();
1606
1607 if (debug_) {
1608 const size_t chkNewNumEnt =
1609 this->getNumEntriesInLocalRow(rowinfo.localRow);
1610 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(chkNewNumEnt != oldNumEnt + numNewInds, std::logic_error,
1611 "chkNewNumEnt = " << chkNewNumEnt
1612 << " != oldNumEnt (= " << oldNumEnt
1613 << ") + numNewInds (= " << numNewInds << ").");
1614 }
1615
1616 return numNewInds;
1617}
1618
1619template <class LocalOrdinal, class GlobalOrdinal, class Node>
1620size_t
1621CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1622 insertGlobalIndicesImpl(const LocalOrdinal lclRow,
1623 const GlobalOrdinal inputGblColInds[],
1624 const size_t numInputInds) {
1625 return this->insertGlobalIndicesImpl(this->getRowInfo(lclRow),
1626 inputGblColInds, numInputInds);
1627}
1628
1629template <class LocalOrdinal, class GlobalOrdinal, class Node>
1630size_t
1631CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1632 insertGlobalIndicesImpl(const RowInfo& rowInfo,
1633 const GlobalOrdinal inputGblColInds[],
1634 const size_t numInputInds,
1635 std::function<void(const size_t, const size_t, const size_t)> fun) {
1636 using Details::verbosePrintArray;
1637 using Kokkos::MemoryUnmanaged;
1638 using Kokkos::subview;
1639 using Kokkos::View;
1640 using Teuchos::ArrayView;
1641 using LO = LocalOrdinal;
1642 using GO = GlobalOrdinal;
1643
1644 const char tfecfFuncName[] = "insertGlobalIndicesImpl: ";
1645 const LO lclRow = static_cast<LO>(rowInfo.localRow);
1646
1647 auto numEntries = rowInfo.numEntries;
1648 using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
1649 inp_view_type inputInds(inputGblColInds, numInputInds);
1650 size_t numInserted;
1651 {
1652 auto gblIndsHostView = this->gblInds_wdv.getHostView(Access::ReadWrite);
1653 numInserted = Details::insertCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
1654 gblIndsHostView,
1655 numEntries, inputInds, fun);
1656 }
1657
1658 const bool insertFailed =
1659 numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
1660 if (insertFailed) {
1661 constexpr size_t ONE(1);
1662 const int myRank = this->getComm()->getRank();
1663 std::ostringstream os;
1664
1665 os << "Proc " << myRank << ": Not enough capacity to insert "
1666 << numInputInds
1667 << " ind" << (numInputInds != ONE ? "ices" : "ex")
1668 << " into local row " << lclRow << ", which currently has "
1669 << rowInfo.numEntries
1670 << " entr" << (rowInfo.numEntries != ONE ? "ies" : "y")
1671 << " and total allocation size " << rowInfo.allocSize
1672 << ". ";
1673 const size_t maxNumToPrint =
1674 Details::Behavior::verbosePrintCountThreshold();
1675 ArrayView<const GO> inputGblColIndsView(inputGblColInds,
1676 numInputInds);
1677 verbosePrintArray(os, inputGblColIndsView,
1678 "Input global "
1679 "column indices",
1680 maxNumToPrint);
1681 os << ", ";
1682 auto curGblColInds = getGlobalIndsViewHost(rowInfo);
1683 ArrayView<const GO> curGblColIndsView(curGblColInds.data(),
1684 rowInfo.numEntries);
1685 verbosePrintArray(os, curGblColIndsView,
1686 "Current global "
1687 "column indices",
1688 maxNumToPrint);
1689 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
1690 }
1691
1692 this->k_numRowEntries_(lclRow) += numInserted;
1693
1694 this->setLocallyModified();
1695 return numInserted;
1696}
1697
1698template <class LocalOrdinal, class GlobalOrdinal, class Node>
1699void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1700 insertLocalIndicesImpl(const LocalOrdinal myRow,
1701 const Teuchos::ArrayView<const LocalOrdinal>& indices,
1702 std::function<void(const size_t, const size_t, const size_t)> fun) {
1703 using Kokkos::MemoryUnmanaged;
1704 using Kokkos::subview;
1705 using Kokkos::View;
1706 using LO = LocalOrdinal;
1707
1708 const char tfecfFuncName[] = "insertLocallIndicesImpl: ";
1709
1710 const RowInfo rowInfo = this->getRowInfo(myRow);
1711
1712 size_t numNewInds = 0;
1713 size_t newNumEntries = 0;
1714
1715 auto numEntries = rowInfo.numEntries;
1716 // Note: Teuchos::ArrayViews are in HostSpace
1717 using inp_view_type = View<const LO*, Kokkos::HostSpace, MemoryUnmanaged>;
1718 inp_view_type inputInds(indices.getRawPtr(), indices.size());
1719 size_t numInserted = 0;
1720 {
1721 auto lclInds = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1722 numInserted = Details::insertCrsIndices(myRow, this->getRowPtrsUnpackedHost(), lclInds,
1723 numEntries, inputInds, fun);
1724 }
1725
1726 const bool insertFailed =
1727 numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
1728 if (insertFailed) {
1729 constexpr size_t ONE(1);
1730 const size_t numInputInds(indices.size());
1731 const int myRank = this->getComm()->getRank();
1732 std::ostringstream os;
1733 os << "On MPI Process " << myRank << ": Not enough capacity to "
1734 "insert "
1735 << numInputInds
1736 << " ind" << (numInputInds != ONE ? "ices" : "ex")
1737 << " into local row " << myRow << ", which currently has "
1738 << rowInfo.numEntries
1739 << " entr" << (rowInfo.numEntries != ONE ? "ies" : "y")
1740 << " and total allocation size " << rowInfo.allocSize << ".";
1741 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
1742 }
1743 numNewInds = numInserted;
1744 newNumEntries = rowInfo.numEntries + numNewInds;
1745
1746 this->k_numRowEntries_(myRow) += numNewInds;
1747 this->setLocallyModified();
1748
1749 if (debug_) {
1750 const size_t chkNewNumEntries = this->getNumEntriesInLocalRow(myRow);
1751 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(chkNewNumEntries != newNumEntries, std::logic_error,
1752 "getNumEntriesInLocalRow(" << myRow << ") = " << chkNewNumEntries
1753 << " != newNumEntries = " << newNumEntries
1754 << ". Please report this bug to the Tpetra developers.");
1755 }
1756}
1757
1758template <class LocalOrdinal, class GlobalOrdinal, class Node>
1759size_t
1760CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1761 findGlobalIndices(const RowInfo& rowInfo,
1762 const Teuchos::ArrayView<const GlobalOrdinal>& indices,
1763 std::function<void(const size_t, const size_t, const size_t)> fun) const {
1764 using GO = GlobalOrdinal;
1765 using Kokkos::MemoryUnmanaged;
1766 using Kokkos::View;
1767
1768 auto invalidCount = Teuchos::OrdinalTraits<size_t>::invalid();
1769
1770 using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
1771 inp_view_type inputInds(indices.getRawPtr(), indices.size());
1772
1773 size_t numFound = 0;
1774 LocalOrdinal lclRow = rowInfo.localRow;
1775 if (this->isLocallyIndexed()) {
1776 if (this->colMap_.is_null())
1777 return invalidCount;
1778 const auto& colMap = *(this->colMap_);
1779 auto map = [&](GO const gblInd) { return colMap.getLocalElement(gblInd); };
1780 if (this->isSorted()) {
1781 numFound = Details::findCrsIndicesSorted(
1782 lclRow,
1783 this->getRowPtrsUnpackedHost(),
1784 rowInfo.numEntries,
1785 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly),
1786 inputInds,
1787 map,
1788 fun);
1789 } else {
1790 numFound = Details::findCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
1791 rowInfo.numEntries,
1792 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly), inputInds, map, fun);
1793 }
1794 } else if (this->isGloballyIndexed()) {
1795 numFound = Details::findCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
1796 rowInfo.numEntries,
1797 gblInds_wdv.getHostView(Access::ReadOnly), inputInds, fun);
1798 }
1799 return numFound;
1800}
1801
1802template <class LocalOrdinal, class GlobalOrdinal, class Node>
1803void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1804 setDomainRangeMaps(const Teuchos::RCP<const map_type>& domainMap,
1805 const Teuchos::RCP<const map_type>& rangeMap) {
1806 // simple pointer comparison for equality
1807 if (domainMap_ != domainMap) {
1808 domainMap_ = domainMap;
1809 importer_ = Teuchos::null;
1810 }
1811 if (rangeMap_ != rangeMap) {
1812 rangeMap_ = rangeMap;
1813 exporter_ = Teuchos::null;
1814 }
1815}
1816
1817template <class LocalOrdinal, class GlobalOrdinal, class Node>
1818void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1819 clearGlobalConstants() {
1820 const auto INV = Teuchos::OrdinalTraits<global_size_t>::invalid();
1821
1822 globalNumEntries_ = INV;
1823 globalMaxNumRowEntries_ = INV;
1824 haveGlobalConstants_ = false;
1825}
1826
1827template <class LocalOrdinal, class GlobalOrdinal, class Node>
1828void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1829 checkInternalState() const {
1830 using Details::ProfilingRegion;
1831 if (debug_) {
1832 using std::endl;
1833 const char tfecfFuncName[] = "checkInternalState: ";
1834 const char suffix[] = " Please report this bug to the Tpetra developers.";
1835
1836 ProfilingRegion("Tpetra::CrsGraph::checkInternalState");
1837
1838 std::unique_ptr<std::string> prefix;
1839 if (verbose_) {
1840 prefix = this->createPrefix("CrsGraph", "checkInternalState");
1841 std::ostringstream os;
1842 os << *prefix << "Start" << endl;
1843 std::cerr << os.str();
1844 }
1845
1846 const global_size_t GSTI = Teuchos::OrdinalTraits<global_size_t>::invalid();
1847 // const size_t STI = Teuchos::OrdinalTraits<size_t>::invalid (); // unused
1848 // check the internal state of this data structure
1849 // this is called by numerous state-changing methods, in a debug build, to ensure that the object
1850 // always remains in a valid state
1851
1852 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->rowMap_.is_null(), std::logic_error,
1853 "Row Map is null." << suffix);
1854 // This may access the row Map, so we need to check first (above)
1855 // whether the row Map is null.
1856 const LocalOrdinal lclNumRows =
1857 static_cast<LocalOrdinal>(this->getLocalNumRows());
1858
1859 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillActive() == this->isFillComplete(), std::logic_error,
1860 "Graph cannot be both fill active and fill complete." << suffix);
1861 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillComplete() &&
1862 (this->colMap_.is_null() ||
1863 this->rangeMap_.is_null() ||
1864 this->domainMap_.is_null()),
1865 std::logic_error,
1866 "Graph is full complete, but at least one of {column, range, domain} "
1867 "Map is null."
1868 << suffix);
1869 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isStorageOptimized() && !this->indicesAreAllocated(),
1870 std::logic_error,
1871 "Storage is optimized, but indices are not "
1872 "allocated, not even trivially."
1873 << suffix);
1874
1875 size_t nodeAllocSize = 0;
1876 try {
1877 nodeAllocSize = this->getLocalAllocationSize();
1878 } catch (std::logic_error& e) {
1879 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1880 "getLocalAllocationSize threw "
1881 "std::logic_error: "
1882 << e.what());
1883 } catch (std::exception& e) {
1884 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1885 "getLocalAllocationSize threw an "
1886 "std::exception: "
1887 << e.what());
1888 } catch (...) {
1889 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
1890 "getLocalAllocationSize threw an exception "
1891 "not a subclass of std::exception.");
1892 }
1893
1894 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isStorageOptimized() &&
1895 nodeAllocSize != this->getLocalNumEntries(),
1896 std::logic_error,
1897 "Storage is optimized, but "
1898 "this->getLocalAllocationSize() = "
1899 << nodeAllocSize
1900 << " != this->getLocalNumEntries() = " << this->getLocalNumEntries()
1901 << "." << suffix);
1902 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_ &&
1903 (this->globalNumEntries_ != GSTI ||
1904 this->globalMaxNumRowEntries_ != GSTI),
1905 std::logic_error,
1906 "Graph claims not to have global constants, but "
1907 "some of the global constants are not marked as invalid."
1908 << suffix);
1909 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->haveGlobalConstants_ &&
1910 (this->globalNumEntries_ == GSTI ||
1911 this->globalMaxNumRowEntries_ == GSTI),
1912 std::logic_error,
1913 "Graph claims to have global constants, but "
1914 "some of them are marked as invalid."
1915 << suffix);
1916 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->haveGlobalConstants_ &&
1917 (this->globalNumEntries_ < this->getLocalNumEntries() ||
1918 this->globalMaxNumRowEntries_ < this->nodeMaxNumRowEntries_),
1919 std::logic_error,
1920 "Graph claims to have global constants, and "
1921 "all of the values of the global constants are valid, but "
1922 "some of the local constants are greater than "
1923 "their corresponding global constants."
1924 << suffix);
1925 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreAllocated() &&
1926 (this->numAllocForAllRows_ != 0 ||
1927 this->k_numAllocPerRow_.extent(0) != 0),
1928 std::logic_error,
1929 "The graph claims that its indices are allocated, but "
1930 "either numAllocForAllRows_ (= "
1931 << this->numAllocForAllRows_ << ") is "
1932 "nonzero, or k_numAllocPerRow_ has nonzero dimension. In other words, "
1933 "the graph is supposed to release its \"allocation specifications\" "
1934 "when it allocates its indices."
1935 << suffix);
1936 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1937 auto rowPtrsUnpacked_dev = this->getRowPtrsUnpackedDevice();
1938 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked_host.extent(0) != rowPtrsUnpacked_dev.extent(0),
1939 std::logic_error,
1940 "The host and device views of k_rowPtrs_ have "
1941 "different sizes; rowPtrsUnpacked_host_ has size "
1942 << rowPtrsUnpacked_host.extent(0)
1943 << ", but rowPtrsUnpacked_dev_ has size "
1944 << rowPtrsUnpacked_dev.extent(0)
1945 << "." << suffix);
1946 if (isGloballyIndexed() && rowPtrsUnpacked_host.extent(0) != 0) {
1947 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(size_t(rowPtrsUnpacked_host.extent(0)) != size_t(lclNumRows + 1),
1948 std::logic_error,
1949 "The graph is globally indexed and "
1950 "k_rowPtrs has nonzero size "
1951 << rowPtrsUnpacked_host.extent(0)
1952 << ", but that size does not equal lclNumRows+1 = "
1953 << (lclNumRows + 1) << "." << suffix);
1954 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked_host(lclNumRows) != size_t(gblInds_wdv.extent(0)),
1955 std::logic_error,
1956 "The graph is globally indexed and "
1957 "k_rowPtrs_ has nonzero size "
1958 << rowPtrsUnpacked_host.extent(0)
1959 << ", but k_rowPtrs_(lclNumRows=" << lclNumRows << ")="
1960 << rowPtrsUnpacked_host(lclNumRows)
1961 << " != gblInds_wdv.extent(0)="
1962 << gblInds_wdv.extent(0) << "." << suffix);
1963 }
1964 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() &&
1965 rowPtrsUnpacked_host.extent(0) != 0 &&
1966 (static_cast<size_t>(rowPtrsUnpacked_host.extent(0)) !=
1967 static_cast<size_t>(lclNumRows + 1) ||
1968 rowPtrsUnpacked_host(lclNumRows) !=
1969 static_cast<size_t>(this->lclIndsUnpacked_wdv.extent(0))),
1970 std::logic_error,
1971 "If k_rowPtrs_ has nonzero size and "
1972 "the graph is locally indexed, then "
1973 "k_rowPtrs_ must have N+1 rows, and "
1974 "k_rowPtrs_(N) must equal lclIndsUnpacked_wdv.extent(0)."
1975 << suffix);
1976
1977 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreAllocated() &&
1978 nodeAllocSize > 0 &&
1979 this->lclIndsUnpacked_wdv.extent(0) == 0 &&
1980 this->gblInds_wdv.extent(0) == 0,
1981 std::logic_error,
1982 "Graph is allocated nontrivially, but "
1983 "but 1-D allocations are not present."
1984 << suffix);
1985
1986 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->indicesAreAllocated() &&
1987 ((rowPtrsUnpacked_host.extent(0) != 0 ||
1988 this->k_numRowEntries_.extent(0) != 0) ||
1989 this->lclIndsUnpacked_wdv.extent(0) != 0 ||
1990 this->gblInds_wdv.extent(0) != 0),
1991 std::logic_error,
1992 "If indices are not allocated, "
1993 "then none of the buffers should be."
1994 << suffix);
1995 // indices may be local or global only if they are allocated
1996 // (numAllocated is redundant; could simply be indicesAreLocal_ ||
1997 // indicesAreGlobal_)
1998 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC((this->indicesAreLocal_ || this->indicesAreGlobal_) &&
1999 !this->indicesAreAllocated_,
2000 std::logic_error,
2001 "Indices may be local or global only if they are "
2002 "allocated."
2003 << suffix);
2004 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreLocal_ && this->indicesAreGlobal_,
2005 std::logic_error, "Indices may not be both local and global." << suffix);
2006 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreLocal_ && gblInds_wdv.extent(0) != 0,
2007 std::logic_error,
2008 "Indices are local, but "
2009 "gblInds_wdv.extent(0) (= "
2010 << gblInds_wdv.extent(0)
2011 << ") != 0. In other words, if indices are local, then "
2012 "allocations of global indices should not be present."
2013 << suffix);
2014 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreGlobal_ && lclIndsUnpacked_wdv.extent(0) != 0,
2015 std::logic_error,
2016 "Indices are global, but "
2017 "lclIndsUnpacked_wdv.extent(0) (= "
2018 << lclIndsUnpacked_wdv.extent(0)
2019 << ") != 0. In other words, if indices are global, "
2020 "then allocations for local indices should not be present."
2021 << suffix);
2022 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreLocal_ && nodeAllocSize > 0 &&
2023 lclIndsUnpacked_wdv.extent(0) == 0 && getLocalNumRows() > 0,
2024 std::logic_error,
2025 "Indices are local and "
2026 "getLocalAllocationSize() = "
2027 << nodeAllocSize << " > 0, but "
2028 "lclIndsUnpacked_wdv.extent(0) = 0 and getLocalNumRows() = "
2029 << getLocalNumRows() << " > 0." << suffix);
2030 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreGlobal_ && nodeAllocSize > 0 &&
2031 gblInds_wdv.extent(0) == 0 && getLocalNumRows() > 0,
2032 std::logic_error,
2033 "Indices are global and "
2034 "getLocalAllocationSize() = "
2035 << nodeAllocSize << " > 0, but "
2036 "gblInds_wdv.extent(0) = 0 and getLocalNumRows() = "
2037 << getLocalNumRows() << " > 0." << suffix);
2038 // check the actual allocations
2039 if (this->indicesAreAllocated() &&
2040 rowPtrsUnpacked_host.extent(0) != 0) {
2041 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(rowPtrsUnpacked_host.extent(0)) !=
2042 this->getLocalNumRows() + 1,
2043 std::logic_error,
2044 "Indices are allocated and "
2045 "k_rowPtrs_ has nonzero length, but rowPtrsUnpacked_host_.extent(0) = "
2046 << rowPtrsUnpacked_host.extent(0) << " != getLocalNumRows()+1 = "
2047 << (this->getLocalNumRows() + 1) << "." << suffix);
2048 const size_t actualNumAllocated =
2049 rowPtrsUnpacked_host(this->getLocalNumRows());
2050 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() &&
2051 static_cast<size_t>(this->lclIndsUnpacked_wdv.extent(0)) != actualNumAllocated,
2052 std::logic_error,
2053 "Graph is locally indexed, indices are "
2054 "are allocated, and k_rowPtrs_ has nonzero length, but "
2055 "lclIndsUnpacked_wdv.extent(0) = "
2056 << this->lclIndsUnpacked_wdv.extent(0)
2057 << " != actualNumAllocated = " << actualNumAllocated << suffix);
2058 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isGloballyIndexed() &&
2059 static_cast<size_t>(this->gblInds_wdv.extent(0)) != actualNumAllocated,
2060 std::logic_error,
2061 "Graph is globally indexed, indices "
2062 "are allocated, and k_rowPtrs_ has nonzero length, but "
2063 "gblInds_wdv.extent(0) = "
2064 << this->gblInds_wdv.extent(0)
2065 << " != actualNumAllocated = " << actualNumAllocated << suffix);
2066 }
2067
2068 if (verbose_) {
2069 std::ostringstream os;
2070 os << *prefix << "Done" << endl;
2071 std::cerr << os.str();
2072 }
2073 }
2074}
2075
2076template <class LocalOrdinal, class GlobalOrdinal, class Node>
2077size_t
2078CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2079 getNumEntriesInGlobalRow(GlobalOrdinal globalRow) const {
2080 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(globalRow);
2081 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
2082 return Teuchos::OrdinalTraits<size_t>::invalid();
2083 } else {
2084 return rowInfo.numEntries;
2085 }
2086}
2087
2088template <class LocalOrdinal, class GlobalOrdinal, class Node>
2089size_t
2090CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2091 getNumEntriesInLocalRow(LocalOrdinal localRow) const {
2092 const RowInfo rowInfo = this->getRowInfo(localRow);
2093 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
2094 return Teuchos::OrdinalTraits<size_t>::invalid();
2095 } else {
2096 return rowInfo.numEntries;
2097 }
2098}
2099
2100template <class LocalOrdinal, class GlobalOrdinal, class Node>
2101size_t
2102CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2103 getNumAllocatedEntriesInGlobalRow(GlobalOrdinal globalRow) const {
2104 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(globalRow);
2105 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
2106 return Teuchos::OrdinalTraits<size_t>::invalid();
2107 } else {
2108 return rowInfo.allocSize;
2109 }
2110}
2111
2112template <class LocalOrdinal, class GlobalOrdinal, class Node>
2113size_t
2114CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2115 getNumAllocatedEntriesInLocalRow(LocalOrdinal localRow) const {
2116 const RowInfo rowInfo = this->getRowInfo(localRow);
2117 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
2118 return Teuchos::OrdinalTraits<size_t>::invalid();
2119 } else {
2120 return rowInfo.allocSize;
2121 }
2122}
2123
2124template <class LocalOrdinal, class GlobalOrdinal, class Node>
2125typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_host_view_type
2130
2131template <class LocalOrdinal, class GlobalOrdinal, class Node>
2132typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_device_view_type
2137
2138template <class LocalOrdinal, class GlobalOrdinal, class Node>
2139typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_inds_host_view_type
2140CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2141 getLocalIndicesHost() const {
2142 return lclIndsPacked_wdv.getHostView(Access::ReadOnly);
2143}
2144
2145template <class LocalOrdinal, class GlobalOrdinal, class Node>
2146typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_inds_device_view_type
2147CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2148 getLocalIndicesDevice() const {
2149 return lclIndsPacked_wdv.getDeviceView(Access::ReadOnly);
2150}
2151
2152template <class LocalOrdinal, class GlobalOrdinal, class Node>
2153void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2154 getLocalRowCopy(LocalOrdinal localRow,
2155 nonconst_local_inds_host_view_type& indices,
2156 size_t& numEntries) const {
2157 using Teuchos::ArrayView;
2158 const char tfecfFuncName[] = "getLocalRowCopy: ";
2159
2160 TEUCHOS_TEST_FOR_EXCEPTION(
2161 isGloballyIndexed() && !hasColMap(), std::runtime_error,
2162 "Tpetra::CrsGraph::getLocalRowCopy: The graph is globally indexed and "
2163 "does not have a column Map yet. That means we don't have local indices "
2164 "for columns yet, so it doesn't make sense to call this method. If the "
2165 "graph doesn't have a column Map yet, you should call fillComplete on "
2166 "it first.");
2167
2168 // This does the right thing (reports an empty row) if the input
2169 // row is invalid.
2170 const RowInfo rowinfo = this->getRowInfo(localRow);
2171 // No side effects on error.
2172 const size_t theNumEntries = rowinfo.numEntries;
2173 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) < theNumEntries, std::runtime_error,
2174 "Specified storage (size==" << indices.size() << ") does not suffice "
2175 "to hold all "
2176 << theNumEntries << " entry/ies for this row.");
2177 numEntries = theNumEntries;
2178
2179 if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid()) {
2180 if (isLocallyIndexed()) {
2181 auto lclInds = getLocalIndsViewHost(rowinfo);
2182 for (size_t j = 0; j < theNumEntries; ++j) {
2183 indices[j] = lclInds(j);
2184 }
2185 } else if (isGloballyIndexed()) {
2186 auto gblInds = getGlobalIndsViewHost(rowinfo);
2187 for (size_t j = 0; j < theNumEntries; ++j) {
2188 indices[j] = colMap_->getLocalElement(gblInds(j));
2189 }
2190 }
2191 }
2192}
2193
2194template <class LocalOrdinal, class GlobalOrdinal, class Node>
2195void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2196 getGlobalRowCopy(GlobalOrdinal globalRow,
2197 nonconst_global_inds_host_view_type& indices,
2198 size_t& numEntries) const {
2199 using Teuchos::ArrayView;
2200 const char tfecfFuncName[] = "getGlobalRowCopy: ";
2201
2202 // This does the right thing (reports an empty row) if the input
2203 // row is invalid.
2204 const RowInfo rowinfo = getRowInfoFromGlobalRowIndex(globalRow);
2205 const size_t theNumEntries = rowinfo.numEntries;
2206 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2207 static_cast<size_t>(indices.size()) < theNumEntries, std::runtime_error,
2208 "Specified storage (size==" << indices.size() << ") does not suffice "
2209 "to hold all "
2210 << theNumEntries << " entry/ies for this row.");
2211 numEntries = theNumEntries; // first side effect
2212
2213 if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid()) {
2214 if (isLocallyIndexed()) {
2215 auto lclInds = getLocalIndsViewHost(rowinfo);
2216 bool err = colMap_->getGlobalElements(lclInds.data(), theNumEntries, indices.data());
2217 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(err, std::runtime_error, "getGlobalElements error");
2218 } else if (isGloballyIndexed()) {
2219 auto gblInds = getGlobalIndsViewHost(rowinfo);
2220 // Kokkos zero-extent views return null from .data(); glibc declares memcpy's
2221 // dst/src params __attribute__((nonnull(1,2))), so UBSan nonnull-arg fires
2222 // on a null pointer even when the byte count is zero.
2223 if (theNumEntries > 0) {
2224 std::memcpy(
2225 (void*)indices.data(),
2226 (const void*)gblInds.data(),
2227 theNumEntries * sizeof(*indices.data()));
2228 }
2229 }
2230 }
2231}
2232
2233template <class LocalOrdinal, class GlobalOrdinal, class Node>
2234void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2235 getLocalRowView(
2236 const LocalOrdinal localRow,
2237 local_inds_host_view_type& indices) const {
2238 const char tfecfFuncName[] = "getLocalRowView: ";
2239
2240 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed(), std::runtime_error,
2241 "The graph's indices are "
2242 "currently stored as global indices, so we cannot return a view with "
2243 "local column indices, whether or not the graph has a column Map. If "
2244 "the graph _does_ have a column Map, use getLocalRowCopy() instead.");
2245
2246 const RowInfo rowInfo = getRowInfo(localRow);
2247 if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid() &&
2248 rowInfo.numEntries > 0) {
2249 indices = lclIndsUnpacked_wdv.getHostSubview(rowInfo.offset1D,
2250 rowInfo.numEntries,
2251 Access::ReadOnly);
2252 } else {
2253 // This does the right thing (reports an empty row) if the input
2254 // row is invalid.
2255 indices = local_inds_host_view_type();
2256 }
2257
2258 if (debug_) {
2259 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) !=
2260 getNumEntriesInLocalRow(localRow),
2261 std::logic_error,
2262 "indices.size() "
2263 "= " << indices.extent(0)
2264 << " != getNumEntriesInLocalRow(localRow=" << localRow << ") = " << getNumEntriesInLocalRow(localRow) << ". Please report this bug to the Tpetra developers.");
2265 }
2266}
2267
2268template <class LocalOrdinal, class GlobalOrdinal, class Node>
2269void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2270 getGlobalRowView(
2271 const GlobalOrdinal globalRow,
2272 global_inds_host_view_type& indices) const {
2273 const char tfecfFuncName[] = "getGlobalRowView: ";
2274
2275 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isLocallyIndexed(), std::runtime_error,
2276 "The graph's indices are "
2277 "currently stored as local indices, so we cannot return a view with "
2278 "global column indices. Use getGlobalRowCopy() instead.");
2279
2280 // This does the right thing (reports an empty row) if the input
2281 // row is invalid.
2282 const RowInfo rowInfo = getRowInfoFromGlobalRowIndex(globalRow);
2283 if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid() &&
2284 rowInfo.numEntries > 0) {
2285 indices = gblInds_wdv.getHostSubview(rowInfo.offset1D,
2286 rowInfo.numEntries,
2287 Access::ReadOnly);
2288 } else {
2289 indices = typename global_inds_dualv_type::t_host::const_type();
2290 }
2291 if (debug_) {
2292 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) !=
2293 getNumEntriesInGlobalRow(globalRow),
2294 std::logic_error, "indices.size() = " << indices.extent(0) << " != getNumEntriesInGlobalRow(globalRow=" << globalRow << ") = " << getNumEntriesInGlobalRow(globalRow) << ". Please report this bug to the Tpetra developers.");
2295 }
2296}
2297
2298template <class LocalOrdinal, class GlobalOrdinal, class Node>
2299void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2300 insertLocalIndices(const LocalOrdinal localRow,
2301 const Teuchos::ArrayView<const LocalOrdinal>& indices) {
2302 const char tfecfFuncName[] = "insertLocalIndices: ";
2303
2304 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive(), std::runtime_error, "Fill must be active.");
2305 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed(), std::runtime_error,
2306 "Graph indices are global; use insertGlobalIndices().");
2307 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error,
2308 "Cannot insert local indices without a column Map.");
2309 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!rowMap_->isNodeLocalElement(localRow), std::runtime_error,
2310 "Local row index " << localRow << " is not in the row Map "
2311 "on the calling process.");
2312 if (!indicesAreAllocated()) {
2313 allocateIndices(LocalIndices, verbose_);
2314 }
2315
2316 if (debug_) {
2317 // In debug mode, if the graph has a column Map, test whether any
2318 // of the given column indices are not in the column Map. Keep
2319 // track of the invalid column indices so we can tell the user
2320 // about them.
2321 if (hasColMap()) {
2322 using std::endl;
2323 using Teuchos::Array;
2324 using Teuchos::toString;
2325 typedef typename Teuchos::ArrayView<const LocalOrdinal>::size_type size_type;
2326
2327 const map_type& colMap = *colMap_;
2328 Array<LocalOrdinal> badColInds;
2329 bool allInColMap = true;
2330 for (size_type k = 0; k < indices.size(); ++k) {
2331 if (!colMap.isNodeLocalElement(indices[k])) {
2332 allInColMap = false;
2333 badColInds.push_back(indices[k]);
2334 }
2335 }
2336 if (!allInColMap) {
2337 std::ostringstream os;
2338 os << "Tpetra::CrsGraph::insertLocalIndices: You attempted to insert "
2339 "entries in owned row "
2340 << localRow << ", at the following column "
2341 "indices: "
2342 << toString(indices) << "." << endl;
2343 os << "Of those, the following indices are not in the column Map on "
2344 "this process: "
2345 << toString(badColInds) << "." << endl
2346 << "Since "
2347 "the graph has a column Map already, it is invalid to insert entries "
2348 "at those locations.";
2349 TEUCHOS_TEST_FOR_EXCEPTION(!allInColMap, std::invalid_argument, os.str());
2350 }
2351 }
2352 }
2353
2354 insertLocalIndicesImpl(localRow, indices);
2355
2356 if (debug_) {
2357 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!indicesAreAllocated() || !isLocallyIndexed(), std::logic_error,
2358 "At the end of insertLocalIndices, ! indicesAreAllocated() || "
2359 "! isLocallyIndexed() is true. Please report this bug to the "
2360 "Tpetra developers.");
2361 }
2362}
2363
2364template <class LocalOrdinal, class GlobalOrdinal, class Node>
2365void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2366 insertLocalIndices(const LocalOrdinal localRow,
2367 const LocalOrdinal numEnt,
2368 const LocalOrdinal inds[]) {
2369 Teuchos::ArrayView<const LocalOrdinal> indsT(inds, numEnt);
2370 this->insertLocalIndices(localRow, indsT);
2371}
2372
2373template <class LocalOrdinal, class GlobalOrdinal, class Node>
2374void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2375 insertGlobalIndices(const GlobalOrdinal gblRow,
2376 const LocalOrdinal numInputInds,
2377 const GlobalOrdinal inputGblColInds[]) {
2378 typedef LocalOrdinal LO;
2379 const char tfecfFuncName[] = "insertGlobalIndices: ";
2380
2381 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed(), std::runtime_error,
2382 "graph indices are local; use insertLocalIndices().");
2383 // This can't really be satisfied for now, because if we are
2384 // fillComplete(), then we are local. In the future, this may
2385 // change. However, the rule that modification require active
2386 // fill will not change.
2387 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->isFillActive(), std::runtime_error,
2388 "You are not allowed to call this method if fill is not active. "
2389 "If fillComplete has been called, you must first call resumeFill "
2390 "before you may insert indices.");
2391 if (!indicesAreAllocated()) {
2392 allocateIndices(GlobalIndices, verbose_);
2393 }
2394 const LO lclRow = this->rowMap_->getLocalElement(gblRow);
2395 if (lclRow != Tpetra::Details::OrdinalTraits<LO>::invalid()) {
2396 if (debug_) {
2397 if (this->hasColMap()) {
2398 using std::endl;
2399 const map_type& colMap = *(this->colMap_);
2400 // In a debug build, keep track of the nonowned ("bad") column
2401 // indices, so that we can display them in the exception
2402 // message. In a release build, just ditch the loop early if
2403 // we encounter a nonowned column index.
2404 std::vector<GlobalOrdinal> badColInds;
2405 bool allInColMap = true;
2406 for (LO k = 0; k < numInputInds; ++k) {
2407 if (!colMap.isNodeGlobalElement(inputGblColInds[k])) {
2408 allInColMap = false;
2409 badColInds.push_back(inputGblColInds[k]);
2410 }
2411 }
2412 if (!allInColMap) {
2413 std::ostringstream os;
2414 os << "You attempted to insert entries in owned row " << gblRow
2415 << ", at the following column indices: [";
2416 for (LO k = 0; k < numInputInds; ++k) {
2417 os << inputGblColInds[k];
2418 if (k + static_cast<LO>(1) < numInputInds) {
2419 os << ",";
2420 }
2421 }
2422 os << "]." << endl
2423 << "Of those, the following indices are not in "
2424 "the column Map on this process: [";
2425 for (size_t k = 0; k < badColInds.size(); ++k) {
2426 os << badColInds[k];
2427 if (k + size_t(1) < badColInds.size()) {
2428 os << ",";
2429 }
2430 }
2431 os << "]." << endl
2432 << "Since the matrix has a column Map already, "
2433 "it is invalid to insert entries at those locations.";
2434 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::invalid_argument, os.str());
2435 }
2436 }
2437 } // debug_
2438 this->insertGlobalIndicesImpl(lclRow, inputGblColInds, numInputInds);
2439 } else { // a nonlocal row
2440 this->insertGlobalIndicesIntoNonownedRows(gblRow, inputGblColInds,
2441 numInputInds);
2442 }
2443}
2444
2445template <class LocalOrdinal, class GlobalOrdinal, class Node>
2446void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2447 insertGlobalIndices(const GlobalOrdinal gblRow,
2448 const Teuchos::ArrayView<const GlobalOrdinal>& inputGblColInds) {
2449 this->insertGlobalIndices(gblRow, inputGblColInds.size(),
2450 inputGblColInds.getRawPtr());
2451}
2452
2453template <class LocalOrdinal, class GlobalOrdinal, class Node>
2454void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2455 insertGlobalIndicesFiltered(const LocalOrdinal lclRow,
2456 const GlobalOrdinal gblColInds[],
2457 const LocalOrdinal numGblColInds) {
2458 typedef LocalOrdinal LO;
2459 typedef GlobalOrdinal GO;
2460 const char tfecfFuncName[] = "insertGlobalIndicesFiltered: ";
2461
2462 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed(), std::runtime_error,
2463 "Graph indices are local; use insertLocalIndices().");
2464 // This can't really be satisfied for now, because if we are
2465 // fillComplete(), then we are local. In the future, this may
2466 // change. However, the rule that modification require active
2467 // fill will not change.
2468 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->isFillActive(), std::runtime_error,
2469 "You are not allowed to call this method if fill is not active. "
2470 "If fillComplete has been called, you must first call resumeFill "
2471 "before you may insert indices.");
2472 if (!indicesAreAllocated()) {
2473 allocateIndices(GlobalIndices, verbose_);
2474 }
2475
2476 Teuchos::ArrayView<const GO> gblColInds_av(gblColInds, numGblColInds);
2477 // If we have a column Map, use it to filter the entries.
2478 if (!colMap_.is_null()) {
2479 const map_type& colMap = *(this->colMap_);
2480
2481 LO curOffset = 0;
2482 while (curOffset < numGblColInds) {
2483 // Find a sequence of input indices that are in the column Map
2484 // on the calling process. Doing a sequence at a time,
2485 // instead of one at a time, amortizes some overhead.
2486 LO endOffset = curOffset;
2487 for (; endOffset < numGblColInds; ++endOffset) {
2488 const LO lclCol = colMap.getLocalElement(gblColInds[endOffset]);
2489 if (lclCol == Tpetra::Details::OrdinalTraits<LO>::invalid()) {
2490 break; // first entry, in current sequence, not in the column Map
2491 }
2492 }
2493 // curOffset, endOffset: half-exclusive range of indices in
2494 // the column Map on the calling process. If endOffset ==
2495 // curOffset, the range is empty.
2496 const LO numIndInSeq = (endOffset - curOffset);
2497 if (numIndInSeq != 0) {
2498 this->insertGlobalIndicesImpl(lclRow, gblColInds + curOffset,
2499 numIndInSeq);
2500 }
2501 // Invariant before this line: Either endOffset ==
2502 // numGblColInds, or gblColInds[endOffset] is not in the
2503 // column Map on the calling process.
2504 curOffset = endOffset + 1;
2505 }
2506 } else {
2507 this->insertGlobalIndicesImpl(lclRow, gblColInds_av.getRawPtr(),
2508 gblColInds_av.size());
2509 }
2510}
2511
2512template <class LocalOrdinal, class GlobalOrdinal, class Node>
2513void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2514 insertGlobalIndicesIntoNonownedRows(const GlobalOrdinal gblRow,
2515 const GlobalOrdinal gblColInds[],
2516 const LocalOrdinal numGblColInds) {
2517 // This creates the std::vector if it doesn't exist yet.
2518 // std::map's operator[] does a lookup each time, so it's better
2519 // to pull nonlocals_[grow] out of the loop.
2520 std::vector<GlobalOrdinal>& nonlocalRow = this->nonlocals_[gblRow];
2521 for (LocalOrdinal k = 0; k < numGblColInds; ++k) {
2522 // FIXME (mfh 20 Jul 2017) Would be better to use a set, in
2523 // order to avoid duplicates. globalAssemble() sorts these
2524 // anyway.
2525 nonlocalRow.push_back(gblColInds[k]);
2526 }
2527}
2528
2529template <class LocalOrdinal, class GlobalOrdinal, class Node>
2530void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2531 removeLocalIndices(LocalOrdinal lrow) {
2532 const char tfecfFuncName[] = "removeLocalIndices: ";
2533 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2534 !isFillActive(), std::runtime_error, "requires that fill is active.");
2535 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2536 isStorageOptimized(), std::runtime_error,
2537 "cannot remove indices after optimizeStorage() has been called.");
2538 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2539 isGloballyIndexed(), std::runtime_error, "graph indices are global.");
2540 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2541 !rowMap_->isNodeLocalElement(lrow), std::runtime_error,
2542 "Local row " << lrow << " is not in the row Map on the calling process.");
2543 if (!indicesAreAllocated()) {
2544 allocateIndices(LocalIndices, verbose_);
2545 }
2546
2547 if (k_numRowEntries_.extent(0) != 0) {
2548 this->k_numRowEntries_(lrow) = 0;
2549 }
2550
2551 if (debug_) {
2552 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(getNumEntriesInLocalRow(lrow) != 0 ||
2553 !indicesAreAllocated() ||
2554 !isLocallyIndexed(),
2555 std::logic_error,
2556 "Violated stated post-conditions. Please contact Tpetra team.");
2557 }
2558}
2559
2560template <class LocalOrdinal, class GlobalOrdinal, class Node>
2561void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2562 setAllIndices(const typename local_graph_device_type::row_map_type& rowPointers,
2563 const typename local_graph_device_type::entries_type::non_const_type& columnIndices) {
2564 using ProfilingRegion = Details::ProfilingRegion;
2565 ProfilingRegion region("Tpetra::CrsGraph::setAllIndices");
2566 const char tfecfFuncName[] = "setAllIndices: ";
2567 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2568 !hasColMap() || getColMap().is_null(), std::runtime_error,
2569 "The graph must have a column Map before you may call this method.");
2570 LocalOrdinal numLocalRows = this->getLocalNumRows();
2571 {
2572 LocalOrdinal rowPtrLen = rowPointers.size();
2573 if (numLocalRows == 0) {
2574 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2575 rowPtrLen != 0 && rowPtrLen != 1,
2576 std::runtime_error, "Have 0 local rows, but rowPointers.size() is neither 0 nor 1.");
2577 } else {
2578 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2579 rowPtrLen != numLocalRows + 1,
2580 std::runtime_error, "rowPointers.size() = " << rowPtrLen << " != this->getLocalNumRows()+1 = " << (numLocalRows + 1) << ".");
2581 }
2582 }
2583
2584 if (debug_) {
2585 using exec_space = typename local_graph_device_type::execution_space;
2586 int columnsOutOfBounds = 0;
2587 local_ordinal_type numLocalCols = this->getLocalNumCols();
2588 Kokkos::parallel_reduce(
2589 Kokkos::RangePolicy<exec_space>(0, columnIndices.extent(0)),
2590 KOKKOS_LAMBDA(const LocalOrdinal i, int& lOutOfBounds) {
2591 if (columnIndices(i) < 0 || columnIndices(i) >= numLocalCols)
2592 lOutOfBounds++;
2593 },
2594 columnsOutOfBounds);
2595 int globalColsOutOfBounds = 0;
2596 auto comm = this->getComm();
2597 Teuchos::reduceAll<int, int>(*comm, Teuchos::REDUCE_MAX, columnsOutOfBounds,
2598 Teuchos::outArg(globalColsOutOfBounds));
2599 if (globalColsOutOfBounds) {
2600 std::string message;
2601 if (columnsOutOfBounds) {
2602 // Only print message from ranks with the problem
2603 message = std::string("ERROR, rank ") + std::to_string(comm->getRank()) + ", CrsGraph::setAllIndices(): provided columnIndices are not all within range [0, getLocalNumCols())!\n";
2604 }
2605 Details::gathervPrint(std::cout, message, *comm);
2606 throw std::invalid_argument("CrsGraph::setAllIndices(): columnIndices are out of the valid range on at least one process.");
2607 }
2608 }
2609
2610 if (debug_ && this->isSorted()) {
2611 // Verify that the local indices are actually sorted
2612 int notSorted = 0;
2613 using exec_space = typename local_graph_device_type::execution_space;
2614 using size_type = typename local_graph_device_type::size_type;
2615 Kokkos::parallel_reduce(
2616 Kokkos::RangePolicy<exec_space>(0, numLocalRows),
2617 KOKKOS_LAMBDA(const LocalOrdinal i, int& lNotSorted) {
2618 size_type rowBegin = rowPointers(i);
2619 size_type rowEnd = rowPointers(i + 1);
2620 for (size_type j = rowBegin + 1; j < rowEnd; j++) {
2621 if (columnIndices(j - 1) > columnIndices(j)) {
2622 lNotSorted = 1;
2623 }
2624 }
2625 },
2626 notSorted);
2627 // All-reduce notSorted to avoid rank divergence
2628 int globalNotSorted = 0;
2629 auto comm = this->getComm();
2630 Teuchos::reduceAll<int, int>(*comm, Teuchos::REDUCE_MAX, notSorted,
2631 Teuchos::outArg(globalNotSorted));
2632 if (globalNotSorted) {
2633 std::string message;
2634 if (notSorted) {
2635 // Only print message from ranks with the problem
2636 message = std::string("ERROR, rank ") + std::to_string(comm->getRank()) + ", CrsGraph::setAllIndices(): provided columnIndices are not sorted!\n";
2637 }
2638 Details::gathervPrint(std::cout, message, *comm);
2639 throw std::invalid_argument("CrsGraph::setAllIndices(): provided columnIndices are not sorted within rows on at least one process.");
2640 }
2641 }
2642
2643 indicesAreAllocated_ = true;
2644 indicesAreLocal_ = true;
2645 lclIndsPacked_wdv = local_inds_wdv_type(columnIndices);
2646 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
2647 setRowPtrs(rowPointers);
2648
2649 set_need_sync_host_uvm_access(); // columnIndices and rowPointers potentially still in a kernel
2650
2651 // Storage MUST be packed, since the interface doesn't give any
2652 // way to indicate any extra space at the end of each row.
2653 storageStatus_ = Details::STORAGE_1D_PACKED;
2654
2655 // These normally get cleared out at the end of allocateIndices.
2656 // It makes sense to clear them out here, because at the end of
2657 // this method, the graph is allocated on the calling process.
2658 numAllocForAllRows_ = 0;
2659 k_numAllocPerRow_ = decltype(k_numAllocPerRow_)();
2660
2661 checkInternalState();
2662}
2663
2664template <class LocalOrdinal, class GlobalOrdinal, class Node>
2665void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2666 setAllIndices(const Teuchos::ArrayRCP<size_t>& rowPointers,
2667 const Teuchos::ArrayRCP<LocalOrdinal>& columnIndices) {
2668 using Kokkos::View;
2669 typedef typename local_graph_device_type::row_map_type row_map_type;
2670 typedef typename row_map_type::array_layout layout_type;
2671 typedef typename row_map_type::non_const_value_type row_offset_type;
2672 typedef View<size_t*, layout_type, Kokkos::HostSpace,
2673 Kokkos::MemoryUnmanaged>
2674 input_view_type;
2675 typedef typename row_map_type::non_const_type nc_row_map_type;
2676
2677 const size_t size = static_cast<size_t>(rowPointers.size());
2678 constexpr bool same = std::is_same<size_t, row_offset_type>::value;
2679 input_view_type ptr_in(rowPointers.getRawPtr(), size);
2680
2681 nc_row_map_type ptr_rot("Tpetra::CrsGraph::ptr", size);
2682
2683 if constexpr (same) { // size_t == row_offset_type
2684 using lexecution_space = typename device_type::execution_space;
2685 Kokkos::deep_copy(lexecution_space(),
2686 ptr_rot,
2687 ptr_in);
2688 } else { // size_t != row_offset_type
2689 // CudaUvmSpace != HostSpace, so this will be false in that case.
2690 constexpr bool inHostMemory =
2691 std::is_same<typename row_map_type::memory_space,
2692 Kokkos::HostSpace>::value;
2693 if (inHostMemory) {
2694 // Copy (with cast from size_t to row_offset_type, with bounds
2695 // checking if necessary) to ptr_rot.
2696 ::Tpetra::Details::copyOffsets(ptr_rot, ptr_in);
2697 } else { // Copy input row offsets to device first.
2698 //
2699 // FIXME (mfh 24 Mar 2015) If CUDA UVM, running in the host's
2700 // execution space would avoid the double copy.
2701 //
2702 View<size_t*, layout_type, device_type> ptr_st("Tpetra::CrsGraph::ptr", size);
2703
2704 // DEEP_COPY REVIEW - NOT TESTED
2705 Kokkos::deep_copy(ptr_st, ptr_in);
2706 // Copy on device (casting from size_t to row_offset_type,
2707 // with bounds checking if necessary) to ptr_rot. This
2708 // executes in the output View's execution space, which is the
2709 // same as execution_space.
2710 ::Tpetra::Details::copyOffsets(ptr_rot, ptr_st);
2711 }
2712 }
2713
2714 Kokkos::View<LocalOrdinal*, layout_type, device_type> k_ind =
2715 Kokkos::Compat::getKokkosViewDeepCopy<device_type>(columnIndices());
2716 setAllIndices(ptr_rot, k_ind);
2717}
2718
2719template <class LocalOrdinal, class GlobalOrdinal, class Node>
2720void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2721 globalAssemble() {
2722 using std::endl;
2723 using Teuchos::Comm;
2724 using Teuchos::outArg;
2725 using Teuchos::RCP;
2726 using Teuchos::rcp;
2727 using Teuchos::REDUCE_MAX;
2728 using Teuchos::REDUCE_MIN;
2729 using Teuchos::reduceAll;
2730 using crs_graph_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
2731 using LO = local_ordinal_type;
2732 using GO = global_ordinal_type;
2733 using size_type = typename Teuchos::Array<GO>::size_type;
2734
2735 const char tfecfFuncName[] = "globalAssemble: "; // for exception macro
2736
2737 Details::ProfilingRegion regionGA("Tpetra::CrsGraph::globalAssemble");
2738
2739 std::unique_ptr<std::string> prefix;
2740 if (verbose_) {
2741 prefix = this->createPrefix("CrsGraph", "globalAssemble");
2742 std::ostringstream os;
2743 os << *prefix << "Start" << endl;
2744 std::cerr << os.str();
2745 }
2746 RCP<const Comm<int>> comm = getComm();
2747
2748 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive(), std::runtime_error,
2749 "Fill must be active before "
2750 "you may call this method.");
2751
2752 const size_t myNumNonlocalRows = this->nonlocals_.size();
2753
2754 // If no processes have nonlocal rows, then we don't have to do
2755 // anything. Checking this is probably cheaper than constructing
2756 // the Map of nonlocal rows (see below) and noticing that it has
2757 // zero global entries.
2758 {
2759 const int iHaveNonlocalRows = (myNumNonlocalRows == 0) ? 0 : 1;
2760 int someoneHasNonlocalRows = 0;
2761 reduceAll<int, int>(*comm, REDUCE_MAX, iHaveNonlocalRows,
2762 outArg(someoneHasNonlocalRows));
2763 if (someoneHasNonlocalRows == 0) {
2764 if (verbose_) {
2765 std::ostringstream os;
2766 os << *prefix << "Done: No nonlocal rows" << endl;
2767 std::cerr << os.str();
2768 }
2769 return;
2770 } else if (verbose_) {
2771 std::ostringstream os;
2772 os << *prefix << "At least 1 process has nonlocal rows"
2773 << endl;
2774 std::cerr << os.str();
2775 }
2776 }
2777
2778 // 1. Create a list of the "nonlocal" rows on each process. this
2779 // requires iterating over nonlocals_, so while we do this,
2780 // deduplicate the entries and get a count for each nonlocal
2781 // row on this process.
2782 // 2. Construct a new row Map corresponding to those rows. This
2783 // Map is likely overlapping. We know that the Map is not
2784 // empty on all processes, because the above all-reduce and
2785 // return exclude that case.
2786
2787 RCP<const map_type> nonlocalRowMap;
2788 // Keep this for CrsGraph's constructor.
2789 Teuchos::Array<size_t> numEntPerNonlocalRow(myNumNonlocalRows);
2790 {
2791 Teuchos::Array<GO> myNonlocalGblRows(myNumNonlocalRows);
2792 size_type curPos = 0;
2793 for (auto mapIter = this->nonlocals_.begin();
2794 mapIter != this->nonlocals_.end();
2795 ++mapIter, ++curPos) {
2796 myNonlocalGblRows[curPos] = mapIter->first;
2797 std::vector<GO>& gblCols = mapIter->second; // by ref; change in place
2798 std::sort(gblCols.begin(), gblCols.end());
2799 auto vecLast = std::unique(gblCols.begin(), gblCols.end());
2800 gblCols.erase(vecLast, gblCols.end());
2801 numEntPerNonlocalRow[curPos] = gblCols.size();
2802 }
2803
2804 // Currently, Map requires that its indexBase be the global min
2805 // of all its global indices. Map won't compute this for us, so
2806 // we must do it. If our process has no nonlocal rows, set the
2807 // "min" to the max possible GO value. This ensures that if
2808 // some process has at least one nonlocal row, then it will pick
2809 // that up as the min. We know that at least one process has a
2810 // nonlocal row, since the all-reduce and return at the top of
2811 // this method excluded that case.
2812 GO myMinNonlocalGblRow = std::numeric_limits<GO>::max();
2813 {
2814 auto iter = std::min_element(myNonlocalGblRows.begin(),
2815 myNonlocalGblRows.end());
2816 if (iter != myNonlocalGblRows.end()) {
2817 myMinNonlocalGblRow = *iter;
2818 }
2819 }
2820 GO gblMinNonlocalGblRow = 0;
2821 reduceAll<int, GO>(*comm, REDUCE_MIN, myMinNonlocalGblRow,
2822 outArg(gblMinNonlocalGblRow));
2823 const GO indexBase = gblMinNonlocalGblRow;
2824 const global_size_t INV = Teuchos::OrdinalTraits<global_size_t>::invalid();
2825 nonlocalRowMap = rcp(new map_type(INV, myNonlocalGblRows(), indexBase, comm));
2826 }
2827
2828 if (verbose_) {
2829 std::ostringstream os;
2830 os << *prefix << "nonlocalRowMap->getIndexBase()="
2831 << nonlocalRowMap->getIndexBase() << endl;
2832 std::cerr << os.str();
2833 }
2834
2835 // 3. Use the column indices for each nonlocal row, as stored in
2836 // nonlocals_, to construct a CrsGraph corresponding to
2837 // nonlocal rows. We need, but we have, exact counts of the
2838 // number of entries in each nonlocal row.
2839
2840 RCP<crs_graph_type> nonlocalGraph =
2841 rcp(new crs_graph_type(nonlocalRowMap, numEntPerNonlocalRow()));
2842 {
2843 size_type curPos = 0;
2844 for (auto mapIter = this->nonlocals_.begin();
2845 mapIter != this->nonlocals_.end();
2846 ++mapIter, ++curPos) {
2847 const GO gblRow = mapIter->first;
2848 std::vector<GO>& gblCols = mapIter->second; // by ref just to avoid copy
2849 const LO numEnt = static_cast<LO>(numEntPerNonlocalRow[curPos]);
2850 nonlocalGraph->insertGlobalIndices(gblRow, numEnt, gblCols.data());
2851 }
2852 }
2853 if (verbose_) {
2854 std::ostringstream os;
2855 os << *prefix << "Built nonlocal graph" << endl;
2856 std::cerr << os.str();
2857 }
2858 // There's no need to fill-complete the nonlocals graph.
2859 // We just use it as a temporary container for the Export.
2860
2861 // 4. If the original row Map is one to one, then we can Export
2862 // directly from nonlocalGraph into this. Otherwise, we have
2863 // to create a temporary graph with a one-to-one row Map,
2864 // Export into that, then Import from the temporary graph into
2865 // *this.
2866
2867 auto origRowMap = this->getRowMap();
2868 const bool origRowMapIsOneToOne = origRowMap->isOneToOne();
2869
2870 if (origRowMapIsOneToOne) {
2871 if (verbose_) {
2872 std::ostringstream os;
2873 os << *prefix << "Original row Map is 1-to-1" << endl;
2874 std::cerr << os.str();
2875 }
2876 export_type exportToOrig(nonlocalRowMap, origRowMap);
2877 this->doExport(*nonlocalGraph, exportToOrig, Tpetra::INSERT);
2878 // We're done at this point!
2879 } else {
2880 if (verbose_) {
2881 std::ostringstream os;
2882 os << *prefix << "Original row Map is NOT 1-to-1" << endl;
2883 std::cerr << os.str();
2884 }
2885 // If you ask a Map whether it is one to one, it does some
2886 // communication and stashes intermediate results for later use
2887 // by createOneToOne. Thus, calling createOneToOne doesn't cost
2888 // much more then the original cost of calling isOneToOne.
2889 auto oneToOneRowMap = Tpetra::createOneToOne(origRowMap);
2890 export_type exportToOneToOne(nonlocalRowMap, oneToOneRowMap);
2891
2892 // Create a temporary graph with the one-to-one row Map.
2893 //
2894 // TODO (mfh 09 Sep 2016) Estimate the number of entries in each
2895 // row, to avoid reallocation during the Export operation.
2896 crs_graph_type oneToOneGraph(oneToOneRowMap, 0);
2897
2898 // Export from graph of nonlocals into the temp one-to-one graph.
2899 if (verbose_) {
2900 std::ostringstream os;
2901 os << *prefix << "Export nonlocal graph" << endl;
2902 std::cerr << os.str();
2903 }
2904 oneToOneGraph.doExport(*nonlocalGraph, exportToOneToOne, Tpetra::INSERT);
2905
2906 // We don't need the graph of nonlocals anymore, so get rid of
2907 // it, to keep the memory high-water mark down.
2908 nonlocalGraph = Teuchos::null;
2909
2910 // Import from the one-to-one graph to the original graph.
2911 import_type importToOrig(oneToOneRowMap, origRowMap);
2912 if (verbose_) {
2913 std::ostringstream os;
2914 os << *prefix << "Import nonlocal graph" << endl;
2915 std::cerr << os.str();
2916 }
2917 this->doImport(oneToOneGraph, importToOrig, Tpetra::INSERT);
2918 }
2919
2920 // It's safe now to clear out nonlocals_, since we've already
2921 // committed side effects to *this. The standard idiom for
2922 // clearing a Container like std::map, is to swap it with an empty
2923 // Container and let the swapped Container fall out of scope.
2924 decltype(this->nonlocals_) newNonlocals;
2925 std::swap(this->nonlocals_, newNonlocals);
2926
2927 checkInternalState();
2928 if (verbose_) {
2929 std::ostringstream os;
2930 os << *prefix << "Done" << endl;
2931 std::cerr << os.str();
2932 }
2933}
2934
2935template <class LocalOrdinal, class GlobalOrdinal, class Node>
2936void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2937 resumeFill(const Teuchos::RCP<Teuchos::ParameterList>& params) {
2938 clearGlobalConstants();
2939 if (params != Teuchos::null) this->setParameterList(params);
2940 // either still sorted/merged or initially sorted/merged
2941 indicesAreSorted_ = true;
2942 noRedundancies_ = true;
2943 fillComplete_ = false;
2944}
2945
2946template <class LocalOrdinal, class GlobalOrdinal, class Node>
2947void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2948 fillComplete(const Teuchos::RCP<Teuchos::ParameterList>& params) {
2949 // If the graph already has domain and range Maps, don't clobber
2950 // them. If it doesn't, use the current row Map for both the
2951 // domain and range Maps.
2952 //
2953 // NOTE (mfh 28 Sep 2014): If the graph was constructed without a
2954 // column Map, and column indices are inserted which are not in
2955 // the row Map on any process, this will cause troubles. However,
2956 // that is not a common case for most applications that we
2957 // encounter, and checking for it might require more
2958 // communication.
2959 Teuchos::RCP<const map_type> domMap = this->getDomainMap();
2960 if (domMap.is_null()) {
2961 domMap = this->getRowMap();
2962 }
2963 Teuchos::RCP<const map_type> ranMap = this->getRangeMap();
2964 if (ranMap.is_null()) {
2965 ranMap = this->getRowMap();
2966 }
2967 this->fillComplete(domMap, ranMap, params);
2968}
2969
2970template <class LocalOrdinal, class GlobalOrdinal, class Node>
2971void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
2972 fillComplete(const Teuchos::RCP<const map_type>& domainMap,
2973 const Teuchos::RCP<const map_type>& rangeMap,
2974 const Teuchos::RCP<Teuchos::ParameterList>& params) {
2975 using std::endl;
2976
2977 const char tfecfFuncName[] = "fillComplete: ";
2978 const bool verbose = verbose_;
2979
2980 Details::ProfilingRegion regionFC("Tpetra::CrsGraph::fillComplete");
2981
2982 std::unique_ptr<std::string> prefix;
2983 if (verbose) {
2984 prefix = this->createPrefix("CrsGraph", "fillComplete");
2985 std::ostringstream os;
2986 os << *prefix << "Start" << endl;
2987 std::cerr << os.str();
2988 }
2989
2990 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive() || isFillComplete(), std::runtime_error,
2991 "Graph fill state must be active (isFillActive() "
2992 "must be true) before calling fillComplete().");
2993
2994 const int numProcs = getComm()->getSize();
2995
2996 //
2997 // Read and set parameters
2998 //
2999
3000 // Does the caller want to sort remote GIDs (within those owned by
3001 // the same process) in makeColMap()?
3002 if (!params.is_null()) {
3003 if (params->isParameter("sort column map ghost gids")) {
3004 sortGhostsAssociatedWithEachProcessor_ =
3005 params->get<bool>("sort column map ghost gids",
3006 sortGhostsAssociatedWithEachProcessor_);
3007 } else if (params->isParameter("Sort column Map ghost GIDs")) {
3008 sortGhostsAssociatedWithEachProcessor_ =
3009 params->get<bool>("Sort column Map ghost GIDs",
3010 sortGhostsAssociatedWithEachProcessor_);
3011 }
3012 }
3013
3014 // If true, the caller promises that no process did nonlocal
3015 // changes since the last call to fillComplete.
3016 bool assertNoNonlocalInserts = false;
3017 if (!params.is_null()) {
3018 assertNoNonlocalInserts =
3019 params->get<bool>("No Nonlocal Changes", assertNoNonlocalInserts);
3020 }
3021
3022 //
3023 // Allocate indices, if they haven't already been allocated
3024 //
3025 if (!indicesAreAllocated()) {
3026 if (hasColMap()) {
3027 // We have a column Map, so use local indices.
3028 allocateIndices(LocalIndices, verbose);
3029 } else {
3030 // We don't have a column Map, so use global indices.
3031 allocateIndices(GlobalIndices, verbose);
3032 }
3033 }
3034
3035 //
3036 // Do global assembly, if requested and if the communicator
3037 // contains more than one process.
3038 //
3039 const bool mayNeedGlobalAssemble = !assertNoNonlocalInserts && numProcs > 1;
3040 if (mayNeedGlobalAssemble) {
3041 // This first checks if we need to do global assembly.
3042 // The check costs a single all-reduce.
3043 globalAssemble();
3044 } else {
3045 const size_t numNonlocals = nonlocals_.size();
3046 if (verbose) {
3047 std::ostringstream os;
3048 os << *prefix << "Do not need to call globalAssemble; "
3049 "assertNoNonlocalInserts="
3050 << (assertNoNonlocalInserts ? "true" : "false")
3051 << "numProcs=" << numProcs
3052 << ", nonlocals_.size()=" << numNonlocals << endl;
3053 std::cerr << os.str();
3054 }
3055 const int lclNeededGlobalAssemble =
3056 (numProcs > 1 && numNonlocals != 0) ? 1 : 0;
3057 if (lclNeededGlobalAssemble != 0 && verbose) {
3058 std::ostringstream os;
3059 os << *prefix;
3060 Details::Impl::verbosePrintMap(
3061 os, nonlocals_.begin(), nonlocals_.end(),
3062 nonlocals_.size(), "nonlocals_");
3063 std::cerr << os.str() << endl;
3064 }
3065
3066 if (debug_) {
3067 auto map = this->getMap();
3068 auto comm = map.is_null() ? Teuchos::null : map->getComm();
3069 int gblNeededGlobalAssemble = lclNeededGlobalAssemble;
3070 if (!comm.is_null()) {
3071 using Teuchos::REDUCE_MAX;
3072 using Teuchos::reduceAll;
3073 reduceAll(*comm, REDUCE_MAX, lclNeededGlobalAssemble,
3074 Teuchos::outArg(gblNeededGlobalAssemble));
3075 }
3076 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(gblNeededGlobalAssemble != 0, std::runtime_error,
3077 "nonlocals_.size()=" << numNonlocals << " != 0 on at "
3078 "least one process in the CrsGraph's communicator. This "
3079 "means either that you incorrectly set the "
3080 "\"No Nonlocal Changes\" fillComplete parameter to true, "
3081 "or that you inserted invalid entries. "
3082 "Rerun with the environment variable TPETRA_VERBOSE="
3083 "CrsGraph set to see the entries of nonlocals_ on every "
3084 "MPI process (WARNING: lots of output).");
3085 } else {
3086 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(lclNeededGlobalAssemble != 0, std::runtime_error,
3087 "nonlocals_.size()=" << numNonlocals << " != 0 on the "
3088 "calling process. This means either that you incorrectly "
3089 "set the \"No Nonlocal Changes\" fillComplete parameter "
3090 "to true, or that you inserted invalid entries. "
3091 "Rerun with the environment "
3092 "variable TPETRA_VERBOSE=CrsGraph set to see the entries "
3093 "of nonlocals_ on every MPI process (WARNING: lots of "
3094 "output).");
3095 }
3096 }
3097
3098 // Set domain and range Map. This may clear the Import / Export
3099 // objects if the new Maps differ from any old ones.
3100 setDomainRangeMaps(domainMap, rangeMap);
3101
3102 // If the graph does not already have a column Map (either from
3103 // the user constructor calling the version of the constructor
3104 // that takes a column Map, or from a previous fillComplete call),
3105 // then create it.
3106 Teuchos::Array<int> remotePIDs(0);
3107 const bool mustBuildColMap = !this->hasColMap();
3108 if (mustBuildColMap) {
3109 this->makeColMap(remotePIDs); // resized on output
3110 }
3111
3112 // Make indices local, if they aren't already.
3113 // The method doesn't do any work if the indices are already local.
3114 const std::pair<size_t, std::string> makeIndicesLocalResult =
3115 this->makeIndicesLocal(verbose);
3116
3117 if (debug_) {
3118 using Details::gathervPrint;
3119 using Teuchos::outArg;
3120 using Teuchos::RCP;
3121 using Teuchos::REDUCE_MIN;
3122 using Teuchos::reduceAll;
3123
3124 RCP<const map_type> map = this->getMap();
3125 RCP<const Teuchos::Comm<int>> comm;
3126 if (!map.is_null()) {
3127 comm = map->getComm();
3128 }
3129 if (comm.is_null()) {
3130 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(makeIndicesLocalResult.first != 0, std::runtime_error,
3131 makeIndicesLocalResult.second);
3132 } else {
3133 const int lclSuccess = (makeIndicesLocalResult.first == 0);
3134 int gblSuccess = 0; // output argument
3135 reduceAll(*comm, REDUCE_MIN, lclSuccess, outArg(gblSuccess));
3136 if (gblSuccess != 1) {
3137 std::ostringstream os;
3138 gathervPrint(os, makeIndicesLocalResult.second, *comm);
3139 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
3140 }
3141 }
3142 } else {
3143 // TODO (mfh 20 Jul 2017) Instead of throwing here, pass along
3144 // the error state to makeImportExport or
3145 // computeGlobalConstants, which may do all-reduces and thus may
3146 // have the opportunity to communicate that error state.
3147 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(makeIndicesLocalResult.first != 0, std::runtime_error,
3148 makeIndicesLocalResult.second);
3149 }
3150
3151 // If this process has no indices, then CrsGraph considers it
3152 // already trivially sorted and merged. Thus, this method need
3153 // not be called on all processes in the row Map's communicator.
3154 this->sortAndMergeAllIndices(this->isSorted(), this->isMerged());
3155
3156 // Make Import and Export objects, if they haven't been made
3157 // already. If we made a column Map above, reuse information from
3158 // that process to avoid communiation in the Import setup.
3159 this->makeImportExport(remotePIDs, mustBuildColMap);
3160
3161 // Create the KokkosSparse::StaticCrsGraph, if it doesn't already exist.
3162 this->fillLocalGraph(params);
3163
3164 const bool callComputeGlobalConstants = params.get() == nullptr ||
3165 params->get("compute global constants", true);
3166 if (callComputeGlobalConstants) {
3167 this->computeGlobalConstants();
3168 } else {
3169 this->computeLocalConstants();
3170 }
3171 this->fillComplete_ = true;
3172 this->checkInternalState();
3173
3174 if (verbose) {
3175 std::ostringstream os;
3176 os << *prefix << "Done" << endl;
3177 std::cerr << os.str();
3178 }
3179}
3180
3181template <class LocalOrdinal, class GlobalOrdinal, class Node>
3182void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3183 expertStaticFillComplete(const Teuchos::RCP<const map_type>& domainMap,
3184 const Teuchos::RCP<const map_type>& rangeMap,
3185 const Teuchos::RCP<const import_type>& importer,
3186 const Teuchos::RCP<const export_type>& exporter,
3187 const Teuchos::RCP<Teuchos::ParameterList>& params) {
3188 const char tfecfFuncName[] = "expertStaticFillComplete: ";
3189 Tpetra::Details::ProfilingRegion prESFC("Tpetra::CrsGraph::expertStaticFillComplete");
3190
3191 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3192 domainMap.is_null() || rangeMap.is_null(),
3193 std::runtime_error, "The input domain Map and range Map must be nonnull.");
3194 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3195 isFillComplete() || !hasColMap(), std::runtime_error,
3196 "You may not "
3197 "call this method unless the graph has a column Map.");
3198 auto rowPtrsUnpackedLength = this->getRowPtrsUnpackedDevice().extent(0);
3199 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3200 getLocalNumRows() > 0 && rowPtrsUnpackedLength == 0,
3201 std::runtime_error, "The calling process has getLocalNumRows() = " << getLocalNumRows() << " > 0 rows, but the row offsets array has not "
3202 "been set.");
3203 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3204 static_cast<size_t>(rowPtrsUnpackedLength) != getLocalNumRows() + 1,
3205 std::runtime_error, "The row offsets array has length " << rowPtrsUnpackedLength << " != getLocalNumRows()+1 = " << (getLocalNumRows() + 1) << ".");
3206
3207 // Note: We don't need to do the following things which are normally done in fillComplete:
3208 // allocateIndices, globalAssemble, makeColMap, makeIndicesLocal, sortAndMergeAllIndices
3209
3210 // Constants from allocateIndices
3211 //
3212 // mfh 08 Aug 2014: numAllocForAllRows_ and k_numAllocPerRow_ go
3213 // away once the graph is allocated. expertStaticFillComplete
3214 // either presumes that the graph is allocated, or "allocates" it.
3215 //
3216 // FIXME (mfh 08 Aug 2014) The goal for the Kokkos refactor
3217 // version of CrsGraph is to allocate in the constructor, not
3218 // lazily on first insert. That will make both
3219 // numAllocForAllRows_ and k_numAllocPerRow_ obsolete.
3220 numAllocForAllRows_ = 0;
3221 k_numAllocPerRow_ = decltype(k_numAllocPerRow_)();
3222 indicesAreAllocated_ = true;
3223
3224 // Constants from makeIndicesLocal
3225 //
3226 // The graph has a column Map, so its indices had better be local.
3227 indicesAreLocal_ = true;
3228 indicesAreGlobal_ = false;
3229
3230 // set domain/range map: may clear the import/export objects
3231 {
3232 Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-Maps");
3233 setDomainRangeMaps(domainMap, rangeMap);
3234 }
3235
3236 // Presume the user sorted and merged the arrays first
3237 indicesAreSorted_ = true;
3238 noRedundancies_ = true;
3239
3240 // makeImportExport won't create a new importer/exporter if I set one here first.
3241
3242 importer_ = Teuchos::null;
3243 exporter_ = Teuchos::null;
3244 if (importer != Teuchos::null) {
3245 Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-mIXcheckI");
3246 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3247 !importer->getSourceMap()->isSameAs(*getDomainMap()) ||
3248 !importer->getTargetMap()->isSameAs(*getColMap()),
3249 std::invalid_argument, ": importer does not match matrix maps.");
3250 importer_ = importer;
3251 }
3252
3253 if (exporter != Teuchos::null) {
3254 Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-mIXcheckE");
3255 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3256 !exporter->getSourceMap()->isSameAs(*getRowMap()) ||
3257 !exporter->getTargetMap()->isSameAs(*getRangeMap()),
3258 std::invalid_argument, ": exporter does not match matrix maps.");
3259 exporter_ = exporter;
3260 }
3261
3262 {
3263 Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-mIXmake");
3264 Teuchos::Array<int> remotePIDs(0); // unused output argument
3265 this->makeImportExport(remotePIDs, false);
3266 }
3267
3268 {
3269 Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-fLG");
3270 this->fillLocalGraph(params);
3271 }
3272
3273 const bool callComputeGlobalConstants = params.get() == nullptr ||
3274 params->get("compute global constants", true);
3275
3276 if (callComputeGlobalConstants) {
3277 Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-cGC (const)");
3278 this->computeGlobalConstants();
3279 } else {
3280 Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-cGC (noconst)");
3281 this->computeLocalConstants();
3282 }
3283
3284 fillComplete_ = true;
3285
3286 checkInternalState();
3287}
3288
3289template <class LocalOrdinal, class GlobalOrdinal, class Node>
3290void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3291 fillLocalGraph(const Teuchos::RCP<Teuchos::ParameterList>& params) {
3292 using ::Tpetra::Details::computeOffsetsFromCounts;
3293 typedef typename local_graph_device_type::row_map_type row_map_type;
3294 typedef typename row_map_type::non_const_type non_const_row_map_type;
3295 typedef typename local_graph_device_type::entries_type::non_const_type lclinds_1d_type;
3296 const char tfecfFuncName[] =
3297 "fillLocalGraph (called from fillComplete or "
3298 "expertStaticFillComplete): ";
3299 const size_t lclNumRows = this->getLocalNumRows();
3300
3301 Details::ProfilingRegion regionFLG("Tpetra::CrsGraph::fillLocalGraph");
3302
3303 // This method's goal is to fill in the two arrays (compressed
3304 // sparse row format) that define the sparse graph's structure.
3305
3306 bool requestOptimizedStorage = true;
3307 if (!params.is_null() && !params->get("Optimize Storage", true)) {
3308 requestOptimizedStorage = false;
3309 }
3310
3311 // The graph's column indices are currently stored in a 1-D
3312 // format, with row offsets in rowPtrsUnpacked_host_ and local column indices
3313 // in k_lclInds1D_.
3314
3315 if (debug_) {
3316 auto rowPtrsUnpacked = this->getRowPtrsUnpackedHost();
3317 // The graph's array of row offsets must already be allocated.
3318 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked.extent(0) == 0, std::logic_error,
3319 "rowPtrsUnpacked_host_ has size zero, but shouldn't");
3320 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked.extent(0) != lclNumRows + 1, std::logic_error,
3321 "rowPtrsUnpacked_host_.extent(0) = "
3322 << rowPtrsUnpacked.extent(0) << " != (lclNumRows + 1) = "
3323 << (lclNumRows + 1) << ".");
3324 const size_t numOffsets = rowPtrsUnpacked.extent(0);
3325 const auto valToCheck = rowPtrsUnpacked(numOffsets - 1);
3326 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numOffsets != 0 &&
3327 lclIndsUnpacked_wdv.extent(0) != valToCheck,
3328 std::logic_error, "numOffsets=" << numOffsets << " != 0 "
3329 " and lclIndsUnpacked_wdv.extent(0)="
3330 << lclIndsUnpacked_wdv.extent(0) << " != rowPtrsUnpacked_host_(" << numOffsets << ")=" << valToCheck << ".");
3331 }
3332
3333 size_t allocSize = 0;
3334 try {
3335 allocSize = this->getLocalAllocationSize();
3336 } catch (std::logic_error& e) {
3337 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
3338 "getLocalAllocationSize threw "
3339 "std::logic_error: "
3340 << e.what());
3341 } catch (std::runtime_error& e) {
3342 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
3343 "getLocalAllocationSize threw "
3344 "std::runtime_error: "
3345 << e.what());
3346 } catch (std::exception& e) {
3347 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
3348 "getLocalAllocationSize threw "
3349 "std::exception: "
3350 << e.what());
3351 } catch (...) {
3352 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
3353 "getLocalAllocationSize threw "
3354 "an exception not a subclass of std::exception.");
3355 }
3356
3357 if (this->getLocalNumEntries() != allocSize) {
3358 // Use the nonconst version of row_map_type for ptr_d, because
3359 // the latter is const and we need to modify ptr_d here.
3360 non_const_row_map_type ptr_d;
3361 row_map_type ptr_d_const;
3362
3363 // The graph's current 1-D storage is "unpacked." This means
3364 // the row offsets may differ from what the final row offsets
3365 // should be. This could happen, for example, if the user set
3366 // an upper bound on the number of entries in each row, but
3367 // didn't fill all those entries.
3368
3369 if (debug_) {
3370 auto rowPtrsUnpacked = this->getRowPtrsUnpackedHost();
3371 if (rowPtrsUnpacked.extent(0) != 0) {
3372 const size_t numOffsets =
3373 static_cast<size_t>(rowPtrsUnpacked.extent(0));
3374 const auto valToCheck = rowPtrsUnpacked(numOffsets - 1);
3375 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != size_t(lclIndsUnpacked_wdv.extent(0)),
3376 std::logic_error,
3377 "(Unpacked branch) Before allocating "
3378 "or packing, k_rowPtrs_("
3379 << (numOffsets - 1) << ")="
3380 << valToCheck << " != lclIndsUnpacked_wdv.extent(0)="
3381 << lclIndsUnpacked_wdv.extent(0) << ".");
3382 }
3383 }
3384
3385 // Pack the row offsets into ptr_d, by doing a sum-scan of the
3386 // array of valid entry counts per row (k_numRowEntries_).
3387
3388 // Total number of entries in the matrix on the calling
3389 // process. We will compute this in the loop below. It's
3390 // cheap to compute and useful as a sanity check.
3391 size_t lclTotalNumEntries = 0;
3392 {
3393 // Allocate the packed row offsets array.
3394 ptr_d =
3395 non_const_row_map_type("Tpetra::CrsGraph::ptr", lclNumRows + 1);
3396 ptr_d_const = ptr_d;
3397
3398 // It's ok that k_numRowEntries_ is a host View; the
3399 // function can handle this.
3400 typename num_row_entries_type::const_type numRowEnt_h = k_numRowEntries_;
3401 if (debug_) {
3402 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(size_t(numRowEnt_h.extent(0)) != lclNumRows,
3403 std::logic_error,
3404 "(Unpacked branch) "
3405 "numRowEnt_h.extent(0)="
3406 << numRowEnt_h.extent(0)
3407 << " != getLocalNumRows()=" << lclNumRows << "");
3408 }
3409
3410 lclTotalNumEntries = computeOffsetsFromCounts(ptr_d, numRowEnt_h);
3411
3412 if (debug_) {
3413 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(ptr_d.extent(0)) != lclNumRows + 1,
3414 std::logic_error,
3415 "(Unpacked branch) After allocating "
3416 "ptr_d, ptr_d.extent(0) = "
3417 << ptr_d.extent(0)
3418 << " != lclNumRows+1 = " << (lclNumRows + 1) << ".");
3419 const auto valToCheck =
3420 ::Tpetra::Details::getEntryOnHost(ptr_d, lclNumRows);
3421 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != lclTotalNumEntries, std::logic_error,
3422 "Tpetra::CrsGraph::fillLocalGraph: In unpacked branch, "
3423 "after filling ptr_d, ptr_d(lclNumRows="
3424 << lclNumRows
3425 << ") = " << valToCheck << " != total number of entries "
3426 "on the calling process = "
3427 << lclTotalNumEntries
3428 << ".");
3429 }
3430 }
3431
3432 // Allocate the array of packed column indices.
3433 lclinds_1d_type ind_d =
3434 lclinds_1d_type("Tpetra::CrsGraph::lclInd", lclTotalNumEntries);
3435
3436 // k_rowPtrs_ and lclIndsUnpacked_wdv are currently unpacked. Pack
3437 // them, using the packed row offsets array ptr_d that we
3438 // created above.
3439 //
3440 // FIXME (mfh 08 Aug 2014) If "Optimize Storage" is false (in
3441 // CrsMatrix?), we need to keep around the unpacked row
3442 // offsets and column indices.
3443
3444 // Pack the column indices from unpacked lclIndsUnpacked_wdv into
3445 // packed ind_d. We will replace lclIndsUnpacked_wdv below.
3446 typedef pack_functor<
3447 typename local_graph_device_type::entries_type::non_const_type,
3448 typename local_inds_dualv_type::t_dev::const_type,
3449 row_map_type,
3450 typename local_graph_device_type::row_map_type>
3451 inds_packer_type;
3452 inds_packer_type f(ind_d,
3453 lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly),
3454 ptr_d, this->getRowPtrsUnpackedDevice());
3455 {
3456 typedef typename decltype(ind_d)::execution_space exec_space;
3457 typedef Kokkos::RangePolicy<exec_space, LocalOrdinal> range_type;
3458 Kokkos::parallel_for(range_type(0, lclNumRows), f);
3459 }
3460
3461 if (debug_) {
3462 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(ptr_d.extent(0) == 0, std::logic_error,
3463 "(\"Optimize Storage\"=true branch) After packing, "
3464 "ptr_d.extent(0)=0.");
3465 if (ptr_d.extent(0) != 0) {
3466 const size_t numOffsets = static_cast<size_t>(ptr_d.extent(0));
3467 const auto valToCheck =
3468 ::Tpetra::Details::getEntryOnHost(ptr_d, numOffsets - 1);
3469 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(valToCheck) != ind_d.extent(0),
3470 std::logic_error,
3471 "(\"Optimize Storage\"=true branch) "
3472 "After packing, ptr_d("
3473 << (numOffsets - 1) << ")="
3474 << valToCheck << " != ind_d.extent(0)="
3475 << ind_d.extent(0) << ".");
3476 }
3477 }
3478 // Build the local graph.
3479 if (requestOptimizedStorage)
3480 setRowPtrs(ptr_d_const);
3481 else
3482 setRowPtrsPacked(ptr_d_const);
3483 lclIndsPacked_wdv = local_inds_wdv_type(ind_d);
3484 } else { // We don't have to pack, so just set the pointers.
3485 // Set both packed and unpacked rowptrs to this
3486 this->setRowPtrs(rowPtrsUnpacked_dev_);
3487 lclIndsPacked_wdv = lclIndsUnpacked_wdv;
3488
3489 if (debug_) {
3490 auto rowPtrsPacked_dev = this->getRowPtrsPackedDevice();
3491 auto rowPtrsPacked_host = this->getRowPtrsPackedHost();
3492 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsPacked_dev.extent(0) == 0, std::logic_error,
3493 "(\"Optimize Storage\"=false branch) "
3494 "rowPtrsPacked_dev_.extent(0) = 0.");
3495 if (rowPtrsPacked_dev.extent(0) != 0) {
3496 const size_t numOffsets =
3497 static_cast<size_t>(rowPtrsPacked_dev.extent(0));
3498 const size_t valToCheck =
3499 rowPtrsPacked_host(numOffsets - 1);
3500 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != size_t(lclIndsPacked_wdv.extent(0)),
3501 std::logic_error,
3502 "(\"Optimize Storage\"=false branch) "
3503 "rowPtrsPacked_dev_("
3504 << (numOffsets - 1) << ")="
3505 << valToCheck
3506 << " != lclIndsPacked_wdv.extent(0)="
3507 << lclIndsPacked_wdv.extent(0) << ".");
3508 }
3509 }
3510 }
3511
3512 if (debug_) {
3513 auto rowPtrsPacked_dev = this->getRowPtrsPackedDevice();
3514 auto rowPtrsPacked_host = this->getRowPtrsPackedHost();
3515 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(rowPtrsPacked_dev.extent(0)) != lclNumRows + 1,
3516 std::logic_error, "After packing, rowPtrsPacked_dev_.extent(0) = " << rowPtrsPacked_dev.extent(0) << " != lclNumRows+1 = " << (lclNumRows + 1) << ".");
3517 if (rowPtrsPacked_dev.extent(0) != 0) {
3518 const size_t numOffsets = static_cast<size_t>(rowPtrsPacked_dev.extent(0));
3519 const auto valToCheck = rowPtrsPacked_host(numOffsets - 1);
3520 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(valToCheck) != lclIndsPacked_wdv.extent(0),
3521 std::logic_error, "After packing, rowPtrsPacked_dev_(" << (numOffsets - 1) << ") = " << valToCheck << " != lclIndsPacked_wdv.extent(0) = " << lclIndsPacked_wdv.extent(0) << ".");
3522 }
3523 }
3524
3525 if (requestOptimizedStorage) {
3526 // With optimized storage, we don't need to store
3527 // the array of row entry counts.
3528
3529 // Free graph data structures that are only needed for
3530 // unpacked 1-D storage.
3531 k_numRowEntries_ = num_row_entries_type();
3532
3533 // Keep the new 1-D packed allocations.
3534 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
3535
3536 storageStatus_ = Details::STORAGE_1D_PACKED;
3537 }
3538
3539 set_need_sync_host_uvm_access(); // make sure kernel setup of indices is fenced before a host access
3540}
3541
3542template <class LocalOrdinal, class GlobalOrdinal, class Node>
3543void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3544 replaceColMap(const Teuchos::RCP<const map_type>& newColMap) {
3545 // NOTE: This safety check matches the code, but not the documentation of Crsgraph
3546 //
3547 // FIXME (mfh 18 Aug 2014) This will break if the calling process
3548 // has no entries, because in that case, currently it is neither
3549 // locally nor globally indexed. This will change once we get rid
3550 // of lazy allocation (so that the constructor allocates indices
3551 // and therefore commits to local vs. global).
3552 const char tfecfFuncName[] = "replaceColMap: ";
3553 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3554 isLocallyIndexed() || isGloballyIndexed(), std::runtime_error,
3555 "Requires matching maps and non-static graph.");
3556 colMap_ = newColMap;
3557}
3558
3559template <class LocalOrdinal, class GlobalOrdinal, class Node>
3560void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3561 reindexColumns(const Teuchos::RCP<const map_type>& newColMap,
3562 const Teuchos::RCP<const import_type>& newImport,
3563 const bool sortIndicesInEachRow) {
3564 using Teuchos::RCP;
3565 using Teuchos::REDUCE_MIN;
3566 using Teuchos::reduceAll;
3567 typedef GlobalOrdinal GO;
3568 typedef LocalOrdinal LO;
3569 using col_inds_type_dev = typename local_inds_dualv_type::t_dev;
3570 const char tfecfFuncName[] = "reindexColumns: ";
3571
3572 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3573 isFillComplete(), std::runtime_error,
3574 "The graph is fill complete "
3575 "(isFillComplete() returns true). You must call resumeFill() before "
3576 "you may call this method.");
3577
3578 // mfh 19 Aug 2014: This method does NOT redistribute data; it
3579 // doesn't claim to do the work of an Import or Export. This
3580 // means that for all processes, the calling process MUST own all
3581 // column indices, in both the old column Map (if it exists) and
3582 // the new column Map. We check this via an all-reduce.
3583 //
3584 // Some processes may be globally indexed, others may be locally
3585 // indexed, and others (that have no graph entries) may be
3586 // neither. This method will NOT change the graph's current
3587 // state. If it's locally indexed, it will stay that way, and
3588 // vice versa. It would easy to add an option to convert indices
3589 // from global to local, so as to save a global-to-local
3590 // conversion pass. However, we don't do this here. The intended
3591 // typical use case is that the graph already has a column Map and
3592 // is locally indexed, and this is the case for which we optimize.
3593
3594 const LO lclNumRows = static_cast<LO>(this->getLocalNumRows());
3595
3596 // Attempt to convert indices to the new column Map's version of
3597 // local. This will fail if on the calling process, the graph has
3598 // indices that are not on that process in the new column Map.
3599 // After the local conversion attempt, we will do an all-reduce to
3600 // see if any processes failed.
3601
3602 // If this is false, then either the graph contains a column index
3603 // which is invalid in the CURRENT column Map, or the graph is
3604 // locally indexed but currently has no column Map. In either
3605 // case, there is no way to convert the current local indices into
3606 // global indices, so that we can convert them into the new column
3607 // Map's local indices. It's possible for this to be true on some
3608 // processes but not others, due to replaceColMap.
3609 bool allCurColIndsValid = true;
3610 // On the calling process, are all valid current column indices
3611 // also in the new column Map on the calling process? In other
3612 // words, does local reindexing suffice, or should the user have
3613 // done an Import or Export instead?
3614 bool localSuffices = true;
3615
3616 {
3617 // Final arrays for the local indices. We will allocate exactly
3618 // one of these ONLY if the graph is locally indexed on the
3619 // calling process, and ONLY if the graph has one or more entries
3620 // (is not empty) on the calling process. In that case, we
3621 // allocate the first (1-D storage) if the graph has a static
3622 // profile, else we allocate the second (2-D storage).
3623 col_inds_type_dev newLclInds1D_dev;
3624
3625 // If indices aren't allocated, that means the calling process
3626 // owns no entries in the graph. Thus, there is nothing to
3627 // convert, and it trivially succeeds locally.
3628 if (indicesAreAllocated()) {
3629 if (isLocallyIndexed()) {
3630 if (hasColMap()) { // locally indexed, and currently has a column Map
3631 const map_type& oldColMap = *(getColMap());
3632
3633 // Allocate storage for the new local indices.
3634 const size_t allocSize = this->getLocalAllocationSize();
3635 auto oldLclInds1D = lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly);
3636 newLclInds1D_dev = col_inds_type_dev("Tpetra::CrsGraph::lclIndsReindexed",
3637 allocSize);
3638 auto oldLclColMap = oldColMap.getLocalMap();
3639 auto newLclColMap = newColMap->getLocalMap();
3640
3641 const auto LO_INVALID = Teuchos::OrdinalTraits<LO>::invalid();
3642 const auto GO_INVALID = Teuchos::OrdinalTraits<GO>::invalid();
3643
3644 const int NOT_ALL_LOCAL_INDICES_ARE_VALID = 1;
3645 const int LOCAL_DOES_NOT_SUFFICE = 2;
3646 int errorStatus = 0;
3647 Kokkos::parallel_reduce(
3648 "Tpetra::CrsGraph::reindexColumns",
3649 Kokkos::RangePolicy<LocalOrdinal, execution_space>(0, allocSize),
3650 KOKKOS_LAMBDA(const LocalOrdinal k, int& result) {
3651 const LocalOrdinal oldLclCol = oldLclInds1D(k);
3652 if (oldLclCol == LO_INVALID) {
3653 result &= NOT_ALL_LOCAL_INDICES_ARE_VALID;
3654 } else {
3655 const GO gblCol = oldLclColMap.getGlobalElement(oldLclCol);
3656 if (gblCol == GO_INVALID) {
3657 result &= LOCAL_DOES_NOT_SUFFICE;
3658 } else {
3659 const LocalOrdinal newLclCol = newLclColMap.getLocalElement(gblCol);
3660 if (newLclCol == LO_INVALID) {
3661 result &= NOT_ALL_LOCAL_INDICES_ARE_VALID;
3662 } else {
3663 newLclInds1D_dev(k) = newLclCol;
3664 }
3665 }
3666 }
3667 },
3668 Kokkos::LOr<int>(errorStatus));
3669 allCurColIndsValid = !(errorStatus & NOT_ALL_LOCAL_INDICES_ARE_VALID);
3670 localSuffices = !(errorStatus & LOCAL_DOES_NOT_SUFFICE);
3671 } else { // locally indexed, but no column Map
3672 // This case is only possible if replaceColMap() was called
3673 // with a null argument on the calling process. It's
3674 // possible, but it means that this method can't possibly
3675 // succeed, since we have no way of knowing how to convert
3676 // the current local indices to global indices.
3677 allCurColIndsValid = false;
3678 }
3679 } else { // globally indexed
3680 // If the graph is globally indexed, we don't need to save
3681 // local indices, but we _do_ need to know whether the current
3682 // global indices are valid in the new column Map. We may
3683 // need to do a getRemoteIndexList call to find this out.
3684 //
3685 // In this case, it doesn't matter whether the graph currently
3686 // has a column Map. We don't need the old column Map to
3687 // convert from global indices to the _new_ column Map's local
3688 // indices. Furthermore, we can use the same code, whether
3689 // the graph is static or dynamic profile.
3690
3691 // Test whether the current global indices are in the new
3692 // column Map on the calling process.
3693 for (LO lclRow = 0; lclRow < lclNumRows; ++lclRow) {
3694 const RowInfo rowInfo = this->getRowInfo(lclRow);
3695 auto oldGblRowView = this->getGlobalIndsViewHost(rowInfo);
3696 for (size_t k = 0; k < rowInfo.numEntries; ++k) {
3697 const GO gblCol = oldGblRowView(k);
3698 if (!newColMap->isNodeGlobalElement(gblCol)) {
3699 localSuffices = false;
3700 break; // Stop at the first invalid index
3701 }
3702 } // for each entry in the current row
3703 } // for each locally owned row
3704 } // locally or globally indexed
3705 } // whether indices are allocated
3706
3707 // Do an all-reduce to check both possible error conditions.
3708 int lclSuccess[2];
3709 lclSuccess[0] = allCurColIndsValid ? 1 : 0;
3710 lclSuccess[1] = localSuffices ? 1 : 0;
3711 int gblSuccess[2];
3712 gblSuccess[0] = 0;
3713 gblSuccess[1] = 0;
3714 RCP<const Teuchos::Comm<int>> comm =
3715 getRowMap().is_null() ? Teuchos::null : getRowMap()->getComm();
3716 if (!comm.is_null()) {
3717 reduceAll<int, int>(*comm, REDUCE_MIN, 2, lclSuccess, gblSuccess);
3718 }
3719
3720 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3721 gblSuccess[0] == 0, std::runtime_error,
3722 "It is not possible to continue."
3723 " The most likely reason is that the graph is locally indexed, but the "
3724 "column Map is missing (null) on some processes, due to a previous call "
3725 "to replaceColMap().");
3726
3727 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3728 gblSuccess[1] == 0, std::runtime_error,
3729 "On some process, the graph "
3730 "contains column indices that are in the old column Map, but not in the "
3731 "new column Map (on that process). This method does NOT redistribute "
3732 "data; it does not claim to do the work of an Import or Export operation."
3733 " This means that for all processess, the calling process MUST own all "
3734 "column indices, in both the old column Map and the new column Map. In "
3735 "this case, you will need to do an Import or Export operation to "
3736 "redistribute data.");
3737
3738 // Commit the results.
3739 if (isLocallyIndexed()) {
3740 lclIndsUnpacked_wdv = local_inds_wdv_type(newLclInds1D_dev);
3741 }
3742 // end of scope for newLclInds1D_dev
3743 // sortAndMergeAllIndices needs host access
3744 }
3745
3746 if (isLocallyIndexed()) {
3747 // We've reindexed, so we don't know if the indices are sorted.
3748 //
3749 // FIXME (mfh 17 Sep 2014) It could make sense to check this,
3750 // since we're already going through all the indices above. We
3751 // could also sort each row in place; that way, we would only
3752 // have to make one pass over the rows.
3753 indicesAreSorted_ = false;
3754 if (sortIndicesInEachRow) {
3755 // NOTE (mfh 17 Sep 2014) The graph must be locally indexed in
3756 // order to call this method.
3757 //
3758 // FIXME (mfh 17 Sep 2014) This violates the strong exception
3759 // guarantee. It would be better to sort the new index arrays
3760 // before committing them.
3761 const bool sorted = false; // need to resort
3762 const bool merged = true; // no need to merge, since no dups
3763 this->sortAndMergeAllIndices(sorted, merged);
3764 }
3765 }
3766 colMap_ = newColMap;
3767
3768 if (newImport.is_null()) {
3769 // FIXME (mfh 19 Aug 2014) Should use the above all-reduce to
3770 // check whether the input Import is null on any process.
3771 //
3772 // If the domain Map hasn't been set yet, we can't compute a new
3773 // Import object. Leave it what it is; it should be null, but
3774 // it doesn't matter. If the domain Map _has_ been set, then
3775 // compute a new Import object if necessary.
3776 if (!domainMap_.is_null()) {
3777 if (!domainMap_->isSameAs(*newColMap)) {
3778 importer_ = Teuchos::rcp(new import_type(domainMap_, newColMap));
3779 } else {
3780 importer_ = Teuchos::null; // don't need an Import
3781 }
3782 }
3783 } else {
3784 // The caller gave us an Import object. Assume that it's valid.
3785 importer_ = newImport;
3786 }
3787}
3788
3789template <class LocalOrdinal, class GlobalOrdinal, class Node>
3790void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3791 replaceDomainMap(const Teuchos::RCP<const map_type>& newDomainMap) {
3792 const char prefix[] = "Tpetra::CrsGraph::replaceDomainMap: ";
3793 TEUCHOS_TEST_FOR_EXCEPTION(
3794 colMap_.is_null(), std::invalid_argument, prefix << "You may not call "
3795 "this method unless the graph already has a column Map.");
3796 TEUCHOS_TEST_FOR_EXCEPTION(
3797 newDomainMap.is_null(), std::invalid_argument,
3798 prefix << "The new domain Map must be nonnull.");
3799
3800 // Create a new importer, if needed
3801 Teuchos::RCP<const import_type> newImporter = Teuchos::null;
3802 if (newDomainMap != colMap_ && (!newDomainMap->isSameAs(*colMap_))) {
3803 newImporter = rcp(new import_type(newDomainMap, colMap_));
3804 }
3805 this->replaceDomainMapAndImporter(newDomainMap, newImporter);
3806}
3807
3808template <class LocalOrdinal, class GlobalOrdinal, class Node>
3809void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3810 replaceDomainMapAndImporter(const Teuchos::RCP<const map_type>& newDomainMap,
3811 const Teuchos::RCP<const import_type>& newImporter) {
3812 const char prefix[] = "Tpetra::CrsGraph::replaceDomainMapAndImporter: ";
3813 TEUCHOS_TEST_FOR_EXCEPTION(
3814 colMap_.is_null(), std::invalid_argument, prefix << "You may not call "
3815 "this method unless the graph already has a column Map.");
3816 TEUCHOS_TEST_FOR_EXCEPTION(
3817 newDomainMap.is_null(), std::invalid_argument,
3818 prefix << "The new domain Map must be nonnull.");
3819
3820 if (debug_) {
3821 if (newImporter.is_null()) {
3822 // It's not a good idea to put expensive operations in a macro
3823 // clause, even if they are side effect - free, because macros
3824 // don't promise that they won't evaluate their arguments more
3825 // than once. It's polite for them to do so, but not required.
3826 const bool colSameAsDom = colMap_->isSameAs(*newDomainMap);
3827 TEUCHOS_TEST_FOR_EXCEPTION(!colSameAsDom, std::invalid_argument,
3828 "If the new Import is null, "
3829 "then the new domain Map must be the same as the current column Map.");
3830 } else {
3831 const bool colSameAsTgt =
3832 colMap_->isSameAs(*(newImporter->getTargetMap()));
3833 const bool newDomSameAsSrc =
3834 newDomainMap->isSameAs(*(newImporter->getSourceMap()));
3835 TEUCHOS_TEST_FOR_EXCEPTION(!colSameAsTgt || !newDomSameAsSrc, std::invalid_argument,
3836 "If the "
3837 "new Import is nonnull, then the current column Map must be the same "
3838 "as the new Import's target Map, and the new domain Map must be the "
3839 "same as the new Import's source Map.");
3840 }
3841 }
3842
3843 domainMap_ = newDomainMap;
3844 importer_ = Teuchos::rcp_const_cast<import_type>(newImporter);
3845}
3846
3847template <class LocalOrdinal, class GlobalOrdinal, class Node>
3848void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3849 replaceRangeMap(const Teuchos::RCP<const map_type>& newRangeMap) {
3850 const char prefix[] = "Tpetra::CrsGraph::replaceRangeMap: ";
3851 TEUCHOS_TEST_FOR_EXCEPTION(
3852 rowMap_.is_null(), std::invalid_argument, prefix << "You may not call "
3853 "this method unless the graph already has a row Map.");
3854 TEUCHOS_TEST_FOR_EXCEPTION(
3855 newRangeMap.is_null(), std::invalid_argument,
3856 prefix << "The new range Map must be nonnull.");
3857
3858 // Create a new exporter, if needed
3859 Teuchos::RCP<const export_type> newExporter = Teuchos::null;
3860 if (newRangeMap != rowMap_ && (!newRangeMap->isSameAs(*rowMap_))) {
3861 newExporter = rcp(new export_type(rowMap_, newRangeMap));
3862 }
3863 this->replaceRangeMapAndExporter(newRangeMap, newExporter);
3864}
3865
3866template <class LocalOrdinal, class GlobalOrdinal, class Node>
3867void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3868 replaceRangeMapAndExporter(const Teuchos::RCP<const map_type>& newRangeMap,
3869 const Teuchos::RCP<const export_type>& newExporter) {
3870 const char prefix[] = "Tpetra::CrsGraph::replaceRangeMapAndExporter: ";
3871 TEUCHOS_TEST_FOR_EXCEPTION(
3872 rowMap_.is_null(), std::invalid_argument, prefix << "You may not call "
3873 "this method unless the graph already has a column Map.");
3874 TEUCHOS_TEST_FOR_EXCEPTION(
3875 newRangeMap.is_null(), std::invalid_argument,
3876 prefix << "The new domain Map must be nonnull.");
3877
3878 if (debug_) {
3879 if (newExporter.is_null()) {
3880 // It's not a good idea to put expensive operations in a macro
3881 // clause, even if they are side effect - free, because macros
3882 // don't promise that they won't evaluate their arguments more
3883 // than once. It's polite for them to do so, but not required.
3884 const bool rowSameAsRange = rowMap_->isSameAs(*newRangeMap);
3885 TEUCHOS_TEST_FOR_EXCEPTION(!rowSameAsRange, std::invalid_argument,
3886 "If the new Export is null, "
3887 "then the new range Map must be the same as the current row Map.");
3888 } else {
3889 const bool newRangeSameAsTgt =
3890 newRangeMap->isSameAs(*(newExporter->getTargetMap()));
3891 const bool rowSameAsSrc =
3892 rowMap_->isSameAs(*(newExporter->getSourceMap()));
3893 TEUCHOS_TEST_FOR_EXCEPTION(!rowSameAsSrc || !newRangeSameAsTgt, std::invalid_argument,
3894 "If the "
3895 "new Export is nonnull, then the current row Map must be the same "
3896 "as the new Export's source Map, and the new range Map must be the "
3897 "same as the new Export's target Map.");
3898 }
3899 }
3900
3901 rangeMap_ = newRangeMap;
3902 exporter_ = Teuchos::rcp_const_cast<export_type>(newExporter);
3903}
3904
3905template <class LocalOrdinal, class GlobalOrdinal, class Node>
3906typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_graph_device_type
3907CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3908 getLocalGraphDevice() const {
3909 return local_graph_device_type(
3910 lclIndsPacked_wdv.getDeviceView(Access::ReadWrite),
3911 this->getRowPtrsPackedDevice());
3912}
3913
3914template <class LocalOrdinal, class GlobalOrdinal, class Node>
3915typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_graph_host_type
3916CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3917 getLocalGraphHost() const {
3918 return local_graph_host_type(
3919 lclIndsPacked_wdv.getHostView(Access::ReadWrite),
3920 this->getRowPtrsPackedHost());
3921}
3922
3923template <class LocalOrdinal, class GlobalOrdinal, class Node>
3924void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3925 computeGlobalConstants() {
3926 using Teuchos::ArrayView;
3927 using Teuchos::outArg;
3928 using Teuchos::reduceAll;
3929 using ::Tpetra::Details::ProfilingRegion;
3930 typedef global_size_t GST;
3931
3932 ProfilingRegion regionCGC("Tpetra::CrsGraph::computeGlobalConstants");
3933
3934 GST lcl, gbl;
3935 std::shared_ptr<Details::CommRequest> req;
3936 if (!this->haveGlobalConstants_) {
3937 lcl = static_cast<GST>(this->getLocalNumEntries());
3938 req = Details::iallreduce(lcl, gbl, Teuchos::REDUCE_SUM, *this->getComm());
3939 }
3940
3941 this->computeLocalConstants();
3942
3943 // Compute global constants from local constants. Processes that
3944 // already have local constants still participate in the
3945 // all-reduces, using their previously computed values.
3946 if (!this->haveGlobalConstants_) {
3947 // Promote all the nodeNum* and nodeMaxNum* quantities from
3948 // size_t to global_size_t, when doing the all-reduces for
3949 // globalNum* / globalMaxNum* results.
3950 //
3951 // FIXME (mfh 07 May 2013) Unfortunately, we either have to do
3952 // this in two all-reduces (one for the sum and the other for
3953 // the max), or use a custom MPI_Op that combines the sum and
3954 // the max. The latter might even be slower than two
3955 // all-reduces on modern network hardware. It would also be a
3956 // good idea to use nonblocking all-reduces (MPI 3), so that we
3957 // don't have to wait around for the first one to finish before
3958 // starting the second one.
3959 const GST lclMaxNumRowEnt = static_cast<GST>(this->nodeMaxNumRowEntries_);
3960 auto req2 = Details::iallreduce(lclMaxNumRowEnt, this->globalMaxNumRowEntries_, Teuchos::REDUCE_MAX, *this->getComm());
3961
3962 req->wait();
3963 this->globalNumEntries_ = gbl;
3964
3965 req2->wait();
3966 this->haveGlobalConstants_ = true;
3967 }
3968}
3969
3970template <class LocalOrdinal, class GlobalOrdinal, class Node>
3971void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3972 computeLocalConstants() {
3973 using ::Tpetra::Details::ProfilingRegion;
3974
3975 ProfilingRegion regionCLC("Tpetra::CrsGraph::computeLocalConstants");
3976 if (this->haveLocalConstants_) {
3977 return;
3978 }
3979
3980 // Reset local properties
3981 this->nodeMaxNumRowEntries_ =
3982 Teuchos::OrdinalTraits<size_t>::invalid();
3983
3984 using LO = local_ordinal_type;
3985
3986 auto ptr = this->getRowPtrsPackedDevice();
3987 const LO lclNumRows = ptr.extent(0) == 0 ? static_cast<LO>(0) : (static_cast<LO>(ptr.extent(0)) - static_cast<LO>(1));
3988
3989 const LO lclMaxNumRowEnt =
3990 ::Tpetra::Details::maxDifference("Tpetra::CrsGraph: nodeMaxNumRowEntries",
3991 ptr, lclNumRows);
3992 this->nodeMaxNumRowEntries_ = static_cast<size_t>(lclMaxNumRowEnt);
3993 this->haveLocalConstants_ = true;
3994}
3995
3996template <class LocalOrdinal, class GlobalOrdinal, class Node>
3997std::pair<size_t, std::string>
3998CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
3999 makeIndicesLocal(const bool verbose) {
4000 using Details::ProfilingRegion;
4001 using std::endl;
4002 using Teuchos::arcp;
4003 using Teuchos::Array;
4004 typedef LocalOrdinal LO;
4005 typedef GlobalOrdinal GO;
4006 typedef device_type DT;
4007 typedef typename local_graph_device_type::row_map_type::non_const_value_type offset_type;
4008 typedef typename num_row_entries_type::non_const_value_type num_ent_type;
4009 const char tfecfFuncName[] = "makeIndicesLocal: ";
4010 ProfilingRegion regionMakeIndicesLocal("Tpetra::CrsGraph::makeIndicesLocal");
4011
4012 std::unique_ptr<std::string> prefix;
4013 if (verbose) {
4014 prefix = this->createPrefix("CrsGraph", "makeIndicesLocal");
4015 std::ostringstream os;
4016 os << *prefix << "lclNumRows: " << getLocalNumRows() << endl;
4017 std::cerr << os.str();
4018 }
4019
4020 // These are somewhat global properties, so it's safe to have
4021 // exception checks for them, rather than returning an error code.
4022 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::logic_error,
4023 "The graph does not have a "
4024 "column Map yet. This method should never be called in that case. "
4025 "Please report this bug to the Tpetra developers.");
4026 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->getColMap().is_null(), std::logic_error,
4027 "The graph claims "
4028 "that it has a column Map, because hasColMap() returns true. However, "
4029 "the result of getColMap() is null. This should never happen. Please "
4030 "report this bug to the Tpetra developers.");
4031
4032 // Return value 1: The number of column indices (counting
4033 // duplicates) that could not be converted to local indices,
4034 // because they were not in the column Map on the calling process.
4035 size_t lclNumErrs = 0;
4036 std::ostringstream errStrm; // for return value 2 (error string)
4037
4038 const LO lclNumRows = static_cast<LO>(this->getLocalNumRows());
4039 const map_type& colMap = *(this->getColMap());
4040
4041 if (this->isGloballyIndexed() && lclNumRows != 0) {
4042 // This is a host-accessible View.
4043 typename num_row_entries_type::const_type h_numRowEnt =
4044 this->k_numRowEntries_;
4045
4046 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
4047
4048 // Allocate space for local indices.
4049 if (rowPtrsUnpacked_host.extent(0) == 0) {
4050 errStrm << "Unpacked row pointers (rowPtrsUnpacked_dev_) has length 0. This should never "
4051 "happen here. Please report this bug to the Tpetra developers."
4052 << endl;
4053 // Need to return early.
4054 return std::make_pair(Tpetra::Details::OrdinalTraits<size_t>::invalid(),
4055 errStrm.str());
4056 }
4057 const auto numEnt = rowPtrsUnpacked_host(lclNumRows);
4058
4059 // mfh 17 Dec 2016: We don't need initial zero-fill of
4060 // lclIndsUnpacked_wdv, because we will fill it below anyway.
4061 // AllowPadding would only help for aligned access (e.g.,
4062 // for vectorization) if we also were to pad each row to the
4063 // same alignment, so we'll skip AllowPadding for now.
4064
4065 // using Kokkos::AllowPadding;
4066 using Kokkos::view_alloc;
4067 using Kokkos::WithoutInitializing;
4068
4069 // When giving the label as an argument to
4070 // Kokkos::view_alloc, the label must be a string and not a
4071 // char*, else the code won't compile. This is because
4072 // view_alloc also allows a raw pointer as its first
4073 // argument. See
4074 // https://github.com/kokkos/kokkos/issues/434. This is a
4075 // large allocation typically, so the overhead of creating
4076 // an std::string is minor.
4077 const std::string label("Tpetra::CrsGraph::lclInd");
4078 if (verbose) {
4079 std::ostringstream os;
4080 os << *prefix << "(Re)allocate lclInd_wdv: old="
4081 << lclIndsUnpacked_wdv.extent(0) << ", new=" << numEnt << endl;
4082 std::cerr << os.str();
4083 }
4084
4085 local_inds_dualv_type lclInds_dualv =
4086 local_inds_dualv_type(view_alloc(label, WithoutInitializing),
4087 numEnt);
4088 lclIndsUnpacked_wdv = local_inds_wdv_type(lclInds_dualv);
4089
4090 auto lclColMap = colMap.getLocalMap();
4091 // This is a "device mirror" of the host View h_numRowEnt.
4092 //
4093 // NOTE (mfh 27 Sep 2016) Currently, the right way to get a
4094 // Device instance is to use its default constructor. See the
4095 // following Kokkos issue:
4096 //
4097 // https://github.com/kokkos/kokkos/issues/442
4098 if (verbose) {
4099 std::ostringstream os;
4100 os << *prefix << "Allocate device mirror k_numRowEnt: "
4101 << h_numRowEnt.extent(0) << endl;
4102 std::cerr << os.str();
4103 }
4104 auto k_numRowEnt =
4105 Kokkos::create_mirror_view_and_copy(device_type(), h_numRowEnt);
4106
4107 using ::Tpetra::Details::convertColumnIndicesFromGlobalToLocal;
4108 lclNumErrs =
4109 convertColumnIndicesFromGlobalToLocal<LO, GO, DT, offset_type, num_ent_type>(
4110 lclIndsUnpacked_wdv.getDeviceView(Access::OverwriteAll),
4111 gblInds_wdv.getDeviceView(Access::ReadOnly),
4112 this->getRowPtrsUnpackedDevice(),
4113 lclColMap,
4114 k_numRowEnt);
4115 if (lclNumErrs != 0) {
4116 const int myRank = [this]() {
4117 auto map = this->getMap();
4118 if (map.is_null()) {
4119 return 0;
4120 } else {
4121 auto comm = map->getComm();
4122 return comm.is_null() ? 0 : comm->getRank();
4123 }
4124 }();
4125 const bool pluralNumErrs = (lclNumErrs != static_cast<size_t>(1));
4126 errStrm << "(Process " << myRank << ") When converting column "
4127 "indices from global to local, we encountered "
4128 << lclNumErrs
4129 << " ind" << (pluralNumErrs ? "ices" : "ex")
4130 << " that do" << (pluralNumErrs ? "" : "es")
4131 << " not live in the column Map on this process." << endl;
4132 }
4133
4134 // We've converted column indices from global to local, so we
4135 // can deallocate the global column indices (which we know are
4136 // in 1-D storage, because the graph has static profile).
4137 if (verbose) {
4138 std::ostringstream os;
4139 os << *prefix << "Free gblInds_wdv: "
4140 << gblInds_wdv.extent(0) << endl;
4141 std::cerr << os.str();
4142 }
4143 gblInds_wdv = global_inds_wdv_type();
4144 } // globallyIndexed() && lclNumRows > 0
4145
4146 this->indicesAreLocal_ = true;
4147 this->indicesAreGlobal_ = false;
4148 this->checkInternalState();
4149
4150 return std::make_pair(lclNumErrs, errStrm.str());
4151}
4152
4153template <class LocalOrdinal, class GlobalOrdinal, class Node>
4154void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4155 makeColMap(Teuchos::Array<int>& remotePIDs) {
4156 using Details::ProfilingRegion;
4157 using std::endl;
4158 const char tfecfFuncName[] = "makeColMap";
4159
4160 ProfilingRegion regionSortAndMerge("Tpetra::CrsGraph::makeColMap");
4161 std::unique_ptr<std::string> prefix;
4162 if (verbose_) {
4163 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4164 std::ostringstream os;
4165 os << *prefix << "Start" << endl;
4166 std::cerr << os.str();
4167 }
4168
4169 // this->colMap_ should be null at this point, but we accept the
4170 // future possibility that it might not be (esp. if we decide
4171 // later to support graph structure changes after first
4172 // fillComplete, which CrsGraph does not currently (as of 12 Feb
4173 // 2017) support).
4174 Teuchos::RCP<const map_type> colMap = this->colMap_;
4175 const bool sortEachProcsGids =
4176 this->sortGhostsAssociatedWithEachProcessor_;
4177
4178 // FIXME (mfh 12 Feb 2017) ::Tpetra::Details::makeColMap returns a
4179 // per-process error code. If an error does occur on a process,
4180 // ::Tpetra::Details::makeColMap does NOT promise that all processes will
4181 // notice that error. This is the caller's responsibility. For
4182 // now, we only propagate (to all processes) and report the error
4183 // in debug mode. In the future, we need to add the local/global
4184 // error handling scheme used in BlockCrsMatrix to this class.
4185 if (debug_) {
4186 using Teuchos::outArg;
4187 using Teuchos::REDUCE_MIN;
4188 using Teuchos::reduceAll;
4189
4190 std::ostringstream errStrm;
4191 const int lclErrCode =
4192 Details::makeColMap(colMap, remotePIDs,
4193 getDomainMap(), *this, sortEachProcsGids, &errStrm);
4194 auto comm = this->getComm();
4195 if (!comm.is_null()) {
4196 const int lclSuccess = (lclErrCode == 0) ? 1 : 0;
4197 int gblSuccess = 0; // output argument
4198 reduceAll<int, int>(*comm, REDUCE_MIN, lclSuccess,
4199 outArg(gblSuccess));
4200 if (gblSuccess != 1) {
4201 std::ostringstream os;
4202 Details::gathervPrint(os, errStrm.str(), *comm);
4203 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
4204 ": An error happened on at "
4205 "least one process in the CrsGraph's communicator. "
4206 "Here are all processes' error messages:"
4207 << std::endl
4208 << os.str());
4209 }
4210 }
4211 } else {
4212 (void)Details::makeColMap(colMap, remotePIDs,
4213 getDomainMap(), *this, sortEachProcsGids, nullptr);
4214 }
4215 // See above. We want to admit the possibility of makeColMap
4216 // actually revising an existing column Map, even though that
4217 // doesn't currently (as of 10 May 2017) happen.
4218 this->colMap_ = colMap;
4219
4220 checkInternalState();
4221 if (verbose_) {
4222 std::ostringstream os;
4223 os << *prefix << "Done" << endl;
4224 std::cerr << os.str();
4225 }
4226}
4227
4228template <class execution_space, class LO, class rowptr_type, class colinds_type, class numRowEntries_type>
4229void prepareSortMergeUnpackedGraph(rowptr_type rowptr, colinds_type colinds, numRowEntries_type numRowEntries) {
4230 using ATS = KokkosKernels::ArithTraits<LO>;
4231 const auto unused = ATS::max();
4232
4233 auto numRows = rowptr.extent(0) - 1;
4234
4235 // make sure that unused entries will get ordered last
4236 Kokkos::parallel_for(
4237 "flag_unused_entries", Kokkos::RangePolicy<execution_space, LO>(0, numRows), KOKKOS_LAMBDA(const LO rlid) {
4238 for (size_t jj = rowptr(rlid) + numRowEntries(rlid); jj < rowptr(rlid + 1); ++jj) {
4239 colinds(jj) = unused;
4240 }
4241 });
4242}
4243
4244template <class execution_space, class LO, class rowptr_type, class colinds_type, class numRowEntries_type>
4245void mergeUnpackedGraph(rowptr_type rowptr, colinds_type colinds, numRowEntries_type numRowEntries) {
4246 auto numRows = rowptr.extent(0) - 1;
4247
4248 // merge
4249 // We cannot use KokkosSparse::sort_and_merge_matrix since we
4250 // do not actually want to change the allocations.
4251
4252 Kokkos::parallel_for(
4253 "merge_entries", Kokkos::RangePolicy<execution_space>(0, numRows), KOKKOS_LAMBDA(const LO rlid) {
4254 auto rowNNZ = numRowEntries(rlid);
4255 if (rowNNZ == 0) {
4256 return;
4257 }
4258 auto rowBegin = rowptr(rlid);
4259 auto pos = rowBegin;
4260 for (size_t offset = rowBegin + 1; offset < rowBegin + rowNNZ; ++offset) {
4261 if ((colinds(offset) != colinds(pos))) {
4262 colinds(++pos) = colinds(offset);
4263 }
4264 }
4265 numRowEntries(rlid) = pos + 1 - rowBegin;
4266 });
4267}
4268
4269template <class LocalOrdinal, class GlobalOrdinal, class Node>
4270void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4271 sortAndMergeAllIndices(const bool sorted, const bool merged) {
4272 using std::endl;
4273 const char tfecfFuncName[] = "sortAndMergeAllIndices";
4274
4275 std::unique_ptr<std::string> prefix;
4276 if (verbose_) {
4277 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4278 std::ostringstream os;
4279 os << *prefix << "Start: "
4280 << "sorted=" << (sorted ? "true" : "false")
4281 << ", merged=" << (merged ? "true" : "false") << endl;
4282 std::cerr << os.str();
4283 }
4284 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isGloballyIndexed(), std::logic_error,
4285 "This method may only be called after makeIndicesLocal.");
4286 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!merged && this->isStorageOptimized(), std::logic_error,
4287 "The graph is already storage optimized, so we shouldn't be "
4288 "merging any indices. "
4289 "Please report this bug to the Tpetra developers.");
4290
4291 if (!sorted || !merged) {
4292 Details::ProfilingRegion regionSortAndMerge("Tpetra::CrsGraph::sortAndMergeAllIndices");
4293
4294 if (storageStatus_ == Details::STORAGE_1D_UNPACKED) {
4295 // We are sorting & merging the unpacked views.
4296 // This means that not all entries are actually in use. We need to take k_numRowEntries_ into account.
4297 auto rowptr = rowPtrsUnpacked_dev_;
4298 auto colinds = lclIndsUnpacked_wdv.getDeviceView(Access::ReadWrite);
4299
4300 // Create a device copy of k_numRowEntries_.
4301 auto k_numRowEntries_d = Kokkos::create_mirror_view_and_copy(execution_space(), k_numRowEntries_);
4302
4303 // set set unused column entries so they get sorted last
4304 prepareSortMergeUnpackedGraph<execution_space, LocalOrdinal>(rowptr, colinds, k_numRowEntries_d);
4305
4306 if (!sorted) {
4307 // For this to work correctly, we require that the unused column entries have been filled
4308 // with indices that get ordered last.
4309 Import_Util::sortCrsEntries(rowptr, colinds);
4310 this->indicesAreSorted_ = true; // we just sorted every row
4311 }
4312 if (!merged) {
4313 mergeUnpackedGraph<execution_space, LocalOrdinal>(rowptr, colinds, k_numRowEntries_d);
4314 Kokkos::deep_copy(k_numRowEntries_, k_numRowEntries_d);
4315 this->noRedundancies_ = true; // we just merged every row
4316 }
4317 } else {
4318 auto rowptr = rowPtrsPacked_dev_;
4319 auto colinds = lclIndsPacked_wdv.getDeviceView(Access::ReadWrite);
4320 if (!sorted && merged) {
4321 Import_Util::sortCrsEntries(rowptr, colinds);
4322 this->indicesAreSorted_ = true; // we just sorted every row
4323 } else {
4324 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
4325 "We should never get here."
4326 "Please report this bug to the Tpetra developers.");
4327 }
4328 }
4329 }
4330
4331 if (verbose_) {
4332 std::ostringstream os;
4333 os << *prefix << "Done" << endl;
4334 std::cerr << os.str();
4335 }
4336}
4337
4338template <class LocalOrdinal, class GlobalOrdinal, class Node>
4339void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4340 makeImportExport(Teuchos::Array<int>& remotePIDs,
4341 const bool useRemotePIDs) {
4342 using Teuchos::ParameterList;
4343 using Teuchos::RCP;
4344 using Teuchos::rcp;
4345 using ::Tpetra::Details::ProfilingRegion;
4346 const char tfecfFuncName[] = "makeImportExport: ";
4347 ProfilingRegion regionMIE("Tpetra::CrsGraph::makeImportExport");
4348
4349 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::logic_error,
4350 "This method may not be called unless the graph has a column Map.");
4351 RCP<ParameterList> params = this->getNonconstParameterList(); // could be null
4352
4353 // Don't do any checks to see if we need to create the Import, if
4354 // it exists already.
4355 //
4356 // FIXME (mfh 25 Mar 2013) This will become incorrect if we
4357 // change CrsGraph in the future to allow changing the column
4358 // Map after fillComplete. For now, the column Map is fixed
4359 // after the first fillComplete call.
4360 if (importer_.is_null()) {
4361 // Create the Import instance if necessary.
4362 if (domainMap_ != colMap_ && (!domainMap_->isSameAs(*colMap_))) {
4363 if (params.is_null() || !params->isSublist("Import")) {
4364 if (useRemotePIDs) {
4365 importer_ = rcp(new import_type(domainMap_, colMap_, remotePIDs));
4366 } else {
4367 importer_ = rcp(new import_type(domainMap_, colMap_));
4368 }
4369 } else {
4370 RCP<ParameterList> importSublist = sublist(params, "Import", true);
4371 if (useRemotePIDs) {
4372 RCP<import_type> newImp =
4373 rcp(new import_type(domainMap_, colMap_, remotePIDs,
4374 importSublist));
4375 importer_ = newImp;
4376 } else {
4377 importer_ = rcp(new import_type(domainMap_, colMap_, importSublist));
4378 }
4379 }
4380 }
4381 }
4382
4383 // Don't do any checks to see if we need to create the Export, if
4384 // it exists already.
4385 if (exporter_.is_null()) {
4386 // Create the Export instance if necessary.
4387 if (rangeMap_ != rowMap_ && !rangeMap_->isSameAs(*rowMap_)) {
4388 if (params.is_null() || !params->isSublist("Export")) {
4389 exporter_ = rcp(new export_type(rowMap_, rangeMap_));
4390 } else {
4391 RCP<ParameterList> exportSublist = sublist(params, "Export", true);
4392 exporter_ = rcp(new export_type(rowMap_, rangeMap_, exportSublist));
4393 }
4394 }
4395 }
4396}
4397
4398template <class LocalOrdinal, class GlobalOrdinal, class Node>
4399std::string
4400CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4401 description() const {
4402 std::ostringstream oss;
4403 oss << dist_object_type::description();
4404 if (isFillComplete()) {
4405 oss << "{status = fill complete"
4406 << ", global rows = " << getGlobalNumRows()
4407 << ", global cols = " << getGlobalNumCols()
4408 << ", global num entries = " << getGlobalNumEntries()
4409 << "}";
4410 } else {
4411 oss << "{status = fill not complete"
4412 << ", global rows = " << getGlobalNumRows()
4413 << "}";
4414 }
4415 return oss.str();
4416}
4417
4418template <class LocalOrdinal, class GlobalOrdinal, class Node>
4419void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4420 describe(Teuchos::FancyOStream& out,
4421 const Teuchos::EVerbosityLevel verbLevel) const {
4422 using std::endl;
4423 using std::setw;
4424 using Teuchos::ArrayView;
4425 using Teuchos::Comm;
4426 using Teuchos::RCP;
4427 using Teuchos::VERB_DEFAULT;
4428 using Teuchos::VERB_EXTREME;
4429 using Teuchos::VERB_HIGH;
4430 using Teuchos::VERB_LOW;
4431 using Teuchos::VERB_MEDIUM;
4432 using Teuchos::VERB_NONE;
4433
4434 Teuchos::EVerbosityLevel vl = verbLevel;
4435 if (vl == VERB_DEFAULT) vl = VERB_LOW;
4436 RCP<const Comm<int>> comm = this->getComm();
4437 const int myImageID = comm->getRank(),
4438 numImages = comm->getSize();
4439 size_t width = 1;
4440 for (size_t dec = 10; dec < getGlobalNumRows(); dec *= 10) {
4441 ++width;
4442 }
4443 width = std::max<size_t>(width, static_cast<size_t>(11)) + 2;
4444 Teuchos::OSTab tab(out);
4445 // none: print nothing
4446 // low: print O(1) info from node 0
4447 // medium: print O(P) info, num entries per node
4448 // high: print O(N) info, num entries per row
4449 // extreme: print O(NNZ) info: print graph indices
4450 //
4451 // for medium and higher, print constituent objects at specified verbLevel
4452 if (vl != VERB_NONE) {
4453 if (myImageID == 0) out << this->description() << std::endl;
4454 // O(1) globals, minus what was already printed by description()
4455 if (isFillComplete() && myImageID == 0) {
4456 out << "Global max number of row entries = " << globalMaxNumRowEntries_ << std::endl;
4457 }
4458 // constituent objects
4459 if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
4460 if (myImageID == 0) out << "\nRow map: " << std::endl;
4461 rowMap_->describe(out, vl);
4462 if (colMap_ != Teuchos::null) {
4463 if (myImageID == 0) out << "\nColumn map: " << std::endl;
4464 colMap_->describe(out, vl);
4465 }
4466 if (domainMap_ != Teuchos::null) {
4467 if (myImageID == 0) out << "\nDomain map: " << std::endl;
4468 domainMap_->describe(out, vl);
4469 }
4470 if (rangeMap_ != Teuchos::null) {
4471 if (myImageID == 0) out << "\nRange map: " << std::endl;
4472 rangeMap_->describe(out, vl);
4473 }
4474 }
4475 // O(P) data
4476 if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
4477 for (int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
4478 if (myImageID == imageCtr) {
4479 out << "Node ID = " << imageCtr << std::endl
4480 << "Node number of entries = " << this->getLocalNumEntries() << std::endl
4481 << "Node max number of entries = " << nodeMaxNumRowEntries_ << std::endl;
4482 if (!indicesAreAllocated()) {
4483 out << "Indices are not allocated." << std::endl;
4484 }
4485 }
4486 comm->barrier();
4487 comm->barrier();
4488 comm->barrier();
4489 }
4490 }
4491 // O(N) and O(NNZ) data
4492 if (vl == VERB_HIGH || vl == VERB_EXTREME) {
4493 for (int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
4494 if (myImageID == imageCtr) {
4495 out << std::setw(width) << "Node ID"
4496 << std::setw(width) << "Global Row"
4497 << std::setw(width) << "Num Entries";
4498 if (vl == VERB_EXTREME) {
4499 out << " Entries";
4500 }
4501 out << std::endl;
4502 const LocalOrdinal lclNumRows =
4503 static_cast<LocalOrdinal>(this->getLocalNumRows());
4504 for (LocalOrdinal r = 0; r < lclNumRows; ++r) {
4505 const RowInfo rowinfo = this->getRowInfo(r);
4506 GlobalOrdinal gid = rowMap_->getGlobalElement(r);
4507 out << std::setw(width) << myImageID
4508 << std::setw(width) << gid
4509 << std::setw(width) << rowinfo.numEntries;
4510 if (vl == VERB_EXTREME) {
4511 out << " ";
4512 if (isGloballyIndexed()) {
4513 auto rowview = gblInds_wdv.getHostView(Access::ReadOnly);
4514 for (size_t j = 0; j < rowinfo.numEntries; ++j) {
4515 GlobalOrdinal colgid = rowview[j + rowinfo.offset1D];
4516 out << colgid << " ";
4517 }
4518 } else if (isLocallyIndexed()) {
4519 auto rowview = lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
4520 for (size_t j = 0; j < rowinfo.numEntries; ++j) {
4521 LocalOrdinal collid = rowview[j + rowinfo.offset1D];
4522 out << colMap_->getGlobalElement(collid) << " ";
4523 }
4524 }
4525 }
4526 out << std::endl;
4527 }
4528 }
4529 comm->barrier();
4530 comm->barrier();
4531 comm->barrier();
4532 }
4533 }
4534 }
4535}
4536
4537template <class LocalOrdinal, class GlobalOrdinal, class Node>
4538bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4539 checkSizes(const SrcDistObject& /* source */) {
4540 // It's not clear what kind of compatibility checks on sizes can
4541 // be performed here. Epetra_CrsGraph doesn't check any sizes for
4542 // compatibility.
4543 return true;
4544}
4545
4546template <class LocalOrdinal, class GlobalOrdinal, class Node>
4547void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4548 copyAndPermute(const SrcDistObject& source,
4549 const size_t numSameIDs,
4550 const Kokkos::DualView<const local_ordinal_type*,
4551 buffer_device_type>& permuteToLIDs,
4552 const Kokkos::DualView<const local_ordinal_type*,
4553 buffer_device_type>& permuteFromLIDs,
4554 const CombineMode /*CM*/) {
4555 using std::endl;
4556 using LO = local_ordinal_type;
4557 using GO = global_ordinal_type;
4558 using this_CRS_type = CrsGraph<LO, GO, node_type>;
4559 const char tfecfFuncName[] = "copyAndPermute: ";
4560 const bool verbose = verbose_;
4561
4562 if (Details::Behavior::useNewCopyAndPermute()) {
4563 const row_graph_type& srcRowGraph = dynamic_cast<const row_graph_type&>(source);
4564 copyAndPermuteNew(srcRowGraph, *this, numSameIDs, permuteToLIDs, permuteFromLIDs, INSERT);
4565 return;
4566 }
4567
4568 Details::ProfilingRegion regionCAP("Tpetra::CrsGraph::copyAndPermute");
4569
4570 std::unique_ptr<std::string> prefix;
4571 if (verbose) {
4572 prefix = this->createPrefix("CrsGraph", "copyAndPermute");
4573 std::ostringstream os;
4574 os << *prefix << endl;
4575 std::cerr << os.str();
4576 }
4577
4578 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(permuteToLIDs.extent(0) != permuteFromLIDs.extent(0),
4579 std::runtime_error, "permuteToLIDs.extent(0) = " << permuteToLIDs.extent(0) << " != permuteFromLIDs.extent(0) = " << permuteFromLIDs.extent(0) << ".");
4580
4581 // We know from checkSizes that the source object is a
4582 // row_graph_type, so we don't need to check again.
4583 const row_graph_type& srcRowGraph =
4584 dynamic_cast<const row_graph_type&>(source);
4585
4586 if (verbose) {
4587 std::ostringstream os;
4588 os << *prefix << "Compute padding" << endl;
4589 std::cerr << os.str();
4590 }
4591 auto padding = computeCrsPadding(srcRowGraph, numSameIDs,
4592 permuteToLIDs, permuteFromLIDs, verbose);
4593 applyCrsPadding(*padding, verbose);
4594
4595 // If the source object is actually a CrsGraph, we can use view
4596 // mode instead of copy mode to access the entries in each row,
4597 // if the graph is not fill complete.
4598 const this_CRS_type* srcCrsGraph =
4599 dynamic_cast<const this_CRS_type*>(&source);
4600
4601 const map_type& srcRowMap = *(srcRowGraph.getRowMap());
4602 const map_type& tgtRowMap = *(getRowMap());
4603 const bool src_filled = srcRowGraph.isFillComplete();
4604 nonconst_global_inds_host_view_type row_copy;
4605 LO myid = 0;
4606
4607 //
4608 // "Copy" part of "copy and permute."
4609 //
4610 if (src_filled || srcCrsGraph == nullptr) {
4611 if (verbose) {
4612 std::ostringstream os;
4613 os << *prefix << "src_filled || srcCrsGraph == nullptr" << endl;
4614 std::cerr << os.str();
4615 }
4616 // If the source graph is fill complete, we can't use view mode,
4617 // because the data might be stored in a different format not
4618 // compatible with the expectations of view mode. Also, if the
4619 // source graph is not a CrsGraph, we can't use view mode,
4620 // because RowGraph only provides copy mode access to the data.
4621 for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
4622 const GO gid = srcRowMap.getGlobalElement(myid);
4623 size_t row_length = srcRowGraph.getNumEntriesInGlobalRow(gid);
4624 Kokkos::resize(row_copy, row_length);
4625 size_t check_row_length = 0;
4626 srcRowGraph.getGlobalRowCopy(gid, row_copy, check_row_length);
4627 this->insertGlobalIndices(gid, row_length, row_copy.data());
4628 }
4629 } else {
4630 if (verbose) {
4631 std::ostringstream os;
4632 os << *prefix << "! src_filled && srcCrsGraph != nullptr" << endl;
4633 std::cerr << os.str();
4634 }
4635 for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
4636 const GO gid = srcRowMap.getGlobalElement(myid);
4637 global_inds_host_view_type row;
4638 srcCrsGraph->getGlobalRowView(gid, row);
4639 this->insertGlobalIndices(gid, row.extent(0), row.data());
4640 }
4641 }
4642
4643 //
4644 // "Permute" part of "copy and permute."
4645 //
4646 auto permuteToLIDs_h = permuteToLIDs.view_host();
4647 auto permuteFromLIDs_h = permuteFromLIDs.view_host();
4648
4649 if (src_filled || srcCrsGraph == nullptr) {
4650 for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
4651 const GO mygid = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
4652 const GO srcgid = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
4653 size_t row_length = srcRowGraph.getNumEntriesInGlobalRow(srcgid);
4654 Kokkos::resize(row_copy, row_length);
4655 size_t check_row_length = 0;
4656 srcRowGraph.getGlobalRowCopy(srcgid, row_copy, check_row_length);
4657 this->insertGlobalIndices(mygid, row_length, row_copy.data());
4658 }
4659 } else {
4660 for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
4661 const GO mygid = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
4662 const GO srcgid = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
4663 global_inds_host_view_type row;
4664 srcCrsGraph->getGlobalRowView(srcgid, row);
4665 this->insertGlobalIndices(mygid, row.extent(0), row.data());
4666 }
4667 }
4668
4669 if (verbose) {
4670 std::ostringstream os;
4671 os << *prefix << "Done" << endl;
4672 std::cerr << os.str();
4673 }
4674}
4675
4676template <class LocalOrdinal, class GlobalOrdinal, class Node>
4677void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4678 applyCrsPadding(const padding_type& padding,
4679 const bool verbose) {
4680 using Details::padCrsArrays;
4681 using Details::ProfilingRegion;
4682 using std::endl;
4683 using LO = local_ordinal_type;
4684 using row_ptrs_type =
4685 typename local_graph_device_type::row_map_type::non_const_type;
4686 using range_policy =
4687 Kokkos::RangePolicy<execution_space, Kokkos::IndexType<LO>>;
4688 const char tfecfFuncName[] = "applyCrsPadding";
4689 ProfilingRegion regionCAP("Tpetra::CrsGraph::applyCrsPadding");
4690
4691 std::unique_ptr<std::string> prefix;
4692 if (verbose) {
4693 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4694 std::ostringstream os;
4695 os << *prefix << "padding: ";
4696 padding.print(os);
4697 os << endl;
4698 std::cerr << os.str();
4699 }
4700 const int myRank = !verbose ? -1 : [&]() {
4701 auto map = this->getMap();
4702 if (map.is_null()) {
4703 return -1;
4704 }
4705 auto comm = map->getComm();
4706 if (comm.is_null()) {
4707 return -1;
4708 }
4709 return comm->getRank();
4710 }();
4711
4712 // FIXME (mfh 10 Feb 2020) We shouldn't actually reallocate
4713 // row_ptrs_beg or allocate row_ptrs_end unless the allocation
4714 // size needs to increase. That should be the job of
4715 // padCrsArrays.
4716
4717 // Assume global indexing we don't have any indices yet
4718 if (!indicesAreAllocated()) {
4719 if (verbose) {
4720 std::ostringstream os;
4721 os << *prefix << "Call allocateIndices" << endl;
4722 std::cerr << os.str();
4723 }
4724 allocateIndices(GlobalIndices, verbose);
4725 }
4726 TEUCHOS_ASSERT(indicesAreAllocated());
4727
4728 // Making copies here because k_rowPtrs_ has a const type. Otherwise, we
4729 // would use it directly.
4730
4731 auto rowPtrsUnpacked_dev = this->getRowPtrsUnpackedDevice();
4732 if (verbose) {
4733 std::ostringstream os;
4734 os << *prefix << "Allocate row_ptrs_beg: "
4735 << rowPtrsUnpacked_dev.extent(0) << endl;
4736 std::cerr << os.str();
4737 }
4738 using Kokkos::view_alloc;
4739 using Kokkos::WithoutInitializing;
4740 row_ptrs_type row_ptrs_beg(
4741 view_alloc("row_ptrs_beg", WithoutInitializing),
4742 rowPtrsUnpacked_dev.extent(0));
4743 // DEEP_COPY REVIEW - DEVICE-TO-DEVICE
4744 Kokkos::deep_copy(execution_space(), row_ptrs_beg, rowPtrsUnpacked_dev);
4745
4746 const size_t N = row_ptrs_beg.extent(0) == 0 ? size_t(0) : size_t(row_ptrs_beg.extent(0) - 1);
4747 if (verbose) {
4748 std::ostringstream os;
4749 os << *prefix << "Allocate row_ptrs_end: " << N << endl;
4750 std::cerr << os.str();
4751 }
4752 row_ptrs_type row_ptrs_end(
4753 view_alloc("row_ptrs_end", WithoutInitializing), N);
4754 row_ptrs_type num_row_entries;
4755
4756 const bool refill_num_row_entries = k_numRowEntries_.extent(0) != 0;
4757
4758 execution_space().fence(); // we need above deep_copy to be done
4759
4760 if (refill_num_row_entries) { // Case 1: Unpacked storage
4761 // We can't assume correct *this capture until C++17, and it's
4762 // likely more efficient just to capture what we need anyway.
4763 num_row_entries =
4764 row_ptrs_type(view_alloc("num_row_entries", WithoutInitializing), N);
4765 Kokkos::deep_copy(num_row_entries, this->k_numRowEntries_);
4766 Kokkos::parallel_for(
4767 "Fill end row pointers", range_policy(0, N),
4768 KOKKOS_LAMBDA(const size_t i) {
4769 row_ptrs_end(i) = row_ptrs_beg(i) + num_row_entries(i);
4770 });
4771 } else {
4772 // FIXME (mfh 10 Feb 2020) Fix padCrsArrays so that if packed
4773 // storage, we don't need row_ptr_end to be separate allocation;
4774 // could just have it alias row_ptr_beg+1.
4775 Kokkos::parallel_for(
4776 "Fill end row pointers", range_policy(0, N),
4777 KOKKOS_LAMBDA(const size_t i) {
4778 row_ptrs_end(i) = row_ptrs_beg(i + 1);
4779 });
4780 }
4781
4782 if (isGloballyIndexed()) {
4783 padCrsArrays(row_ptrs_beg, row_ptrs_end, gblInds_wdv,
4784 padding, myRank, verbose);
4785 } else {
4786 padCrsArrays(row_ptrs_beg, row_ptrs_end, lclIndsUnpacked_wdv,
4787 padding, myRank, verbose);
4788 }
4789
4790 if (refill_num_row_entries) {
4791 Kokkos::parallel_for(
4792 "Fill num entries", range_policy(0, N),
4793 KOKKOS_LAMBDA(const size_t i) {
4794 num_row_entries(i) = row_ptrs_end(i) - row_ptrs_beg(i);
4795 });
4796 Kokkos::deep_copy(this->k_numRowEntries_, num_row_entries);
4797 }
4798 if (verbose) {
4799 std::ostringstream os;
4800 os << *prefix << "Reassign k_rowPtrs_; old size: "
4801 << rowPtrsUnpacked_dev.extent(0) << ", new size: "
4802 << row_ptrs_beg.extent(0) << endl;
4803 std::cerr << os.str();
4804 TEUCHOS_ASSERT(rowPtrsUnpacked_dev.extent(0) == row_ptrs_beg.extent(0));
4805 }
4806
4807 setRowPtrsUnpacked(row_ptrs_beg);
4808}
4809
4810template <class LocalOrdinal, class GlobalOrdinal, class Node>
4811std::unique_ptr<
4812 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
4813CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4814 computeCrsPadding(
4815 const RowGraph<LocalOrdinal, GlobalOrdinal, Node>& source,
4816 const size_t numSameIDs,
4817 const Kokkos::DualView<const local_ordinal_type*,
4818 buffer_device_type>& permuteToLIDs,
4819 const Kokkos::DualView<const local_ordinal_type*,
4820 buffer_device_type>& permuteFromLIDs,
4821 const bool verbose) const {
4822 using LO = local_ordinal_type;
4823 using std::endl;
4824
4825 std::unique_ptr<std::string> prefix;
4826 if (verbose) {
4827 prefix = this->createPrefix("CrsGraph",
4828 "computeCrsPadding(same & permute)");
4829 std::ostringstream os;
4830 os << *prefix << "{numSameIDs: " << numSameIDs
4831 << ", numPermutes: " << permuteFromLIDs.extent(0) << "}"
4832 << endl;
4833 std::cerr << os.str();
4834 }
4835
4836 const int myRank = [&]() {
4837 auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
4838 return comm.is_null() ? -1 : comm->getRank();
4839 }();
4840 std::unique_ptr<padding_type> padding(
4841 new padding_type(myRank, numSameIDs,
4842 permuteFromLIDs.extent(0)));
4843
4844 computeCrsPaddingForSameIDs(*padding, source,
4845 static_cast<LO>(numSameIDs));
4846 computeCrsPaddingForPermutedIDs(*padding, source, permuteToLIDs,
4847 permuteFromLIDs);
4848 return padding;
4849}
4850
4851template <class LocalOrdinal, class GlobalOrdinal, class Node>
4852void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4853 computeCrsPaddingForSameIDs(
4854 padding_type& padding,
4855 const RowGraph<local_ordinal_type, global_ordinal_type,
4856 node_type>& source,
4857 const local_ordinal_type numSameIDs) const {
4858 using LO = local_ordinal_type;
4859 using GO = global_ordinal_type;
4860 using Details::Impl::getRowGraphGlobalRow;
4861 using std::endl;
4862 const char tfecfFuncName[] = "computeCrsPaddingForSameIds";
4863
4864 std::unique_ptr<std::string> prefix;
4865 const bool verbose = verbose_;
4866 if (verbose) {
4867 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4868 std::ostringstream os;
4869 os << *prefix << "numSameIDs: " << numSameIDs << endl;
4870 std::cerr << os.str();
4871 }
4872
4873 if (numSameIDs == 0) {
4874 return;
4875 }
4876
4877 const map_type& srcRowMap = *(source.getRowMap());
4878 const map_type& tgtRowMap = *rowMap_;
4879 using this_CRS_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
4880 const this_CRS_type* srcCrs = dynamic_cast<const this_CRS_type*>(&source);
4881 const bool src_is_unique =
4882 srcCrs == nullptr ? false : srcCrs->isMerged();
4883 const bool tgt_is_unique = this->isMerged();
4884
4885 std::vector<GO> srcGblColIndsScratch;
4886 std::vector<GO> tgtGblColIndsScratch;
4887
4888 execute_sync_host_uvm_access(); // protect host UVM access
4889 for (LO lclRowInd = 0; lclRowInd < numSameIDs; ++lclRowInd) {
4890 const GO srcGblRowInd = srcRowMap.getGlobalElement(lclRowInd);
4891 const GO tgtGblRowInd = tgtRowMap.getGlobalElement(lclRowInd);
4892 auto srcGblColInds = getRowGraphGlobalRow(
4893 srcGblColIndsScratch, source, srcGblRowInd);
4894 auto tgtGblColInds = getRowGraphGlobalRow(
4895 tgtGblColIndsScratch, *this, tgtGblRowInd);
4896 padding.update_same(lclRowInd, tgtGblColInds.getRawPtr(),
4897 tgtGblColInds.size(), tgt_is_unique,
4898 srcGblColInds.getRawPtr(),
4899 srcGblColInds.size(), src_is_unique);
4900 }
4901 if (verbose) {
4902 std::ostringstream os;
4903 os << *prefix << "Done" << endl;
4904 std::cerr << os.str();
4905 }
4906}
4907
4908template <class LocalOrdinal, class GlobalOrdinal, class Node>
4909void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4910 computeCrsPaddingForPermutedIDs(
4911 padding_type& padding,
4912 const RowGraph<local_ordinal_type, global_ordinal_type,
4913 node_type>& source,
4914 const Kokkos::DualView<const local_ordinal_type*,
4915 buffer_device_type>& permuteToLIDs,
4916 const Kokkos::DualView<const local_ordinal_type*,
4917 buffer_device_type>& permuteFromLIDs) const {
4918 using LO = local_ordinal_type;
4919 using GO = global_ordinal_type;
4920 using Details::Impl::getRowGraphGlobalRow;
4921 using std::endl;
4922 const char tfecfFuncName[] = "computeCrsPaddingForPermutedIds";
4923
4924 std::unique_ptr<std::string> prefix;
4925 const bool verbose = verbose_;
4926 if (verbose) {
4927 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
4928 std::ostringstream os;
4929 os << *prefix << "permuteToLIDs.extent(0): "
4930 << permuteToLIDs.extent(0)
4931 << ", permuteFromLIDs.extent(0): "
4932 << permuteFromLIDs.extent(0) << endl;
4933 std::cerr << os.str();
4934 }
4935
4936 if (permuteToLIDs.extent(0) == 0) {
4937 return;
4938 }
4939
4940 const map_type& srcRowMap = *(source.getRowMap());
4941 const map_type& tgtRowMap = *rowMap_;
4942 using this_CRS_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
4943 const this_CRS_type* srcCrs = dynamic_cast<const this_CRS_type*>(&source);
4944 const bool src_is_unique =
4945 srcCrs == nullptr ? false : srcCrs->isMerged();
4946 const bool tgt_is_unique = this->isMerged();
4947
4948 TEUCHOS_ASSERT(!permuteToLIDs.need_sync_host());
4949 auto permuteToLIDs_h = permuteToLIDs.view_host();
4950 TEUCHOS_ASSERT(!permuteFromLIDs.need_sync_host());
4951 auto permuteFromLIDs_h = permuteFromLIDs.view_host();
4952
4953 std::vector<GO> srcGblColIndsScratch;
4954 std::vector<GO> tgtGblColIndsScratch;
4955 const LO numPermutes = static_cast<LO>(permuteToLIDs_h.extent(0));
4956
4957 execute_sync_host_uvm_access(); // protect host UVM access
4958 for (LO whichPermute = 0; whichPermute < numPermutes; ++whichPermute) {
4959 const LO srcLclRowInd = permuteFromLIDs_h[whichPermute];
4960 const GO srcGblRowInd = srcRowMap.getGlobalElement(srcLclRowInd);
4961 auto srcGblColInds = getRowGraphGlobalRow(
4962 srcGblColIndsScratch, source, srcGblRowInd);
4963 const LO tgtLclRowInd = permuteToLIDs_h[whichPermute];
4964 const GO tgtGblRowInd = tgtRowMap.getGlobalElement(tgtLclRowInd);
4965 auto tgtGblColInds = getRowGraphGlobalRow(
4966 tgtGblColIndsScratch, *this, tgtGblRowInd);
4967 padding.update_permute(whichPermute, tgtLclRowInd,
4968 tgtGblColInds.getRawPtr(),
4969 tgtGblColInds.size(), tgt_is_unique,
4970 srcGblColInds.getRawPtr(),
4971 srcGblColInds.size(), src_is_unique);
4972 }
4973
4974 if (verbose) {
4975 std::ostringstream os;
4976 os << *prefix << "Done" << endl;
4977 std::cerr << os.str();
4978 }
4979}
4980
4981template <class LocalOrdinal, class GlobalOrdinal, class Node>
4982std::unique_ptr<
4983 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
4984CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4985 computeCrsPaddingForImports(
4986 const Kokkos::DualView<const local_ordinal_type*,
4987 buffer_device_type>& importLIDs,
4988 Kokkos::DualView<packet_type*, buffer_device_type> imports,
4989 Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
4990 const bool verbose) const {
4991 using Details::Impl::getRowGraphGlobalRow;
4992 using std::endl;
4993 using LO = local_ordinal_type;
4994 using GO = global_ordinal_type;
4995 const char tfecfFuncName[] = "computeCrsPaddingForImports";
4996
4997 std::unique_ptr<std::string> prefix;
4998 if (verbose) {
4999 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
5000 std::ostringstream os;
5001 os << *prefix << "importLIDs.extent(0): "
5002 << importLIDs.extent(0)
5003 << ", imports.extent(0): "
5004 << imports.extent(0)
5005 << ", numPacketsPerLID.extent(0): "
5006 << numPacketsPerLID.extent(0) << endl;
5007 std::cerr << os.str();
5008 }
5009
5010 const LO numImports = static_cast<LO>(importLIDs.extent(0));
5011 const int myRank = [&]() {
5012 auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
5013 return comm.is_null() ? -1 : comm->getRank();
5014 }();
5015 std::unique_ptr<padding_type> padding(
5016 new padding_type(myRank, numImports));
5017
5018 if (imports.need_sync_host()) {
5019 imports.sync_host();
5020 }
5021 auto imports_h = imports.view_host();
5022 if (numPacketsPerLID.need_sync_host()) {
5023 numPacketsPerLID.sync_host();
5024 }
5025 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5026
5027 TEUCHOS_ASSERT(!importLIDs.need_sync_host());
5028 auto importLIDs_h = importLIDs.view_host();
5029
5030 const map_type& tgtRowMap = *rowMap_;
5031 // Always merge source column indices, since isMerged() is
5032 // per-process state, and we don't know its value on other
5033 // processes that sent us data.
5034 constexpr bool src_is_unique = false;
5035 const bool tgt_is_unique = isMerged();
5036
5037 std::vector<GO> tgtGblColIndsScratch;
5038 size_t offset = 0;
5039 execute_sync_host_uvm_access(); // protect host UVM access
5040 for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
5041 // CrsGraph packs just global column indices, while CrsMatrix
5042 // packs bytes (first the number of entries in the row, then the
5043 // global column indices, then other stuff like the matrix
5044 // values in that row).
5045 const LO origSrcNumEnt =
5046 static_cast<LO>(numPacketsPerLID_h[whichImport]);
5047 GO* const srcGblColInds = imports_h.data() + offset;
5048
5049 const LO tgtLclRowInd = importLIDs_h[whichImport];
5050 const GO tgtGblRowInd =
5051 tgtRowMap.getGlobalElement(tgtLclRowInd);
5052 auto tgtGblColInds = getRowGraphGlobalRow(
5053 tgtGblColIndsScratch, *this, tgtGblRowInd);
5054 const size_t origTgtNumEnt(tgtGblColInds.size());
5055
5056 padding->update_import(whichImport, tgtLclRowInd,
5057 tgtGblColInds.getRawPtr(),
5058 origTgtNumEnt, tgt_is_unique,
5059 srcGblColInds,
5060 origSrcNumEnt, src_is_unique);
5061 offset += origSrcNumEnt;
5062 }
5063
5064 if (verbose) {
5065 std::ostringstream os;
5066 os << *prefix << "Done" << endl;
5067 std::cerr << os.str();
5068 }
5069 return padding;
5070}
5071
5072template <class LocalOrdinal, class GlobalOrdinal, class Node>
5073std::unique_ptr<
5074 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
5075CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5076 computePaddingForCrsMatrixUnpack(
5077 const Kokkos::DualView<const local_ordinal_type*,
5078 buffer_device_type>& importLIDs,
5079 Kokkos::DualView<char*, buffer_device_type> imports,
5080 Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
5081 const bool verbose) const {
5082 using Details::PackTraits;
5083 using Details::Impl::getRowGraphGlobalRow;
5084 using std::endl;
5085 using LO = local_ordinal_type;
5086 using GO = global_ordinal_type;
5087 const char tfecfFuncName[] = "computePaddingForCrsMatrixUnpack";
5088
5089 std::unique_ptr<std::string> prefix;
5090 if (verbose) {
5091 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
5092 std::ostringstream os;
5093 os << *prefix << "importLIDs.extent(0): "
5094 << importLIDs.extent(0)
5095 << ", imports.extent(0): "
5096 << imports.extent(0)
5097 << ", numPacketsPerLID.extent(0): "
5098 << numPacketsPerLID.extent(0) << endl;
5099 std::cerr << os.str();
5100 }
5101 const bool extraVerbose =
5102 verbose && Details::Behavior::verbose("CrsPadding");
5103
5104 const LO numImports = static_cast<LO>(importLIDs.extent(0));
5105 TEUCHOS_ASSERT(LO(numPacketsPerLID.extent(0)) >= numImports);
5106 const int myRank = [&]() {
5107 auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
5108 return comm.is_null() ? -1 : comm->getRank();
5109 }();
5110 std::unique_ptr<padding_type> padding(
5111 new padding_type(myRank, numImports));
5112
5113 if (imports.need_sync_host()) {
5114 imports.sync_host();
5115 }
5116 auto imports_h = imports.view_host();
5117 if (numPacketsPerLID.need_sync_host()) {
5118 numPacketsPerLID.sync_host();
5119 }
5120 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5121
5122 TEUCHOS_ASSERT(!importLIDs.need_sync_host());
5123 auto importLIDs_h = importLIDs.view_host();
5124
5125 const map_type& tgtRowMap = *rowMap_;
5126 // Always merge source column indices, since isMerged() is
5127 // per-process state, and we don't know its value on other
5128 // processes that sent us data.
5129 constexpr bool src_is_unique = false;
5130 const bool tgt_is_unique = isMerged();
5131
5132 std::vector<GO> srcGblColIndsScratch;
5133 std::vector<GO> tgtGblColIndsScratch;
5134 size_t offset = 0;
5135 execute_sync_host_uvm_access(); // protect host UVM access
5136 for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
5137 // CrsGraph packs just global column indices, while CrsMatrix
5138 // packs bytes (first the number of entries in the row, then the
5139 // global column indices, then other stuff like the matrix
5140 // values in that row).
5141 const size_t numBytes = numPacketsPerLID_h[whichImport];
5142 if (extraVerbose) {
5143 std::ostringstream os;
5144 os << *prefix << "whichImport=" << whichImport
5145 << ", numImports=" << numImports
5146 << ", numBytes=" << numBytes << endl;
5147 std::cerr << os.str();
5148 }
5149 if (numBytes == 0) {
5150 continue; // special case: no entries to unpack for this row
5151 }
5152 LO origSrcNumEnt = 0;
5153 const size_t numEntBeg = offset;
5154 const size_t numEntLen =
5155 PackTraits<LO>::packValueCount(origSrcNumEnt);
5156 TEUCHOS_ASSERT(numBytes >= numEntLen);
5157 TEUCHOS_ASSERT(imports_h.extent(0) >= numEntBeg + numEntLen);
5158 PackTraits<LO>::unpackValue(origSrcNumEnt,
5159 imports_h.data() + numEntBeg);
5160 if (extraVerbose) {
5161 std::ostringstream os;
5162 os << *prefix << "whichImport=" << whichImport
5163 << ", numImports=" << numImports
5164 << ", origSrcNumEnt=" << origSrcNumEnt << endl;
5165 std::cerr << os.str();
5166 }
5167 TEUCHOS_ASSERT(origSrcNumEnt >= LO(0));
5168 TEUCHOS_ASSERT(numBytes >= size_t(numEntLen + origSrcNumEnt * sizeof(GO)));
5169 const size_t gidsBeg = numEntBeg + numEntLen;
5170 if (srcGblColIndsScratch.size() < size_t(origSrcNumEnt)) {
5171 srcGblColIndsScratch.resize(origSrcNumEnt);
5172 }
5173 GO* const srcGblColInds = srcGblColIndsScratch.data();
5174 PackTraits<GO>::unpackArray(srcGblColInds,
5175 imports_h.data() + gidsBeg,
5176 origSrcNumEnt);
5177 const LO tgtLclRowInd = importLIDs_h[whichImport];
5178 const GO tgtGblRowInd =
5179 tgtRowMap.getGlobalElement(tgtLclRowInd);
5180 auto tgtGblColInds = getRowGraphGlobalRow(
5181 tgtGblColIndsScratch, *this, tgtGblRowInd);
5182 const size_t origNumTgtEnt(tgtGblColInds.size());
5183
5184 if (extraVerbose) {
5185 std::ostringstream os;
5186 os << *prefix << "whichImport=" << whichImport
5187 << ", numImports=" << numImports
5188 << ": Call padding->update_import" << endl;
5189 std::cerr << os.str();
5190 }
5191 padding->update_import(whichImport, tgtLclRowInd,
5192 tgtGblColInds.getRawPtr(),
5193 origNumTgtEnt, tgt_is_unique,
5194 srcGblColInds,
5195 origSrcNumEnt, src_is_unique);
5196 offset += numBytes;
5197 }
5198
5199 if (verbose) {
5200 std::ostringstream os;
5201 os << *prefix << "Done" << endl;
5202 std::cerr << os.str();
5203 }
5204 return padding;
5205}
5206
5207template <class LocalOrdinal, class GlobalOrdinal, class Node>
5208void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5209 packAndPrepare(const SrcDistObject& source,
5210 const Kokkos::DualView<const local_ordinal_type*,
5211 buffer_device_type>& exportLIDs,
5212 Kokkos::DualView<packet_type*,
5213 buffer_device_type>& exports,
5214 Kokkos::DualView<size_t*,
5215 buffer_device_type>
5216 numPacketsPerLID,
5217 size_t& constantNumPackets) {
5218 using Tpetra::Details::ProfilingRegion;
5219 using GO = global_ordinal_type;
5220 using std::endl;
5221 using crs_graph_type =
5222 CrsGraph<local_ordinal_type, global_ordinal_type, node_type>;
5223 const char tfecfFuncName[] = "packAndPrepare: ";
5224 ProfilingRegion region_papn("Tpetra::CrsGraph::packAndPrepare");
5225
5226 const bool verbose = verbose_;
5227 std::unique_ptr<std::string> prefix;
5228 if (verbose) {
5229 prefix = this->createPrefix("CrsGraph", "packAndPrepare");
5230 std::ostringstream os;
5231 os << *prefix << "Start" << endl;
5232 std::cerr << os.str();
5233 }
5234
5235 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(exportLIDs.extent(0) != numPacketsPerLID.extent(0),
5236 std::runtime_error,
5237 "exportLIDs.extent(0) = " << exportLIDs.extent(0)
5238 << " != numPacketsPerLID.extent(0) = " << numPacketsPerLID.extent(0)
5239 << ".");
5240 const row_graph_type* srcRowGraphPtr =
5241 dynamic_cast<const row_graph_type*>(&source);
5242 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(srcRowGraphPtr == nullptr, std::invalid_argument,
5243 "Source of an Export "
5244 "or Import operation to a CrsGraph must be a RowGraph with the same "
5245 "template parameters.");
5246 // We don't check whether src_graph has had fillComplete called,
5247 // because it doesn't matter whether the *source* graph has been
5248 // fillComplete'd. The target graph can not be fillComplete'd yet.
5249 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillComplete(), std::runtime_error,
5250 "The target graph of an Import or Export must not be fill complete.");
5251
5252 const crs_graph_type* srcCrsGraphPtr =
5253 dynamic_cast<const crs_graph_type*>(&source);
5254
5255 if (srcCrsGraphPtr == nullptr) {
5256 using Teuchos::ArrayView;
5257 using LO = local_ordinal_type;
5258
5259 if (verbose) {
5260 std::ostringstream os;
5261 os << *prefix << "Source is a RowGraph but not a CrsGraph"
5262 << endl;
5263 std::cerr << os.str();
5264 }
5265 // RowGraph::pack serves the "old" DistObject interface. It
5266 // takes Teuchos::ArrayView and Teuchos::Array&. The latter
5267 // entails deep-copying the exports buffer on output. RowGraph
5268 // is a convenience interface when not a CrsGraph, so we accept
5269 // the performance hit.
5270 TEUCHOS_ASSERT(!exportLIDs.need_sync_host());
5271 auto exportLIDs_h = exportLIDs.view_host();
5272 ArrayView<const LO> exportLIDs_av(exportLIDs_h.data(),
5273 exportLIDs_h.extent(0));
5274 Teuchos::Array<GO> exports_a;
5275
5276 numPacketsPerLID.clear_sync_state();
5277 numPacketsPerLID.modify_host();
5278 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5279 ArrayView<size_t> numPacketsPerLID_av(numPacketsPerLID_h.data(),
5280 numPacketsPerLID_h.extent(0));
5281 srcRowGraphPtr->pack(exportLIDs_av, exports_a, numPacketsPerLID_av,
5282 constantNumPackets);
5283 const size_t newSize = static_cast<size_t>(exports_a.size());
5284 if (static_cast<size_t>(exports.extent(0)) != newSize) {
5285 using exports_dv_type = Kokkos::DualView<packet_type*, buffer_device_type>;
5286 exports = exports_dv_type("exports", newSize);
5287 }
5288 Kokkos::View<const packet_type*, Kokkos::HostSpace,
5289 Kokkos::MemoryUnmanaged>
5290 exports_a_h(exports_a.getRawPtr(), newSize);
5291 exports.clear_sync_state();
5292 exports.modify_host();
5293 // DEEP_COPY REVIEW - NOT TESTED
5294 Kokkos::deep_copy(exports.view_host(), exports_a_h);
5295 }
5296 // packCrsGraphNew requires k_rowPtrsPacked_ to be set
5297 else if (!getColMap().is_null() &&
5298 (this->getRowPtrsPackedDevice().extent(0) != 0 ||
5299 getRowMap()->getLocalNumElements() == 0)) {
5300 if (verbose) {
5301 std::ostringstream os;
5302 os << *prefix << "packCrsGraphNew path" << endl;
5303 std::cerr << os.str();
5304 }
5305 using export_pids_type =
5306 Kokkos::DualView<const int*, buffer_device_type>;
5307 export_pids_type exportPIDs; // not filling it; needed for syntax
5308 using LO = local_ordinal_type;
5309 using NT = node_type;
5310 using Tpetra::Details::packCrsGraphNew;
5311 packCrsGraphNew<LO, GO, NT>(*srcCrsGraphPtr, exportLIDs, exportPIDs,
5312 exports, numPacketsPerLID,
5313 constantNumPackets, false);
5314 } else {
5315 srcCrsGraphPtr->packFillActiveNew(exportLIDs, exports, numPacketsPerLID,
5316 constantNumPackets);
5317 }
5318
5319 if (verbose) {
5320 std::ostringstream os;
5321 os << *prefix << "Done" << endl;
5322 std::cerr << os.str();
5323 }
5324}
5325
5326template <class LocalOrdinal, class GlobalOrdinal, class Node>
5327void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5328 pack(const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
5329 Teuchos::Array<GlobalOrdinal>& exports,
5330 const Teuchos::ArrayView<size_t>& numPacketsPerLID,
5331 size_t& constantNumPackets) const {
5332 auto col_map = this->getColMap();
5333 // packCrsGraph requires k_rowPtrsPacked to be set
5334 if (!col_map.is_null() && (this->getRowPtrsPackedDevice().extent(0) != 0 || getRowMap()->getLocalNumElements() == 0)) {
5335 using Tpetra::Details::packCrsGraph;
5336 packCrsGraph<LocalOrdinal, GlobalOrdinal, Node>(*this, exports, numPacketsPerLID,
5337 exportLIDs, constantNumPackets);
5338 } else {
5339 this->packFillActive(exportLIDs, exports, numPacketsPerLID,
5340 constantNumPackets);
5341 }
5342}
5343
5344template <class LocalOrdinal, class GlobalOrdinal, class Node>
5345void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5346 packFillActive(const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
5347 Teuchos::Array<GlobalOrdinal>& exports,
5348 const Teuchos::ArrayView<size_t>& numPacketsPerLID,
5349 size_t& constantNumPackets) const {
5350 using std::endl;
5351 using LO = LocalOrdinal;
5352 using GO = GlobalOrdinal;
5353 using host_execution_space =
5354 typename Kokkos::View<size_t*, device_type>::
5355 host_mirror_type::execution_space;
5356 const char tfecfFuncName[] = "packFillActive: ";
5357 const bool verbose = verbose_;
5358
5359 const auto numExportLIDs = exportLIDs.size();
5360 std::unique_ptr<std::string> prefix;
5361 if (verbose) {
5362 prefix = this->createPrefix("CrsGraph", "allocateIndices");
5363 std::ostringstream os;
5364 os << *prefix << "numExportLIDs=" << numExportLIDs << endl;
5365 std::cerr << os.str();
5366 }
5367 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numExportLIDs != numPacketsPerLID.size(), std::runtime_error,
5368 "exportLIDs.size() = " << numExportLIDs << " != numPacketsPerLID.size()"
5369 " = "
5370 << numPacketsPerLID.size() << ".");
5371
5372 const map_type& rowMap = *(this->getRowMap());
5373 const map_type* const colMapPtr = this->colMap_.getRawPtr();
5374 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() && colMapPtr == nullptr, std::logic_error,
5375 "This graph claims to be locally indexed, but its column Map is nullptr. "
5376 "This should never happen. Please report this bug to the Tpetra "
5377 "developers.");
5378
5379 // We may pack different amounts of data for different rows.
5380 constantNumPackets = 0;
5381
5382 // mfh 20 Sep 2017: Teuchos::ArrayView isn't thread safe (well,
5383 // it might be now, but we might as well be safe).
5384 size_t* const numPacketsPerLID_raw = numPacketsPerLID.getRawPtr();
5385 const LO* const exportLIDs_raw = exportLIDs.getRawPtr();
5386
5387 // Count the total number of packets (column indices, in the case
5388 // of a CrsGraph) to pack. While doing so, set
5389 // numPacketsPerLID[i] to the number of entries owned by the
5390 // calling process in (local) row exportLIDs[i] of the graph, that
5391 // the caller wants us to send out.
5392 Kokkos::RangePolicy<host_execution_space, LO> inputRange(0, numExportLIDs);
5393 size_t totalNumPackets = 0;
5394 size_t errCount = 0;
5395 // lambdas turn what they capture const, so we can't
5396 // atomic_add(&errCount,1). Instead, we need a View to modify.
5397 typedef Kokkos::Device<host_execution_space, Kokkos::HostSpace>
5398 host_device_type;
5399 Kokkos::View<size_t, host_device_type> errCountView(&errCount);
5400 constexpr size_t ONE = 1;
5401
5402 execute_sync_host_uvm_access(); // protect host UVM access
5403 Kokkos::parallel_reduce(
5404 "Tpetra::CrsGraph::pack: totalNumPackets",
5405 inputRange,
5406 [=, *this](const LO& i, size_t& curTotalNumPackets) {
5407 const GO gblRow = rowMap.getGlobalElement(exportLIDs_raw[i]);
5408 if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid()) {
5409 Kokkos::atomic_add(&errCountView(), ONE);
5410 numPacketsPerLID_raw[i] = 0;
5411 } else {
5412 const size_t numEnt = this->getNumEntriesInGlobalRow(gblRow);
5413 numPacketsPerLID_raw[i] = numEnt;
5414 curTotalNumPackets += numEnt;
5415 }
5416 },
5417 totalNumPackets);
5418
5419 if (verbose) {
5420 std::ostringstream os;
5421 os << *prefix << "totalNumPackets=" << totalNumPackets << endl;
5422 std::cerr << os.str();
5423 }
5424 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
5425 "totalNumPackets count encountered "
5426 "one or more errors! errCount = "
5427 << errCount
5428 << ", totalNumPackets = " << totalNumPackets << ".");
5429 errCount = 0;
5430
5431 // Allocate space for all the column indices to pack.
5432 exports.resize(totalNumPackets);
5433
5434 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->supportsRowViews(), std::logic_error,
5435 "this->supportsRowViews() returns false; this should never happen. "
5436 "Please report this bug to the Tpetra developers.");
5437
5438 // Loop again over the rows to export, and pack rows of indices
5439 // into the output buffer.
5440
5441 if (verbose) {
5442 std::ostringstream os;
5443 os << *prefix << "Pack into exports" << endl;
5444 std::cerr << os.str();
5445 }
5446
5447 // Teuchos::ArrayView may not be thread safe, or may not be
5448 // efficiently thread safe. Better to use the raw pointer.
5449 GO* const exports_raw = exports.getRawPtr();
5450 errCount = 0;
5451 Kokkos::parallel_scan("Tpetra::CrsGraph::pack: pack from views",
5452 inputRange, [=, &prefix, *this](const LO i, size_t& exportsOffset, const bool final) {
5453 const size_t curOffset = exportsOffset;
5454 const GO gblRow = rowMap.getGlobalElement(exportLIDs_raw[i]);
5455 const RowInfo rowInfo =
5456 this->getRowInfoFromGlobalRowIndex(gblRow);
5457
5458 using TDO = Tpetra::Details::OrdinalTraits<size_t>;
5459 if (rowInfo.localRow == TDO::invalid()) {
5460 if (verbose) {
5461 std::ostringstream os;
5462 os << *prefix << ": INVALID rowInfo: i=" << i
5463 << ", lclRow=" << exportLIDs_raw[i] << endl;
5464 std::cerr << os.str();
5465 }
5466 Kokkos::atomic_add(&errCountView(), ONE);
5467 } else if (curOffset + rowInfo.numEntries > totalNumPackets) {
5468 if (verbose) {
5469 std::ostringstream os;
5470 os << *prefix << ": UH OH! For i=" << i << ", lclRow="
5471 << exportLIDs_raw[i] << ", gblRow=" << gblRow << ", curOffset "
5472 "(= "
5473 << curOffset << ") + numEnt (= " << rowInfo.numEntries
5474 << ") > totalNumPackets (= " << totalNumPackets << ")."
5475 << endl;
5476 std::cerr << os.str();
5477 }
5478 Kokkos::atomic_add(&errCountView(), ONE);
5479 } else {
5480 const LO numEnt = static_cast<LO>(rowInfo.numEntries);
5481 if (this->isLocallyIndexed()) {
5482 auto lclColInds = getLocalIndsViewHost(rowInfo);
5483 if (final) {
5484 for (LO k = 0; k < numEnt; ++k) {
5485 const LO lclColInd = lclColInds(k);
5486 const GO gblColInd = colMapPtr->getGlobalElement(lclColInd);
5487 // Pack it, even if it's wrong. Let the receiving
5488 // process deal with it. Otherwise, we'll miss out
5489 // on any correct data.
5490 exports_raw[curOffset + k] = gblColInd;
5491 } // for each entry in the row
5492 } // final pass?
5493 exportsOffset = curOffset + numEnt;
5494 } else if (this->isGloballyIndexed()) {
5495 auto gblColInds = getGlobalIndsViewHost(rowInfo);
5496 if (final) {
5497 for (LO k = 0; k < numEnt; ++k) {
5498 const GO gblColInd = gblColInds(k);
5499 // Pack it, even if it's wrong. Let the receiving
5500 // process deal with it. Otherwise, we'll miss out
5501 // on any correct data.
5502 exports_raw[curOffset + k] = gblColInd;
5503 } // for each entry in the row
5504 } // final pass?
5505 exportsOffset = curOffset + numEnt;
5506 }
5507 // If neither globally nor locally indexed, then the graph
5508 // has no entries in this row (or indeed, in any row on this
5509 // process) to pack.
5510 }
5511 });
5512
5513 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
5514 "Packing encountered "
5515 "one or more errors! errCount = "
5516 << errCount
5517 << ", totalNumPackets = " << totalNumPackets << ".");
5518
5519 if (verbose) {
5520 std::ostringstream os;
5521 os << *prefix << "Done" << endl;
5522 std::cerr << os.str();
5523 }
5524}
5525
5526template <class LocalOrdinal, class GlobalOrdinal, class Node>
5527void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5528 packFillActiveNew(const Kokkos::DualView<const local_ordinal_type*,
5529 buffer_device_type>& exportLIDs,
5530 Kokkos::DualView<packet_type*,
5531 buffer_device_type>& exports,
5532 Kokkos::DualView<size_t*,
5533 buffer_device_type>
5534 numPacketsPerLID,
5535 size_t& constantNumPackets) const {
5536 using std::endl;
5537 using LO = local_ordinal_type;
5538 using GO = global_ordinal_type;
5539 using host_execution_space = typename Kokkos::View<size_t*,
5540 device_type>::host_mirror_type::execution_space;
5541 using host_device_type =
5542 Kokkos::Device<host_execution_space, Kokkos::HostSpace>;
5543 using exports_dv_type =
5544 Kokkos::DualView<packet_type*, buffer_device_type>;
5545 const char tfecfFuncName[] = "packFillActiveNew: ";
5546 const bool verbose = verbose_;
5547
5548 const auto numExportLIDs = exportLIDs.extent(0);
5549 std::unique_ptr<std::string> prefix;
5550 if (verbose) {
5551 prefix = this->createPrefix("CrsGraph", "packFillActiveNew");
5552 std::ostringstream os;
5553 os << *prefix << "numExportLIDs: " << numExportLIDs
5554 << ", numPacketsPerLID.extent(0): "
5555 << numPacketsPerLID.extent(0) << endl;
5556 std::cerr << os.str();
5557 }
5558 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numExportLIDs != numPacketsPerLID.extent(0), std::runtime_error,
5559 "exportLIDs.extent(0) = " << numExportLIDs
5560 << " != numPacketsPerLID.extent(0) = "
5561 << numPacketsPerLID.extent(0) << ".");
5562 TEUCHOS_ASSERT(!exportLIDs.need_sync_host());
5563 auto exportLIDs_h = exportLIDs.view_host();
5564
5565 const map_type& rowMap = *(this->getRowMap());
5566 const map_type* const colMapPtr = this->colMap_.getRawPtr();
5567 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() && colMapPtr == nullptr, std::logic_error,
5568 "This graph claims to be locally indexed, but its column Map is nullptr. "
5569 "This should never happen. Please report this bug to the Tpetra "
5570 "developers.");
5571
5572 // We may pack different amounts of data for different rows.
5573 constantNumPackets = 0;
5574
5575 numPacketsPerLID.clear_sync_state();
5576 numPacketsPerLID.modify_host();
5577 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5578
5579 // Count the total number of packets (column indices, in the case
5580 // of a CrsGraph) to pack. While doing so, set
5581 // numPacketsPerLID[i] to the number of entries owned by the
5582 // calling process in (local) row exportLIDs[i] of the graph, that
5583 // the caller wants us to send out.
5584 using range_type = Kokkos::RangePolicy<host_execution_space, LO>;
5585 range_type inputRange(0, numExportLIDs);
5586 size_t totalNumPackets = 0;
5587 size_t errCount = 0;
5588 // lambdas turn what they capture const, so we can't
5589 // atomic_add(&errCount,1). Instead, we need a View to modify.
5590 Kokkos::View<size_t, host_device_type> errCountView(&errCount);
5591 constexpr size_t ONE = 1;
5592
5593 if (verbose) {
5594 std::ostringstream os;
5595 os << *prefix << "Compute totalNumPackets" << endl;
5596 std::cerr << os.str();
5597 }
5598
5599 execute_sync_host_uvm_access(); // protect host UVM access
5600 totalNumPackets = 0;
5601 for (size_t i = 0; i < numExportLIDs; ++i) {
5602 const LO lclRow = exportLIDs_h[i];
5603 const GO gblRow = rowMap.getGlobalElement(lclRow);
5604 if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid()) {
5605 if (verbose) {
5606 std::ostringstream os;
5607 os << *prefix << "For i=" << i << ", lclRow=" << lclRow
5608 << " not in row Map on this process" << endl;
5609 std::cerr << os.str();
5610 }
5611 Kokkos::atomic_add(&errCountView(), ONE);
5612 numPacketsPerLID_h(i) = 0;
5613 } else {
5614 const size_t numEnt = this->getNumEntriesInGlobalRow(gblRow);
5615 numPacketsPerLID_h(i) = numEnt;
5616 totalNumPackets += numEnt;
5617 }
5618 }
5619
5620 if (verbose) {
5621 std::ostringstream os;
5622 os << *prefix << "totalNumPackets: " << totalNumPackets
5623 << ", errCount: " << errCount << endl;
5624 std::cerr << os.str();
5625 }
5626 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
5627 "totalNumPackets count encountered "
5628 "one or more errors! totalNumPackets: "
5629 << totalNumPackets
5630 << ", errCount: " << errCount << ".");
5631
5632 // Allocate space for all the column indices to pack.
5633 if (size_t(exports.extent(0)) < totalNumPackets) {
5634 // FIXME (mfh 09 Apr 2019) Create without initializing.
5635 exports = exports_dv_type("exports", totalNumPackets);
5636 }
5637
5638 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->supportsRowViews(), std::logic_error,
5639 "this->supportsRowViews() returns false; this should never happen. "
5640 "Please report this bug to the Tpetra developers.");
5641
5642 // Loop again over the rows to export, and pack rows of indices
5643 // into the output buffer.
5644
5645 if (verbose) {
5646 std::ostringstream os;
5647 os << *prefix << "Pack into exports buffer" << endl;
5648 std::cerr << os.str();
5649 }
5650
5651 exports.clear_sync_state();
5652 exports.modify_host();
5653 auto exports_h = exports.view_host();
5654
5655 errCount = 0;
5656
5657 // The following parallel_scan needs const host access to lclIndsUnpacked_wdv
5658 // (if locally indexed) or gblInds_wdv (if globally indexed).
5659 if (isLocallyIndexed())
5660 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
5661 else if (isGloballyIndexed())
5662 gblInds_wdv.getHostView(Access::ReadOnly);
5663
5664 Details::disableWDVTracking();
5665 Kokkos::parallel_scan("Tpetra::CrsGraph::packFillActiveNew: Pack exports",
5666 inputRange, [=, &prefix, *this](const LO i, size_t& exportsOffset, const bool final) {
5667 const size_t curOffset = exportsOffset;
5668 const LO lclRow = exportLIDs_h(i);
5669 const GO gblRow = rowMap.getGlobalElement(lclRow);
5670 if (gblRow == Details::OrdinalTraits<GO>::invalid()) {
5671 if (verbose) {
5672 std::ostringstream os;
5673 os << *prefix << "For i=" << i << ", lclRow=" << lclRow
5674 << " not in row Map on this process" << endl;
5675 std::cerr << os.str();
5676 }
5677 Kokkos::atomic_add(&errCountView(), ONE);
5678 return;
5679 }
5680
5681 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(gblRow);
5682 if (rowInfo.localRow == Details::OrdinalTraits<size_t>::invalid()) {
5683 if (verbose) {
5684 std::ostringstream os;
5685 os << *prefix << "For i=" << i << ", lclRow=" << lclRow
5686 << ", gblRow=" << gblRow << ": invalid rowInfo"
5687 << endl;
5688 std::cerr << os.str();
5689 }
5690 Kokkos::atomic_add(&errCountView(), ONE);
5691 return;
5692 }
5693
5694 if (curOffset + rowInfo.numEntries > totalNumPackets) {
5695 if (verbose) {
5696 std::ostringstream os;
5697 os << *prefix << "For i=" << i << ", lclRow=" << lclRow
5698 << ", gblRow=" << gblRow << ", curOffset (= "
5699 << curOffset << ") + numEnt (= " << rowInfo.numEntries
5700 << ") > totalNumPackets (= " << totalNumPackets
5701 << ")." << endl;
5702 std::cerr << os.str();
5703 }
5704 Kokkos::atomic_add(&errCountView(), ONE);
5705 return;
5706 }
5707
5708 const LO numEnt = static_cast<LO>(rowInfo.numEntries);
5709 if (this->isLocallyIndexed()) {
5710 auto lclColInds = getLocalIndsViewHost(rowInfo);
5711 if (final) {
5712 for (LO k = 0; k < numEnt; ++k) {
5713 const LO lclColInd = lclColInds(k);
5714 const GO gblColInd = colMapPtr->getGlobalElement(lclColInd);
5715 // Pack it, even if it's wrong. Let the receiving
5716 // process deal with it. Otherwise, we'll miss out
5717 // on any correct data.
5718 exports_h(curOffset + k) = gblColInd;
5719 } // for each entry in the row
5720 } // final pass?
5721 exportsOffset = curOffset + numEnt;
5722 } else if (this->isGloballyIndexed()) {
5723 auto gblColInds = getGlobalIndsViewHost(rowInfo);
5724 if (final) {
5725 for (LO k = 0; k < numEnt; ++k) {
5726 const GO gblColInd = gblColInds(k);
5727 // Pack it, even if it's wrong. Let the receiving
5728 // process deal with it. Otherwise, we'll miss out
5729 // on any correct data.
5730 exports_h(curOffset + k) = gblColInd;
5731 } // for each entry in the row
5732 } // final pass?
5733 exportsOffset = curOffset + numEnt;
5734 }
5735 // If neither globally nor locally indexed, then the graph
5736 // has no entries in this row (or indeed, in any row on this
5737 // process) to pack.
5738 });
5739 Details::enableWDVTracking();
5740
5741 // TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5742 // (errCount != 0, std::logic_error, "Packing encountered "
5743 // "one or more errors! errCount = " << errCount
5744 // << ", totalNumPackets = " << totalNumPackets << ".");
5745
5746 if (verbose) {
5747 std::ostringstream os;
5748 os << *prefix << "errCount=" << errCount << "; Done" << endl;
5749 std::cerr << os.str();
5750 }
5751}
5752
5753template <class LocalOrdinal, class GlobalOrdinal, class Node>
5754void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5755 unpackAndCombine(const Kokkos::DualView<const local_ordinal_type*,
5756 buffer_device_type>& importLIDs,
5757 Kokkos::DualView<packet_type*,
5758 buffer_device_type>
5759 imports,
5760 Kokkos::DualView<size_t*,
5761 buffer_device_type>
5762 numPacketsPerLID,
5763 const size_t /* constantNumPackets */,
5764 const CombineMode /* combineMode */) {
5765 using Details::ProfilingRegion;
5766 using std::endl;
5767 using LO = local_ordinal_type;
5768 using GO = global_ordinal_type;
5769 const char tfecfFuncName[] = "unpackAndCombine";
5770
5771 ProfilingRegion regionCGC("Tpetra::CrsGraph::unpackAndCombine");
5772 const bool verbose = verbose_;
5773
5774 std::unique_ptr<std::string> prefix;
5775 if (verbose) {
5776 prefix = this->createPrefix("CrsGraph", tfecfFuncName);
5777 std::ostringstream os;
5778 os << *prefix << "Start" << endl;
5779 std::cerr << os.str();
5780 }
5781 {
5782 auto padding = computeCrsPaddingForImports(
5783 importLIDs, imports, numPacketsPerLID, verbose);
5784 applyCrsPadding(*padding, verbose);
5785 if (verbose) {
5786 std::ostringstream os;
5787 os << *prefix << "Done computing & applying padding" << endl;
5788 std::cerr << os.str();
5789 }
5790 }
5791
5792 // FIXME (mfh 02 Apr 2012) REPLACE combine mode has a perfectly
5793 // reasonable meaning, whether or not the matrix is fill complete.
5794 // It's just more work to implement.
5795
5796 // We are not checking the value of the CombineMode input
5797 // argument. For CrsGraph, we only support import/export
5798 // operations if fillComplete has not yet been called. Any
5799 // incoming column-indices are inserted into the target graph. In
5800 // this context, CombineMode values of ADD vs INSERT are
5801 // equivalent. What is the meaning of REPLACE for CrsGraph? If a
5802 // duplicate column-index is inserted, it will be compressed out
5803 // when fillComplete is called.
5804 //
5805 // Note: I think REPLACE means that an existing row is replaced by
5806 // the imported row, i.e., the existing indices are cleared. CGB,
5807 // 6/17/2010
5808
5809 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(importLIDs.extent(0) != numPacketsPerLID.extent(0),
5810 std::runtime_error, ": importLIDs.extent(0) = " << importLIDs.extent(0) << " != numPacketsPerLID.extent(0) = " << numPacketsPerLID.extent(0) << ".");
5811 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isFillComplete(), std::runtime_error,
5812 ": Import or Export operations are not allowed on a target "
5813 "CrsGraph that is fillComplete.");
5814
5815 const size_t numImportLIDs(importLIDs.extent(0));
5816 if (numPacketsPerLID.need_sync_host()) {
5817 numPacketsPerLID.sync_host();
5818 }
5819 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5820 if (imports.need_sync_host()) {
5821 imports.sync_host();
5822 }
5823 auto imports_h = imports.view_host();
5824 TEUCHOS_ASSERT(!importLIDs.need_sync_host());
5825 auto importLIDs_h = importLIDs.view_host();
5826
5827 // If we're inserting in local indices, let's pre-allocate
5828 Teuchos::Array<LO> lclColInds;
5829 if (isLocallyIndexed()) {
5830 if (verbose) {
5831 std::ostringstream os;
5832 os << *prefix << "Preallocate local indices scratch" << endl;
5833 std::cerr << os.str();
5834 }
5835 size_t maxNumInserts = 0;
5836 for (size_t i = 0; i < numImportLIDs; ++i) {
5837 maxNumInserts = std::max(maxNumInserts, numPacketsPerLID_h[i]);
5838 }
5839 if (verbose) {
5840 std::ostringstream os;
5841 os << *prefix << "Local indices scratch size: "
5842 << maxNumInserts << endl;
5843 std::cerr << os.str();
5844 }
5845 lclColInds.resize(maxNumInserts);
5846 } else {
5847 if (verbose) {
5848 std::ostringstream os;
5849 os << *prefix;
5850 if (isGloballyIndexed()) {
5851 os << "Graph is globally indexed";
5852 } else {
5853 os << "Graph is neither locally nor globally indexed";
5854 }
5855 os << endl;
5856 std::cerr << os.str();
5857 }
5858 }
5859
5860 TEUCHOS_ASSERT(!rowMap_.is_null());
5861 const map_type& rowMap = *rowMap_;
5862
5863 try {
5864 size_t importsOffset = 0;
5865 for (size_t i = 0; i < numImportLIDs; ++i) {
5866 if (verbose) {
5867 std::ostringstream os;
5868 os << *prefix << "i=" << i << ", numImportLIDs="
5869 << numImportLIDs << endl;
5870 std::cerr << os.str();
5871 }
5872 // We can only unpack into owned rows, since we only have
5873 // local row indices.
5874 const LO lclRow = importLIDs_h[i];
5875 const GO gblRow = rowMap.getGlobalElement(lclRow);
5876 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(gblRow == Teuchos::OrdinalTraits<GO>::invalid(),
5877 std::logic_error, "importLIDs[i=" << i << "]=" << lclRow << " is not in the row Map on the calling "
5878 "process.");
5879 const LO numEnt = numPacketsPerLID_h[i];
5880 const GO* const gblColInds = (numEnt == 0) ? nullptr : imports_h.data() + importsOffset;
5881 if (!isLocallyIndexed()) {
5882 insertGlobalIndicesFiltered(lclRow, gblColInds, numEnt);
5883 } else {
5884 // FIXME (mfh 09 Feb 2020) Now would be a good time to do
5885 // column Map filtering.
5886 for (LO j = 0; j < numEnt; j++) {
5887 lclColInds[j] = colMap_->getLocalElement(gblColInds[j]);
5888 }
5889 insertLocalIndices(lclRow, numEnt, lclColInds.data());
5890 }
5891 importsOffset += numEnt;
5892 }
5893 } catch (std::exception& e) {
5894 TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
5895 "Tpetra::CrsGraph::unpackAndCombine: Insert loop threw an "
5896 "exception: "
5897 << endl
5898 << e.what());
5899 }
5900
5901 if (verbose) {
5902 std::ostringstream os;
5903 os << *prefix << "Done" << endl;
5904 std::cerr << os.str();
5905 }
5906}
5907
5908template <class LocalOrdinal, class GlobalOrdinal, class Node>
5909void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5910 removeEmptyProcessesInPlace(const Teuchos::RCP<const map_type>& newMap) {
5911 using Teuchos::Comm;
5912 using Teuchos::null;
5913 using Teuchos::ParameterList;
5914 using Teuchos::RCP;
5915
5916 // We'll set all the state "transactionally," so that this method
5917 // satisfies the strong exception guarantee. This object's state
5918 // won't be modified until the end of this method.
5919 RCP<const map_type> rowMap, domainMap, rangeMap, colMap;
5920 RCP<import_type> importer;
5921 RCP<export_type> exporter;
5922
5923 rowMap = newMap;
5924 RCP<const Comm<int>> newComm =
5925 (newMap.is_null()) ? null : newMap->getComm();
5926
5927 if (!domainMap_.is_null()) {
5928 if (domainMap_.getRawPtr() == rowMap_.getRawPtr()) {
5929 // Common case: original domain and row Maps are identical.
5930 // In that case, we need only replace the original domain Map
5931 // with the new Map. This ensures that the new domain and row
5932 // Maps _stay_ identical.
5933 domainMap = newMap;
5934 } else {
5935 domainMap = domainMap_->replaceCommWithSubset(newComm);
5936 }
5937 }
5938 if (!rangeMap_.is_null()) {
5939 if (rangeMap_.getRawPtr() == rowMap_.getRawPtr()) {
5940 // Common case: original range and row Maps are identical. In
5941 // that case, we need only replace the original range Map with
5942 // the new Map. This ensures that the new range and row Maps
5943 // _stay_ identical.
5944 rangeMap = newMap;
5945 } else {
5946 rangeMap = rangeMap_->replaceCommWithSubset(newComm);
5947 }
5948 }
5949 if (!colMap_.is_null()) {
5950 colMap = colMap_->replaceCommWithSubset(newComm);
5951 }
5952
5953 // (Re)create the Export and / or Import if necessary.
5954 if (!newComm.is_null()) {
5955 RCP<ParameterList> params = this->getNonconstParameterList(); // could be null
5956 //
5957 // The operations below are collective on the new communicator.
5958 //
5959 // (Re)create the Export object if necessary. If I haven't
5960 // called fillComplete yet, I don't have a rangeMap, so I must
5961 // first check if the _original_ rangeMap is not null. Ditto
5962 // for the Import object and the domain Map.
5963 if (!rangeMap_.is_null() &&
5964 rangeMap != rowMap &&
5965 !rangeMap->isSameAs(*rowMap)) {
5966 if (params.is_null() || !params->isSublist("Export")) {
5967 exporter = rcp(new export_type(rowMap, rangeMap));
5968 } else {
5969 RCP<ParameterList> exportSublist = sublist(params, "Export", true);
5970 exporter = rcp(new export_type(rowMap, rangeMap, exportSublist));
5971 }
5972 }
5973 // (Re)create the Import object if necessary.
5974 if (!domainMap_.is_null() &&
5975 domainMap != colMap &&
5976 !domainMap->isSameAs(*colMap)) {
5977 if (params.is_null() || !params->isSublist("Import")) {
5978 importer = rcp(new import_type(domainMap, colMap));
5979 } else {
5980 RCP<ParameterList> importSublist = sublist(params, "Import", true);
5981 importer = rcp(new import_type(domainMap, colMap, importSublist));
5982 }
5983 }
5984 } // if newComm is not null
5985
5986 // Defer side effects until the end. If no destructors throw
5987 // exceptions (they shouldn't anyway), then this method satisfies
5988 // the strong exception guarantee.
5989 exporter_ = exporter;
5990 importer_ = importer;
5991 rowMap_ = rowMap;
5992 // mfh 31 Mar 2013: DistObject's map_ is the row Map of a CrsGraph
5993 // or CrsMatrix. CrsGraph keeps a redundant pointer (rowMap_) to
5994 // the same object. We might want to get rid of this redundant
5995 // pointer sometime, but for now, we'll leave it alone and just
5996 // set map_ to the same object.
5997 this->map_ = rowMap;
5998 domainMap_ = domainMap;
5999 rangeMap_ = rangeMap;
6000 colMap_ = colMap;
6001}
6002
6003template <class LocalOrdinal, class GlobalOrdinal, class Node>
6004void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6005 getLocalDiagOffsets(const Kokkos::View<size_t*, device_type, Kokkos::MemoryUnmanaged>& offsets) const {
6006 using std::endl;
6007 using LO = LocalOrdinal;
6008 using GO = GlobalOrdinal;
6009 const char tfecfFuncName[] = "getLocalDiagOffsets: ";
6010 const bool verbose = verbose_;
6011
6012 std::unique_ptr<std::string> prefix;
6013 if (verbose) {
6014 prefix = this->createPrefix("CrsGraph", "getLocalDiagOffsets");
6015 std::ostringstream os;
6016 os << *prefix << "offsets.extent(0)=" << offsets.extent(0)
6017 << endl;
6018 std::cerr << os.str();
6019 }
6020
6021 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error, "The graph must have a column Map.");
6022 const LO lclNumRows = static_cast<LO>(this->getLocalNumRows());
6023 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<LO>(offsets.extent(0)) < lclNumRows,
6024 std::invalid_argument, "offsets.extent(0) = " << offsets.extent(0) << " < getLocalNumRows() = " << lclNumRows << ".");
6025
6026 const map_type& rowMap = *(this->getRowMap());
6027 const map_type& colMap = *(this->getColMap());
6028
6029 // We only use these in debug mode, but since debug mode is a
6030 // run-time option, they need to exist here. That's why we create
6031 // the vector with explicit size zero, to avoid overhead if debug
6032 // mode is off.
6033 bool allRowMapDiagEntriesInColMap = true;
6034 bool allDiagEntriesFound = true;
6035 bool allOffsetsCorrect = true;
6036 bool noOtherWeirdness = true;
6037 using wrong_offsets_type = std::vector<std::pair<LO, size_t>>;
6038 wrong_offsets_type wrongOffsets(0);
6039
6040 // mfh 12 Mar 2016: LocalMap works on (CUDA) device. It has just
6041 // the subset of Map functionality that we need below.
6042 auto lclRowMap = rowMap.getLocalMap();
6043 auto lclColMap = colMap.getLocalMap();
6044
6045 // FIXME (mfh 16 Dec 2015) It's easy to thread-parallelize this
6046 // setup, at least on the host. For CUDA, we have to use LocalMap
6047 // (that comes from each of the two Maps).
6048
6049 const bool sorted = this->isSorted();
6050 if (isFillComplete()) {
6051 auto lclGraph = this->getLocalGraphDevice();
6052 ::Tpetra::Details::getGraphDiagOffsets(offsets, lclRowMap, lclColMap,
6053 lclGraph.row_map,
6054 lclGraph.entries, sorted);
6055 } else {
6056 // NOTE (mfh 22 Feb 2017): We have to run this code on host,
6057 // since the graph is not fill complete. The previous version
6058 // of this code assumed UVM; this version does not.
6059 auto offsets_h = Kokkos::create_mirror_view(offsets);
6060
6061 for (LO lclRowInd = 0; lclRowInd < lclNumRows; ++lclRowInd) {
6062 // Find the diagonal entry. Since the row Map and column Map
6063 // may differ, we have to compare global row and column
6064 // indices, not local.
6065 const GO gblRowInd = lclRowMap.getGlobalElement(lclRowInd);
6066 const GO gblColInd = gblRowInd;
6067 const LO lclColInd = lclColMap.getLocalElement(gblColInd);
6068
6069 if (lclColInd == Tpetra::Details::OrdinalTraits<LO>::invalid()) {
6070 allRowMapDiagEntriesInColMap = false;
6071 offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid();
6072 } else {
6073 const RowInfo rowInfo = this->getRowInfo(lclRowInd);
6074 if (static_cast<LO>(rowInfo.localRow) == lclRowInd &&
6075 rowInfo.numEntries > 0) {
6076 auto colInds = this->getLocalIndsViewHost(rowInfo);
6077 const size_t hint = 0; // not needed for this algorithm
6078 const size_t offset =
6079 KokkosSparse::findRelOffset(colInds, rowInfo.numEntries,
6080 lclColInd, hint, sorted);
6081 offsets_h(lclRowInd) = offset;
6082
6083 if (debug_) {
6084 // Now that we have what we think is an offset, make sure
6085 // that it really does point to the diagonal entry. Offsets
6086 // are _relative_ to each row, not absolute (for the whole
6087 // (local) graph).
6088 typename local_inds_dualv_type::t_host::const_type lclColInds;
6089 try {
6090 lclColInds = this->getLocalIndsViewHost(rowInfo);
6091 } catch (...) {
6092 noOtherWeirdness = false;
6093 }
6094 // Don't continue with error checking if the above failed.
6095 if (noOtherWeirdness) {
6096 const size_t numEnt = lclColInds.extent(0);
6097 if (offset >= numEnt) {
6098 // Offsets are relative to each row, so this means that
6099 // the offset is out of bounds.
6100 allOffsetsCorrect = false;
6101 wrongOffsets.push_back(std::make_pair(lclRowInd, offset));
6102 } else {
6103 const LO actualLclColInd = lclColInds(offset);
6104 const GO actualGblColInd = lclColMap.getGlobalElement(actualLclColInd);
6105 if (actualGblColInd != gblColInd) {
6106 allOffsetsCorrect = false;
6107 wrongOffsets.push_back(std::make_pair(lclRowInd, offset));
6108 }
6109 }
6110 }
6111 } // debug_
6112 } else { // either row is empty, or something went wrong w/ getRowInfo()
6113 offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid();
6114 allDiagEntriesFound = false;
6115 }
6116 } // whether lclColInd is a valid local column index
6117 } // for each local row
6118 // DEEP_COPY REVIEW - NOT TESTED
6119 Kokkos::deep_copy(offsets, offsets_h);
6120 } // whether the graph is fill complete
6121
6122 if (verbose && wrongOffsets.size() != 0) {
6123 std::ostringstream os;
6124 os << *prefix << "Wrong offsets: [";
6125 for (size_t k = 0; k < wrongOffsets.size(); ++k) {
6126 os << "(" << wrongOffsets[k].first << ","
6127 << wrongOffsets[k].second << ")";
6128 if (k + 1 < wrongOffsets.size()) {
6129 os << ", ";
6130 }
6131 }
6132 os << "]" << endl;
6133 std::cerr << os.str();
6134 }
6135
6136 if (debug_) {
6137 using std::endl;
6138 using Teuchos::reduceAll;
6139 Teuchos::RCP<const Teuchos::Comm<int>> comm = this->getComm();
6140 const bool localSuccess =
6141 allRowMapDiagEntriesInColMap && allDiagEntriesFound && allOffsetsCorrect;
6142 const int numResults = 5;
6143 int lclResults[5];
6144 lclResults[0] = allRowMapDiagEntriesInColMap ? 1 : 0;
6145 lclResults[1] = allDiagEntriesFound ? 1 : 0;
6146 lclResults[2] = allOffsetsCorrect ? 1 : 0;
6147 lclResults[3] = noOtherWeirdness ? 1 : 0;
6148 // min-all-reduce will compute least rank of all the processes
6149 // that didn't succeed.
6150 lclResults[4] = !localSuccess ? comm->getRank() : comm->getSize();
6151
6152 int gblResults[5];
6153 gblResults[0] = 0;
6154 gblResults[1] = 0;
6155 gblResults[2] = 0;
6156 gblResults[3] = 0;
6157 gblResults[4] = 0;
6158 reduceAll<int, int>(*comm, Teuchos::REDUCE_MIN,
6159 numResults, lclResults, gblResults);
6160
6161 if (gblResults[0] != 1 || gblResults[1] != 1 || gblResults[2] != 1 || gblResults[3] != 1) {
6162 std::ostringstream os; // build error message
6163 os << "Issue(s) that we noticed (on Process " << gblResults[4] << ", "
6164 "possibly among others): "
6165 << endl;
6166 if (gblResults[0] == 0) {
6167 os << " - The column Map does not contain at least one diagonal entry "
6168 "of the graph."
6169 << endl;
6170 }
6171 if (gblResults[1] == 0) {
6172 os << " - On one or more processes, some row does not contain a "
6173 "diagonal entry."
6174 << endl;
6175 }
6176 if (gblResults[2] == 0) {
6177 os << " - On one or more processes, some offsets are incorrect."
6178 << endl;
6179 }
6180 if (gblResults[3] == 0) {
6181 os << " - One or more processes had some other error."
6182 << endl;
6183 }
6184 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
6185 }
6186 } // debug_
6187}
6188
6189template <class LocalOrdinal, class GlobalOrdinal, class Node>
6190void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6191 getLocalOffRankOffsets(offset_device_view_type& offsets) const {
6192 using std::endl;
6193 const char tfecfFuncName[] = "getLocalOffRankOffsets: ";
6194 const bool verbose = verbose_;
6195
6196 std::unique_ptr<std::string> prefix;
6197 if (verbose) {
6198 prefix = this->createPrefix("CrsGraph", "getLocalOffRankOffsets");
6199 std::ostringstream os;
6200 os << *prefix << "offsets.extent(0)=" << offsets.extent(0)
6201 << endl;
6202 std::cerr << os.str();
6203 }
6204
6205 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error, "The graph must have a column Map.");
6206 // Instead of throwing, we could also copy the rowPtr to k_offRankOffsets_.
6207
6208 const size_t lclNumRows = this->getLocalNumRows();
6209
6210 if (haveLocalOffRankOffsets_ && k_offRankOffsets_.extent(0) == lclNumRows + 1) {
6211 offsets = k_offRankOffsets_;
6212 return;
6213 }
6214 haveLocalOffRankOffsets_ = false;
6215
6216 const map_type& colMap = *(this->getColMap());
6217 const map_type& domMap = *(this->getDomainMap());
6218
6219 // mfh 12 Mar 2016: LocalMap works on (CUDA) device. It has just
6220 // the subset of Map functionality that we need below.
6221 auto lclColMap = colMap.getLocalMap();
6222 auto lclDomMap = domMap.getLocalMap();
6223
6224 // FIXME (mfh 16 Dec 2015) It's easy to thread-parallelize this
6225 // setup, at least on the host. For CUDA, we have to use LocalMap
6226 // (that comes from each of the two Maps).
6227
6228 TEUCHOS_ASSERT(this->isSorted());
6229 if (isFillComplete()) {
6230 k_offRankOffsets_ = offset_device_view_type(Kokkos::ViewAllocateWithoutInitializing("offRankOffset"), lclNumRows + 1);
6231 auto lclGraph = this->getLocalGraphDevice();
6232 ::Tpetra::Details::getGraphOffRankOffsets(k_offRankOffsets_,
6233 lclColMap, lclDomMap,
6234 lclGraph);
6235 offsets = k_offRankOffsets_;
6236 haveLocalOffRankOffsets_ = true;
6237 } else {
6238 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error, "Can't get off-rank offsets for non-fill-complete graph");
6239 }
6240}
6241
6242namespace { // (anonymous)
6243
6244// mfh 21 Jan 2016: This is useful for getLocalDiagOffsets (see
6245// below). The point is to avoid the deep copy between the input
6246// Teuchos::ArrayRCP and the internally used Kokkos::View. We
6247// can't use UVM to avoid the deep copy with CUDA, because the
6248// ArrayRCP is a host pointer, while the input to the graph's
6249// getLocalDiagOffsets method is a device pointer. Assigning a
6250// host pointer to a device pointer is incorrect unless the host
6251// pointer points to host pinned memory. The goal is to get rid
6252// of the Teuchos::ArrayRCP overload anyway, so we accept the deep
6253// copy for backwards compatibility.
6254//
6255// We have to use template magic because
6256// "staticGraph_->getLocalDiagOffsets(offsetsHosts)" won't compile
6257// if device_type::memory_space is not Kokkos::HostSpace (as is
6258// the case with CUDA).
6259
6260template <class DeviceType,
6261 const bool memSpaceIsHostSpace =
6262 std::is_same<typename DeviceType::memory_space,
6263 Kokkos::HostSpace>::value>
6264struct HelpGetLocalDiagOffsets {};
6265
6266template <class DeviceType>
6267struct HelpGetLocalDiagOffsets<DeviceType, true> {
6268 typedef DeviceType device_type;
6269 typedef Kokkos::View<size_t*, Kokkos::HostSpace,
6270 Kokkos::MemoryUnmanaged>
6271 device_offsets_type;
6272 typedef Kokkos::View<size_t*, Kokkos::HostSpace,
6273 Kokkos::MemoryUnmanaged>
6274 host_offsets_type;
6275
6276 static device_offsets_type
6277 getDeviceOffsets(const host_offsets_type& hostOffsets) {
6278 // Host and device are the same; no need to allocate a
6279 // temporary device View.
6280 return hostOffsets;
6281 }
6282
6283 static void
6284 copyBackIfNeeded(const host_offsets_type& /* hostOffsets */,
6285 const device_offsets_type& /* deviceOffsets */) { /* copy back not needed; host and device are the same */
6286 }
6287};
6288
6289template <class DeviceType>
6290struct HelpGetLocalDiagOffsets<DeviceType, false> {
6291 typedef DeviceType device_type;
6292 // We have to do a deep copy, since host memory space != device
6293 // memory space. Thus, the device View is managed (we need to
6294 // allocate a temporary device View).
6295 typedef Kokkos::View<size_t*, device_type> device_offsets_type;
6296 typedef Kokkos::View<size_t*, Kokkos::HostSpace,
6297 Kokkos::MemoryUnmanaged>
6298 host_offsets_type;
6299
6300 static device_offsets_type
6301 getDeviceOffsets(const host_offsets_type& hostOffsets) {
6302 // Host memory space != device memory space, so we must
6303 // allocate a temporary device View for the graph.
6304 return device_offsets_type("offsets", hostOffsets.extent(0));
6305 }
6306
6307 static void
6308 copyBackIfNeeded(const host_offsets_type& hostOffsets,
6309 const device_offsets_type& deviceOffsets) {
6310 // DEEP_COPY REVIEW - NOT TESTED
6311 Kokkos::deep_copy(hostOffsets, deviceOffsets);
6312 }
6313};
6314} // namespace
6315
6316template <class LocalOrdinal, class GlobalOrdinal, class Node>
6317void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6318 getLocalDiagOffsets(Teuchos::ArrayRCP<size_t>& offsets) const {
6319 typedef LocalOrdinal LO;
6320 const char tfecfFuncName[] = "getLocalDiagOffsets: ";
6321 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::runtime_error,
6322 "The graph does not yet have a column Map.");
6323 const LO myNumRows = static_cast<LO>(this->getLocalNumRows());
6324 if (static_cast<LO>(offsets.size()) != myNumRows) {
6325 // NOTE (mfh 21 Jan 2016) This means that the method does not
6326 // satisfy the strong exception guarantee (no side effects
6327 // unless successful).
6328 offsets.resize(myNumRows);
6329 }
6330
6331 // mfh 21 Jan 2016: This method unfortunately takes a
6332 // Teuchos::ArrayRCP, which is host memory. The graph wants a
6333 // device pointer. We can't access host memory from the device;
6334 // that's the wrong direction for UVM. (It's the right direction
6335 // for inefficient host pinned memory, but we don't want to use
6336 // that here.) Thus, if device memory space != host memory space,
6337 // we allocate and use a temporary device View to get the offsets.
6338 // If the two spaces are equal, the template magic makes the deep
6339 // copy go away.
6340 typedef HelpGetLocalDiagOffsets<device_type> helper_type;
6341 typedef typename helper_type::host_offsets_type host_offsets_type;
6342 // Unmanaged host View that views the output array.
6343 host_offsets_type hostOffsets(offsets.getRawPtr(), myNumRows);
6344 // Allocate temp device View if host != device, else reuse host array.
6345 auto deviceOffsets = helper_type::getDeviceOffsets(hostOffsets);
6346 // NOT recursion; this calls the overload that takes a device View.
6347 this->getLocalDiagOffsets(deviceOffsets);
6348 helper_type::copyBackIfNeeded(hostOffsets, deviceOffsets);
6349}
6350
6351template <class LocalOrdinal, class GlobalOrdinal, class Node>
6352bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6353 supportsRowViews() const {
6354 return true;
6355}
6356
6357template <class LocalOrdinal, class GlobalOrdinal, class Node>
6358void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6359 transferAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6360 const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node>& rowTransfer,
6361 const Teuchos::RCP<const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node>>& domainTransfer,
6362 const Teuchos::RCP<const map_type>& domainMap,
6363 const Teuchos::RCP<const map_type>& rangeMap,
6364 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6365 using std::string;
6366 using Teuchos::ArrayRCP;
6367 using Teuchos::ArrayView;
6368 using Teuchos::Comm;
6369 using Teuchos::ParameterList;
6370 using Teuchos::rcp;
6371 using Teuchos::RCP;
6372 using Tpetra::Details::getArrayViewFromDualView;
6373 using Tpetra::Details::packCrsGraphWithOwningPIDs;
6374 using Tpetra::Details::unpackAndCombineIntoCrsArrays;
6375 using Tpetra::Details::unpackAndCombineWithOwningPIDsCount;
6376
6377 using LO = LocalOrdinal;
6378 using GO = GlobalOrdinal;
6379 using NT = node_type;
6380 using this_CRS_type = CrsGraph<LO, GO, NT>;
6381 using ivector_type = Vector<int, LO, GO, NT>;
6382
6383 const char* prefix = "Tpetra::CrsGraph::transferAndFillComplete: ";
6384
6385 auto MM = rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC Pack-1"));
6386
6387 // Make sure that the input argument rowTransfer is either an
6388 // Import or an Export. Import and Export are the only two
6389 // subclasses of Transfer that we defined, but users might
6390 // (unwisely, for now at least) decide to implement their own
6391 // subclasses. Exclude this possibility.
6392 const import_type* xferAsImport = dynamic_cast<const import_type*>(&rowTransfer);
6393 const export_type* xferAsExport = dynamic_cast<const export_type*>(&rowTransfer);
6394 TEUCHOS_TEST_FOR_EXCEPTION(
6395 xferAsImport == nullptr && xferAsExport == nullptr, std::invalid_argument,
6396 prefix << "The 'rowTransfer' input argument must be either an Import or "
6397 "an Export, and its template parameters must match the corresponding "
6398 "template parameters of the CrsGraph.");
6399
6400 // Make sure that the input argument domainTransfer is either an
6401 // Import or an Export. Import and Export are the only two
6402 // subclasses of Transfer that we defined, but users might
6403 // (unwisely, for now at least) decide to implement their own
6404 // subclasses. Exclude this possibility.
6405 Teuchos::RCP<const import_type> xferDomainAsImport =
6406 Teuchos::rcp_dynamic_cast<const import_type>(domainTransfer);
6407 Teuchos::RCP<const export_type> xferDomainAsExport =
6408 Teuchos::rcp_dynamic_cast<const export_type>(domainTransfer);
6409
6410 if (!domainTransfer.is_null()) {
6411 TEUCHOS_TEST_FOR_EXCEPTION(
6412 (xferDomainAsImport.is_null() && xferDomainAsExport.is_null()), std::invalid_argument,
6413 prefix << "The 'domainTransfer' input argument must be either an "
6414 "Import or an Export, and its template parameters must match the "
6415 "corresponding template parameters of the CrsGraph.");
6416
6417 TEUCHOS_TEST_FOR_EXCEPTION(
6418 (xferAsImport != nullptr || !xferDomainAsImport.is_null()) &&
6419 ((xferAsImport != nullptr && xferDomainAsImport.is_null()) ||
6420 (xferAsImport == nullptr && !xferDomainAsImport.is_null())),
6421 std::invalid_argument,
6422 prefix << "The 'rowTransfer' and 'domainTransfer' input arguments "
6423 "must be of the same type (either Import or Export).");
6424
6425 TEUCHOS_TEST_FOR_EXCEPTION(
6426 (xferAsExport != nullptr || !xferDomainAsExport.is_null()) &&
6427 ((xferAsExport != nullptr && xferDomainAsExport.is_null()) ||
6428 (xferAsExport == nullptr && !xferDomainAsExport.is_null())),
6429 std::invalid_argument,
6430 prefix << "The 'rowTransfer' and 'domainTransfer' input arguments "
6431 "must be of the same type (either Import or Export).");
6432
6433 } // domainTransfer != null
6434
6435 // FIXME (mfh 15 May 2014) Wouldn't communication still be needed,
6436 // if the source Map is not distributed but the target Map is?
6437 const bool communication_needed = rowTransfer.getSourceMap()->isDistributed();
6438
6439 //
6440 // Get the caller's parameters
6441 //
6442
6443 bool reverseMode = false; // Are we in reverse mode?
6444 bool restrictComm = false; // Do we need to restrict the communicator?
6445 RCP<ParameterList> graphparams; // parameters for the destination graph
6446 if (!params.is_null()) {
6447 reverseMode = params->get("Reverse Mode", reverseMode);
6448 restrictComm = params->get("Restrict Communicator", restrictComm);
6449 graphparams = sublist(params, "CrsGraph");
6450 }
6451
6452 // Get the new domain and range Maps. We need some of them for error
6453 // checking, now that we have the reverseMode parameter.
6454 RCP<const map_type> MyRowMap = reverseMode ? rowTransfer.getSourceMap() : rowTransfer.getTargetMap();
6455 RCP<const map_type> MyColMap; // create this below
6456 RCP<const map_type> MyDomainMap = !domainMap.is_null() ? domainMap : getDomainMap();
6457 RCP<const map_type> MyRangeMap = !rangeMap.is_null() ? rangeMap : getRangeMap();
6458 RCP<const map_type> BaseRowMap = MyRowMap;
6459 RCP<const map_type> BaseDomainMap = MyDomainMap;
6460
6461 // If the user gave us a nonnull destGraph, then check whether it's
6462 // "pristine." That means that it has no entries.
6463 //
6464 // FIXME (mfh 15 May 2014) If this is not true on all processes,
6465 // then this exception test may hang. It would be better to
6466 // forward an error flag to the next communication phase.
6467 if (!destGraph.is_null()) {
6468 // FIXME (mfh 15 May 2014): The Epetra idiom for checking
6469 // whether a graph or matrix has no entries on the calling
6470 // process, is that it is neither locally nor globally indexed.
6471 // This may change eventually with the Kokkos refactor version
6472 // of Tpetra, so it would be better just to check the quantity
6473 // of interest directly. Note that with the Kokkos refactor
6474 // version of Tpetra, asking for the total number of entries in
6475 // a graph or matrix that is not fill complete might require
6476 // computation (kernel launch), since it is not thread scalable
6477 // to update a count every time an entry is inserted.
6478 const bool NewFlag =
6479 !destGraph->isLocallyIndexed() && !destGraph->isGloballyIndexed();
6480 TEUCHOS_TEST_FOR_EXCEPTION(!NewFlag, std::invalid_argument,
6481 prefix << "The input argument 'destGraph' is only allowed to be nonnull, "
6482 "if its graph is empty (neither locally nor globally indexed).");
6483
6484 // FIXME (mfh 15 May 2014) At some point, we want to change
6485 // graphs and matrices so that their DistObject Map
6486 // (this->getMap()) may differ from their row Map. This will
6487 // make redistribution for 2-D distributions more efficient. I
6488 // hesitate to change this check, because I'm not sure how much
6489 // the code here depends on getMap() and getRowMap() being the
6490 // same.
6491 TEUCHOS_TEST_FOR_EXCEPTION(
6492 !destGraph->getRowMap()->isSameAs(*MyRowMap), std::invalid_argument,
6493 prefix << "The (row) Map of the input argument 'destGraph' is not the "
6494 "same as the (row) Map specified by the input argument 'rowTransfer'.");
6495
6496 TEUCHOS_TEST_FOR_EXCEPTION(
6497 !destGraph->checkSizes(*this), std::invalid_argument,
6498 prefix << "You provided a nonnull destination graph, but checkSizes() "
6499 "indicates that it is not a legal legal target for redistribution from "
6500 "the source graph (*this). This may mean that they do not have the "
6501 "same dimensions.");
6502 }
6503
6504 // If forward mode (the default), then *this's (row) Map must be
6505 // the same as the source Map of the Transfer. If reverse mode,
6506 // then *this's (row) Map must be the same as the target Map of
6507 // the Transfer.
6508 //
6509 // FIXME (mfh 15 May 2014) At some point, we want to change graphs
6510 // and matrices so that their DistObject Map (this->getMap()) may
6511 // differ from their row Map. This will make redistribution for
6512 // 2-D distributions more efficient. I hesitate to change this
6513 // check, because I'm not sure how much the code here depends on
6514 // getMap() and getRowMap() being the same.
6515 TEUCHOS_TEST_FOR_EXCEPTION(
6516 !(reverseMode || getRowMap()->isSameAs(*rowTransfer.getSourceMap())),
6517 std::invalid_argument, prefix << "rowTransfer->getSourceMap() must match this->getRowMap() in forward mode.");
6518
6519 TEUCHOS_TEST_FOR_EXCEPTION(
6520 !(!reverseMode || getRowMap()->isSameAs(*rowTransfer.getTargetMap())),
6521 std::invalid_argument, prefix << "rowTransfer->getTargetMap() must match this->getRowMap() in reverse mode.");
6522
6523 // checks for domainTransfer
6524 TEUCHOS_TEST_FOR_EXCEPTION(
6525 !xferDomainAsImport.is_null() && !xferDomainAsImport->getTargetMap()->isSameAs(*domainMap),
6526 std::invalid_argument,
6527 prefix << "The target map of the 'domainTransfer' input argument must be "
6528 "the same as the rebalanced domain map 'domainMap'");
6529
6530 TEUCHOS_TEST_FOR_EXCEPTION(
6531 !xferDomainAsExport.is_null() && !xferDomainAsExport->getSourceMap()->isSameAs(*domainMap),
6532 std::invalid_argument,
6533 prefix << "The source map of the 'domainTransfer' input argument must be "
6534 "the same as the rebalanced domain map 'domainMap'");
6535
6536 // The basic algorithm here is:
6537 //
6538 // 1. Call the moral equivalent of "Distor.do" to handle the import.
6539 // 2. Copy all the Imported and Copy/Permuted data into the raw
6540 // CrsGraph pointers, still using GIDs.
6541 // 3. Call an optimized version of MakeColMap that avoids the
6542 // Directory lookups (since the importer knows who owns all the
6543 // GIDs) AND reindexes to LIDs.
6544 // 4. Call expertStaticFillComplete()
6545
6546 // Get information from the Importer
6547 const size_t NumSameIDs = rowTransfer.getNumSameIDs();
6548 ArrayView<const LO> ExportLIDs = reverseMode ? rowTransfer.getRemoteLIDs() : rowTransfer.getExportLIDs();
6549 ArrayView<const LO> RemoteLIDs = reverseMode ? rowTransfer.getExportLIDs() : rowTransfer.getRemoteLIDs();
6550 ArrayView<const LO> PermuteToLIDs = reverseMode ? rowTransfer.getPermuteFromLIDs() : rowTransfer.getPermuteToLIDs();
6551 ArrayView<const LO> PermuteFromLIDs = reverseMode ? rowTransfer.getPermuteToLIDs() : rowTransfer.getPermuteFromLIDs();
6552 Distributor& Distor = rowTransfer.getDistributor();
6553
6554 // Owning PIDs
6555 Teuchos::Array<int> SourcePids;
6556 Teuchos::Array<int> TargetPids;
6557 int MyPID = getComm()->getRank();
6558
6559 // Temp variables for sub-communicators
6560 RCP<const map_type> ReducedRowMap, ReducedColMap,
6561 ReducedDomainMap, ReducedRangeMap;
6562 RCP<const Comm<int>> ReducedComm;
6563
6564 // If the user gave us a null destGraph, then construct the new
6565 // destination graph. We will replace its column Map later.
6566 if (destGraph.is_null()) {
6567 destGraph = rcp(new this_CRS_type(MyRowMap, 0, graphparams));
6568 }
6569
6570 /***************************************************/
6571 /***** 1) First communicator restriction phase ****/
6572 /***************************************************/
6573 if (restrictComm) {
6574 ReducedRowMap = MyRowMap->removeEmptyProcesses();
6575 ReducedComm = ReducedRowMap.is_null() ? Teuchos::null : ReducedRowMap->getComm();
6576 destGraph->removeEmptyProcessesInPlace(ReducedRowMap);
6577
6578 ReducedDomainMap = MyRowMap.getRawPtr() == MyDomainMap.getRawPtr() ? ReducedRowMap : MyDomainMap->replaceCommWithSubset(ReducedComm);
6579 ReducedRangeMap = MyRowMap.getRawPtr() == MyRangeMap.getRawPtr() ? ReducedRowMap : MyRangeMap->replaceCommWithSubset(ReducedComm);
6580
6581 // Reset the "my" maps
6582 MyRowMap = ReducedRowMap;
6583 MyDomainMap = ReducedDomainMap;
6584 MyRangeMap = ReducedRangeMap;
6585
6586 // Update my PID, if we've restricted the communicator
6587 if (!ReducedComm.is_null()) {
6588 MyPID = ReducedComm->getRank();
6589 } else {
6590 MyPID = -2; // For debugging
6591 }
6592 } else {
6593 ReducedComm = MyRowMap->getComm();
6594 }
6595
6596 /***************************************************/
6597 /***** 2) From Tpera::DistObject::doTransfer() ****/
6598 /***************************************************/
6599 MM = Teuchos::null;
6600 MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC ImportSetup"));
6601 // Get the owning PIDs
6602 RCP<const import_type> MyImporter = getImporter();
6603
6604 // check whether domain maps of source graph and base domain map is the same
6605 bool bSameDomainMap = BaseDomainMap->isSameAs(*getDomainMap());
6606
6607 if (!restrictComm && !MyImporter.is_null() && bSameDomainMap) {
6608 // Same domain map as source graph
6609 //
6610 // NOTE: This won't work for restrictComm (because the Import
6611 // doesn't know the restricted PIDs), though writing an
6612 // optimized version for that case would be easy (Import an
6613 // IntVector of the new PIDs). Might want to add this later.
6614 Import_Util::getPids(*MyImporter, SourcePids, false);
6615 } else if (restrictComm && !MyImporter.is_null() && bSameDomainMap) {
6616 // Same domain map as source graph (restricted communicator)
6617 // We need one import from the domain to the column map
6618 ivector_type SourceDomain_pids(getDomainMap(), true);
6619 ivector_type SourceCol_pids(getColMap());
6620 // SourceDomain_pids contains the restricted pids
6621 SourceDomain_pids.putScalar(MyPID);
6622
6623 SourceCol_pids.doImport(SourceDomain_pids, *MyImporter, INSERT);
6624 SourcePids.resize(getColMap()->getLocalNumElements());
6625 SourceCol_pids.get1dCopy(SourcePids());
6626 } else if (MyImporter.is_null() && bSameDomainMap) {
6627 // Graph has no off-process entries
6628 SourcePids.resize(getColMap()->getLocalNumElements());
6629 SourcePids.assign(getColMap()->getLocalNumElements(), MyPID);
6630 } else if (!MyImporter.is_null() &&
6631 !domainTransfer.is_null()) {
6632 // general implementation for rectangular matrices with
6633 // domain map different than SourceGraph domain map.
6634 // User has to provide a DomainTransfer object. We need
6635 // to communications (import/export)
6636
6637 // TargetDomain_pids lives on the rebalanced new domain map
6638 ivector_type TargetDomain_pids(domainMap);
6639 TargetDomain_pids.putScalar(MyPID);
6640
6641 // SourceDomain_pids lives on the non-rebalanced old domain map
6642 ivector_type SourceDomain_pids(getDomainMap());
6643
6644 // SourceCol_pids lives on the non-rebalanced old column map
6645 ivector_type SourceCol_pids(getColMap());
6646
6647 if (!reverseMode && !xferDomainAsImport.is_null()) {
6648 SourceDomain_pids.doExport(TargetDomain_pids, *xferDomainAsImport, INSERT);
6649 } else if (reverseMode && !xferDomainAsExport.is_null()) {
6650 SourceDomain_pids.doExport(TargetDomain_pids, *xferDomainAsExport, INSERT);
6651 } else if (!reverseMode && !xferDomainAsExport.is_null()) {
6652 SourceDomain_pids.doImport(TargetDomain_pids, *xferDomainAsExport, INSERT);
6653 } else if (reverseMode && !xferDomainAsImport.is_null()) {
6654 SourceDomain_pids.doImport(TargetDomain_pids, *xferDomainAsImport, INSERT);
6655 } else {
6656 TEUCHOS_TEST_FOR_EXCEPTION(
6657 true, std::logic_error,
6658 prefix << "Should never get here! Please report this bug to a Tpetra developer.");
6659 }
6660 SourceCol_pids.doImport(SourceDomain_pids, *MyImporter, INSERT);
6661 SourcePids.resize(getColMap()->getLocalNumElements());
6662 SourceCol_pids.get1dCopy(SourcePids());
6663 } else if (BaseDomainMap->isSameAs(*BaseRowMap) &&
6664 getDomainMap()->isSameAs(*getRowMap())) {
6665 // We can use the rowTransfer + SourceGraph's Import to find out who owns what.
6666 ivector_type TargetRow_pids(domainMap);
6667 ivector_type SourceRow_pids(getRowMap());
6668 ivector_type SourceCol_pids(getColMap());
6669
6670 TargetRow_pids.putScalar(MyPID);
6671 if (!reverseMode && xferAsImport != nullptr) {
6672 SourceRow_pids.doExport(TargetRow_pids, *xferAsImport, INSERT);
6673 } else if (reverseMode && xferAsExport != nullptr) {
6674 SourceRow_pids.doExport(TargetRow_pids, *xferAsExport, INSERT);
6675 } else if (!reverseMode && xferAsExport != nullptr) {
6676 SourceRow_pids.doImport(TargetRow_pids, *xferAsExport, INSERT);
6677 } else if (reverseMode && xferAsImport != nullptr) {
6678 SourceRow_pids.doImport(TargetRow_pids, *xferAsImport, INSERT);
6679 } else {
6680 TEUCHOS_TEST_FOR_EXCEPTION(
6681 true, std::logic_error,
6682 prefix << "Should never get here! Please report this bug to a Tpetra developer.");
6683 }
6684 SourceCol_pids.doImport(SourceRow_pids, *MyImporter, INSERT);
6685 SourcePids.resize(getColMap()->getLocalNumElements());
6686 SourceCol_pids.get1dCopy(SourcePids());
6687 } else {
6688 TEUCHOS_TEST_FOR_EXCEPTION(
6689 true, std::invalid_argument,
6690 prefix << "This method only allows either domainMap == getDomainMap(), "
6691 "or (domainMap == rowTransfer.getTargetMap() and getDomainMap() == getRowMap()).");
6692 }
6693
6694 // Tpetra-specific stuff
6695 size_t constantNumPackets = destGraph->constantNumberOfPackets();
6696 if (constantNumPackets == 0) {
6697 destGraph->reallocArraysForNumPacketsPerLid(ExportLIDs.size(),
6698 RemoteLIDs.size());
6699 } else {
6700 // There are a constant number of packets per element. We
6701 // already know (from the number of "remote" (incoming)
6702 // elements) how many incoming elements we expect, so we can
6703 // resize the buffer accordingly.
6704 const size_t rbufLen = RemoteLIDs.size() * constantNumPackets;
6705 destGraph->reallocImportsIfNeeded(rbufLen, false, nullptr);
6706 }
6707
6708 {
6709 // packAndPrepare* methods modify numExportPacketsPerLID_.
6710 destGraph->numExportPacketsPerLID_.modify_host();
6711 Teuchos::ArrayView<size_t> numExportPacketsPerLID =
6712 getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
6713
6714 // Pack & Prepare w/ owning PIDs
6715 packCrsGraphWithOwningPIDs(*this, destGraph->exports_,
6716 numExportPacketsPerLID, ExportLIDs,
6717 SourcePids, constantNumPackets);
6718 }
6719
6720 // Do the exchange of remote data.
6721 MM = Teuchos::null;
6722 MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC Transfer"));
6723
6724 if (communication_needed) {
6725 if (reverseMode) {
6726 if (constantNumPackets == 0) { // variable number of packets per LID
6727 // Make sure that host has the latest version, since we're
6728 // using the version on host. If host has the latest
6729 // version, syncing to host does nothing.
6730 destGraph->numExportPacketsPerLID_.sync_host();
6731 Teuchos::ArrayView<const size_t> numExportPacketsPerLID =
6732 getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
6733 destGraph->numImportPacketsPerLID_.sync_host();
6734 Teuchos::ArrayView<size_t> numImportPacketsPerLID =
6735 getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
6736
6737 Distor.doReversePostsAndWaits(destGraph->numExportPacketsPerLID_.view_host(), 1,
6738 destGraph->numImportPacketsPerLID_.view_host());
6739 size_t totalImportPackets = 0;
6740 for (Array_size_type i = 0; i < numImportPacketsPerLID.size(); ++i) {
6741 totalImportPackets += numImportPacketsPerLID[i];
6742 }
6743
6744 // Reallocation MUST go before setting the modified flag,
6745 // because it may clear out the flags.
6746 destGraph->reallocImportsIfNeeded(totalImportPackets, false, nullptr);
6747 destGraph->imports_.modify_host();
6748 auto hostImports = destGraph->imports_.view_host();
6749 // This is a legacy host pack/unpack path, so use the host
6750 // version of exports_.
6751 destGraph->exports_.sync_host();
6752 auto hostExports = destGraph->exports_.view_host();
6753 Distor.doReversePostsAndWaits(hostExports,
6754 numExportPacketsPerLID,
6755 hostImports,
6756 numImportPacketsPerLID);
6757 } else { // constant number of packets per LI
6758 destGraph->imports_.modify_host();
6759 auto hostImports = destGraph->imports_.view_host();
6760 // This is a legacy host pack/unpack path, so use the host
6761 // version of exports_.
6762 destGraph->exports_.sync_host();
6763 auto hostExports = destGraph->exports_.view_host();
6764 Distor.doReversePostsAndWaits(hostExports,
6765 constantNumPackets,
6766 hostImports);
6767 }
6768 } else { // forward mode (the default)
6769 if (constantNumPackets == 0) { // variable number of packets per LID
6770 // Make sure that host has the latest version, since we're
6771 // using the version on host. If host has the latest
6772 // version, syncing to host does nothing.
6773 destGraph->numExportPacketsPerLID_.sync_host();
6774 destGraph->numImportPacketsPerLID_.sync_host();
6775 Distor.doPostsAndWaits(destGraph->numExportPacketsPerLID_.view_host(), 1,
6776 destGraph->numImportPacketsPerLID_.view_host());
6777
6778 Teuchos::ArrayView<const size_t> numImportPacketsPerLID =
6779 getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
6780 size_t totalImportPackets = 0;
6781 for (Array_size_type i = 0; i < numImportPacketsPerLID.size(); ++i) {
6782 totalImportPackets += numImportPacketsPerLID[i];
6783 }
6784
6785 // Reallocation MUST go before setting the modified flag,
6786 // because it may clear out the flags.
6787 destGraph->reallocImportsIfNeeded(totalImportPackets, false, nullptr);
6788 destGraph->imports_.modify_host();
6789 auto hostImports = destGraph->imports_.view_host();
6790 // This is a legacy host pack/unpack path, so use the host
6791 // version of exports_.
6792 destGraph->exports_.sync_host();
6793 auto hostExports = destGraph->exports_.view_host();
6794 Teuchos::ArrayView<const size_t> numExportPacketsPerLID =
6795 getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
6796 Distor.doPostsAndWaits(hostExports, numExportPacketsPerLID, hostImports, numImportPacketsPerLID);
6797 } else { // constant number of packets per LID
6798 destGraph->imports_.modify_host();
6799 auto hostImports = destGraph->imports_.view_host();
6800 // This is a legacy host pack/unpack path, so use the host
6801 // version of exports_.
6802 destGraph->exports_.sync_host();
6803 auto hostExports = destGraph->exports_.view_host();
6804 Distor.doPostsAndWaits(hostExports, constantNumPackets, hostImports);
6805 }
6806 }
6807 }
6808
6809 /*********************************************************************/
6810 /**** 3) Copy all of the Same/Permute/Remote data into CSR_arrays ****/
6811 /*********************************************************************/
6812
6813 MM = Teuchos::null;
6814 MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC Unpack-1"));
6815
6816 // Backwards compatibility measure. We'll use this again below.
6817 destGraph->numImportPacketsPerLID_.sync_host();
6818 Teuchos::ArrayView<const size_t> numImportPacketsPerLID =
6819 getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
6820 destGraph->imports_.sync_host();
6821 Teuchos::ArrayView<const packet_type> hostImports =
6822 getArrayViewFromDualView(destGraph->imports_);
6823 size_t mynnz =
6824 unpackAndCombineWithOwningPIDsCount(*this, RemoteLIDs, hostImports,
6825 numImportPacketsPerLID,
6826 constantNumPackets, INSERT,
6827 NumSameIDs, PermuteToLIDs, PermuteFromLIDs);
6828 size_t N = BaseRowMap->getLocalNumElements();
6829
6830 // Allocations
6831 ArrayRCP<size_t> CSR_rowptr(N + 1);
6832 ArrayRCP<GO> CSR_colind_GID;
6833 ArrayRCP<LO> CSR_colind_LID;
6834 CSR_colind_GID.resize(mynnz);
6835
6836 // If LO and GO are the same, we can reuse memory when
6837 // converting the column indices from global to local indices.
6838 if (typeid(LO) == typeid(GO)) {
6839 CSR_colind_LID = Teuchos::arcp_reinterpret_cast<LO>(CSR_colind_GID);
6840 } else {
6841 CSR_colind_LID.resize(mynnz);
6842 }
6843
6844 // FIXME (mfh 15 May 2014) Why can't we abstract this out as an
6845 // unpackAndCombine method on a "CrsArrays" object? This passing
6846 // in a huge list of arrays is icky. Can't we have a bit of an
6847 // abstraction? Implementing a concrete DistObject subclass only
6848 // takes five methods.
6849 unpackAndCombineIntoCrsArrays(*this, RemoteLIDs, hostImports,
6850 numImportPacketsPerLID, constantNumPackets,
6851 INSERT, NumSameIDs, PermuteToLIDs,
6852 PermuteFromLIDs, N, mynnz, MyPID,
6853 CSR_rowptr(), CSR_colind_GID(),
6854 SourcePids(), TargetPids);
6855
6856 /**************************************************************/
6857 /**** 4) Call Optimized MakeColMap w/ no Directory Lookups ****/
6858 /**************************************************************/
6859 MM = Teuchos::null;
6860 MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC Unpack-2"));
6861 // Call an optimized version of makeColMap that avoids the
6862 // Directory lookups (since the Import object knows who owns all
6863 // the GIDs).
6864 Teuchos::Array<int> RemotePids;
6865 Import_Util::lowCommunicationMakeColMapAndReindex(CSR_rowptr(),
6866 CSR_colind_LID(),
6867 CSR_colind_GID(),
6868 BaseDomainMap,
6869 TargetPids, RemotePids,
6870 MyColMap,
6871 params);
6872
6873 /*******************************************************/
6874 /**** 4) Second communicator restriction phase ****/
6875 /*******************************************************/
6876 if (restrictComm) {
6877 ReducedColMap = (MyRowMap.getRawPtr() == MyColMap.getRawPtr()) ? ReducedRowMap : MyColMap->replaceCommWithSubset(ReducedComm);
6878 MyColMap = ReducedColMap; // Reset the "my" maps
6879 }
6880
6881 // Replace the col map
6882 destGraph->replaceColMap(MyColMap);
6883
6884 // Short circuit if the processor is no longer in the communicator
6885 //
6886 // NOTE: Epetra replaces modifies all "removed" processes so they
6887 // have a dummy (serial) Map that doesn't touch the original
6888 // communicator. Duplicating that here might be a good idea.
6889 if (ReducedComm.is_null()) {
6890 return;
6891 }
6892
6893 /***************************************************/
6894 /**** 5) Sort ****/
6895 /***************************************************/
6896 if ((!reverseMode && xferAsImport != nullptr) ||
6897 (reverseMode && xferAsExport != nullptr)) {
6898 Import_Util::sortCrsEntries(CSR_rowptr(),
6899 CSR_colind_LID());
6900 } else if ((!reverseMode && xferAsExport != nullptr) ||
6901 (reverseMode && xferAsImport != nullptr)) {
6902 Import_Util::sortAndMergeCrsEntries(CSR_rowptr(),
6903 CSR_colind_LID());
6904 if (CSR_rowptr[N] != mynnz) {
6905 CSR_colind_LID.resize(CSR_rowptr[N]);
6906 }
6907 } else {
6908 TEUCHOS_TEST_FOR_EXCEPTION(
6909 true, std::logic_error,
6910 prefix << "Should never get here! Please report this bug to a Tpetra developer.");
6911 }
6912 /***************************************************/
6913 /**** 6) Reset the colmap and the arrays ****/
6914 /***************************************************/
6915
6916 // Call constructor for the new graph (restricted as needed)
6917 //
6918 destGraph->setAllIndices(CSR_rowptr, CSR_colind_LID);
6919
6920 /***************************************************/
6921 /**** 7) Build Importer & Call ESFC ****/
6922 /***************************************************/
6923 // Pre-build the importer using the existing PIDs
6924 Teuchos::ParameterList esfc_params;
6925 MM = Teuchos::null;
6926 MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC CreateImporter"));
6927 RCP<import_type> MyImport = rcp(new import_type(MyDomainMap, MyColMap, RemotePids));
6928 MM = Teuchos::null;
6929 MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC ESFC"));
6930
6931 if (!params.is_null())
6932 esfc_params.set("compute global constants", params->get("compute global constants", true));
6933
6934 destGraph->expertStaticFillComplete(MyDomainMap, MyRangeMap,
6935 MyImport, Teuchos::null, rcp(&esfc_params, false));
6936}
6937
6938template <class LocalOrdinal, class GlobalOrdinal, class Node>
6939void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6940 importAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6941 const import_type& importer,
6942 const Teuchos::RCP<const map_type>& domainMap,
6943 const Teuchos::RCP<const map_type>& rangeMap,
6944 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6945 transferAndFillComplete(destGraph, importer, Teuchos::null, domainMap, rangeMap, params);
6946}
6947
6948template <class LocalOrdinal, class GlobalOrdinal, class Node>
6949void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6950 importAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6951 const import_type& rowImporter,
6952 const import_type& domainImporter,
6953 const Teuchos::RCP<const map_type>& domainMap,
6954 const Teuchos::RCP<const map_type>& rangeMap,
6955 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6956 transferAndFillComplete(destGraph, rowImporter, Teuchos::rcpFromRef(domainImporter), domainMap, rangeMap, params);
6957}
6958
6959template <class LocalOrdinal, class GlobalOrdinal, class Node>
6960void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6961 exportAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6962 const export_type& exporter,
6963 const Teuchos::RCP<const map_type>& domainMap,
6964 const Teuchos::RCP<const map_type>& rangeMap,
6965 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6966 transferAndFillComplete(destGraph, exporter, Teuchos::null, domainMap, rangeMap, params);
6967}
6968
6969template <class LocalOrdinal, class GlobalOrdinal, class Node>
6970void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6971 exportAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
6972 const export_type& rowExporter,
6973 const export_type& domainExporter,
6974 const Teuchos::RCP<const map_type>& domainMap,
6975 const Teuchos::RCP<const map_type>& rangeMap,
6976 const Teuchos::RCP<Teuchos::ParameterList>& params) const {
6977 transferAndFillComplete(destGraph, rowExporter, Teuchos::rcpFromRef(domainExporter), domainMap, rangeMap, params);
6978}
6979
6980template <class LocalOrdinal, class GlobalOrdinal, class Node>
6981void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
6982 swap(CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& graph) {
6983 std::swap(graph.need_sync_host_uvm_access, this->need_sync_host_uvm_access);
6984
6985 std::swap(graph.rowMap_, this->rowMap_);
6986 std::swap(graph.colMap_, this->colMap_);
6987 std::swap(graph.rangeMap_, this->rangeMap_);
6988 std::swap(graph.domainMap_, this->domainMap_);
6989
6990 std::swap(graph.importer_, this->importer_);
6991 std::swap(graph.exporter_, this->exporter_);
6992
6993 std::swap(graph.nodeMaxNumRowEntries_, this->nodeMaxNumRowEntries_);
6994
6995 std::swap(graph.globalNumEntries_, this->globalNumEntries_);
6996 std::swap(graph.globalMaxNumRowEntries_, this->globalMaxNumRowEntries_);
6997
6998 std::swap(graph.numAllocForAllRows_, this->numAllocForAllRows_);
6999
7000 std::swap(graph.rowPtrsPacked_dev_, this->rowPtrsPacked_dev_);
7001 std::swap(graph.rowPtrsPacked_host_, this->rowPtrsPacked_host_);
7002
7003 std::swap(graph.rowPtrsUnpacked_dev_, this->rowPtrsUnpacked_dev_);
7004 std::swap(graph.rowPtrsUnpacked_host_, this->rowPtrsUnpacked_host_);
7005 std::swap(graph.packedUnpackedRowPtrsMatch_, this->packedUnpackedRowPtrsMatch_);
7006
7007 std::swap(graph.k_offRankOffsets_, this->k_offRankOffsets_);
7008
7009 std::swap(graph.lclIndsUnpacked_wdv, this->lclIndsUnpacked_wdv);
7010 std::swap(graph.gblInds_wdv, this->gblInds_wdv);
7011 std::swap(graph.lclIndsPacked_wdv, this->lclIndsPacked_wdv);
7012
7013 std::swap(graph.storageStatus_, this->storageStatus_);
7014
7015 std::swap(graph.indicesAreAllocated_, this->indicesAreAllocated_);
7016 std::swap(graph.indicesAreLocal_, this->indicesAreLocal_);
7017 std::swap(graph.indicesAreGlobal_, this->indicesAreGlobal_);
7018 std::swap(graph.fillComplete_, this->fillComplete_);
7019 std::swap(graph.indicesAreSorted_, this->indicesAreSorted_);
7020 std::swap(graph.noRedundancies_, this->noRedundancies_);
7021 std::swap(graph.haveLocalConstants_, this->haveLocalConstants_);
7022 std::swap(graph.haveGlobalConstants_, this->haveGlobalConstants_);
7023 std::swap(graph.haveLocalOffRankOffsets_, this->haveLocalOffRankOffsets_);
7024
7025 std::swap(graph.sortGhostsAssociatedWithEachProcessor_, this->sortGhostsAssociatedWithEachProcessor_);
7026
7027 std::swap(graph.k_numAllocPerRow_, this->k_numAllocPerRow_); // View
7028 std::swap(graph.k_numRowEntries_, this->k_numRowEntries_); // View
7029 std::swap(graph.nonlocals_, this->nonlocals_); // std::map
7030}
7031
7032template <class LocalOrdinal, class GlobalOrdinal, class Node>
7033bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
7034 isIdenticalTo(const CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& graph) const {
7035 auto compare_nonlocals = [&](const nonlocals_type& m1, const nonlocals_type& m2) {
7036 bool output = true;
7037 output = m1.size() == m2.size() ? output : false;
7038 for (auto& it_m : m1) {
7039 size_t key = it_m.first;
7040 output = m2.find(key) != m2.end() ? output : false;
7041 if (output) {
7042 auto v1 = m1.find(key)->second;
7043 auto v2 = m2.find(key)->second;
7044 std::sort(v1.begin(), v1.end());
7045 std::sort(v2.begin(), v2.end());
7046
7047 output = v1.size() == v2.size() ? output : false;
7048 for (size_t i = 0; output && i < v1.size(); i++) {
7049 output = v1[i] == v2[i] ? output : false;
7050 }
7051 }
7052 }
7053 return output;
7054 };
7055
7056 bool output = true;
7057
7058 output = this->rowMap_->isSameAs(*(graph.rowMap_)) ? output : false;
7059 output = this->colMap_->isSameAs(*(graph.colMap_)) ? output : false;
7060 output = this->rangeMap_->isSameAs(*(graph.rangeMap_)) ? output : false;
7061 output = this->domainMap_->isSameAs(*(graph.domainMap_)) ? output : false;
7062
7063 output = this->nodeMaxNumRowEntries_ == graph.nodeMaxNumRowEntries_ ? output : false;
7064
7065 output = this->globalNumEntries_ == graph.globalNumEntries_ ? output : false;
7066 output = this->globalMaxNumRowEntries_ == graph.globalMaxNumRowEntries_ ? output : false;
7067
7068 output = this->numAllocForAllRows_ == graph.numAllocForAllRows_ ? output : false;
7069
7070 output = this->storageStatus_ == graph.storageStatus_ ? output : false; // EStorageStatus is an enum
7071
7072 output = this->indicesAreAllocated_ == graph.indicesAreAllocated_ ? output : false;
7073 output = this->indicesAreLocal_ == graph.indicesAreLocal_ ? output : false;
7074 output = this->indicesAreGlobal_ == graph.indicesAreGlobal_ ? output : false;
7075 output = this->fillComplete_ == graph.fillComplete_ ? output : false;
7076 output = this->indicesAreSorted_ == graph.indicesAreSorted_ ? output : false;
7077 output = this->noRedundancies_ == graph.noRedundancies_ ? output : false;
7078 output = this->haveLocalConstants_ == graph.haveLocalConstants_ ? output : false;
7079 output = this->haveGlobalConstants_ == graph.haveGlobalConstants_ ? output : false;
7080 output = this->haveLocalOffRankOffsets_ == graph.haveLocalOffRankOffsets_ ? output : false;
7081 output = this->sortGhostsAssociatedWithEachProcessor_ == graph.sortGhostsAssociatedWithEachProcessor_ ? output : false;
7082
7083 // Compare nonlocals_ -- std::map<GlobalOrdinal, std::vector<GlobalOrdinal> >
7084 // nonlocals_ isa std::map<GO, std::vector<GO> >
7085 output = compare_nonlocals(this->nonlocals_, graph.nonlocals_) ? output : false;
7086
7087 // Compare k_numAllocPerRow_ isa Kokkos::View::host_mirror_type
7088 // - since this is a host_mirror_type type, it should be in host memory already
7089 output = this->k_numAllocPerRow_.extent(0) == graph.k_numAllocPerRow_.extent(0) ? output : false;
7090 if (output && this->k_numAllocPerRow_.extent(0) > 0) {
7091 for (size_t i = 0; output && i < this->k_numAllocPerRow_.extent(0); i++)
7092 output = this->k_numAllocPerRow_(i) == graph.k_numAllocPerRow_(i) ? output : false;
7093 }
7094
7095 // Compare k_numRowEntries_ isa Kokkos::View::host_mirror_type
7096 // - since this is a host_mirror_type type, it should be in host memory already
7097 output = this->k_numRowEntries_.extent(0) == graph.k_numRowEntries_.extent(0) ? output : false;
7098 if (output && this->k_numRowEntries_.extent(0) > 0) {
7099 for (size_t i = 0; output && i < this->k_numRowEntries_.extent(0); i++)
7100 output = this->k_numRowEntries_(i) == graph.k_numRowEntries_(i) ? output : false;
7101 }
7102
7103 // Compare this->k_rowPtrs_ isa Kokkos::View<LocalOrdinal*, ...>
7104 {
7105 auto rowPtrsThis = this->getRowPtrsUnpackedHost();
7106 auto rowPtrsGraph = graph.getRowPtrsUnpackedHost();
7107 output = rowPtrsThis.extent(0) == rowPtrsGraph.extent(0) ? output : false;
7108 for (size_t i = 0; output && i < rowPtrsThis.extent(0); i++)
7109 output = rowPtrsThis(i) == rowPtrsGraph(i) ? output : false;
7110 }
7111
7112 // Compare lclIndsUnpacked_wdv isa Kokkos::View<LocalOrdinal*, ...>
7113 output = this->lclIndsUnpacked_wdv.extent(0) == graph.lclIndsUnpacked_wdv.extent(0) ? output : false;
7114 if (output && this->lclIndsUnpacked_wdv.extent(0) > 0) {
7115 auto indThis = this->lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
7116 auto indGraph = graph.lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
7117 for (size_t i = 0; output && i < indThis.extent(0); i++)
7118 output = indThis(i) == indGraph(i) ? output : false;
7119 }
7120
7121 // Compare gblInds_wdv isa Kokkos::View<GlobalOrdinal*, ...>
7122 output = this->gblInds_wdv.extent(0) == graph.gblInds_wdv.extent(0) ? output : false;
7123 if (output && this->gblInds_wdv.extent(0) > 0) {
7124 auto indtThis = this->gblInds_wdv.getHostView(Access::ReadOnly);
7125 auto indtGraph = graph.gblInds_wdv.getHostView(Access::ReadOnly);
7126 for (size_t i = 0; output && i < indtThis.extent(0); i++)
7127 output = indtThis(i) == indtGraph(i) ? output : false;
7128 }
7129
7130 // Check lclGraph_ isa
7131 // KokkosSparse::StaticCrsGraph<LocalOrdinal, Kokkos::LayoutLeft, execution_space>
7132 // KokkosSparse::StaticCrsGraph has 3 data members in it:
7133 // Kokkos::View<size_type*, ...> row_map
7134 // (local_graph_device_type::row_map_type)
7135 // Kokkos::View<data_type*, ...> entries
7136 // (local_graph_device_type::entries_type)
7137 // Kokkos::View<size_type*, ...> row_block_offsets
7138 // (local_graph_device_type::row_block_type)
7139 // There is currently no KokkosSparse::StaticCrsGraph comparison function
7140 // that's built-in, so we will just compare
7141 // the three data items here. This can be replaced if Kokkos ever
7142 // puts in its own comparison routine.
7143 local_graph_host_type thisLclGraph = this->getLocalGraphHost();
7144 local_graph_host_type graphLclGraph = graph.getLocalGraphHost();
7145
7146 output = thisLclGraph.row_map.extent(0) == graphLclGraph.row_map.extent(0)
7147 ? output
7148 : false;
7149 if (output && thisLclGraph.row_map.extent(0) > 0) {
7150 auto lclGraph_rowmap_host_this = thisLclGraph.row_map;
7151 auto lclGraph_rowmap_host_graph = graphLclGraph.row_map;
7152 for (size_t i = 0; output && i < lclGraph_rowmap_host_this.extent(0); i++)
7153 output = lclGraph_rowmap_host_this(i) == lclGraph_rowmap_host_graph(i)
7154 ? output
7155 : false;
7156 }
7157
7158 output = thisLclGraph.entries.extent(0) == graphLclGraph.entries.extent(0)
7159 ? output
7160 : false;
7161 if (output && thisLclGraph.entries.extent(0) > 0) {
7162 auto lclGraph_entries_host_this = thisLclGraph.entries;
7163 auto lclGraph_entries_host_graph = graphLclGraph.entries;
7164 for (size_t i = 0; output && i < lclGraph_entries_host_this.extent(0); i++)
7165 output = lclGraph_entries_host_this(i) == lclGraph_entries_host_graph(i)
7166 ? output
7167 : false;
7168 }
7169
7170 output =
7171 thisLclGraph.row_block_offsets.extent(0) ==
7172 graphLclGraph.row_block_offsets.extent(0)
7173 ? output
7174 : false;
7175 if (output && thisLclGraph.row_block_offsets.extent(0) > 0) {
7176 auto lclGraph_rbo_host_this = thisLclGraph.row_block_offsets;
7177 auto lclGraph_rbo_host_graph = graphLclGraph.row_block_offsets;
7178 for (size_t i = 0; output && i < lclGraph_rbo_host_this.extent(0); i++)
7179 output = lclGraph_rbo_host_this(i) == lclGraph_rbo_host_graph(i)
7180 ? output
7181 : false;
7182 }
7183
7184 // For Importer and Exporter, we don't need to explicitly check them since
7185 // they will be consistent with the maps.
7186 // Note: importer_ isa Teuchos::RCP<const import_type>
7187 // exporter_ isa Teuchos::RCP<const export_type>
7188
7189 return output;
7190}
7191
7192template <class LocalOrdinal, class GlobalOrdinal, class Node>
7193void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::insertGlobalIndicesDevice(
7194 const CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& srcCrsGraph,
7195 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& tgtCrsGraph,
7196 const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteToLIDs,
7197 const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteFromLIDs,
7198 LocalOrdinal loopEnd) {
7199 using crs_graph_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
7200 using LO = LocalOrdinal;
7201 using GO = GlobalOrdinal;
7202 typedef typename crs_graph_type::global_inds_device_view_type::non_const_value_type global_inds_device_value_t;
7203 typedef typename Node::execution_space exec_space;
7204 typedef Kokkos::RangePolicy<exec_space, LO> range_type;
7205
7206 const LocalOrdinal LINV = Teuchos::OrdinalTraits<LocalOrdinal>::invalid();
7207 const GlobalOrdinal GINV = Teuchos::OrdinalTraits<GlobalOrdinal>::invalid();
7208
7209 using local_map_type = typename crs_graph_type::map_type::local_map_type;
7210 local_map_type srcRowMapLocal = srcCrsGraph.getRowMap()->getLocalMap();
7211 local_map_type srcColMapLocal = srcCrsGraph.getColMap()->getLocalMap();
7212 local_map_type tgtRowMapLocal = tgtCrsGraph.getRowMap()->getLocalMap();
7213
7214 auto tgtLocalRowPtrsDevice = tgtCrsGraph.getRowPtrsUnpackedDevice();
7215 auto tgtGlobalColInds = tgtCrsGraph.gblInds_wdv.getDeviceView(Access::ReadWrite);
7216 auto srcLocalRowPtrsDevice = srcCrsGraph.getLocalRowPtrsDevice();
7217 auto srcLocalColIndsDevice = srcCrsGraph.lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly);
7218
7219 typename crs_graph_type::num_row_entries_type::non_const_type h_numRowEnt = tgtCrsGraph.k_numRowEntries_;
7220
7221 auto k_numRowEnt = Kokkos::create_mirror_view_and_copy(device_type(), h_numRowEnt);
7222
7223 const bool sorted = false;
7224
7225 bool hasMap = permuteFromLIDs.extent(0) > 0;
7226 auto permuteToLIDs_d = permuteToLIDs.view_device();
7227 auto permuteFromLIDs_d = permuteFromLIDs.view_device();
7228
7229#ifdef CRSGRAPH_INNER_ABORT
7230#undef CRSGRAPH_INNER_ABORT
7231#endif
7232
7233#ifdef KOKKOS_ENABLE_SYCL
7234#define CRSGRAPH_INNER_ABORT(lin) \
7235 do { \
7236 sycl::ext::oneapi::experimental::printf("ERROR: Tpetra_CrsGraph_def.hpp:%d", lin); \
7237 Kokkos::abort("error"); \
7238 } while (0)
7239#else
7240#define CRSGRAPH_INNER_ABORT(lin) \
7241 do { \
7242 printf("ERROR: Tpetra_CrsGraph_def.hpp:%d", lin); \
7243 Kokkos::abort("error"); \
7244 } while (0)
7245#endif
7246
7247 Kokkos::parallel_for(
7248 "Tpetra_CrsGraph::copyAndPermuteNew",
7249 range_type(0, loopEnd),
7250 KOKKOS_LAMBDA(const LO sourceLID) {
7251 auto srcLid = sourceLID;
7252 auto tgtLid = sourceLID;
7253 if (hasMap) {
7254 srcLid = permuteFromLIDs_d(srcLid);
7255 tgtLid = permuteToLIDs_d(tgtLid);
7256 }
7257 auto srcGid = srcRowMapLocal.getGlobalElement(srcLid);
7258 if (srcGid == GINV) CRSGRAPH_INNER_ABORT(__LINE__);
7259 auto tgtGid = tgtRowMapLocal.getGlobalElement(tgtLid);
7260 auto tgtLocalRow = tgtRowMapLocal.getLocalElement(tgtGid);
7261 if (tgtLocalRow == LINV) CRSGRAPH_INNER_ABORT(__LINE__);
7262 if (tgtLocalRow != tgtLid) CRSGRAPH_INNER_ABORT(__LINE__);
7263 auto tgtNumEntries = k_numRowEnt(tgtLocalRow);
7264
7265 // FIXME no auto use
7266 auto start = srcLocalRowPtrsDevice(srcLid);
7267 auto end = srcLocalRowPtrsDevice(srcLid + 1);
7268 auto rowLength = (end - start);
7269
7270 auto tstart = tgtLocalRowPtrsDevice(tgtLocalRow);
7271 auto tend = tstart + tgtNumEntries;
7272 auto tend1 = tgtLocalRowPtrsDevice(tgtLocalRow + 1);
7273
7274 const size_t num_avail = (tend1 < tend) ? size_t(0) : tend1 - tend;
7275 size_t num_inserted = 0;
7276
7277 global_inds_device_value_t* tgtGlobalColIndsPtr = tgtGlobalColInds.data();
7278
7279 size_t hint = 0;
7280 for (size_t j = 0; j < rowLength; j++) {
7281 auto ci = srcLocalColIndsDevice(start + j);
7282 GO gi = srcColMapLocal.getGlobalElement(ci);
7283 if (gi == GINV) CRSGRAPH_INNER_ABORT(__LINE__);
7284 auto numInTgtRow = (tend - tstart);
7285
7286 const size_t offset = KokkosSparse::findRelOffset(
7287 tgtGlobalColIndsPtr + tstart, numInTgtRow, gi, hint, sorted);
7288
7289 if (offset == numInTgtRow) {
7290 if (num_inserted >= num_avail) { // not enough room
7291 Kokkos::abort("num_avail");
7292 }
7293 tgtGlobalColIndsPtr[tstart + offset] = gi;
7294 ++tend;
7295 hint = offset + 1;
7296 ++num_inserted;
7297 }
7298 }
7299 k_numRowEnt(tgtLocalRow) += num_inserted;
7300 return size_t(0);
7301 });
7302 Kokkos::deep_copy(tgtCrsGraph.k_numRowEntries_, k_numRowEnt);
7303 tgtCrsGraph.setLocallyModified();
7304}
7305
7306template <class LocalOrdinal, class GlobalOrdinal, class Node>
7307void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::copyAndPermuteNew(
7308 const row_graph_type& srcRowGraph,
7309 row_graph_type& tgtRowGraph,
7310 const size_t numSameIDs,
7311 const Kokkos::DualView<const local_ordinal_type*,
7312 buffer_device_type>& permuteToLIDs,
7313 const Kokkos::DualView<const local_ordinal_type*,
7314 buffer_device_type>& permuteFromLIDs,
7315 const CombineMode CM) {
7316 using std::endl;
7317 using LO = local_ordinal_type;
7318 using GO = global_ordinal_type;
7319 const char tfecfFuncName[] = "copyAndPermuteNew: ";
7320 const bool verbose = verbose_;
7321
7322 Details::ProfilingRegion regionCAP("Tpetra::CrsGraph::copyAndPermuteNew");
7323 std::unique_ptr<std::string> prefix;
7324 if (verbose) {
7325 prefix = this->createPrefix("CrsGraph", "copyAndPermuteNew");
7326 std::ostringstream os;
7327 os << *prefix << endl;
7328 std::cerr << os.str();
7329 }
7330
7331 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
7332 permuteToLIDs.extent(0) != permuteFromLIDs.extent(0),
7333 std::runtime_error,
7334 "permuteToLIDs.extent(0) = " << permuteToLIDs.extent(0) << " != permuteFromLIDs.extent(0) = " << permuteFromLIDs.extent(0) << ".");
7335
7336 if (verbose) {
7337 std::ostringstream os;
7338 os << *prefix << "Compute padding" << endl;
7339 std::cerr << os.str();
7340 }
7341
7342 using crs_graph_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
7343 const crs_graph_type* srcCrsGraphPtr = dynamic_cast<const crs_graph_type*>(&srcRowGraph);
7344 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
7345 !srcCrsGraphPtr, std::runtime_error, "error srcGraph type= " << typeid(srcRowGraph).name());
7346 const crs_graph_type& srcCrsGraph = *srcCrsGraphPtr;
7347
7348 crs_graph_type* tgtCrsGraphPtr = dynamic_cast<crs_graph_type*>(&tgtRowGraph);
7349 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
7350 !tgtCrsGraphPtr, std::runtime_error, "error tgtGraph type= " << typeid(tgtRowGraph).name());
7351
7352 crs_graph_type& tgtCrsGraph = *tgtCrsGraphPtr;
7353 auto padding = tgtCrsGraph.computeCrsPadding(
7354 srcRowGraph, numSameIDs, permuteToLIDs, permuteFromLIDs, verbose);
7355 tgtCrsGraph.applyCrsPadding(*padding, verbose);
7356
7357 const map_type& srcRowMap = *(srcRowGraph.getRowMap());
7358 const map_type& tgtRowMap = *(tgtRowGraph.getRowMap());
7359 const bool src_filled = srcRowGraph.isFillComplete();
7360 nonconst_global_inds_host_view_type row_copy;
7361 LO myid = 0;
7362
7363 //
7364 // "Copy" part of "copy and permute."
7365 //
7366 LO numSameIDs_as_LID = static_cast<LO>(numSameIDs);
7367
7368 if (src_filled || srcCrsGraphPtr == nullptr) {
7369 if (verbose) {
7370 std::ostringstream os;
7371 os << *prefix << "src_filled || srcCrsGraph == nullptr" << endl;
7372 std::cerr << os.str();
7373 }
7374 // If the source graph is fill complete, we can't use view mode,
7375 // because the data might be stored in a different format not
7376 // compatible with the expectations of view mode. Also, if the
7377 // source graph is not a CrsGraph, we can't use view mode,
7378 // because RowGraph only provides copy mode access to the data.
7379 Kokkos::DualView<const local_ordinal_type*, buffer_device_type> noPermute;
7380 insertGlobalIndicesDevice(srcCrsGraph, tgtCrsGraph,
7381 noPermute, noPermute,
7382 numSameIDs_as_LID);
7383 } else {
7384 if (verbose) {
7385 std::ostringstream os;
7386 os << *prefix << "! src_filled && srcCrsGraph != nullptr" << endl;
7387 std::cerr << os.str();
7388 }
7389 for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
7390 const GO gid = srcRowMap.getGlobalElement(myid);
7391 global_inds_host_view_type row;
7392 srcCrsGraph.getGlobalRowView(gid, row);
7393 tgtCrsGraph.insertGlobalIndices(gid, row.extent(0), row.data());
7394 }
7395 }
7396
7397 //
7398 // "Permute" part of "copy and permute."
7399 //
7400 auto permuteToLIDs_h = permuteToLIDs.view_host();
7401 auto permuteFromLIDs_h = permuteFromLIDs.view_host();
7402 auto permuteToLIDs_d = permuteToLIDs.view_device();
7403 auto permuteFromLIDs_d = permuteFromLIDs.view_device();
7404
7405 if (src_filled || srcCrsGraphPtr == nullptr) {
7406 insertGlobalIndicesDevice(
7407 srcCrsGraph,
7408 tgtCrsGraph,
7409 permuteToLIDs,
7410 permuteFromLIDs, // note reversed arg order, tgt, then src
7411 static_cast<LO>(permuteToLIDs_h.extent(0)));
7412 } else {
7413 for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
7414 const GO mygid = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
7415 const GO srcgid = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
7416 global_inds_host_view_type row;
7417 srcCrsGraph.getGlobalRowView(srcgid, row);
7418 tgtCrsGraph.insertGlobalIndices(mygid, row.extent(0), row.data());
7419 }
7420 }
7421
7422 if (verbose) {
7423 std::ostringstream os;
7424 os << *prefix << "Done" << endl;
7425 std::cerr << os.str();
7426 }
7427}
7428
7429} // namespace Tpetra
7430
7431//
7432// Explicit instantiation macros
7433//
7434// Must be expanded from within the Tpetra namespace!
7435//
7436
7437#define TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE) \
7438 template <> \
7439 Teuchos::RCP<CrsGraph<LO, GO, NODE>> \
7440 importAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph, \
7441 const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7442 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7443 CrsGraph<LO, GO, NODE>::node_type>& importer, \
7444 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7445 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7446 CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
7447 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7448 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7449 CrsGraph<LO, GO, NODE>::node_type>>& rangeMap, \
7450 const Teuchos::RCP<Teuchos::ParameterList>& params);
7451
7452#define TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE) \
7453 template <> \
7454 Teuchos::RCP<CrsGraph<LO, GO, NODE>> \
7455 importAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph, \
7456 const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7457 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7458 CrsGraph<LO, GO, NODE>::node_type>& rowImporter, \
7459 const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7460 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7461 CrsGraph<LO, GO, NODE>::node_type>& domainImporter, \
7462 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7463 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7464 CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
7465 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7466 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7467 CrsGraph<LO, GO, NODE>::node_type>>& rangeMap, \
7468 const Teuchos::RCP<Teuchos::ParameterList>& params);
7469
7470#define TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE) \
7471 template <> \
7472 Teuchos::RCP<CrsGraph<LO, GO, NODE>> \
7473 exportAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph, \
7474 const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7475 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7476 CrsGraph<LO, GO, NODE>::node_type>& exporter, \
7477 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7478 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7479 CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
7480 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7481 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7482 CrsGraph<LO, GO, NODE>::node_type>>& rangeMap, \
7483 const Teuchos::RCP<Teuchos::ParameterList>& params);
7484
7485#define TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE) \
7486 template <> \
7487 Teuchos::RCP<CrsGraph<LO, GO, NODE>> \
7488 exportAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph, \
7489 const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7490 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7491 CrsGraph<LO, GO, NODE>::node_type>& rowExporter, \
7492 const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7493 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7494 CrsGraph<LO, GO, NODE>::node_type>& domainExporter, \
7495 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7496 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7497 CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
7498 const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type, \
7499 CrsGraph<LO, GO, NODE>::global_ordinal_type, \
7500 CrsGraph<LO, GO, NODE>::node_type>>& rangeMap, \
7501 const Teuchos::RCP<Teuchos::ParameterList>& params);
7502
7503#define TPETRA_CRSGRAPH_INSTANT(LO, GO, NODE) \
7504 template class CrsGraph<LO, GO, NODE>; \
7505 TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE) \
7506 TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE) \
7507 TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE) \
7508 TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE)
7509
7510#endif // TPETRA_CRSGRAPH_DEF_HPP
Tpetra_Details_Behavior.hpp
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Tpetra_Details_Profiling.hpp
Declaration of Tpetra::Details::Profiling, a scope guard for Kokkos Profiling.
Tpetra_Details_computeOffsets.hpp
Declare and define the functions Tpetra::Details::computeOffsetsFromCounts and Tpetra::computeOffsets...
Tpetra_Details_copyOffsets.hpp
Declare and define Tpetra::Details::copyOffsets, an implementation detail of Tpetra (in particular,...
Tpetra_Details_crsUtils.hpp
Functions for manipulating CRS arrays.
Tpetra_Details_gathervPrint.hpp
Declaration of a function that prints strings from each process.
Tpetra_Details_getEntryOnHost.hpp
Declaration and definition of Tpetra::Details::getEntryOnHost.
Tpetra_Details_iallreduce.hpp
Declaration of Tpetra::Details::iallreduce.
Tpetra_Import_Util2.hpp
Utility functions for packing and unpacking sparse matrix entries.
Tpetra::Import_Util::sortAndMergeCrsEntries
void sortAndMergeCrsEntries(const Teuchos::ArrayView< size_t > &CRS_rowptr, const Teuchos::ArrayView< Ordinal > &CRS_colind, const Teuchos::ArrayView< Scalar > &CRS_vals)
Sort and merge the entries of the (raw CSR) matrix by column index within each row.
Definition Tpetra_Import_Util2.hpp:523
Tpetra::Import_Util::lowCommunicationMakeColMapAndReindex
void lowCommunicationMakeColMapAndReindex(const Teuchos::ArrayView< const size_t > &rowptr, const Teuchos::ArrayView< LocalOrdinal > &colind_LID, const Teuchos::ArrayView< GlobalOrdinal > &colind_GID, const Teuchos::RCP< const Tpetra::Map< LocalOrdinal, GlobalOrdinal, Node > > &domainMapRCP, const Teuchos::ArrayView< const int > &owningPIDs, Teuchos::Array< int > &remotePIDs, Teuchos::RCP< const Tpetra::Map< LocalOrdinal, GlobalOrdinal, Node > > &colMap, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
lowCommunicationMakeColMapAndReindex
Definition Tpetra_Import_Util2.hpp:856
Tpetra::Import_Util::sortCrsEntries
void sortCrsEntries(const Teuchos::ArrayView< size_t > &CRS_rowptr, const Teuchos::ArrayView< Ordinal > &CRS_colind, const Teuchos::ArrayView< Scalar > &CRS_vals)
Sort the entries of the (raw CSR) matrix by column index within each row.
Definition Tpetra_Import_Util2.hpp:467
Tpetra_Import_Util.hpp
Internal functions and macros designed for use with Tpetra::Import and Tpetra::Export objects.
Tpetra::Import_Util::getPids
void getPids(const Tpetra::Import< LocalOrdinal, GlobalOrdinal, Node > &Importer, Teuchos::Array< int > &pids, bool use_minus_one_for_local)
Like getPidGidPairs, but just gets the PIDs, ordered by the column Map.
Definition Tpetra_Import_Util.hpp:110
Tpetra_Util.hpp
Stand-alone utility functions and macros.
Tpetra::CrsGraph
A distributed graph accessed by rows (adjacency lists) and stored sparsely.
Definition Tpetra_CrsGraph_decl.hpp:196
Tpetra::CrsGraph::isMerged
bool isMerged() const
Whether duplicate column indices in each row have been merged.
Definition Tpetra_CrsGraph_def.hpp:1129
Tpetra::CrsGraph::unpackAndCombine
virtual void unpackAndCombine(const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &importLIDs, Kokkos::DualView< packet_type *, buffer_device_type > imports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, const size_t constantNumPackets, const CombineMode combineMode) override
Definition Tpetra_CrsGraph_def.hpp:5755
Tpetra::CrsGraph::getLocalIndsViewDevice
local_inds_dualv_type::t_dev::const_type getLocalIndsViewDevice(const RowInfo &rowinfo) const
Get a const, locally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myRo...
Definition Tpetra_CrsGraph_def.hpp:1367
Tpetra::CrsGraph::globalMaxNumRowEntries_
global_size_t globalMaxNumRowEntries_
Global maximum of the number of entries in each row.
Definition Tpetra_CrsGraph_decl.hpp:2194
Tpetra::CrsGraph::reindexColumns
void reindexColumns(const Teuchos::RCP< const map_type > &newColMap, const Teuchos::RCP< const import_type > &newImport=Teuchos::null, const bool sortIndicesInEachRow=true)
Reindex the column indices in place, and replace the column Map. Optionally, replace the Import objec...
Definition Tpetra_CrsGraph_def.hpp:3561
Tpetra::CrsGraph::getGlobalIndsViewHost
global_inds_dualv_type::t_host::const_type getGlobalIndsViewHost(const RowInfo &rowinfo) const
Get a const, globally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myR...
Definition Tpetra_CrsGraph_def.hpp:1354
Tpetra::CrsGraph::getNumEntriesInLocalRow
size_t getNumEntriesInLocalRow(local_ordinal_type localRow) const override
Get the number of entries in the given row (local index).
Definition Tpetra_CrsGraph_def.hpp:2091
Tpetra::CrsGraph::getColMap
Teuchos::RCP< const map_type > getColMap() const override
Returns the Map that describes the column distribution in this graph.
Definition Tpetra_CrsGraph_def.hpp:898
Tpetra::CrsGraph::getValidParameters
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const override
Default parameter list suitable for validation.
Definition Tpetra_CrsGraph_def.hpp:807
Tpetra::CrsGraph::storageStatus_
Details::EStorageStatus storageStatus_
Status of the graph's storage, when not in a fill-complete state.
Definition Tpetra_CrsGraph_decl.hpp:2497
Tpetra::CrsGraph::import_type
::Tpetra::Import< LocalOrdinal, GlobalOrdinal, Node > import_type
The Import specialization used by this class.
Definition Tpetra_CrsGraph_decl.hpp:234
Tpetra::CrsGraph::packet_type
global_ordinal_type packet_type
Type of each entry of the DistObject communication buffer.
Definition Tpetra_CrsGraph_decl.hpp:1214
Tpetra::CrsGraph::global_ordinal_type
GlobalOrdinal global_ordinal_type
The type of the graph's global indices.
Definition Tpetra_CrsGraph_decl.hpp:211
Tpetra::CrsGraph::insertGlobalIndicesIntoNonownedRows
void insertGlobalIndicesIntoNonownedRows(const global_ordinal_type gblRow, const global_ordinal_type gblColInds[], const local_ordinal_type numGblColInds)
Implementation of insertGlobalIndices for nonowned rows.
Definition Tpetra_CrsGraph_def.hpp:2514
Tpetra::CrsGraph::rangeMap_
Teuchos::RCP< const map_type > rangeMap_
The Map describing the range of the (matrix corresponding to the) graph.
Definition Tpetra_CrsGraph_decl.hpp:2158
Tpetra::CrsGraph::makeIndicesLocal
std::pair< size_t, std::string > makeIndicesLocal(const bool verbose=false)
Convert column indices from global to local.
Definition Tpetra_CrsGraph_def.hpp:3999
Tpetra::CrsGraph::getLocalIndicesDevice
local_inds_device_view_type getLocalIndicesDevice() const
Get a device view of the packed column indicies.
Definition Tpetra_CrsGraph_def.hpp:2148
Tpetra::CrsGraph::getGlobalNumEntries
global_size_t getGlobalNumEntries() const override
Returns the global number of entries in the graph.
Definition Tpetra_CrsGraph_def.hpp:952
Tpetra::CrsGraph::isIdenticalTo
bool isIdenticalTo(const CrsGraph< LocalOrdinal, GlobalOrdinal, Node > &graph) const
Create a cloned CrsGraph for a different Node type.
Definition Tpetra_CrsGraph_def.hpp:7034
Tpetra::CrsGraph::getComm
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const override
Returns the communicator.
Definition Tpetra_CrsGraph_def.hpp:1104
Tpetra::CrsGraph::lclIndsUnpacked_wdv
local_inds_wdv_type lclIndsUnpacked_wdv
Local ordinals of column indices for all rows Valid when isLocallyIndexed is true If OptimizedStorage...
Definition Tpetra_CrsGraph_decl.hpp:2322
Tpetra::CrsGraph::globalAssemble
void globalAssemble()
Communicate nonlocal contributions to other processes.
Definition Tpetra_CrsGraph_def.hpp:2721
Tpetra::CrsGraph::getRowInfoFromGlobalRowIndex
RowInfo getRowInfoFromGlobalRowIndex(const global_ordinal_type gblRow) const
Get information about the locally owned row with global index gblRow.
Definition Tpetra_CrsGraph_def.hpp:1432
Tpetra::CrsGraph::getLocalDiagOffsets
void getLocalDiagOffsets(const Kokkos::View< size_t *, device_type, Kokkos::MemoryUnmanaged > &offsets) const
Get offsets of the diagonal entries in the graph.
Definition Tpetra_CrsGraph_def.hpp:6005
Tpetra::CrsGraph::findGlobalIndices
size_t findGlobalIndices(const RowInfo &rowInfo, const Teuchos::ArrayView< const global_ordinal_type > &indices, std::function< void(const size_t, const size_t, const size_t)> fun) const
Finds indices in the given row.
Definition Tpetra_CrsGraph_def.hpp:1761
Tpetra::CrsGraph::fillComplete
void fillComplete(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Tell the graph that you are done changing its structure.
Definition Tpetra_CrsGraph_def.hpp:2972
Tpetra::CrsGraph::gblInds_wdv
global_inds_wdv_type gblInds_wdv
Global ordinals of column indices for all rows.
Definition Tpetra_CrsGraph_decl.hpp:2352
Tpetra::CrsGraph::nodeMaxNumRowEntries_
size_t nodeMaxNumRowEntries_
Local maximum of the number of entries in each row.
Definition Tpetra_CrsGraph_decl.hpp:2181
Tpetra::CrsGraph::local_graph_device_type
KokkosSparse::StaticCrsGraph< local_ordinal_type, Kokkos::LayoutLeft, device_type, void, size_t > local_graph_device_type
The type of the part of the sparse graph on each MPI process.
Definition Tpetra_CrsGraph_decl.hpp:224
Tpetra::CrsGraph::importer_
Teuchos::RCP< const import_type > importer_
The Import from the domain Map to the column Map.
Definition Tpetra_CrsGraph_decl.hpp:2168
Tpetra::CrsGraph::k_numRowEntries_
num_row_entries_type k_numRowEntries_
The number of local entries in each locally owned row.
Definition Tpetra_CrsGraph_decl.hpp:2477
Tpetra::CrsGraph::getRowPtrsUnpackedDevice
const row_ptrs_device_view_type & getRowPtrsUnpackedDevice() const
Get the unpacked row pointers on device.
Definition Tpetra_CrsGraph_decl.hpp:2230
Tpetra::CrsGraph::numAllocForAllRows_
size_t numAllocForAllRows_
The maximum number of entries to allow in each locally owned row.
Definition Tpetra_CrsGraph_decl.hpp:2425
Tpetra::CrsGraph::hasColMap
bool hasColMap() const override
Whether the graph has a column Map.
Definition Tpetra_CrsGraph_def.hpp:932
Tpetra::CrsGraph::local_ordinal_type
LocalOrdinal local_ordinal_type
The type of the graph's local indices.
Definition Tpetra_CrsGraph_decl.hpp:209
Tpetra::CrsGraph::description
std::string description() const override
Return a one-line human-readable description of this object.
Definition Tpetra_CrsGraph_def.hpp:4401
Tpetra::CrsGraph::isStorageOptimized
bool isStorageOptimized() const
Returns true if storage has been optimized.
Definition Tpetra_CrsGraph_def.hpp:938
Tpetra::CrsGraph::getGlobalRowCopy
void getGlobalRowCopy(global_ordinal_type gblRow, nonconst_global_inds_host_view_type &gblColInds, size_t &numColInds) const override
Get a copy of the given row, using global indices.
Definition Tpetra_CrsGraph_def.hpp:2196
Tpetra::CrsGraph::removeLocalIndices
void removeLocalIndices(local_ordinal_type localRow)
Remove all graph indices from the specified local row.
Definition Tpetra_CrsGraph_def.hpp:2531
Tpetra::CrsGraph::importAndFillComplete
void importAndFillComplete(Teuchos::RCP< CrsGraph< local_ordinal_type, global_ordinal_type, Node > > &destGraph, const import_type &importer, const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null) const
Import from this to the given destination graph, and make the result fill complete.
Definition Tpetra_CrsGraph_def.hpp:6940
Tpetra::CrsGraph::getGlobalNumRows
global_size_t getGlobalNumRows() const override
Returns the number of global rows in the graph.
Definition Tpetra_CrsGraph_def.hpp:851
Tpetra::CrsGraph::getDomainMap
Teuchos::RCP< const map_type > getDomainMap() const override
Returns the Map associated with the domain of this graph.
Definition Tpetra_CrsGraph_def.hpp:905
Tpetra::CrsGraph::replaceRangeMapAndExporter
void replaceRangeMapAndExporter(const Teuchos::RCP< const map_type > &newRangeMap, const Teuchos::RCP< const export_type > &newExporter)
Replace the current Range Map and Export with the given parameters.
Definition Tpetra_CrsGraph_def.hpp:3868
Tpetra::CrsGraph::computeLocalConstants
void computeLocalConstants()
Compute local constants, if they have not yet been computed.
Definition Tpetra_CrsGraph_def.hpp:3972
Tpetra::CrsGraph::describe
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const override
Print this object to the given output stream with the given verbosity level.
Definition Tpetra_CrsGraph_def.hpp:4420
Tpetra::CrsGraph::local_graph_host_type
typename local_graph_device_type::host_mirror_type local_graph_host_type
The type of the part of the sparse graph on each MPI process.
Definition Tpetra_CrsGraph_decl.hpp:229
Tpetra::CrsGraph::setParameterList
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &params) override
Set the given list of parameters (must be nonnull).
Definition Tpetra_CrsGraph_def.hpp:841
Tpetra::CrsGraph::resumeFill
void resumeFill(const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Resume fill operations.
Definition Tpetra_CrsGraph_def.hpp:2937
Tpetra::CrsGraph::insertIndices
size_t insertIndices(RowInfo &rowInfo, const SLocalGlobalViews &newInds, const ELocalGlobal lg, const ELocalGlobal I)
Insert indices into the given row.
Definition Tpetra_CrsGraph_def.hpp:1525
Tpetra::CrsGraph::device_type
typename Node::device_type device_type
This class' Kokkos device type.
Definition Tpetra_CrsGraph_decl.hpp:213
Tpetra::CrsGraph::insertGlobalIndicesFiltered
void insertGlobalIndicesFiltered(const local_ordinal_type lclRow, const global_ordinal_type gblColInds[], const local_ordinal_type numGblColInds)
Like insertGlobalIndices(), but with column Map filtering.
Definition Tpetra_CrsGraph_def.hpp:2455
Tpetra::CrsGraph::copyAndPermute
virtual void copyAndPermute(const SrcDistObject &source, const size_t numSameIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteToLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteFromLIDs, const CombineMode CM) override
Definition Tpetra_CrsGraph_def.hpp:4548
Tpetra::CrsGraph::getRowInfo
RowInfo getRowInfo(const local_ordinal_type myRow) const
Get information about the locally owned row with local index myRow.
Definition Tpetra_CrsGraph_def.hpp:1392
Tpetra::CrsGraph::getGlobalIndsViewDevice
global_inds_dualv_type::t_dev::const_type getGlobalIndsViewDevice(const RowInfo &rowinfo) const
Get a const, globally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myR...
Definition Tpetra_CrsGraph_def.hpp:1380
Tpetra::CrsGraph::colMap_
Teuchos::RCP< const map_type > colMap_
The Map describing the distribution of columns of the graph.
Definition Tpetra_CrsGraph_decl.hpp:2156
Tpetra::CrsGraph::noRedundancies_
bool noRedundancies_
Whether the graph's indices are non-redundant (merged) in each row, on this process.
Definition Tpetra_CrsGraph_decl.hpp:2509
Tpetra::CrsGraph::getLocalRowPtrsHost
row_ptrs_host_view_type getLocalRowPtrsHost() const
Get a host view of the packed row offsets.
Definition Tpetra_CrsGraph_def.hpp:2127
Tpetra::CrsGraph::isSorted
bool isSorted() const
Whether graph indices in all rows are known to be sorted.
Definition Tpetra_CrsGraph_def.hpp:1123
Tpetra::CrsGraph::buffer_device_type
typename dist_object_type::buffer_device_type buffer_device_type
Kokkos::Device specialization for communication buffers.
Definition Tpetra_CrsGraph_decl.hpp:1212
Tpetra::CrsGraph::setAllIndices
void setAllIndices(const typename local_graph_device_type::row_map_type &rowPointers, const typename local_graph_device_type::entries_type::non_const_type &columnIndices)
Set the graph's data directly, using 1-D storage.
Definition Tpetra_CrsGraph_def.hpp:2562
Tpetra::CrsGraph::insertLocalIndices
void insertLocalIndices(const local_ordinal_type localRow, const Teuchos::ArrayView< const local_ordinal_type > &indices)
Insert local indices into the graph.
Definition Tpetra_CrsGraph_def.hpp:2300
Tpetra::CrsGraph::getLocalIndicesHost
local_inds_host_view_type getLocalIndicesHost() const
Get a host view of the packed column indicies.
Definition Tpetra_CrsGraph_def.hpp:2141
Tpetra::CrsGraph::supportsRowViews
bool supportsRowViews() const override
Whether this class implements getLocalRowView() and getGlobalRowView() (it does).
Definition Tpetra_CrsGraph_def.hpp:6353
Tpetra::CrsGraph::getNumEntriesInGlobalRow
size_t getNumEntriesInGlobalRow(global_ordinal_type globalRow) const override
Returns the current number of entries on this node in the specified global row.
Definition Tpetra_CrsGraph_def.hpp:2079
Tpetra::CrsGraph::isFillComplete
bool isFillComplete() const override
Whether fillComplete() has been called and the graph is in compute mode.
Definition Tpetra_CrsGraph_def.hpp:1041
Tpetra::CrsGraph::setDomainRangeMaps
void setDomainRangeMaps(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap)
Definition Tpetra_CrsGraph_def.hpp:1804
Tpetra::CrsGraph::swap
void swap(CrsGraph< local_ordinal_type, global_ordinal_type, Node > &graph)
Swaps the data from *this with the data and maps from graph.
Definition Tpetra_CrsGraph_def.hpp:6982
Tpetra::CrsGraph::map_type
::Tpetra::Map< LocalOrdinal, GlobalOrdinal, Node > map_type
The Map specialization used by this class.
Definition Tpetra_CrsGraph_decl.hpp:232
Tpetra::CrsGraph::getLocalRowCopy
void getLocalRowCopy(local_ordinal_type lclRow, nonconst_local_inds_host_view_type &lclColInds, size_t &numColInds) const override
Get a copy of the given row, using local indices.
Definition Tpetra_CrsGraph_def.hpp:2154
Tpetra::CrsGraph::getGlobalRowView
void getGlobalRowView(const global_ordinal_type gblRow, global_inds_host_view_type &gblColInds) const override
Get a const view of the given global row's global column indices.
Definition Tpetra_CrsGraph_def.hpp:2270
Tpetra::CrsGraph::getRowPtrsUnpackedHost
const row_ptrs_host_view_type & getRowPtrsUnpackedHost() const
Get the unpacked row pointers on host. Lazily make a copy from device.
Definition Tpetra_CrsGraph_decl.hpp:2235
Tpetra::CrsGraph::exportAndFillComplete
void exportAndFillComplete(Teuchos::RCP< CrsGraph< local_ordinal_type, global_ordinal_type, Node > > &destGraph, const export_type &exporter, const Teuchos::RCP< const map_type > &domainMap=Teuchos::null, const Teuchos::RCP< const map_type > &rangeMap=Teuchos::null, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null) const
Export from this to the given destination graph, and make the result fill complete.
Definition Tpetra_CrsGraph_def.hpp:6961
Tpetra::CrsGraph::haveGlobalConstants_
bool haveGlobalConstants_
Whether all processes have computed global constants.
Definition Tpetra_CrsGraph_decl.hpp:2513
Tpetra::CrsGraph::getGlobalMaxNumRowEntries
size_t getGlobalMaxNumRowEntries() const override
Maximum number of entries in any row of the graph, over all processes in the graph's communicator.
Definition Tpetra_CrsGraph_def.hpp:1023
Tpetra::CrsGraph::checkInternalState
void checkInternalState() const
Throw an exception if the internal state is not consistent.
Definition Tpetra_CrsGraph_def.hpp:1829
Tpetra::CrsGraph::getRangeMap
Teuchos::RCP< const map_type > getRangeMap() const override
Returns the Map associated with the domain of this graph.
Definition Tpetra_CrsGraph_def.hpp:912
Tpetra::CrsGraph::expertStaticFillComplete
void expertStaticFillComplete(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< const import_type > &importer=Teuchos::null, const Teuchos::RCP< const export_type > &exporter=Teuchos::null, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Perform a fillComplete on a graph that already has data, via setAllIndices().
Definition Tpetra_CrsGraph_def.hpp:3183
Tpetra::CrsGraph::sortGhostsAssociatedWithEachProcessor_
bool sortGhostsAssociatedWithEachProcessor_
Whether to require makeColMap() (and therefore fillComplete()) to order column Map GIDs associated wi...
Definition Tpetra_CrsGraph_decl.hpp:2536
Tpetra::CrsGraph::getNumAllocatedEntriesInGlobalRow
size_t getNumAllocatedEntriesInGlobalRow(global_ordinal_type globalRow) const
Current number of allocated entries in the given row on the calling (MPI) process,...
Definition Tpetra_CrsGraph_def.hpp:2103
Tpetra::CrsGraph::getExporter
Teuchos::RCP< const export_type > getExporter() const override
Returns the exporter associated with this graph.
Definition Tpetra_CrsGraph_def.hpp:926
Tpetra::CrsGraph::makeImportExport
void makeImportExport(Teuchos::Array< int > &remotePIDs, const bool useRemotePIDs)
Make the Import and Export objects, if needed.
Definition Tpetra_CrsGraph_def.hpp:4340
Tpetra::CrsGraph::getIndexBase
global_ordinal_type getIndexBase() const override
Returns the index base for global indices for this graph.
Definition Tpetra_CrsGraph_def.hpp:1111
Tpetra::CrsGraph::getLocalRowPtrsDevice
row_ptrs_device_view_type getLocalRowPtrsDevice() const
Get a device view of the packed row offsets.
Definition Tpetra_CrsGraph_def.hpp:2134
Tpetra::CrsGraph::getLocalIndsViewHost
local_inds_dualv_type::t_host::const_type getLocalIndsViewHost(const RowInfo &rowinfo) const
Get a const, locally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myRo...
Definition Tpetra_CrsGraph_def.hpp:1328
Tpetra::CrsGraph::isFillActive
bool isFillActive() const
Whether resumeFill() has been called and the graph is in edit mode.
Definition Tpetra_CrsGraph_def.hpp:1047
Tpetra::CrsGraph::getRowMap
Teuchos::RCP< const map_type > getRowMap() const override
Returns the Map that describes the row distribution in this graph.
Definition Tpetra_CrsGraph_def.hpp:891
Tpetra::CrsGraph::globalNumEntries_
global_size_t globalNumEntries_
Global number of entries in the graph.
Definition Tpetra_CrsGraph_decl.hpp:2187
Tpetra::CrsGraph::insertGlobalIndicesImpl
size_t insertGlobalIndicesImpl(const local_ordinal_type lclRow, const global_ordinal_type inputGblColInds[], const size_t numInputInds)
Insert global indices, using an input local row index.
Definition Tpetra_CrsGraph_def.hpp:1622
Tpetra::CrsGraph::export_type
::Tpetra::Export< LocalOrdinal, GlobalOrdinal, Node > export_type
The Export specialization used by this class.
Definition Tpetra_CrsGraph_decl.hpp:236
Tpetra::CrsGraph::getLocalNumEntries
size_t getLocalNumEntries() const override
The local number of entries in the graph.
Definition Tpetra_CrsGraph_def.hpp:964
Tpetra::CrsGraph::getImporter
Teuchos::RCP< const import_type > getImporter() const override
Returns the importer associated with this graph.
Definition Tpetra_CrsGraph_def.hpp:919
Tpetra::CrsGraph::lclIndsPacked_wdv
local_inds_wdv_type lclIndsPacked_wdv
Local ordinals of column indices for all rows Valid when isLocallyIndexed is true Built during fillCo...
Definition Tpetra_CrsGraph_decl.hpp:2337
Tpetra::CrsGraph::domainMap_
Teuchos::RCP< const map_type > domainMap_
The Map describing the domain of the (matrix corresponding to the) graph.
Definition Tpetra_CrsGraph_decl.hpp:2160
Tpetra::CrsGraph::getRowPtrsPackedHost
const row_ptrs_host_view_type & getRowPtrsPackedHost() const
Get the packed row pointers on host. Lazily make a copy from device.
Definition Tpetra_CrsGraph_decl.hpp:2271
Tpetra::CrsGraph::getLocalNumCols
size_t getLocalNumCols() const override
Returns the number of columns connected to the locally owned rows of this graph.
Definition Tpetra_CrsGraph_def.hpp:877
Tpetra::CrsGraph::nonlocals_
nonlocals_type nonlocals_
Nonlocal data given to insertGlobalIndices.
Definition Tpetra_CrsGraph_decl.hpp:2520
Tpetra::CrsGraph::pack
virtual void pack(const Teuchos::ArrayView< const local_ordinal_type > &exportLIDs, Teuchos::Array< global_ordinal_type > &exports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, size_t &constantNumPackets) const override
Definition Tpetra_CrsGraph_def.hpp:5328
Tpetra::CrsGraph::getLocalOffRankOffsets
void getLocalOffRankOffsets(offset_device_view_type &offsets) const
Get offsets of the off-rank entries in the graph.
Definition Tpetra_CrsGraph_def.hpp:6191
Tpetra::CrsGraph::getGlobalNumCols
global_size_t getGlobalNumCols() const override
Returns the number of global columns in the graph.
Definition Tpetra_CrsGraph_def.hpp:858
Tpetra::CrsGraph::k_numAllocPerRow_
Kokkos::View< constsize_t *, device_type >::host_mirror_type k_numAllocPerRow_
The maximum number of entries to allow in each locally owned row, per row.
Definition Tpetra_CrsGraph_decl.hpp:2414
Tpetra::CrsGraph::indicesAreSorted_
bool indicesAreSorted_
Whether the graph's indices are sorted in each row, on this process.
Definition Tpetra_CrsGraph_decl.hpp:2506
Tpetra::CrsGraph::node_type
Node node_type
This class' Kokkos Node type.
Definition Tpetra_CrsGraph_decl.hpp:221
Tpetra::CrsGraph::exporter_
Teuchos::RCP< const export_type > exporter_
The Export from the row Map to the range Map.
Definition Tpetra_CrsGraph_decl.hpp:2175
Tpetra::CrsGraph::insertGlobalIndices
void insertGlobalIndices(const global_ordinal_type globalRow, const Teuchos::ArrayView< const global_ordinal_type > &indices)
Insert global indices into the graph.
Definition Tpetra_CrsGraph_def.hpp:2447
Tpetra::CrsGraph::getLocalIndsViewHostNonConst
local_inds_dualv_type::t_host getLocalIndsViewHostNonConst(const RowInfo &rowinfo)
Get a ReadWrite locally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(m...
Definition Tpetra_CrsGraph_def.hpp:1341
Tpetra::CrsGraph::replaceDomainMap
void replaceDomainMap(const Teuchos::RCP< const map_type > &newDomainMap)
Replace the current domain Map with the given objects.
Definition Tpetra_CrsGraph_def.hpp:3791
Tpetra::CrsGraph::computeGlobalConstants
void computeGlobalConstants()
Compute global constants, if they have not yet been computed.
Definition Tpetra_CrsGraph_def.hpp:3925
Tpetra::CrsGraph::getNumAllocatedEntriesInLocalRow
size_t getNumAllocatedEntriesInLocalRow(local_ordinal_type localRow) const
Current number of allocated entries in the given row on the calling (MPI) process,...
Definition Tpetra_CrsGraph_def.hpp:2115
Tpetra::CrsGraph::local_inds_device_view_type
typename row_graph_type::local_inds_device_view_type local_inds_device_view_type
The Kokkos::View type for views of local ordinals on device and host.
Definition Tpetra_CrsGraph_decl.hpp:259
Tpetra::CrsGraph::k_offRankOffsets_
offset_device_view_type k_offRankOffsets_
The offsets for off-rank entries.
Definition Tpetra_CrsGraph_decl.hpp:2484
Tpetra::CrsGraph::replaceDomainMapAndImporter
void replaceDomainMapAndImporter(const Teuchos::RCP< const map_type > &newDomainMap, const Teuchos::RCP< const import_type > &newImporter)
Replace the current domain Map and Import with the given parameters.
Definition Tpetra_CrsGraph_def.hpp:3810
Tpetra::CrsGraph::setLocallyModified
void setLocallyModified()
Report that we made a local modification to its structure.
Definition Tpetra_CrsGraph_def.hpp:1135
Tpetra::CrsGraph::getLocalAllocationSize
size_t getLocalAllocationSize() const
The local number of indices allocated for the graph, over all rows on the calling (MPI) process.
Definition Tpetra_CrsGraph_def.hpp:1066
Tpetra::CrsGraph::replaceRangeMap
void replaceRangeMap(const Teuchos::RCP< const map_type > &newRangeMap)
Replace the current Range Map with the given objects.
Definition Tpetra_CrsGraph_def.hpp:3849
Tpetra::CrsGraph::rowMap_
Teuchos::RCP< const map_type > rowMap_
The Map describing the distribution of rows of the graph.
Definition Tpetra_CrsGraph_decl.hpp:2154
Tpetra::CrsGraph::getRowPtrsPackedDevice
const row_ptrs_device_view_type & getRowPtrsPackedDevice() const
Get the packed row pointers on device.
Definition Tpetra_CrsGraph_decl.hpp:2266
Tpetra::CrsGraph::removeEmptyProcessesInPlace
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap) override
Remove processes owning zero rows from the Maps and their communicator.
Definition Tpetra_CrsGraph_def.hpp:5910
Tpetra::CrsGraph::getLocalRowView
void getLocalRowView(const LocalOrdinal lclRow, local_inds_host_view_type &lclColInds) const override
Get a const view of the given local row's local column indices.
Definition Tpetra_CrsGraph_def.hpp:2235
Tpetra::CrsGraph::isGloballyIndexed
bool isGloballyIndexed() const override
Whether the graph's column indices are stored as global indices.
Definition Tpetra_CrsGraph_def.hpp:1059
Tpetra::CrsGraph::isLocallyIndexed
bool isLocallyIndexed() const override
Whether the graph's column indices are stored as local indices.
Definition Tpetra_CrsGraph_def.hpp:1053
Tpetra::CrsGraph::getLocalMaxNumRowEntries
size_t getLocalMaxNumRowEntries() const override
Maximum number of entries in any row of the graph, on this process.
Definition Tpetra_CrsGraph_def.hpp:1035
Tpetra::CrsGraph::checkSizes
virtual bool checkSizes(const SrcDistObject &source) override
Compare the source and target (this) objects for compatibility.
Definition Tpetra_CrsGraph_def.hpp:4539
Tpetra::CrsGraph::getLocalGraphDevice
local_graph_device_type getLocalGraphDevice() const
Get the local graph.
Definition Tpetra_CrsGraph_def.hpp:3908
Tpetra::CrsGraph::getLocalNumRows
size_t getLocalNumRows() const override
Returns the number of graph rows owned on the calling node.
Definition Tpetra_CrsGraph_def.hpp:870
Tpetra::CrsGraph::replaceColMap
void replaceColMap(const Teuchos::RCP< const map_type > &newColMap)
Replace the graph's current column Map with the given Map.
Definition Tpetra_CrsGraph_def.hpp:3544
Tpetra::CrsGraph::haveLocalConstants_
bool haveLocalConstants_
Whether this process has computed local constants.
Definition Tpetra_CrsGraph_decl.hpp:2511
Tpetra::CrsMatrix::getGlobalRowView
void getGlobalRowView(GlobalOrdinal GlobalRow, global_inds_host_view_type &indices, values_host_view_type &values) const override
Get a constant, nonpersisting view of a row of this matrix, using global row and column indices.
Definition Tpetra_CrsMatrix_def.hpp:3276
Tpetra::CrsMatrix::isFillComplete
bool isFillComplete() const override
Whether the matrix is fill complete.
Definition Tpetra_CrsMatrix_def.hpp:826
Tpetra::Details::Behavior::useNewCopyAndPermute
static bool useNewCopyAndPermute()
Use new implementation of copyAndPermute.
Definition Tpetra_Details_Behavior.cpp:549
Tpetra::Details::Behavior::debug
static bool debug()
Whether Tpetra is in debug mode.
Definition Tpetra_Details_Behavior.cpp:238
Tpetra::Details::Behavior::verbose
static bool verbose()
Whether Tpetra is in verbose mode.
Definition Tpetra_Details_Behavior.cpp:247
Tpetra::Details::Behavior::verbosePrintCountThreshold
static size_t verbosePrintCountThreshold()
Number of entries below which arrays, lists, etc. will be printed in debug mode.
Definition Tpetra_Details_Behavior.cpp:300
Tpetra::Details::LocalMap
"Local" part of Map suitable for Kokkos kernels.
Definition Tpetra_Details_LocalMap.hpp:39
Tpetra::DistObject< GlobalOrdinal, LocalOrdinal, GlobalOrdinal, Node >::doImport
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, const CombineMode CM, const bool restrictedMode=false)
Tpetra::DistObject< GlobalOrdinal, LocalOrdinal, GlobalOrdinal, Node >::doExport
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, const CombineMode CM, const bool restrictedMode=false)
Tpetra::Distributor
Sets up and executes a communication plan for a Tpetra DistObject.
Definition Tpetra_Distributor.hpp:95
Tpetra::Map::getGlobalElement
global_ordinal_type getGlobalElement(local_ordinal_type localIndex) const
The global index corresponding to the given local index.
Definition Tpetra_Map_def.hpp:1212
Tpetra::Map::isNodeLocalElement
bool isNodeLocalElement(local_ordinal_type localIndex) const
Whether the given local index is valid for this Map on the calling process.
Definition Tpetra_Map_def.hpp:1261
Tpetra::Map::getLocalElement
local_ordinal_type getLocalElement(global_ordinal_type globalIndex) const
The local index corresponding to the given global index.
Definition Tpetra_Map_def.hpp:1190
Tpetra::Map::isNodeGlobalElement
bool isNodeGlobalElement(global_ordinal_type globalIndex) const
Whether the given global index is owned by this Map on the calling process.
Definition Tpetra_Map_def.hpp:1271
Tpetra::Map::getLocalMap
local_map_type getLocalMap() const
Get the LocalMap for Kokkos-Kernels.
Definition Tpetra_Map_def.hpp:1289
Tpetra::RowGraph
An abstract interface for graphs accessed by rows.
Definition Tpetra_RowGraph_decl.hpp:38
Tpetra::RowGraph::isFillComplete
virtual bool isFillComplete() const =0
Whether fillComplete() has been called (without an intervening resumeFill()).
Tpetra::RowGraph::getRowMap
virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getRowMap() const =0
The Map that describes this graph's distribution of rows over processes.
Tpetra::RowGraph::getGlobalRowCopy
virtual void getGlobalRowCopy(GlobalOrdinal gblRow, nonconst_global_inds_host_view_type &gblColInds, size_t &numColInds) const =0
Get a copy of the global column indices in a given row of the graph.
Tpetra::RowGraph::getNumEntriesInGlobalRow
virtual size_t getNumEntriesInGlobalRow(GlobalOrdinal globalRow) const =0
Returns the current number of entries on this node in the specified global row.
Tpetra::SrcDistObject
Abstract base class for objects that can be the source of an Import or Export operation.
Definition Tpetra_SrcDistObject.hpp:57
Tpetra::Vector
A distributed dense vector.
Definition Tpetra_Vector_decl.hpp:44
Details
Implementation details of Tpetra.
Tpetra::Details
Nonmember function that computes a residual Computes R = B - A * X.
Definition Tpetra_KokkosRefactor_Details_MultiVectorLocalDeepCopy.hpp:17
Tpetra::Details::padCrsArrays
void padCrsArrays(const RowPtr &rowPtrBeg, const RowPtr &rowPtrEnd, Indices &indices_wdv, const Padding &padding, const int my_rank, const bool verbose)
Determine if the row pointers and indices arrays need to be resized to accommodate new entries....
Definition Tpetra_Details_crsUtils.hpp:565
Tpetra::Details::verbosePrintArray
void verbosePrintArray(std::ostream &out, const ArrayType &x, const char name[], const size_t maxNumToPrint)
Print min(x.size(), maxNumToPrint) entries of x.
Definition Tpetra_Util.hpp:1019
Tpetra::Details::disableWDVTracking
void disableWDVTracking()
Disable WrappedDualView reference-count tracking and syncing. Call this before entering a host-parall...
Definition Tpetra_Details_WrappedDualView.cpp:23
Tpetra::Details::copyOffsets
void copyOffsets(const OutputViewType &dst, const InputViewType &src)
Copy row offsets (in a sparse graph or matrix) from src to dst. The offsets may have different types.
Definition Tpetra_Details_copyOffsets.hpp:490
Tpetra::Details::unpackAndCombineIntoCrsArrays
void unpackAndCombineIntoCrsArrays(const CrsGraph< LO, GO, NT > &sourceGraph, const Teuchos::ArrayView< const LO > &importLIDs, const Teuchos::ArrayView< const typename CrsGraph< LO, GO, NT >::packet_type > &imports, const Teuchos::ArrayView< const size_t > &numPacketsPerLID, const size_t constantNumPackets, const CombineMode combineMode, const size_t numSameIDs, const Teuchos::ArrayView< const LO > &permuteToLIDs, const Teuchos::ArrayView< const LO > &permuteFromLIDs, size_t TargetNumRows, size_t TargetNumNonzeros, const int MyTargetPID, const Teuchos::ArrayView< size_t > &CRS_rowptr, const Teuchos::ArrayView< GO > &CRS_colind, const Teuchos::ArrayView< const int > &SourcePids, Teuchos::Array< int > &TargetPids)
unpackAndCombineIntoCrsArrays
Tpetra::Details::enableWDVTracking
void enableWDVTracking()
Enable WrappedDualView reference-count tracking and syncing. Call this after exiting a host-parallel ...
Definition Tpetra_Details_WrappedDualView.cpp:17
Tpetra::Details::packCrsGraph
void packCrsGraph(const CrsGraph< LO, GO, NT > &sourceGraph, Teuchos::Array< typename CrsGraph< LO, GO, NT >::packet_type > &exports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, const Teuchos::ArrayView< const LO > &exportLIDs, size_t &constantNumPackets)
Pack specified entries of the given local sparse graph for communication.
Definition Tpetra_Details_packCrsGraph_def.hpp:665
Tpetra::Details::makeColMap
int makeColMap(Teuchos::RCP< const Tpetra::Map< LO, GO, NT > > &colMap, Teuchos::Array< int > &remotePIDs, const Teuchos::RCP< const Tpetra::Map< LO, GO, NT > > &domMap, const CrsGraph< LO, GO, NT > &graph, const bool sortEachProcsGids=true, std::ostream *errStrm=NULL)
Make the graph's column Map.
Definition Tpetra_Details_makeColMap_def.hpp:355
Tpetra::Details::unpackAndCombineWithOwningPIDsCount
size_t unpackAndCombineWithOwningPIDsCount(const CrsGraph< LO, GO, NT > &sourceGraph, const Teuchos::ArrayView< const LO > &importLIDs, const Teuchos::ArrayView< const typename CrsGraph< LO, GO, NT >::packet_type > &imports, const Teuchos::ArrayView< const size_t > &numPacketsPerLID, size_t constantNumPackets, CombineMode combineMode, size_t numSameIDs, const Teuchos::ArrayView< const LO > &permuteToLIDs, const Teuchos::ArrayView< const LO > &permuteFromLIDs)
Special version of Tpetra::Details::unpackCrsGraphAndCombine that also unpacks owning process ranks.
Tpetra::Details::getArrayViewFromDualView
Teuchos::ArrayView< typename DualViewType::t_dev::value_type > getArrayViewFromDualView(const DualViewType &x)
Get a Teuchos::ArrayView which views the host Kokkos::View of the input 1-D Kokkos::DualView.
Definition Tpetra_Util.hpp:939
Tpetra::Details::insertCrsIndices
size_t insertCrsIndices(typename Pointers::value_type const row, Pointers const &rowPtrs, InOutIndices &curIndices, size_t &numAssigned, InIndices const &newIndices, std::function< void(const size_t, const size_t, const size_t)> cb=std::function< void(const size_t, const size_t, const size_t)>())
Insert new indices in to current list of indices.
Definition Tpetra_Details_crsUtils.hpp:641
Tpetra::Details::packCrsGraphNew
void packCrsGraphNew(const CrsGraph< LO, GO, NT > &sourceGraph, const Kokkos::DualView< const LO *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exportLIDs, const Kokkos::DualView< const int *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exportPIDs, Kokkos::DualView< typename CrsGraph< LO, GO, NT >::packet_type *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exports, Kokkos::DualView< size_t *, typename CrsGraph< LO, GO, NT >::buffer_device_type > numPacketsPerLID, size_t &constantNumPackets, const bool pack_pids)
Pack specified entries of the given local sparse graph for communication, for "new" DistObject interf...
Definition Tpetra_Details_packCrsGraph_def.hpp:749
Tpetra::Details::convertColumnIndicesFromGlobalToLocal
OffsetType convertColumnIndicesFromGlobalToLocal(const Kokkos::View< LO *, DT > &lclColInds, const Kokkos::View< const GO *, DT > &gblColInds, const Kokkos::View< const OffsetType *, DT > &ptr, const LocalMap< LO, GO, DT > &lclColMap, const Kokkos::View< const NumEntType *, DT > &numRowEnt)
Convert a CrsGraph's global column indices into local column indices.
Definition Tpetra_CrsGraph_def.hpp:184
Tpetra::Details::createPrefix
std::unique_ptr< std::string > createPrefix(const int myRank, const char prefix[])
Create string prefix for each line of verbose output.
Definition Tpetra_Util.cpp:69
Tpetra::Details::computeOffsetsFromCounts
OffsetsViewType::non_const_value_type computeOffsetsFromCounts(const ExecutionSpace &execSpace, const OffsetsViewType &ptr, const CountsViewType &counts)
Compute offsets from counts.
Definition Tpetra_Details_computeOffsets.hpp:209
Tpetra::Details::computeOffsetsFromConstantCount
OffsetsViewType::non_const_value_type computeOffsetsFromConstantCount(const OffsetsViewType &ptr, const CountType count)
Compute offsets from a constant count.
Definition Tpetra_Details_computeOffsets.hpp:318
Tpetra::Details::findCrsIndices
size_t findCrsIndices(typename Pointers::value_type const row, Pointers const &rowPtrs, const size_t curNumEntries, Indices1 const &curIndices, Indices2 const &newIndices, Callback &&cb)
Finds offsets in to current list of indices.
Definition Tpetra_Details_crsUtils.hpp:708
Tpetra::Details::packCrsGraphWithOwningPIDs
void packCrsGraphWithOwningPIDs(const CrsGraph< LO, GO, NT > &sourceGraph, Kokkos::DualView< typename CrsGraph< LO, GO, NT >::packet_type *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exports_dv, const Teuchos::ArrayView< size_t > &numPacketsPerLID, const Teuchos::ArrayView< const LO > &exportLIDs, const Teuchos::ArrayView< const int > &sourcePIDs, size_t &constantNumPackets)
Pack specified entries of the given local sparse graph for communication.
Definition Tpetra_Details_packCrsGraph_def.hpp:835
Tpetra::Details::gathervPrint
void gathervPrint(std::ostream &out, const std::string &s, const Teuchos::Comm< int > &comm)
On Process 0 in the given communicator, print strings from each process in that communicator,...
Definition Tpetra_Details_gathervPrint.cpp:17
Tpetra
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Tpetra::Array_size_type
Teuchos_Ordinal Array_size_type
Size type for Teuchos Array objects.
Definition Tpetra_ConfigDefs.hpp:28
Tpetra::global_size_t
size_t global_size_t
Global size_t object.
Definition Tpetra_ConfigDefs.hpp:77
Tpetra::createOneToOne
Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > createOneToOne(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &M)
Nonmember constructor for a contiguous Map with user-defined weights and a user-specified,...
Tpetra::CombineMode
CombineMode
Rule for combining data in an Import or Export.
Definition Tpetra_CombineMode.hpp:65
Tpetra::INSERT
@ INSERT
Insert new values that don't currently exist.
Definition Tpetra_CombineMode.hpp:67
Tpetra::Details::PackTraits
Traits class for packing / unpacking data of type T.
Definition Tpetra_Details_PackTraits.hpp:29
Tpetra::RowInfo
Allocation information for a locally owned row in a CrsGraph or CrsMatrix.
Definition Tpetra_CrsGraph_decl.hpp:79
Generated by doxygen 1.15.0