Tpetra_CrsGraph_def.hpp

Go to the documentation of this file.

// @HEADER
// *****************************************************************************
//          Tpetra: Templated Linear Algebra Services Package
//
// Copyright 2008 NTESS and the Tpetra contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER
 
#ifndef TPETRA_CRSGRAPH_DEF_HPP
#define TPETRA_CRSGRAPH_DEF_HPP
 
 
#include <memory>
#include "Tpetra_Details_iallreduce.hpp"
#ifdef KOKKOS_ENABLE_SYCL
#include <sycl/sycl.hpp>
#endif
 
#include "Tpetra_Details_Behavior.hpp"
#include "Tpetra_Details_computeOffsets.hpp"
#include "Tpetra_Details_copyOffsets.hpp"
#include "Tpetra_Details_gathervPrint.hpp"
#include "Tpetra_Details_getGraphDiagOffsets.hpp"
#include "Tpetra_Details_getGraphOffRankOffsets.hpp"
#include "Tpetra_Details_makeColMap.hpp"
#include "Tpetra_Details_Profiling.hpp"
#include "Tpetra_Details_getEntryOnHost.hpp"
#include "Tpetra_Details_crsUtils.hpp"
#include "Tpetra_Distributor.hpp"
#include "Teuchos_SerialDenseMatrix.hpp"
#include "Tpetra_Vector.hpp"
#include "Tpetra_Import_Util.hpp"
#include "Tpetra_Import_Util2.hpp"
#include "Tpetra_Details_packCrsGraph.hpp"
#include "Tpetra_Details_unpackCrsGraphAndCombine.hpp"
#include "Tpetra_Details_CrsPadding.hpp"
#include "Tpetra_Util.hpp"
#include <algorithm>
#include <limits>
#include <map>
#include <sstream>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
 
namespace Tpetra {
namespace Details {
namespace Impl {
 
template <class MapIter>
void verbosePrintMap(std::ostream& out,
                     MapIter beg,
                     MapIter end,
                     const size_t numEnt,
                     const char mapName[]) {
  using ::Tpetra::Details::Behavior;
  using ::Tpetra::Details::verbosePrintArray;
 
  out << mapName << ": {";
  const size_t maxNumToPrint =
      Behavior::verbosePrintCountThreshold();
  if (maxNumToPrint == 0) {
    if (numEnt != 0) {
      out << "...";
    }
  } else {
    const size_t numToPrint = numEnt > maxNumToPrint ? maxNumToPrint : numEnt;
    size_t count            = 0;
    for (MapIter it = beg; it != end; ++it) {
      out << "(" << (*it).first << ", ";
      verbosePrintArray(out, (*it).second, "gblColInds",
                        maxNumToPrint);
      out << ")";
      if (count + size_t(1) < numToPrint) {
        out << ", ";
      }
      ++count;
    }
    if (count < numEnt) {
      out << ", ...";
    }
  }
  out << "}";
}
 
template <class LO, class GO, class Node>
Teuchos::ArrayView<GO>
getRowGraphGlobalRow(
    std::vector<GO>& gblColIndsStorage,
    const RowGraph<LO, GO, Node>& graph,
    const GO gblRowInd) {
  size_t origNumEnt = graph.getNumEntriesInGlobalRow(gblRowInd);
  if (gblColIndsStorage.size() < origNumEnt) {
    gblColIndsStorage.resize(origNumEnt);
  }
  typename CrsGraph<LO, GO, Node>::nonconst_global_inds_host_view_type gblColInds(gblColIndsStorage.data(),
                                                                                  origNumEnt);
  graph.getGlobalRowCopy(gblRowInd, gblColInds, origNumEnt);
  Teuchos::ArrayView<GO> retval(gblColIndsStorage.data(), origNumEnt);
  return retval;
}
 
template <class LO, class GO, class DT, class OffsetType, class NumEntType>
class ConvertColumnIndicesFromGlobalToLocal {
 public:
  ConvertColumnIndicesFromGlobalToLocal(const ::Kokkos::View<LO*, DT>& lclColInds,
                                        const ::Kokkos::View<const GO*, DT>& gblColInds,
                                        const ::Kokkos::View<const OffsetType*, DT>& ptr,
                                        const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
                                        const ::Kokkos::View<const NumEntType*, DT>& numRowEnt)
    : lclColInds_(lclColInds)
    , gblColInds_(gblColInds)
    , ptr_(ptr)
    , lclColMap_(lclColMap)
    , numRowEnt_(numRowEnt) {}
 
  KOKKOS_FUNCTION void
  operator()(const LO& lclRow, OffsetType& curNumBad) const {
    const OffsetType offset = ptr_(lclRow);
    // NOTE (mfh 26 Jun 2016) It's always legal to cast the number
    // of entries in a row to LO, as long as the row doesn't have
    // too many duplicate entries.
    const LO numEnt = static_cast<LO>(numRowEnt_(lclRow));
    for (LO j = 0; j < numEnt; ++j) {
      const GO gid            = gblColInds_(offset + j);
      const LO lid            = lclColMap_.getLocalElement(gid);
      lclColInds_(offset + j) = lid;
      if (lid == ::Tpetra::Details::OrdinalTraits<LO>::invalid()) {
        ++curNumBad;
      }
    }
  }
 
  static OffsetType
  run(const ::Kokkos::View<LO*, DT>& lclColInds,
      const ::Kokkos::View<const GO*, DT>& gblColInds,
      const ::Kokkos::View<const OffsetType*, DT>& ptr,
      const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
      const ::Kokkos::View<const NumEntType*, DT>& numRowEnt) {
    typedef ::Kokkos::RangePolicy<typename DT::execution_space, LO> range_type;
    typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> functor_type;
 
    const LO lclNumRows = ptr.extent(0) == 0 ? static_cast<LO>(0) : static_cast<LO>(ptr.extent(0) - 1);
    OffsetType numBad   = 0;
    // Count of "bad" column indices is a reduction over rows.
    ::Kokkos::parallel_reduce(range_type(0, lclNumRows),
                              functor_type(lclColInds, gblColInds, ptr,
                                           lclColMap, numRowEnt),
                              numBad);
    return numBad;
  }
 
 private:
  ::Kokkos::View<LO*, DT> lclColInds_;
  ::Kokkos::View<const GO*, DT> gblColInds_;
  ::Kokkos::View<const OffsetType*, DT> ptr_;
  ::Tpetra::Details::LocalMap<LO, GO, DT> lclColMap_;
  ::Kokkos::View<const NumEntType*, DT> numRowEnt_;
};
 
}  // namespace Impl
 
template <class LO, class GO, class DT, class OffsetType, class NumEntType>
OffsetType
convertColumnIndicesFromGlobalToLocal(const Kokkos::View<LO*, DT>& lclColInds,
                                      const Kokkos::View<const GO*, DT>& gblColInds,
                                      const Kokkos::View<const OffsetType*, DT>& ptr,
                                      const LocalMap<LO, GO, DT>& lclColMap,
                                      const Kokkos::View<const NumEntType*, DT>& numRowEnt) {
  using Impl::ConvertColumnIndicesFromGlobalToLocal;
  typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> impl_type;
  return impl_type::run(lclColInds, gblColInds, ptr, lclColMap, numRowEnt);
}
 
template <class ViewType, class LO>
class MaxDifference {
 public:
  MaxDifference(const ViewType& ptr)
    : ptr_(ptr) {}
 
  KOKKOS_INLINE_FUNCTION void init(LO& dst) const {
    dst = 0;
  }
 
  KOKKOS_INLINE_FUNCTION void
  join(LO& dst, const LO& src) const {
    dst = (src > dst) ? src : dst;
  }
 
  KOKKOS_INLINE_FUNCTION void
  operator()(const LO lclRow, LO& maxNumEnt) const {
    const LO numEnt = static_cast<LO>(ptr_(lclRow + 1) - ptr_(lclRow));
    maxNumEnt       = (numEnt > maxNumEnt) ? numEnt : maxNumEnt;
  }
 
 private:
  typename ViewType::const_type ptr_;
};
 
template <class ViewType, class LO>
typename ViewType::non_const_value_type
maxDifference(const char kernelLabel[],
              const ViewType& ptr,
              const LO lclNumRows) {
  if (lclNumRows == 0) {
    // mfh 07 May 2018: Weirdly, I need this special case,
    // otherwise I get the wrong answer.
    return static_cast<LO>(0);
  } else {
    using execution_space = typename ViewType::execution_space;
    using range_type      = Kokkos::RangePolicy<execution_space, LO>;
    LO theMaxNumEnt{0};
    Kokkos::parallel_reduce(kernelLabel,
                            range_type(0, lclNumRows),
                            MaxDifference<ViewType, LO>(ptr),
                            theMaxNumEnt);
    return theMaxNumEnt;
  }
}
 
}  // namespace Details
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getDebug() {
  return Details::Behavior::debug("CrsGraph");
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getVerbose() {
  return Details::Behavior::verbose("CrsGraph");
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const size_t maxNumEntriesPerRow,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , numAllocForAllRows_(maxNumEntriesPerRow) {
  const char tfecfFuncName[] =
      "CrsGraph(rowMap,maxNumEntriesPerRow,params): ";
  staticAssertions();
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid(),
                                        std::invalid_argument,
                                        "The allocation hint maxNumEntriesPerRow must be "
                                        "a valid size_t value, which in this case means it must not be "
                                        "Teuchos::OrdinalTraits<size_t>::invalid().");
  resumeFill(params);
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const size_t maxNumEntriesPerRow,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , colMap_(colMap)
  , numAllocForAllRows_(maxNumEntriesPerRow) {
  const char tfecfFuncName[] =
      "CrsGraph(rowMap,colMap,maxNumEntriesPerRow,params): ";
  staticAssertions();
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid(),
      std::invalid_argument,
      "The allocation hint maxNumEntriesPerRow must be "
      "a valid size_t value, which in this case means it must not be "
      "Teuchos::OrdinalTraits<size_t>::invalid().");
  resumeFill(params);
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::ArrayView<const size_t>& numEntPerRow,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , numAllocForAllRows_(0) {
  const char tfecfFuncName[] =
      "CrsGraph(rowMap,numEntPerRow,params): ";
  staticAssertions();
 
  const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      static_cast<size_t>(numEntPerRow.size()) != lclNumRows,
      std::invalid_argument, "numEntPerRow has length " << numEntPerRow.size() << " != the local number of rows " << lclNumRows << " as specified by "
                                                                                                                                   "the input row Map.");
 
  if (debug_) {
    for (size_t r = 0; r < lclNumRows; ++r) {
      const size_t curRowCount = numEntPerRow[r];
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
                                            std::invalid_argument, "numEntPerRow(" << r << ") "
                                                                                           "specifies an invalid number of entries "
                                                                                           "(Teuchos::OrdinalTraits<size_t>::invalid()).");
    }
  }
 
  // Deep-copy the (host-accessible) input into k_numAllocPerRow_.
  // The latter is a const View, so we have to copy into a nonconst
  // View first, then assign.
  typedef decltype(k_numAllocPerRow_) out_view_type;
  typedef typename out_view_type::non_const_type nc_view_type;
  typedef Kokkos::View<const size_t*,
                       typename nc_view_type::array_layout,
                       Kokkos::HostSpace,
                       Kokkos::MemoryUnmanaged>
      in_view_type;
  in_view_type numAllocPerRowIn(numEntPerRow.getRawPtr(), lclNumRows);
  nc_view_type numAllocPerRowOut("Tpetra::CrsGraph::numAllocPerRow",
                                 lclNumRows);
  // DEEP_COPY REVIEW - HOST-TO-HOSTMIRROR
  using exec_space = typename nc_view_type::execution_space;
  Kokkos::deep_copy(exec_space(), numAllocPerRowOut, numAllocPerRowIn);
  k_numAllocPerRow_ = numAllocPerRowOut;
 
  resumeFill(params);
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , k_numAllocPerRow_(numEntPerRow.view_host())
  , numAllocForAllRows_(0) {
  const char tfecfFuncName[] =
      "CrsGraph(rowMap,numEntPerRow,params): ";
  staticAssertions();
 
  const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      static_cast<size_t>(numEntPerRow.extent(0)) != lclNumRows,
      std::invalid_argument, "numEntPerRow has length " << numEntPerRow.extent(0) << " != the local number of rows " << lclNumRows << " as specified by "
                                                                                                                                      "the input row Map.");
 
  if (debug_) {
    for (size_t r = 0; r < lclNumRows; ++r) {
      const size_t curRowCount = numEntPerRow.view_host()(r);
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
                                            std::invalid_argument, "numEntPerRow(" << r << ") "
                                                                                           "specifies an invalid number of entries "
                                                                                           "(Teuchos::OrdinalTraits<size_t>::invalid()).");
    }
  }
 
  resumeFill(params);
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , colMap_(colMap)
  , k_numAllocPerRow_(numEntPerRow.view_host())
  , numAllocForAllRows_(0) {
  const char tfecfFuncName[] =
      "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
  staticAssertions();
 
  const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      static_cast<size_t>(numEntPerRow.extent(0)) != lclNumRows,
      std::invalid_argument, "numEntPerRow has length " << numEntPerRow.extent(0) << " != the local number of rows " << lclNumRows << " as specified by "
                                                                                                                                      "the input row Map.");
 
  if (debug_) {
    for (size_t r = 0; r < lclNumRows; ++r) {
      const size_t curRowCount = numEntPerRow.view_host()(r);
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
                                            std::invalid_argument, "numEntPerRow(" << r << ") "
                                                                                           "specifies an invalid number of entries "
                                                                                           "(Teuchos::OrdinalTraits<size_t>::invalid()).");
    }
  }
 
  resumeFill(params);
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const Teuchos::ArrayView<const size_t>& numEntPerRow,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , colMap_(colMap)
  , numAllocForAllRows_(0) {
  const char tfecfFuncName[] =
      "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
  staticAssertions();
 
  const size_t lclNumRows = rowMap.is_null() ? static_cast<size_t>(0) : rowMap->getLocalNumElements();
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      static_cast<size_t>(numEntPerRow.size()) != lclNumRows,
      std::invalid_argument, "numEntPerRow has length " << numEntPerRow.size() << " != the local number of rows " << lclNumRows << " as specified by "
                                                                                                                                   "the input row Map.");
 
  if (debug_) {
    for (size_t r = 0; r < lclNumRows; ++r) {
      const size_t curRowCount = numEntPerRow[r];
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(curRowCount == Teuchos::OrdinalTraits<size_t>::invalid(),
                                            std::invalid_argument, "numEntPerRow(" << r << ") "
                                                                                           "specifies an invalid number of entries "
                                                                                           "(Teuchos::OrdinalTraits<size_t>::invalid()).");
    }
  }
 
  // Deep-copy the (host-accessible) input into k_numAllocPerRow_.
  // The latter is a const View, so we have to copy into a nonconst
  // View first, then assign.
  typedef decltype(k_numAllocPerRow_) out_view_type;
  typedef typename out_view_type::non_const_type nc_view_type;
  typedef Kokkos::View<const size_t*,
                       typename nc_view_type::array_layout,
                       Kokkos::HostSpace,
                       Kokkos::MemoryUnmanaged>
      in_view_type;
  in_view_type numAllocPerRowIn(numEntPerRow.getRawPtr(), lclNumRows);
  nc_view_type numAllocPerRowOut("Tpetra::CrsGraph::numAllocPerRow",
                                 lclNumRows);
  // DEEP_COPY REVIEW - HOST-TO-HOSTMIRROR
  using exec_space = typename nc_view_type::execution_space;
  Kokkos::deep_copy(exec_space(), numAllocPerRowOut, numAllocPerRowIn);
  k_numAllocPerRow_ = numAllocPerRowOut;
 
  resumeFill(params);
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(CrsGraph<local_ordinal_type, global_ordinal_type, node_type>& originalGraph,
             const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , colMap_(originalGraph.colMap_)
  , numAllocForAllRows_(originalGraph.numAllocForAllRows_)
  , storageStatus_(originalGraph.storageStatus_)
  , indicesAreAllocated_(originalGraph.indicesAreAllocated_)
  , indicesAreLocal_(originalGraph.indicesAreLocal_)
  , indicesAreSorted_(originalGraph.indicesAreSorted_) {
  staticAssertions();
 
  int numRows        = rowMap->getLocalNumElements();
  size_t numNonZeros = originalGraph.getRowPtrsPackedHost()(numRows);
  auto rowsToUse     = Kokkos::pair<size_t, size_t>(0, numRows + 1);
 
  this->setRowPtrsUnpacked(Kokkos::subview(originalGraph.getRowPtrsUnpackedDevice(), rowsToUse));
  this->setRowPtrsPacked(Kokkos::subview(originalGraph.getRowPtrsPackedDevice(), rowsToUse));
 
  if (indicesAreLocal_) {
    lclIndsUnpacked_wdv = local_inds_wdv_type(originalGraph.lclIndsUnpacked_wdv, 0, numNonZeros);
    lclIndsPacked_wdv   = local_inds_wdv_type(originalGraph.lclIndsPacked_wdv, 0, numNonZeros);
  } else {
    gblInds_wdv = global_inds_wdv_type(originalGraph.gblInds_wdv, 0, numNonZeros);
  }
 
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const typename local_graph_device_type::row_map_type& rowPointers,
             const typename local_graph_device_type::entries_type::non_const_type& columnIndices,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , colMap_(colMap)
  , numAllocForAllRows_(0)
  , storageStatus_(Details::STORAGE_1D_PACKED)
  , indicesAreAllocated_(true)
  , indicesAreLocal_(true) {
  staticAssertions();
  if (!params.is_null() && params->isParameter("sorted") &&
      !params->get<bool>("sorted")) {
    indicesAreSorted_ = false;
  } else {
    indicesAreSorted_ = true;
  }
  setAllIndices(rowPointers, columnIndices);
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const Teuchos::ArrayRCP<size_t>& rowPointers,
             const Teuchos::ArrayRCP<LocalOrdinal>& columnIndices,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : dist_object_type(rowMap)
  , rowMap_(rowMap)
  , colMap_(colMap)
  , numAllocForAllRows_(0)
  , storageStatus_(Details::STORAGE_1D_PACKED)
  , indicesAreAllocated_(true)
  , indicesAreLocal_(true) {
  staticAssertions();
  if (!params.is_null() && params->isParameter("sorted") &&
      !params->get<bool>("sorted")) {
    indicesAreSorted_ = false;
  } else {
    indicesAreSorted_ = true;
  }
  setAllIndices(rowPointers, columnIndices);
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const local_graph_device_type& k_local_graph_,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : CrsGraph(k_local_graph_,
             rowMap,
             colMap,
             Teuchos::null,
             Teuchos::null,
             params) {}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const local_graph_device_type& k_local_graph_,
             const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const Teuchos::RCP<const map_type>& domainMap,
             const Teuchos::RCP<const map_type>& rangeMap,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
  , rowMap_(rowMap)
  , colMap_(colMap)
  , numAllocForAllRows_(0)
  , storageStatus_(Details::STORAGE_1D_PACKED)
  , indicesAreAllocated_(true)
  , indicesAreLocal_(true) {
  staticAssertions();
  const char tfecfFuncName[] = "CrsGraph(Kokkos::LocalStaticCrsGraph,Map,Map,Map,Map)";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      colMap.is_null(), std::runtime_error,
      ": The input column Map must be nonnull.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      k_local_graph_.numRows() != rowMap->getLocalNumElements(),
      std::runtime_error,
      ": The input row Map and the input local graph need to have the same "
      "number of rows.  The row Map claims "
          << rowMap->getLocalNumElements()
          << " row(s), but the local graph claims " << k_local_graph_.numRows()
          << " row(s).");
 
  // NOTE (mfh 17 Mar 2014) getLocalNumRows() returns
  // rowMap_->getLocalNumElements(), but it doesn't have to.
  // TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
  //   k_local_graph_.numRows () != getLocalNumRows (), std::runtime_error,
  //   ": The input row Map and the input local graph need to have the same "
  //   "number of rows.  The row Map claims " << getLocalNumRows () << " row(s), "
  //   "but the local graph claims " << k_local_graph_.numRows () << " row(s).");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      lclIndsUnpacked_wdv.extent(0) != 0 || gblInds_wdv.extent(0) != 0, std::logic_error,
      ": cannot have 1D data structures allocated.");
 
  if (!params.is_null() && params->isParameter("sorted") &&
      !params->get<bool>("sorted")) {
    indicesAreSorted_ = false;
  } else {
    indicesAreSorted_ = true;
  }
 
  setDomainRangeMaps(domainMap.is_null() ? rowMap_ : domainMap,
                     rangeMap.is_null() ? rowMap_ : rangeMap);
  Teuchos::Array<int> remotePIDs(0);  // unused output argument
  this->makeImportExport(remotePIDs, false);
 
  lclIndsPacked_wdv   = local_inds_wdv_type(k_local_graph_.entries);
  lclIndsUnpacked_wdv = lclIndsPacked_wdv;
  this->setRowPtrs(k_local_graph_.row_map);
 
  set_need_sync_host_uvm_access();  // lclGraph_ potentially still in a kernel
 
  const bool callComputeGlobalConstants = params.get() == nullptr ||
                                          params->get("compute global constants", true);
 
  if (callComputeGlobalConstants) {
    this->computeGlobalConstants();
  }
  this->fillComplete_ = true;
  this->checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const local_graph_device_type& lclGraph,
             const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const Teuchos::RCP<const map_type>& domainMap,
             const Teuchos::RCP<const map_type>& rangeMap,
             const Teuchos::RCP<const import_type>& importer,
             const Teuchos::RCP<const export_type>& exporter,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
  , rowMap_(rowMap)
  , colMap_(colMap)
  , rangeMap_(rangeMap.is_null() ? rowMap : rangeMap)
  , domainMap_(domainMap.is_null() ? rowMap : domainMap)
  , importer_(importer)
  , exporter_(exporter)
  , numAllocForAllRows_(0)
  , storageStatus_(Details::STORAGE_1D_PACKED)
  , indicesAreAllocated_(true)
  , indicesAreLocal_(true) {
  staticAssertions();
  const char tfecfFuncName[] =
      "Tpetra::CrsGraph(local_graph_device_type,"
      "Map,Map,Map,Map,Import,Export,params): ";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(colMap.is_null(), std::runtime_error,
                                        "The input column Map must be nonnull.");
 
  lclIndsPacked_wdv   = local_inds_wdv_type(lclGraph.entries);
  lclIndsUnpacked_wdv = lclIndsPacked_wdv;
  setRowPtrs(lclGraph.row_map);
 
  set_need_sync_host_uvm_access();  // lclGraph_ potentially still in a kernel
 
  if (!params.is_null() && params->isParameter("sorted") &&
      !params->get<bool>("sorted")) {
    indicesAreSorted_ = false;
  } else {
    indicesAreSorted_ = true;
  }
 
  const bool callComputeGlobalConstants =
      params.get() == nullptr ||
      params->get("compute global constants", true);
  if (callComputeGlobalConstants) {
    this->computeGlobalConstants();
  }
  fillComplete_ = true;
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    CrsGraph(const row_ptrs_device_view_type& rowPointers,
             const local_inds_wdv_type& columnIndices,
             const Teuchos::RCP<const map_type>& rowMap,
             const Teuchos::RCP<const map_type>& colMap,
             const Teuchos::RCP<const map_type>& domainMap,
             const Teuchos::RCP<const map_type>& rangeMap,
             const Teuchos::RCP<const import_type>& importer,
             const Teuchos::RCP<const export_type>& exporter,
             const Teuchos::RCP<Teuchos::ParameterList>& params)
  : DistObject<GlobalOrdinal, LocalOrdinal, GlobalOrdinal, node_type>(rowMap)
  , rowMap_(rowMap)
  , colMap_(colMap)
  , rangeMap_(rangeMap.is_null() ? rowMap : rangeMap)
  , domainMap_(domainMap.is_null() ? rowMap : domainMap)
  , importer_(importer)
  , exporter_(exporter)
  , numAllocForAllRows_(0)
  , storageStatus_(Details::STORAGE_1D_PACKED)
  , indicesAreAllocated_(true)
  , indicesAreLocal_(true) {
  staticAssertions();
  const char tfecfFuncName[] =
      "Tpetra::CrsGraph(row_ptrs_device_view_type,local_inds_wdv_type"
      "Map,Map,Map,Map,Import,Export,params): ";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(colMap.is_null(), std::runtime_error,
                                        "The input column Map must be nonnull.");
 
  lclIndsPacked_wdv   = columnIndices;
  lclIndsUnpacked_wdv = lclIndsPacked_wdv;
  setRowPtrs(rowPointers);
 
  set_need_sync_host_uvm_access();  // lclGraph_ potentially still in a kernel
 
  if (!params.is_null() && params->isParameter("sorted") &&
      !params->get<bool>("sorted")) {
    indicesAreSorted_ = false;
  } else {
    indicesAreSorted_ = true;
  }
 
  const bool callComputeGlobalConstants =
      params.get() == nullptr ||
      params->get("compute global constants", true);
  if (callComputeGlobalConstants) {
    this->computeGlobalConstants();
  }
  fillComplete_ = true;
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<const Teuchos::ParameterList>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getValidParameters() const {
  using Teuchos::ParameterList;
  using Teuchos::parameterList;
  using Teuchos::RCP;
 
  RCP<ParameterList> params = parameterList("Tpetra::CrsGraph");
 
  // Make a sublist for the Import.
  RCP<ParameterList> importSublist = parameterList("Import");
 
  // FIXME (mfh 02 Apr 2012) We should really have the Import and
  // Export objects fill in these lists.  However, we don't want to
  // create an Import or Export unless we need them.  For now, we
  // know that the Import and Export just pass the list directly to
  // their Distributor, so we can create a Distributor here
  // (Distributor's constructor is a lightweight operation) and have
  // it fill in the list.
 
  // Fill in Distributor default parameters by creating a
  // Distributor and asking it to do the work.
  Distributor distributor(rowMap_->getComm(), importSublist);
  params->set("Import", *importSublist, "How the Import performs communication.");
 
  // Make a sublist for the Export.  For now, it's a clone of the
  // Import sublist.  It's not a shallow copy, though, since we
  // might like the Import to do communication differently than the
  // Export.
  params->set("Export", *importSublist, "How the Export performs communication.");
 
  return params;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    setParameterList(const Teuchos::RCP<Teuchos::ParameterList>& params) {
  Teuchos::RCP<const Teuchos::ParameterList> validParams =
      getValidParameters();
  params->validateParametersAndSetDefaults(*validParams);
  this->setMyParamList(params);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
global_size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getGlobalNumRows() const {
  return rowMap_->getGlobalNumElements();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
global_size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getGlobalNumCols() const {
  const char tfecfFuncName[] = "getGlobalNumCols: ";
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      !isFillComplete() || getDomainMap().is_null(), std::runtime_error,
      "The graph does not have a domain Map.  You may not call this method in "
      "that case.");
  return getDomainMap()->getGlobalNumElements();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalNumRows() const {
  return this->rowMap_.is_null() ? static_cast<size_t>(0) : this->rowMap_->getLocalNumElements();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalNumCols() const {
  const char tfecfFuncName[] = "getLocalNumCols: ";
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      !hasColMap(), std::runtime_error,
      "The graph does not have a column Map.  You may not call this method "
      "unless the graph has a column Map.  This requires either that a custom "
      "column Map was given to the constructor, or that fillComplete() has "
      "been called.");
  return colMap_.is_null() ? static_cast<size_t>(0) : colMap_->getLocalNumElements();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getRowMap() const {
  return rowMap_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getColMap() const {
  return colMap_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getDomainMap() const {
  return domainMap_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getRangeMap() const {
  return rangeMap_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::import_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getImporter() const {
  return importer_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::export_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getExporter() const {
  return exporter_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    hasColMap() const {
  return !colMap_.is_null();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    isStorageOptimized() const {
  // FIXME (mfh 07 Aug 2014) Why wouldn't storage be optimized if
  // getLocalNumRows() is zero?
 
  const bool isOpt = indicesAreAllocated_ &&
                     k_numRowEntries_.extent(0) == 0 &&
                     getLocalNumRows() > 0;
 
  return isOpt;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
global_size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getGlobalNumEntries() const {
  const char tfecfFuncName[] = "getGlobalNumEntries: ";
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_, std::logic_error,
                                        "The graph does not have global constants computed, "
                                        "but the user has requested them.");
 
  return globalNumEntries_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalNumEntries() const {
  const char tfecfFuncName[] = "getLocalNumEntries: ";
  typedef LocalOrdinal LO;
 
  Details::ProfilingRegion regionGLNE("Tpetra::CrsGraph::getLocalNumEntries");
 
  if (this->indicesAreAllocated_) {
    const LO lclNumRows = this->getLocalNumRows();
    if (lclNumRows == 0) {
      return static_cast<size_t>(0);
    } else {
      // Avoid the "*this capture" issue by creating a local Kokkos::View.
      auto numEntPerRow        = this->k_numRowEntries_;
      const LO numNumEntPerRow = numEntPerRow.extent(0);
      if (numNumEntPerRow == 0) {
        if (static_cast<LO>(this->getRowPtrsPackedDevice().extent(0)) <
            static_cast<LO>(lclNumRows + 1)) {
          return static_cast<size_t>(0);
        } else {
          // indices are allocated and k_numRowEntries_ is not allocated,
          // so we have packed storage and the length of lclIndsPacked_wdv
          // must be the number of local entries.
          if (debug_) {
            TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->getRowPtrsPackedHost()(lclNumRows) != lclIndsPacked_wdv.extent(0), std::logic_error,
                                                  "Final entry of packed host rowptrs doesn't match the length of lclIndsPacked");
          }
          return lclIndsPacked_wdv.extent(0);
        }
      } else {  // k_numRowEntries_ is populated
        // k_numRowEntries_ is actually be a host View, so we run
        // the sum in its native execution space.  This also means
        // that we can use explicit capture (which could perhaps
        // improve build time) instead of KOKKOS_LAMBDA, and avoid
        // any CUDA build issues with trying to run a __device__ -
        // only function on host.
        typedef typename num_row_entries_type::execution_space
            host_exec_space;
        typedef Kokkos::RangePolicy<host_exec_space, LO> range_type;
 
        const LO upperLoopBound = lclNumRows < numNumEntPerRow ? lclNumRows : numNumEntPerRow;
        size_t nodeNumEnt       = 0;
        Kokkos::parallel_reduce(
            "Tpetra::CrsGraph::getNumNodeEntries",
            range_type(0, upperLoopBound),
            [=](const LO& k, size_t& lclSum) {
              lclSum += numEntPerRow(k);
            },
            nodeNumEnt);
        return nodeNumEnt;
      }
    }
  } else {  // nothing allocated on this process, so no entries
    return static_cast<size_t>(0);
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
global_size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getGlobalMaxNumRowEntries() const {
  const char tfecfFuncName[] = "getGlobalMaxNumRowEntries: ";
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_, std::logic_error,
                                        "The graph does not have global constants computed, "
                                        "but the user has requested them.");
 
  return globalMaxNumRowEntries_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalMaxNumRowEntries() const {
  return nodeMaxNumRowEntries_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    isFillComplete() const {
  return fillComplete_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    isFillActive() const {
  return !fillComplete_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    isLocallyIndexed() const {
  return indicesAreLocal_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    isGloballyIndexed() const {
  return indicesAreGlobal_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalAllocationSize() const {
  typedef LocalOrdinal LO;
 
  if (this->indicesAreAllocated_) {
    const LO lclNumRows = this->getLocalNumRows();
    if (lclNumRows == 0) {
      return static_cast<size_t>(0);
    } else if (storageStatus_ == Details::STORAGE_1D_PACKED) {
      if (static_cast<LO>(this->getRowPtrsPackedDevice().extent(0)) <
          static_cast<LO>(lclNumRows + 1)) {
        return static_cast<size_t>(0);
      } else {
        if (this->isLocallyIndexed())
          return lclIndsPacked_wdv.extent(0);
        else
          return gblInds_wdv.extent(0);
      }
    } else if (storageStatus_ == Details::STORAGE_1D_UNPACKED) {
      auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
      if (rowPtrsUnpacked_host.extent(0) == 0) {
        return static_cast<size_t>(0);
      } else {
        if (this->isLocallyIndexed())
          return lclIndsUnpacked_wdv.extent(0);
        else
          return gblInds_wdv.extent(0);
      }
    } else {
      return static_cast<size_t>(0);
    }
  } else {
    return Tpetra::Details::OrdinalTraits<size_t>::invalid();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<const Teuchos::Comm<int>>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getComm() const {
  return this->rowMap_.is_null() ? Teuchos::null : this->rowMap_->getComm();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
GlobalOrdinal
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getIndexBase() const {
  return rowMap_->getIndexBase();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    indicesAreAllocated() const {
  return indicesAreAllocated_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    isSorted() const {
  return indicesAreSorted_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    isMerged() const {
  return noRedundancies_;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    setLocallyModified() {
  // FIXME (mfh 07 May 2013) How do we know that the change
  // introduced a redundancy, or even that it invalidated the sorted
  // order of indices?  CrsGraph has always made this conservative
  // guess.  It could be a bit costly to check at insertion time,
  // though.
  indicesAreSorted_ = false;
  noRedundancies_   = false;
 
  // We've modified the graph, so we'll have to recompute local
  // constants like the number of diagonal entries on this process.
  haveLocalConstants_ = false;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    allocateIndices(const ELocalGlobal lg, const bool verbose) {
  using std::endl;
  using Teuchos::arcp;
  using Teuchos::Array;
  using Teuchos::ArrayRCP;
  typedef Teuchos::ArrayRCP<size_t>::size_type size_type;
  typedef typename local_graph_device_type::row_map_type::non_const_type
      non_const_row_map_type;
  const char tfecfFuncName[] = "allocateIndices: ";
  const char suffix[] =
      "  Please report this bug to the Tpetra developers.";
 
  Details::ProfilingRegion profRegion("Tpetra::CrsGraph::allocateIndices");
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "Start: lg="
       << (lg == GlobalIndices ? "GlobalIndices" : "LocalIndices")
       << ", numRows: " << this->getLocalNumRows() << endl;
    std::cerr << os.str();
  }
 
  // This is a protected function, only callable by us.  If it was
  // called incorrectly, it is our fault.  That's why the tests
  // below throw std::logic_error instead of std::invalid_argument.
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isLocallyIndexed() && lg == GlobalIndices, std::logic_error,
                                        ": The graph is locally indexed, but Tpetra code is calling "
                                        "this method with lg=GlobalIndices."
                                            << suffix);
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed() && lg == LocalIndices, std::logic_error,
                                        ": The graph is globally indexed, but Tpetra code is calling "
                                        "this method with lg=LocalIndices."
                                            << suffix);
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreAllocated(), std::logic_error,
                                        ": The graph's "
                                        "indices are already allocated, but Tpetra is calling "
                                        "allocateIndices again."
                                            << suffix);
  const size_t numRows = this->getLocalNumRows();
 
  //
  //  STATIC ALLOCATION PROFILE
  //
  size_type numInds = 0;
  {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Allocate k_rowPtrs: " << (numRows + 1) << endl;
      std::cerr << os.str();
    }
    non_const_row_map_type k_rowPtrs("Tpetra::CrsGraph::ptr", numRows + 1);
 
    if (this->k_numAllocPerRow_.extent(0) != 0) {
      // It's OK to throw std::invalid_argument here, because we
      // haven't incurred any side effects yet.  Throwing that
      // exception (and not, say, std::logic_error) implies that the
      // instance can recover.
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->k_numAllocPerRow_.extent(0) != numRows,
                                            std::invalid_argument,
                                            "k_numAllocPerRow_ is allocated, that is, "
                                            "has nonzero length "
                                                << this->k_numAllocPerRow_.extent(0)
                                                << ", but its length != numRows = " << numRows << ".");
 
      // k_numAllocPerRow_ is a host View, but k_rowPtrs (the thing
      // we want to compute here) lives on device.  That's OK;
      // computeOffsetsFromCounts can handle this case.
      using Details::computeOffsetsFromCounts;
 
      // FIXME (mfh 27 Jun 2016) Currently, computeOffsetsFromCounts
      // doesn't attempt to check its input for "invalid" flag
      // values.  For now, we omit that feature of the sequential
      // code disabled below.
      numInds = computeOffsetsFromCounts(k_rowPtrs, k_numAllocPerRow_);
    } else {
      // It's OK to throw std::invalid_argument here, because we
      // haven't incurred any side effects yet.  Throwing that
      // exception (and not, say, std::logic_error) implies that the
      // instance can recover.
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->numAllocForAllRows_ ==
                                                Tpetra::Details::OrdinalTraits<size_t>::invalid(),
                                            std::invalid_argument,
                                            "numAllocForAllRows_ has an invalid value, "
                                            "namely Tpetra::Details::OrdinalTraits<size_t>::invalid() = "
                                                << Tpetra::Details::OrdinalTraits<size_t>::invalid() << ".");
 
      using Details::computeOffsetsFromConstantCount;
      numInds = computeOffsetsFromConstantCount(k_rowPtrs, this->numAllocForAllRows_);
    }
    // "Commit" the resulting row offsets.
    setRowPtrsUnpacked(k_rowPtrs);
  }
  if (debug_) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numInds != size_type(this->getRowPtrsUnpackedHost()(numRows)), std::logic_error,
                                          ": Number of indices produced by computeOffsetsFrom[Constant]Counts "
                                          "does not match final entry of rowptrs unpacked");
  }
 
  if (lg == LocalIndices) {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Allocate local column indices "
                       "lclIndsUnpacked_wdv: "
         << numInds << endl;
      std::cerr << os.str();
    }
    lclIndsUnpacked_wdv = local_inds_wdv_type(
        local_inds_dualv_type("Tpetra::CrsGraph::lclInd", numInds));
  } else {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Allocate global column indices "
                       "gblInds_wdv: "
         << numInds << endl;
      std::cerr << os.str();
    }
    gblInds_wdv = global_inds_wdv_type(
        global_inds_dualv_type("Tpetra::CrsGraph::gblInd", numInds));
  }
  storageStatus_ = Details::STORAGE_1D_UNPACKED;
 
  this->indicesAreLocal_  = (lg == LocalIndices);
  this->indicesAreGlobal_ = (lg == GlobalIndices);
 
  if (numRows > 0) {  // reallocate k_numRowEntries_ & fill w/ 0s
    using Kokkos::ViewAllocateWithoutInitializing;
    const char label[] = "Tpetra::CrsGraph::numRowEntries";
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Allocate k_numRowEntries_: " << numRows
         << endl;
      std::cerr << os.str();
    }
    num_row_entries_type numRowEnt(ViewAllocateWithoutInitializing(label), numRows);
    // DEEP_COPY REVIEW - VALUE-TO-HOSTMIRROR
    Kokkos::deep_copy(execution_space(), numRowEnt, static_cast<size_t>(0));  // fill w/ 0s
    Kokkos::fence("CrsGraph::allocateIndices");                               // TODO: Need to understand downstream failure points and move this fence.
    this->k_numRowEntries_ = numRowEnt;                                       // "commit" our allocation
  }
 
  // Once indices are allocated, CrsGraph needs to free this information.
  this->numAllocForAllRows_  = 0;
  this->k_numAllocPerRow_    = decltype(k_numAllocPerRow_)();
  this->indicesAreAllocated_ = true;
 
  try {
    this->checkInternalState();
  } catch (std::logic_error& e) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
                                          "At end of allocateIndices, "
                                          "checkInternalState threw std::logic_error: "
                                              << e.what());
  } catch (std::exception& e) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                          "At end of allocateIndices, "
                                          "checkInternalState threw std::exception: "
                                              << e.what());
  } catch (...) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                          "At end of allocateIndices, "
                                          "checkInternalState threw an exception "
                                          "not a subclass of std::exception.");
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    local_inds_dualv_type::t_host::const_type
    CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
        getLocalIndsViewHost(const RowInfo& rowinfo) const {
  if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
    return typename local_inds_dualv_type::t_host::const_type();
  else
    return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
                                              rowinfo.allocSize,
                                              Access::ReadOnly);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    local_inds_dualv_type::t_host
    CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
        getLocalIndsViewHostNonConst(const RowInfo& rowinfo) {
  if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
    return typename local_inds_dualv_type::t_host();
  else
    return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
                                              rowinfo.allocSize,
                                              Access::ReadWrite);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    global_inds_dualv_type::t_host::const_type
    CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
        getGlobalIndsViewHost(const RowInfo& rowinfo) const {
  if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
    return typename global_inds_dualv_type::t_host::const_type();
  else
    return gblInds_wdv.getHostSubview(rowinfo.offset1D,
                                      rowinfo.allocSize,
                                      Access::ReadOnly);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    local_inds_dualv_type::t_dev::const_type
    CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
        getLocalIndsViewDevice(const RowInfo& rowinfo) const {
  if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
    return typename local_inds_dualv_type::t_dev::const_type();
  else
    return lclIndsUnpacked_wdv.getDeviceSubview(rowinfo.offset1D,
                                                rowinfo.allocSize,
                                                Access::ReadOnly);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    global_inds_dualv_type::t_dev::const_type
    CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
        getGlobalIndsViewDevice(const RowInfo& rowinfo) const {
  if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
    return typename global_inds_dualv_type::t_dev::const_type();
  else
    return gblInds_wdv.getDeviceSubview(rowinfo.offset1D,
                                        rowinfo.allocSize,
                                        Access::ReadOnly);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
RowInfo
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getRowInfo(const LocalOrdinal myRow) const {
  const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid();
  RowInfo ret;
  if (this->rowMap_.is_null() || !this->rowMap_->isNodeLocalElement(myRow)) {
    ret.localRow   = STINV;
    ret.allocSize  = 0;
    ret.numEntries = 0;
    ret.offset1D   = STINV;
    return ret;
  }
 
  ret.localRow = static_cast<size_t>(myRow);
  if (this->indicesAreAllocated()) {
    auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
    // Offsets tell us the allocation size in this case.
    if (rowPtrsUnpacked_host.extent(0) == 0) {
      ret.offset1D  = 0;
      ret.allocSize = 0;
    } else {
      ret.offset1D  = rowPtrsUnpacked_host(myRow);
      ret.allocSize = rowPtrsUnpacked_host(myRow + 1) - rowPtrsUnpacked_host(myRow);
    }
 
    ret.numEntries = (this->k_numRowEntries_.extent(0) == 0) ? ret.allocSize : this->k_numRowEntries_(myRow);
  } else {  // haven't performed allocation yet; probably won't hit this code
    // FIXME (mfh 07 Aug 2014) We want graph's constructors to
    // allocate, rather than doing lazy allocation at first insert.
    // This will make k_numAllocPerRow_ obsolete.
    ret.allocSize  = (this->k_numAllocPerRow_.extent(0) != 0) ? this->k_numAllocPerRow_(myRow) :  // this is a host View
                        this->numAllocForAllRows_;
    ret.numEntries = 0;
    ret.offset1D   = STINV;
  }
 
  return ret;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
RowInfo
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getRowInfoFromGlobalRowIndex(const GlobalOrdinal gblRow) const {
  const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid();
  RowInfo ret;
  if (this->rowMap_.is_null()) {
    ret.localRow   = STINV;
    ret.allocSize  = 0;
    ret.numEntries = 0;
    ret.offset1D   = STINV;
    return ret;
  }
  const LocalOrdinal myRow = this->rowMap_->getLocalElement(gblRow);
  if (myRow == Teuchos::OrdinalTraits<LocalOrdinal>::invalid()) {
    ret.localRow   = STINV;
    ret.allocSize  = 0;
    ret.numEntries = 0;
    ret.offset1D   = STINV;
    return ret;
  }
 
  ret.localRow = static_cast<size_t>(myRow);
  if (this->indicesAreAllocated()) {
    // graph data structures have the info that we need
    //
    // if static graph, offsets tell us the allocation size
    auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
    if (rowPtrsUnpacked_host.extent(0) == 0) {
      ret.offset1D  = 0;
      ret.allocSize = 0;
    } else {
      ret.offset1D  = rowPtrsUnpacked_host(myRow);
      ret.allocSize = rowPtrsUnpacked_host(myRow + 1) - rowPtrsUnpacked_host(myRow);
    }
 
    ret.numEntries = (this->k_numRowEntries_.extent(0) == 0) ? ret.allocSize : this->k_numRowEntries_(myRow);
  } else {  // haven't performed allocation yet; probably won't hit this code
    // FIXME (mfh 07 Aug 2014) We want graph's constructors to
    // allocate, rather than doing lazy allocation at first insert.
    // This will make k_numAllocPerRow_ obsolete.
    ret.allocSize  = (this->k_numAllocPerRow_.extent(0) != 0) ? this->k_numAllocPerRow_(myRow) :  // this is a host View
                        this->numAllocForAllRows_;
    ret.numEntries = 0;
    ret.offset1D   = STINV;
  }
 
  return ret;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    staticAssertions() const {
  using Teuchos::OrdinalTraits;
  typedef LocalOrdinal LO;
  typedef GlobalOrdinal GO;
  typedef global_size_t GST;
 
  // Assumption: sizeof(GlobalOrdinal) >= sizeof(LocalOrdinal):
  //     This is so that we can store local indices in the memory
  //     formerly occupied by global indices.
  static_assert(sizeof(GlobalOrdinal) >= sizeof(LocalOrdinal),
                "Tpetra::CrsGraph: sizeof(GlobalOrdinal) must be >= sizeof(LocalOrdinal).");
  // Assumption: max(size_t) >= max(LocalOrdinal)
  //     This is so that we can represent any LocalOrdinal as a size_t.
  static_assert(sizeof(size_t) >= sizeof(LocalOrdinal),
                "Tpetra::CrsGraph: sizeof(size_t) must be >= sizeof(LocalOrdinal).");
  static_assert(sizeof(GST) >= sizeof(size_t),
                "Tpetra::CrsGraph: sizeof(Tpetra::global_size_t) must be >= sizeof(size_t).");
 
  // FIXME (mfh 30 Sep 2015) We're not using
  // Teuchos::CompileTimeAssert any more.  Can we do these checks
  // with static_assert?
 
  // can't call max() with CompileTimeAssert, because it isn't a
  // constant expression; will need to make this a runtime check
  const char msg[] =
      "Tpetra::CrsGraph: Object cannot be created with the "
      "given template arguments: size assumptions are not valid.";
  TEUCHOS_TEST_FOR_EXCEPTION(
      static_cast<size_t>(Teuchos::OrdinalTraits<LO>::max()) > Teuchos::OrdinalTraits<size_t>::max(),
      std::runtime_error, msg);
  TEUCHOS_TEST_FOR_EXCEPTION(
      static_cast<GST>(Teuchos::OrdinalTraits<LO>::max()) > static_cast<GST>(Teuchos::OrdinalTraits<GO>::max()),
      std::runtime_error, msg);
  TEUCHOS_TEST_FOR_EXCEPTION(
      static_cast<size_t>(Teuchos::OrdinalTraits<GO>::max()) > Teuchos::OrdinalTraits<GST>::max(),
      std::runtime_error, msg);
  TEUCHOS_TEST_FOR_EXCEPTION(
      Teuchos::OrdinalTraits<size_t>::max() > Teuchos::OrdinalTraits<GST>::max(),
      std::runtime_error, msg);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertIndices(RowInfo& rowinfo,
                  const SLocalGlobalViews& newInds,
                  const ELocalGlobal lg,
                  const ELocalGlobal I) {
  using Teuchos::ArrayView;
  typedef LocalOrdinal LO;
  typedef GlobalOrdinal GO;
  const char tfecfFuncName[] = "insertIndices: ";
 
  size_t oldNumEnt = 0;
  if (debug_) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(lg != GlobalIndices && lg != LocalIndices, std::invalid_argument,
                                          "lg must be either GlobalIndices or LocalIndices.");
    oldNumEnt = this->getNumEntriesInLocalRow(rowinfo.localRow);
  }
 
  size_t numNewInds = 0;
  if (lg == GlobalIndices) {  // input indices are global
    ArrayView<const GO> new_ginds = newInds.ginds;
    numNewInds                    = new_ginds.size();
    if (I == GlobalIndices) {  // store global indices
      auto gind_view = gblInds_wdv.getHostView(Access::ReadWrite);
      if (debug_) {
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(gind_view.size()) <
                                                  rowinfo.numEntries + numNewInds,
                                              std::logic_error,
                                              "gind_view.size() = " << gind_view.size()
                                                                    << " < rowinfo.numEntries (= " << rowinfo.numEntries
                                                                    << ") + numNewInds (= " << numNewInds << ").");
      }
      GO* const gblColInds_out = gind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
      for (size_t k = 0; k < numNewInds; ++k) {
        gblColInds_out[k] = new_ginds[k];
      }
    } else if (I == LocalIndices) {  // store local indices
      auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
      if (debug_) {
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(lind_view.size()) <
                                                  rowinfo.numEntries + numNewInds,
                                              std::logic_error,
                                              "lind_view.size() = " << lind_view.size()
                                                                    << " < rowinfo.numEntries (= " << rowinfo.numEntries
                                                                    << ") + numNewInds (= " << numNewInds << ").");
      }
      LO* const lclColInds_out = lind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
      for (size_t k = 0; k < numNewInds; ++k) {
        lclColInds_out[k] = colMap_->getLocalElement(new_ginds[k]);
      }
    }
  } else if (lg == LocalIndices) {  // input indices are local
    ArrayView<const LO> new_linds = newInds.linds;
    numNewInds                    = new_linds.size();
    if (I == LocalIndices) {  // store local indices
      auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
      if (debug_) {
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(lind_view.size()) <
                                                  rowinfo.numEntries + numNewInds,
                                              std::logic_error,
                                              "lind_view.size() = " << lind_view.size()
                                                                    << " < rowinfo.numEntries (= " << rowinfo.numEntries
                                                                    << ") + numNewInds (= " << numNewInds << ").");
      }
      LO* const lclColInds_out = lind_view.data() + rowinfo.offset1D + rowinfo.numEntries;
      for (size_t k = 0; k < numNewInds; ++k) {
        lclColInds_out[k] = new_linds[k];
      }
    } else if (I == GlobalIndices) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
                                            "The case where the input indices are local "
                                            "and the indices to write are global (lg=LocalIndices, I="
                                            "GlobalIndices) is not implemented, because it does not make sense."
                                                << std::endl
                                                << "If you have correct local column indices, that "
                                                   "means the graph has a column Map.  In that case, you should be "
                                                   "storing local indices.");
    }
  }
 
  rowinfo.numEntries += numNewInds;
  this->k_numRowEntries_(rowinfo.localRow) += numNewInds;
  this->setLocallyModified();
 
  if (debug_) {
    const size_t chkNewNumEnt =
        this->getNumEntriesInLocalRow(rowinfo.localRow);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(chkNewNumEnt != oldNumEnt + numNewInds, std::logic_error,
                                          "chkNewNumEnt = " << chkNewNumEnt
                                                            << " != oldNumEnt (= " << oldNumEnt
                                                            << ") + numNewInds (= " << numNewInds << ").");
  }
 
  return numNewInds;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertGlobalIndicesImpl(const LocalOrdinal lclRow,
                            const GlobalOrdinal inputGblColInds[],
                            const size_t numInputInds) {
  return this->insertGlobalIndicesImpl(this->getRowInfo(lclRow),
                                       inputGblColInds, numInputInds);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertGlobalIndicesImpl(const RowInfo& rowInfo,
                            const GlobalOrdinal inputGblColInds[],
                            const size_t numInputInds,
                            std::function<void(const size_t, const size_t, const size_t)> fun) {
  using Details::verbosePrintArray;
  using Kokkos::MemoryUnmanaged;
  using Kokkos::subview;
  using Kokkos::View;
  using Teuchos::ArrayView;
  using LO = LocalOrdinal;
  using GO = GlobalOrdinal;
 
  const char tfecfFuncName[] = "insertGlobalIndicesImpl: ";
  const LO lclRow            = static_cast<LO>(rowInfo.localRow);
 
  auto numEntries     = rowInfo.numEntries;
  using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
  inp_view_type inputInds(inputGblColInds, numInputInds);
  size_t numInserted;
  {
    auto gblIndsHostView = this->gblInds_wdv.getHostView(Access::ReadWrite);
    numInserted          = Details::insertCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
                                                     gblIndsHostView,
                                                     numEntries, inputInds, fun);
  }
 
  const bool insertFailed =
      numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
  if (insertFailed) {
    constexpr size_t ONE(1);
    const int myRank = this->getComm()->getRank();
    std::ostringstream os;
 
    os << "Proc " << myRank << ": Not enough capacity to insert "
       << numInputInds
       << " ind" << (numInputInds != ONE ? "ices" : "ex")
       << " into local row " << lclRow << ", which currently has "
       << rowInfo.numEntries
       << " entr" << (rowInfo.numEntries != ONE ? "ies" : "y")
       << " and total allocation size " << rowInfo.allocSize
       << ".  ";
    const size_t maxNumToPrint =
        Details::Behavior::verbosePrintCountThreshold();
    ArrayView<const GO> inputGblColIndsView(inputGblColInds,
                                            numInputInds);
    verbosePrintArray(os, inputGblColIndsView,
                      "Input global "
                      "column indices",
                      maxNumToPrint);
    os << ", ";
    auto curGblColInds = getGlobalIndsViewHost(rowInfo);
    ArrayView<const GO> curGblColIndsView(curGblColInds.data(),
                                          rowInfo.numEntries);
    verbosePrintArray(os, curGblColIndsView,
                      "Current global "
                      "column indices",
                      maxNumToPrint);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
  }
 
  this->k_numRowEntries_(lclRow) += numInserted;
 
  this->setLocallyModified();
  return numInserted;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertLocalIndicesImpl(const LocalOrdinal myRow,
                           const Teuchos::ArrayView<const LocalOrdinal>& indices,
                           std::function<void(const size_t, const size_t, const size_t)> fun) {
  using Kokkos::MemoryUnmanaged;
  using Kokkos::subview;
  using Kokkos::View;
  using LO = LocalOrdinal;
 
  const char tfecfFuncName[] = "insertLocallIndicesImpl: ";
 
  const RowInfo rowInfo = this->getRowInfo(myRow);
 
  size_t numNewInds    = 0;
  size_t newNumEntries = 0;
 
  auto numEntries = rowInfo.numEntries;
  // Note: Teuchos::ArrayViews are in HostSpace
  using inp_view_type = View<const LO*, Kokkos::HostSpace, MemoryUnmanaged>;
  inp_view_type inputInds(indices.getRawPtr(), indices.size());
  size_t numInserted = 0;
  {
    auto lclInds = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
    numInserted  = Details::insertCrsIndices(myRow, this->getRowPtrsUnpackedHost(), lclInds,
                                             numEntries, inputInds, fun);
  }
 
  const bool insertFailed =
      numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
  if (insertFailed) {
    constexpr size_t ONE(1);
    const size_t numInputInds(indices.size());
    const int myRank = this->getComm()->getRank();
    std::ostringstream os;
    os << "On MPI Process " << myRank << ": Not enough capacity to "
                                         "insert "
       << numInputInds
       << " ind" << (numInputInds != ONE ? "ices" : "ex")
       << " into local row " << myRow << ", which currently has "
       << rowInfo.numEntries
       << " entr" << (rowInfo.numEntries != ONE ? "ies" : "y")
       << " and total allocation size " << rowInfo.allocSize << ".";
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
  }
  numNewInds    = numInserted;
  newNumEntries = rowInfo.numEntries + numNewInds;
 
  this->k_numRowEntries_(myRow) += numNewInds;
  this->setLocallyModified();
 
  if (debug_) {
    const size_t chkNewNumEntries = this->getNumEntriesInLocalRow(myRow);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(chkNewNumEntries != newNumEntries, std::logic_error,
                                          "getNumEntriesInLocalRow(" << myRow << ") = " << chkNewNumEntries
                                                                     << " != newNumEntries = " << newNumEntries
                                                                     << ".  Please report this bug to the Tpetra developers.");
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    findGlobalIndices(const RowInfo& rowInfo,
                      const Teuchos::ArrayView<const GlobalOrdinal>& indices,
                      std::function<void(const size_t, const size_t, const size_t)> fun) const {
  using GO = GlobalOrdinal;
  using Kokkos::MemoryUnmanaged;
  using Kokkos::View;
 
  auto invalidCount = Teuchos::OrdinalTraits<size_t>::invalid();
 
  using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
  inp_view_type inputInds(indices.getRawPtr(), indices.size());
 
  size_t numFound     = 0;
  LocalOrdinal lclRow = rowInfo.localRow;
  if (this->isLocallyIndexed()) {
    if (this->colMap_.is_null())
      return invalidCount;
    const auto& colMap = *(this->colMap_);
    auto map           = [&](GO const gblInd) { return colMap.getLocalElement(gblInd); };
    if (this->isSorted()) {
      numFound = Details::findCrsIndicesSorted(
          lclRow,
          this->getRowPtrsUnpackedHost(),
          rowInfo.numEntries,
          lclIndsUnpacked_wdv.getHostView(Access::ReadOnly),
          inputInds,
          map,
          fun);
    } else {
      numFound = Details::findCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
                                         rowInfo.numEntries,
                                         lclIndsUnpacked_wdv.getHostView(Access::ReadOnly), inputInds, map, fun);
    }
  } else if (this->isGloballyIndexed()) {
    numFound = Details::findCrsIndices(lclRow, this->getRowPtrsUnpackedHost(),
                                       rowInfo.numEntries,
                                       gblInds_wdv.getHostView(Access::ReadOnly), inputInds, fun);
  }
  return numFound;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    setDomainRangeMaps(const Teuchos::RCP<const map_type>& domainMap,
                       const Teuchos::RCP<const map_type>& rangeMap) {
  // simple pointer comparison for equality
  if (domainMap_ != domainMap) {
    domainMap_ = domainMap;
    importer_  = Teuchos::null;
  }
  if (rangeMap_ != rangeMap) {
    rangeMap_ = rangeMap;
    exporter_ = Teuchos::null;
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    clearGlobalConstants() {
  const auto INV = Teuchos::OrdinalTraits<global_size_t>::invalid();
 
  globalNumEntries_       = INV;
  globalMaxNumRowEntries_ = INV;
  haveGlobalConstants_    = false;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    checkInternalState() const {
  using Details::ProfilingRegion;
  if (debug_) {
    using std::endl;
    const char tfecfFuncName[] = "checkInternalState: ";
    const char suffix[]        = "  Please report this bug to the Tpetra developers.";
 
    ProfilingRegion("Tpetra::CrsGraph::checkInternalState");
 
    std::unique_ptr<std::string> prefix;
    if (verbose_) {
      prefix = this->createPrefix("CrsGraph", "checkInternalState");
      std::ostringstream os;
      os << *prefix << "Start" << endl;
      std::cerr << os.str();
    }
 
    const global_size_t GSTI = Teuchos::OrdinalTraits<global_size_t>::invalid();
    // const size_t         STI = Teuchos::OrdinalTraits<size_t>::invalid (); // unused
    //  check the internal state of this data structure
    //  this is called by numerous state-changing methods, in a debug build, to ensure that the object
    //  always remains in a valid state
 
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->rowMap_.is_null(), std::logic_error,
                                          "Row Map is null." << suffix);
    // This may access the row Map, so we need to check first (above)
    // whether the row Map is null.
    const LocalOrdinal lclNumRows =
        static_cast<LocalOrdinal>(this->getLocalNumRows());
 
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillActive() == this->isFillComplete(), std::logic_error,
                                          "Graph cannot be both fill active and fill complete." << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillComplete() &&
                                              (this->colMap_.is_null() ||
                                               this->rangeMap_.is_null() ||
                                               this->domainMap_.is_null()),
                                          std::logic_error,
                                          "Graph is full complete, but at least one of {column, range, domain} "
                                          "Map is null."
                                              << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isStorageOptimized() && !this->indicesAreAllocated(),
                                          std::logic_error,
                                          "Storage is optimized, but indices are not "
                                          "allocated, not even trivially."
                                              << suffix);
 
    size_t nodeAllocSize = 0;
    try {
      nodeAllocSize = this->getLocalAllocationSize();
    } catch (std::logic_error& e) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                            "getLocalAllocationSize threw "
                                            "std::logic_error: "
                                                << e.what());
    } catch (std::exception& e) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                            "getLocalAllocationSize threw an "
                                            "std::exception: "
                                                << e.what());
    } catch (...) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                            "getLocalAllocationSize threw an exception "
                                            "not a subclass of std::exception.");
    }
 
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isStorageOptimized() &&
                                              nodeAllocSize != this->getLocalNumEntries(),
                                          std::logic_error,
                                          "Storage is optimized, but "
                                          "this->getLocalAllocationSize() = "
                                              << nodeAllocSize
                                              << " != this->getLocalNumEntries() = " << this->getLocalNumEntries()
                                              << "." << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->haveGlobalConstants_ &&
                                              (this->globalNumEntries_ != GSTI ||
                                               this->globalMaxNumRowEntries_ != GSTI),
                                          std::logic_error,
                                          "Graph claims not to have global constants, but "
                                          "some of the global constants are not marked as invalid."
                                              << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->haveGlobalConstants_ &&
                                              (this->globalNumEntries_ == GSTI ||
                                               this->globalMaxNumRowEntries_ == GSTI),
                                          std::logic_error,
                                          "Graph claims to have global constants, but "
                                          "some of them are marked as invalid."
                                              << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->haveGlobalConstants_ &&
                                              (this->globalNumEntries_ < this->getLocalNumEntries() ||
                                               this->globalMaxNumRowEntries_ < this->nodeMaxNumRowEntries_),
                                          std::logic_error,
                                          "Graph claims to have global constants, and "
                                          "all of the values of the global constants are valid, but "
                                          "some of the local constants are greater than "
                                          "their corresponding global constants."
                                              << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreAllocated() &&
                                              (this->numAllocForAllRows_ != 0 ||
                                               this->k_numAllocPerRow_.extent(0) != 0),
                                          std::logic_error,
                                          "The graph claims that its indices are allocated, but "
                                          "either numAllocForAllRows_ (= "
                                              << this->numAllocForAllRows_ << ") is "
                                                                              "nonzero, or k_numAllocPerRow_ has nonzero dimension.  In other words, "
                                                                              "the graph is supposed to release its \"allocation specifications\" "
                                                                              "when it allocates its indices."
                                              << suffix);
    auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
    auto rowPtrsUnpacked_dev  = this->getRowPtrsUnpackedDevice();
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked_host.extent(0) != rowPtrsUnpacked_dev.extent(0),
                                          std::logic_error,
                                          "The host and device views of k_rowPtrs_ have "
                                          "different sizes; rowPtrsUnpacked_host_ has size  "
                                              << rowPtrsUnpacked_host.extent(0)
                                              << ", but rowPtrsUnpacked_dev_ has size "
                                              << rowPtrsUnpacked_dev.extent(0)
                                              << "." << suffix);
    if (isGloballyIndexed() && rowPtrsUnpacked_host.extent(0) != 0) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(size_t(rowPtrsUnpacked_host.extent(0)) != size_t(lclNumRows + 1),
                                            std::logic_error,
                                            "The graph is globally indexed and "
                                            "k_rowPtrs has nonzero size "
                                                << rowPtrsUnpacked_host.extent(0)
                                                << ", but that size does not equal lclNumRows+1 = "
                                                << (lclNumRows + 1) << "." << suffix);
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked_host(lclNumRows) != size_t(gblInds_wdv.extent(0)),
                                            std::logic_error,
                                            "The graph is globally indexed and "
                                            "k_rowPtrs_ has nonzero size "
                                                << rowPtrsUnpacked_host.extent(0)
                                                << ", but k_rowPtrs_(lclNumRows=" << lclNumRows << ")="
                                                << rowPtrsUnpacked_host(lclNumRows)
                                                << " != gblInds_wdv.extent(0)="
                                                << gblInds_wdv.extent(0) << "." << suffix);
    }
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() &&
                                              rowPtrsUnpacked_host.extent(0) != 0 &&
                                              (static_cast<size_t>(rowPtrsUnpacked_host.extent(0)) !=
                                                   static_cast<size_t>(lclNumRows + 1) ||
                                               rowPtrsUnpacked_host(lclNumRows) !=
                                                   static_cast<size_t>(this->lclIndsUnpacked_wdv.extent(0))),
                                          std::logic_error,
                                          "If k_rowPtrs_ has nonzero size and "
                                          "the graph is locally indexed, then "
                                          "k_rowPtrs_ must have N+1 rows, and "
                                          "k_rowPtrs_(N) must equal lclIndsUnpacked_wdv.extent(0)."
                                              << suffix);
 
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreAllocated() &&
                                              nodeAllocSize > 0 &&
                                              this->lclIndsUnpacked_wdv.extent(0) == 0 &&
                                              this->gblInds_wdv.extent(0) == 0,
                                          std::logic_error,
                                          "Graph is allocated nontrivially, but "
                                          "but 1-D allocations are not present."
                                              << suffix);
 
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->indicesAreAllocated() &&
                                              ((rowPtrsUnpacked_host.extent(0) != 0 ||
                                                this->k_numRowEntries_.extent(0) != 0) ||
                                               this->lclIndsUnpacked_wdv.extent(0) != 0 ||
                                               this->gblInds_wdv.extent(0) != 0),
                                          std::logic_error,
                                          "If indices are not allocated, "
                                          "then none of the buffers should be."
                                              << suffix);
    // indices may be local or global only if they are allocated
    // (numAllocated is redundant; could simply be indicesAreLocal_ ||
    // indicesAreGlobal_)
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC((this->indicesAreLocal_ || this->indicesAreGlobal_) &&
                                              !this->indicesAreAllocated_,
                                          std::logic_error,
                                          "Indices may be local or global only if they are "
                                          "allocated."
                                              << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->indicesAreLocal_ && this->indicesAreGlobal_,
                                          std::logic_error, "Indices may not be both local and global." << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreLocal_ && gblInds_wdv.extent(0) != 0,
                                          std::logic_error,
                                          "Indices are local, but "
                                          "gblInds_wdv.extent(0) (= "
                                              << gblInds_wdv.extent(0)
                                              << ") != 0.  In other words, if indices are local, then "
                                                 "allocations of global indices should not be present."
                                              << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreGlobal_ && lclIndsUnpacked_wdv.extent(0) != 0,
                                          std::logic_error,
                                          "Indices are global, but "
                                          "lclIndsUnpacked_wdv.extent(0) (= "
                                              << lclIndsUnpacked_wdv.extent(0)
                                              << ") != 0.  In other words, if indices are global, "
                                                 "then allocations for local indices should not be present."
                                              << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreLocal_ && nodeAllocSize > 0 &&
                                              lclIndsUnpacked_wdv.extent(0) == 0 && getLocalNumRows() > 0,
                                          std::logic_error,
                                          "Indices are local and "
                                          "getLocalAllocationSize() = "
                                              << nodeAllocSize << " > 0, but "
                                                                  "lclIndsUnpacked_wdv.extent(0) = 0 and getLocalNumRows() = "
                                              << getLocalNumRows() << " > 0." << suffix);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(indicesAreGlobal_ && nodeAllocSize > 0 &&
                                              gblInds_wdv.extent(0) == 0 && getLocalNumRows() > 0,
                                          std::logic_error,
                                          "Indices are global and "
                                          "getLocalAllocationSize() = "
                                              << nodeAllocSize << " > 0, but "
                                                                  "gblInds_wdv.extent(0) = 0 and getLocalNumRows() = "
                                              << getLocalNumRows() << " > 0." << suffix);
    // check the actual allocations
    if (this->indicesAreAllocated() &&
        rowPtrsUnpacked_host.extent(0) != 0) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(rowPtrsUnpacked_host.extent(0)) !=
                                                this->getLocalNumRows() + 1,
                                            std::logic_error,
                                            "Indices are allocated and "
                                            "k_rowPtrs_ has nonzero length, but rowPtrsUnpacked_host_.extent(0) = "
                                                << rowPtrsUnpacked_host.extent(0) << " != getLocalNumRows()+1 = "
                                                << (this->getLocalNumRows() + 1) << "." << suffix);
      const size_t actualNumAllocated =
          rowPtrsUnpacked_host(this->getLocalNumRows());
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() &&
                                                static_cast<size_t>(this->lclIndsUnpacked_wdv.extent(0)) != actualNumAllocated,
                                            std::logic_error,
                                            "Graph is locally indexed, indices are "
                                            "are allocated, and k_rowPtrs_ has nonzero length, but "
                                            "lclIndsUnpacked_wdv.extent(0) = "
                                                << this->lclIndsUnpacked_wdv.extent(0)
                                                << " != actualNumAllocated = " << actualNumAllocated << suffix);
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isGloballyIndexed() &&
                                                static_cast<size_t>(this->gblInds_wdv.extent(0)) != actualNumAllocated,
                                            std::logic_error,
                                            "Graph is globally indexed, indices "
                                            "are allocated, and k_rowPtrs_ has nonzero length, but "
                                            "gblInds_wdv.extent(0) = "
                                                << this->gblInds_wdv.extent(0)
                                                << " != actualNumAllocated = " << actualNumAllocated << suffix);
    }
 
    if (verbose_) {
      std::ostringstream os;
      os << *prefix << "Done" << endl;
      std::cerr << os.str();
    }
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getNumEntriesInGlobalRow(GlobalOrdinal globalRow) const {
  const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(globalRow);
  if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
    return Teuchos::OrdinalTraits<size_t>::invalid();
  } else {
    return rowInfo.numEntries;
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getNumEntriesInLocalRow(LocalOrdinal localRow) const {
  const RowInfo rowInfo = this->getRowInfo(localRow);
  if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
    return Teuchos::OrdinalTraits<size_t>::invalid();
  } else {
    return rowInfo.numEntries;
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getNumAllocatedEntriesInGlobalRow(GlobalOrdinal globalRow) const {
  const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(globalRow);
  if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
    return Teuchos::OrdinalTraits<size_t>::invalid();
  } else {
    return rowInfo.allocSize;
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
size_t
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getNumAllocatedEntriesInLocalRow(LocalOrdinal localRow) const {
  const RowInfo rowInfo = this->getRowInfo(localRow);
  if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid()) {
    return Teuchos::OrdinalTraits<size_t>::invalid();
  } else {
    return rowInfo.allocSize;
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_host_view_type
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalRowPtrsHost() const {
  return getRowPtrsPackedHost();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_device_view_type
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalRowPtrsDevice() const {
  return getRowPtrsPackedDevice();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_inds_host_view_type
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalIndicesHost() const {
  return lclIndsPacked_wdv.getHostView(Access::ReadOnly);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_inds_device_view_type
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalIndicesDevice() const {
  return lclIndsPacked_wdv.getDeviceView(Access::ReadOnly);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalRowCopy(LocalOrdinal localRow,
                    nonconst_local_inds_host_view_type& indices,
                    size_t& numEntries) const {
  using Teuchos::ArrayView;
  const char tfecfFuncName[] = "getLocalRowCopy: ";
 
  TEUCHOS_TEST_FOR_EXCEPTION(
      isGloballyIndexed() && !hasColMap(), std::runtime_error,
      "Tpetra::CrsGraph::getLocalRowCopy: The graph is globally indexed and "
      "does not have a column Map yet.  That means we don't have local indices "
      "for columns yet, so it doesn't make sense to call this method.  If the "
      "graph doesn't have a column Map yet, you should call fillComplete on "
      "it first.");
 
  // This does the right thing (reports an empty row) if the input
  // row is invalid.
  const RowInfo rowinfo = this->getRowInfo(localRow);
  // No side effects on error.
  const size_t theNumEntries = rowinfo.numEntries;
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) < theNumEntries, std::runtime_error,
                                        "Specified storage (size==" << indices.size() << ") does not suffice "
                                                                                         "to hold all "
                                                                    << theNumEntries << " entry/ies for this row.");
  numEntries = theNumEntries;
 
  if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid()) {
    if (isLocallyIndexed()) {
      auto lclInds = getLocalIndsViewHost(rowinfo);
      for (size_t j = 0; j < theNumEntries; ++j) {
        indices[j] = lclInds(j);
      }
    } else if (isGloballyIndexed()) {
      auto gblInds = getGlobalIndsViewHost(rowinfo);
      for (size_t j = 0; j < theNumEntries; ++j) {
        indices[j] = colMap_->getLocalElement(gblInds(j));
      }
    }
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getGlobalRowCopy(GlobalOrdinal globalRow,
                     nonconst_global_inds_host_view_type& indices,
                     size_t& numEntries) const {
  using Teuchos::ArrayView;
  const char tfecfFuncName[] = "getGlobalRowCopy: ";
 
  // This does the right thing (reports an empty row) if the input
  // row is invalid.
  const RowInfo rowinfo      = getRowInfoFromGlobalRowIndex(globalRow);
  const size_t theNumEntries = rowinfo.numEntries;
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      static_cast<size_t>(indices.size()) < theNumEntries, std::runtime_error,
      "Specified storage (size==" << indices.size() << ") does not suffice "
                                                       "to hold all "
                                  << theNumEntries << " entry/ies for this row.");
  numEntries = theNumEntries;  // first side effect
 
  if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid()) {
    if (isLocallyIndexed()) {
      auto lclInds = getLocalIndsViewHost(rowinfo);
      bool err     = colMap_->getGlobalElements(lclInds.data(), theNumEntries, indices.data());
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(err, std::runtime_error, "getGlobalElements error");
    } else if (isGloballyIndexed()) {
      auto gblInds = getGlobalIndsViewHost(rowinfo);
      // Kokkos zero-extent views return null from .data(); glibc declares memcpy's
      // dst/src params __attribute__((nonnull(1,2))), so UBSan nonnull-arg fires
      // on a null pointer even when the byte count is zero.
      if (theNumEntries > 0) {
        std::memcpy(
            (void*)indices.data(),
            (const void*)gblInds.data(),
            theNumEntries * sizeof(*indices.data()));
      }
    }
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalRowView(
        const LocalOrdinal localRow,
        local_inds_host_view_type& indices) const {
  const char tfecfFuncName[] = "getLocalRowView: ";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed(), std::runtime_error,
                                        "The graph's indices are "
                                        "currently stored as global indices, so we cannot return a view with "
                                        "local column indices, whether or not the graph has a column Map.  If "
                                        "the graph _does_ have a column Map, use getLocalRowCopy() instead.");
 
  const RowInfo rowInfo = getRowInfo(localRow);
  if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid() &&
      rowInfo.numEntries > 0) {
    indices = lclIndsUnpacked_wdv.getHostSubview(rowInfo.offset1D,
                                                 rowInfo.numEntries,
                                                 Access::ReadOnly);
  } else {
    // This does the right thing (reports an empty row) if the input
    // row is invalid.
    indices = local_inds_host_view_type();
  }
 
  if (debug_) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) !=
                                              getNumEntriesInLocalRow(localRow),
                                          std::logic_error,
                                          "indices.size() "
                                          "= " << indices.extent(0)
                                               << " != getNumEntriesInLocalRow(localRow=" << localRow << ") = " << getNumEntriesInLocalRow(localRow) << ".  Please report this bug to the Tpetra developers.");
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getGlobalRowView(
        const GlobalOrdinal globalRow,
        global_inds_host_view_type& indices) const {
  const char tfecfFuncName[] = "getGlobalRowView: ";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isLocallyIndexed(), std::runtime_error,
                                        "The graph's indices are "
                                        "currently stored as local indices, so we cannot return a view with "
                                        "global column indices.  Use getGlobalRowCopy() instead.");
 
  // This does the right thing (reports an empty row) if the input
  // row is invalid.
  const RowInfo rowInfo = getRowInfoFromGlobalRowIndex(globalRow);
  if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid() &&
      rowInfo.numEntries > 0) {
    indices = gblInds_wdv.getHostSubview(rowInfo.offset1D,
                                         rowInfo.numEntries,
                                         Access::ReadOnly);
  } else {
    indices = typename global_inds_dualv_type::t_host::const_type();
  }
  if (debug_) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(indices.size()) !=
                                              getNumEntriesInGlobalRow(globalRow),
                                          std::logic_error, "indices.size() = " << indices.extent(0) << " != getNumEntriesInGlobalRow(globalRow=" << globalRow << ") = " << getNumEntriesInGlobalRow(globalRow) << ".  Please report this bug to the Tpetra developers.");
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertLocalIndices(const LocalOrdinal localRow,
                       const Teuchos::ArrayView<const LocalOrdinal>& indices) {
  const char tfecfFuncName[] = "insertLocalIndices: ";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive(), std::runtime_error, "Fill must be active.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isGloballyIndexed(), std::runtime_error,
                                        "Graph indices are global; use insertGlobalIndices().");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error,
                                        "Cannot insert local indices without a column Map.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!rowMap_->isNodeLocalElement(localRow), std::runtime_error,
                                        "Local row index " << localRow << " is not in the row Map "
                                                                          "on the calling process.");
  if (!indicesAreAllocated()) {
    allocateIndices(LocalIndices, verbose_);
  }
 
  if (debug_) {
    // In debug mode, if the graph has a column Map, test whether any
    // of the given column indices are not in the column Map.  Keep
    // track of the invalid column indices so we can tell the user
    // about them.
    if (hasColMap()) {
      using std::endl;
      using Teuchos::Array;
      using Teuchos::toString;
      typedef typename Teuchos::ArrayView<const LocalOrdinal>::size_type size_type;
 
      const map_type& colMap = *colMap_;
      Array<LocalOrdinal> badColInds;
      bool allInColMap = true;
      for (size_type k = 0; k < indices.size(); ++k) {
        if (!colMap.isNodeLocalElement(indices[k])) {
          allInColMap = false;
          badColInds.push_back(indices[k]);
        }
      }
      if (!allInColMap) {
        std::ostringstream os;
        os << "Tpetra::CrsGraph::insertLocalIndices: You attempted to insert "
              "entries in owned row "
           << localRow << ", at the following column "
                          "indices: "
           << toString(indices) << "." << endl;
        os << "Of those, the following indices are not in the column Map on "
              "this process: "
           << toString(badColInds) << "." << endl
           << "Since "
              "the graph has a column Map already, it is invalid to insert entries "
              "at those locations.";
        TEUCHOS_TEST_FOR_EXCEPTION(!allInColMap, std::invalid_argument, os.str());
      }
    }
  }
 
  insertLocalIndicesImpl(localRow, indices);
 
  if (debug_) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!indicesAreAllocated() || !isLocallyIndexed(), std::logic_error,
                                          "At the end of insertLocalIndices, ! indicesAreAllocated() || "
                                          "! isLocallyIndexed() is true.  Please report this bug to the "
                                          "Tpetra developers.");
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertLocalIndices(const LocalOrdinal localRow,
                       const LocalOrdinal numEnt,
                       const LocalOrdinal inds[]) {
  Teuchos::ArrayView<const LocalOrdinal> indsT(inds, numEnt);
  this->insertLocalIndices(localRow, indsT);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertGlobalIndices(const GlobalOrdinal gblRow,
                        const LocalOrdinal numInputInds,
                        const GlobalOrdinal inputGblColInds[]) {
  typedef LocalOrdinal LO;
  const char tfecfFuncName[] = "insertGlobalIndices: ";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed(), std::runtime_error,
                                        "graph indices are local; use insertLocalIndices().");
  // This can't really be satisfied for now, because if we are
  // fillComplete(), then we are local.  In the future, this may
  // change.  However, the rule that modification require active
  // fill will not change.
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->isFillActive(), std::runtime_error,
                                        "You are not allowed to call this method if fill is not active.  "
                                        "If fillComplete has been called, you must first call resumeFill "
                                        "before you may insert indices.");
  if (!indicesAreAllocated()) {
    allocateIndices(GlobalIndices, verbose_);
  }
  const LO lclRow = this->rowMap_->getLocalElement(gblRow);
  if (lclRow != Tpetra::Details::OrdinalTraits<LO>::invalid()) {
    if (debug_) {
      if (this->hasColMap()) {
        using std::endl;
        const map_type& colMap = *(this->colMap_);
        // In a debug build, keep track of the nonowned ("bad") column
        // indices, so that we can display them in the exception
        // message.  In a release build, just ditch the loop early if
        // we encounter a nonowned column index.
        std::vector<GlobalOrdinal> badColInds;
        bool allInColMap = true;
        for (LO k = 0; k < numInputInds; ++k) {
          if (!colMap.isNodeGlobalElement(inputGblColInds[k])) {
            allInColMap = false;
            badColInds.push_back(inputGblColInds[k]);
          }
        }
        if (!allInColMap) {
          std::ostringstream os;
          os << "You attempted to insert entries in owned row " << gblRow
             << ", at the following column indices: [";
          for (LO k = 0; k < numInputInds; ++k) {
            os << inputGblColInds[k];
            if (k + static_cast<LO>(1) < numInputInds) {
              os << ",";
            }
          }
          os << "]." << endl
             << "Of those, the following indices are not in "
                "the column Map on this process: [";
          for (size_t k = 0; k < badColInds.size(); ++k) {
            os << badColInds[k];
            if (k + size_t(1) < badColInds.size()) {
              os << ",";
            }
          }
          os << "]." << endl
             << "Since the matrix has a column Map already, "
                "it is invalid to insert entries at those locations.";
          TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::invalid_argument, os.str());
        }
      }
    }  // debug_
    this->insertGlobalIndicesImpl(lclRow, inputGblColInds, numInputInds);
  } else {  // a nonlocal row
    this->insertGlobalIndicesIntoNonownedRows(gblRow, inputGblColInds,
                                              numInputInds);
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertGlobalIndices(const GlobalOrdinal gblRow,
                        const Teuchos::ArrayView<const GlobalOrdinal>& inputGblColInds) {
  this->insertGlobalIndices(gblRow, inputGblColInds.size(),
                            inputGblColInds.getRawPtr());
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertGlobalIndicesFiltered(const LocalOrdinal lclRow,
                                const GlobalOrdinal gblColInds[],
                                const LocalOrdinal numGblColInds) {
  typedef LocalOrdinal LO;
  typedef GlobalOrdinal GO;
  const char tfecfFuncName[] = "insertGlobalIndicesFiltered: ";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed(), std::runtime_error,
                                        "Graph indices are local; use insertLocalIndices().");
  // This can't really be satisfied for now, because if we are
  // fillComplete(), then we are local.  In the future, this may
  // change.  However, the rule that modification require active
  // fill will not change.
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->isFillActive(), std::runtime_error,
                                        "You are not allowed to call this method if fill is not active.  "
                                        "If fillComplete has been called, you must first call resumeFill "
                                        "before you may insert indices.");
  if (!indicesAreAllocated()) {
    allocateIndices(GlobalIndices, verbose_);
  }
 
  Teuchos::ArrayView<const GO> gblColInds_av(gblColInds, numGblColInds);
  // If we have a column Map, use it to filter the entries.
  if (!colMap_.is_null()) {
    const map_type& colMap = *(this->colMap_);
 
    LO curOffset = 0;
    while (curOffset < numGblColInds) {
      // Find a sequence of input indices that are in the column Map
      // on the calling process.  Doing a sequence at a time,
      // instead of one at a time, amortizes some overhead.
      LO endOffset = curOffset;
      for (; endOffset < numGblColInds; ++endOffset) {
        const LO lclCol = colMap.getLocalElement(gblColInds[endOffset]);
        if (lclCol == Tpetra::Details::OrdinalTraits<LO>::invalid()) {
          break;  // first entry, in current sequence, not in the column Map
        }
      }
      // curOffset, endOffset: half-exclusive range of indices in
      // the column Map on the calling process.  If endOffset ==
      // curOffset, the range is empty.
      const LO numIndInSeq = (endOffset - curOffset);
      if (numIndInSeq != 0) {
        this->insertGlobalIndicesImpl(lclRow, gblColInds + curOffset,
                                      numIndInSeq);
      }
      // Invariant before this line: Either endOffset ==
      // numGblColInds, or gblColInds[endOffset] is not in the
      // column Map on the calling process.
      curOffset = endOffset + 1;
    }
  } else {
    this->insertGlobalIndicesImpl(lclRow, gblColInds_av.getRawPtr(),
                                  gblColInds_av.size());
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    insertGlobalIndicesIntoNonownedRows(const GlobalOrdinal gblRow,
                                        const GlobalOrdinal gblColInds[],
                                        const LocalOrdinal numGblColInds) {
  // This creates the std::vector if it doesn't exist yet.
  // std::map's operator[] does a lookup each time, so it's better
  // to pull nonlocals_[grow] out of the loop.
  std::vector<GlobalOrdinal>& nonlocalRow = this->nonlocals_[gblRow];
  for (LocalOrdinal k = 0; k < numGblColInds; ++k) {
    // FIXME (mfh 20 Jul 2017) Would be better to use a set, in
    // order to avoid duplicates.  globalAssemble() sorts these
    // anyway.
    nonlocalRow.push_back(gblColInds[k]);
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    removeLocalIndices(LocalOrdinal lrow) {
  const char tfecfFuncName[] = "removeLocalIndices: ";
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      !isFillActive(), std::runtime_error, "requires that fill is active.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      isStorageOptimized(), std::runtime_error,
      "cannot remove indices after optimizeStorage() has been called.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      isGloballyIndexed(), std::runtime_error, "graph indices are global.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      !rowMap_->isNodeLocalElement(lrow), std::runtime_error,
      "Local row " << lrow << " is not in the row Map on the calling process.");
  if (!indicesAreAllocated()) {
    allocateIndices(LocalIndices, verbose_);
  }
 
  if (k_numRowEntries_.extent(0) != 0) {
    this->k_numRowEntries_(lrow) = 0;
  }
 
  if (debug_) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(getNumEntriesInLocalRow(lrow) != 0 ||
                                              !indicesAreAllocated() ||
                                              !isLocallyIndexed(),
                                          std::logic_error,
                                          "Violated stated post-conditions. Please contact Tpetra team.");
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    setAllIndices(const typename local_graph_device_type::row_map_type& rowPointers,
                  const typename local_graph_device_type::entries_type::non_const_type& columnIndices) {
  using ProfilingRegion = Details::ProfilingRegion;
  ProfilingRegion region("Tpetra::CrsGraph::setAllIndices");
  const char tfecfFuncName[] = "setAllIndices: ";
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      !hasColMap() || getColMap().is_null(), std::runtime_error,
      "The graph must have a column Map before you may call this method.");
  LocalOrdinal numLocalRows = this->getLocalNumRows();
  {
    LocalOrdinal rowPtrLen = rowPointers.size();
    if (numLocalRows == 0) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
          rowPtrLen != 0 && rowPtrLen != 1,
          std::runtime_error, "Have 0 local rows, but rowPointers.size() is neither 0 nor 1.");
    } else {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
          rowPtrLen != numLocalRows + 1,
          std::runtime_error, "rowPointers.size() = " << rowPtrLen << " != this->getLocalNumRows()+1 = " << (numLocalRows + 1) << ".");
    }
  }
 
  if (debug_) {
    using exec_space                = typename local_graph_device_type::execution_space;
    int columnsOutOfBounds          = 0;
    local_ordinal_type numLocalCols = this->getLocalNumCols();
    Kokkos::parallel_reduce(
        Kokkos::RangePolicy<exec_space>(0, columnIndices.extent(0)),
        KOKKOS_LAMBDA(const LocalOrdinal i, int& lOutOfBounds) {
          if (columnIndices(i) < 0 || columnIndices(i) >= numLocalCols)
            lOutOfBounds++;
        },
        columnsOutOfBounds);
    int globalColsOutOfBounds = 0;
    auto comm                 = this->getComm();
    Teuchos::reduceAll<int, int>(*comm, Teuchos::REDUCE_MAX, columnsOutOfBounds,
                                 Teuchos::outArg(globalColsOutOfBounds));
    if (globalColsOutOfBounds) {
      std::string message;
      if (columnsOutOfBounds) {
        // Only print message from ranks with the problem
        message = std::string("ERROR, rank ") + std::to_string(comm->getRank()) + ", CrsGraph::setAllIndices(): provided columnIndices are not all within range [0, getLocalNumCols())!\n";
      }
      Details::gathervPrint(std::cout, message, *comm);
      throw std::invalid_argument("CrsGraph::setAllIndices(): columnIndices are out of the valid range on at least one process.");
    }
  }
 
  if (debug_ && this->isSorted()) {
    // Verify that the local indices are actually sorted
    int notSorted    = 0;
    using exec_space = typename local_graph_device_type::execution_space;
    using size_type  = typename local_graph_device_type::size_type;
    Kokkos::parallel_reduce(
        Kokkos::RangePolicy<exec_space>(0, numLocalRows),
        KOKKOS_LAMBDA(const LocalOrdinal i, int& lNotSorted) {
          size_type rowBegin = rowPointers(i);
          size_type rowEnd   = rowPointers(i + 1);
          for (size_type j = rowBegin + 1; j < rowEnd; j++) {
            if (columnIndices(j - 1) > columnIndices(j)) {
              lNotSorted = 1;
            }
          }
        },
        notSorted);
    // All-reduce notSorted to avoid rank divergence
    int globalNotSorted = 0;
    auto comm           = this->getComm();
    Teuchos::reduceAll<int, int>(*comm, Teuchos::REDUCE_MAX, notSorted,
                                 Teuchos::outArg(globalNotSorted));
    if (globalNotSorted) {
      std::string message;
      if (notSorted) {
        // Only print message from ranks with the problem
        message = std::string("ERROR, rank ") + std::to_string(comm->getRank()) + ", CrsGraph::setAllIndices(): provided columnIndices are not sorted!\n";
      }
      Details::gathervPrint(std::cout, message, *comm);
      throw std::invalid_argument("CrsGraph::setAllIndices(): provided columnIndices are not sorted within rows on at least one process.");
    }
  }
 
  indicesAreAllocated_ = true;
  indicesAreLocal_     = true;
  indicesAreSorted_    = true;
  noRedundancies_      = true;
  lclIndsPacked_wdv    = local_inds_wdv_type(columnIndices);
  lclIndsUnpacked_wdv  = lclIndsPacked_wdv;
  setRowPtrs(rowPointers);
 
  set_need_sync_host_uvm_access();  // columnIndices and rowPointers potentially still in a kernel
 
  // Storage MUST be packed, since the interface doesn't give any
  // way to indicate any extra space at the end of each row.
  storageStatus_ = Details::STORAGE_1D_PACKED;
 
  // These normally get cleared out at the end of allocateIndices.
  // It makes sense to clear them out here, because at the end of
  // this method, the graph is allocated on the calling process.
  numAllocForAllRows_ = 0;
  k_numAllocPerRow_   = decltype(k_numAllocPerRow_)();
 
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    setAllIndices(const Teuchos::ArrayRCP<size_t>& rowPointers,
                  const Teuchos::ArrayRCP<LocalOrdinal>& columnIndices) {
  using Kokkos::View;
  typedef typename local_graph_device_type::row_map_type row_map_type;
  typedef typename row_map_type::array_layout layout_type;
  typedef typename row_map_type::non_const_value_type row_offset_type;
  typedef View<size_t*, layout_type, Kokkos::HostSpace,
               Kokkos::MemoryUnmanaged>
      input_view_type;
  typedef typename row_map_type::non_const_type nc_row_map_type;
 
  const size_t size   = static_cast<size_t>(rowPointers.size());
  constexpr bool same = std::is_same<size_t, row_offset_type>::value;
  input_view_type ptr_in(rowPointers.getRawPtr(), size);
 
  nc_row_map_type ptr_rot("Tpetra::CrsGraph::ptr", size);
 
  if constexpr (same) {  // size_t == row_offset_type
    using lexecution_space = typename device_type::execution_space;
    Kokkos::deep_copy(lexecution_space(),
                      ptr_rot,
                      ptr_in);
  } else {  // size_t != row_offset_type
    // CudaUvmSpace != HostSpace, so this will be false in that case.
    constexpr bool inHostMemory =
        std::is_same<typename row_map_type::memory_space,
                     Kokkos::HostSpace>::value;
    if (inHostMemory) {
      // Copy (with cast from size_t to row_offset_type, with bounds
      // checking if necessary) to ptr_rot.
      ::Tpetra::Details::copyOffsets(ptr_rot, ptr_in);
    } else {  // Copy input row offsets to device first.
      //
      // FIXME (mfh 24 Mar 2015) If CUDA UVM, running in the host's
      // execution space would avoid the double copy.
      //
      View<size_t*, layout_type, device_type> ptr_st("Tpetra::CrsGraph::ptr", size);
 
      // DEEP_COPY REVIEW - NOT TESTED
      Kokkos::deep_copy(ptr_st, ptr_in);
      // Copy on device (casting from size_t to row_offset_type,
      // with bounds checking if necessary) to ptr_rot.  This
      // executes in the output View's execution space, which is the
      // same as execution_space.
      ::Tpetra::Details::copyOffsets(ptr_rot, ptr_st);
    }
  }
 
  Kokkos::View<LocalOrdinal*, layout_type, device_type> k_ind =
      Kokkos::Compat::getKokkosViewDeepCopy<device_type>(columnIndices());
  setAllIndices(ptr_rot, k_ind);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    globalAssemble() {
  using std::endl;
  using Teuchos::Comm;
  using Teuchos::outArg;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using Teuchos::REDUCE_MAX;
  using Teuchos::REDUCE_MIN;
  using Teuchos::reduceAll;
  using crs_graph_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
  using LO             = local_ordinal_type;
  using GO             = global_ordinal_type;
  using size_type      = typename Teuchos::Array<GO>::size_type;
 
  const char tfecfFuncName[] = "globalAssemble: ";  // for exception macro
 
  Details::ProfilingRegion regionGA("Tpetra::CrsGraph::globalAssemble");
 
  std::unique_ptr<std::string> prefix;
  if (verbose_) {
    prefix = this->createPrefix("CrsGraph", "globalAssemble");
    std::ostringstream os;
    os << *prefix << "Start" << endl;
    std::cerr << os.str();
  }
  RCP<const Comm<int>> comm = getComm();
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive(), std::runtime_error,
                                        "Fill must be active before "
                                        "you may call this method.");
 
  const size_t myNumNonlocalRows = this->nonlocals_.size();
 
  // If no processes have nonlocal rows, then we don't have to do
  // anything.  Checking this is probably cheaper than constructing
  // the Map of nonlocal rows (see below) and noticing that it has
  // zero global entries.
  {
    const int iHaveNonlocalRows = (myNumNonlocalRows == 0) ? 0 : 1;
    int someoneHasNonlocalRows  = 0;
    reduceAll<int, int>(*comm, REDUCE_MAX, iHaveNonlocalRows,
                        outArg(someoneHasNonlocalRows));
    if (someoneHasNonlocalRows == 0) {
      if (verbose_) {
        std::ostringstream os;
        os << *prefix << "Done: No nonlocal rows" << endl;
        std::cerr << os.str();
      }
      return;
    } else if (verbose_) {
      std::ostringstream os;
      os << *prefix << "At least 1 process has nonlocal rows"
         << endl;
      std::cerr << os.str();
    }
  }
 
  // 1. Create a list of the "nonlocal" rows on each process.  this
  //    requires iterating over nonlocals_, so while we do this,
  //    deduplicate the entries and get a count for each nonlocal
  //    row on this process.
  // 2. Construct a new row Map corresponding to those rows.  This
  //    Map is likely overlapping.  We know that the Map is not
  //    empty on all processes, because the above all-reduce and
  //    return exclude that case.
 
  RCP<const map_type> nonlocalRowMap;
  // Keep this for CrsGraph's constructor.
  Teuchos::Array<size_t> numEntPerNonlocalRow(myNumNonlocalRows);
  {
    Teuchos::Array<GO> myNonlocalGblRows(myNumNonlocalRows);
    size_type curPos = 0;
    for (auto mapIter = this->nonlocals_.begin();
         mapIter != this->nonlocals_.end();
         ++mapIter, ++curPos) {
      myNonlocalGblRows[curPos] = mapIter->first;
      std::vector<GO>& gblCols  = mapIter->second;  // by ref; change in place
      std::sort(gblCols.begin(), gblCols.end());
      auto vecLast = std::unique(gblCols.begin(), gblCols.end());
      gblCols.erase(vecLast, gblCols.end());
      numEntPerNonlocalRow[curPos] = gblCols.size();
    }
 
    // Currently, Map requires that its indexBase be the global min
    // of all its global indices.  Map won't compute this for us, so
    // we must do it.  If our process has no nonlocal rows, set the
    // "min" to the max possible GO value.  This ensures that if
    // some process has at least one nonlocal row, then it will pick
    // that up as the min.  We know that at least one process has a
    // nonlocal row, since the all-reduce and return at the top of
    // this method excluded that case.
    GO myMinNonlocalGblRow = std::numeric_limits<GO>::max();
    {
      auto iter = std::min_element(myNonlocalGblRows.begin(),
                                   myNonlocalGblRows.end());
      if (iter != myNonlocalGblRows.end()) {
        myMinNonlocalGblRow = *iter;
      }
    }
    GO gblMinNonlocalGblRow = 0;
    reduceAll<int, GO>(*comm, REDUCE_MIN, myMinNonlocalGblRow,
                       outArg(gblMinNonlocalGblRow));
    const GO indexBase      = gblMinNonlocalGblRow;
    const global_size_t INV = Teuchos::OrdinalTraits<global_size_t>::invalid();
    nonlocalRowMap          = rcp(new map_type(INV, myNonlocalGblRows(), indexBase, comm));
  }
 
  if (verbose_) {
    std::ostringstream os;
    os << *prefix << "nonlocalRowMap->getIndexBase()="
       << nonlocalRowMap->getIndexBase() << endl;
    std::cerr << os.str();
  }
 
  // 3. Use the column indices for each nonlocal row, as stored in
  //    nonlocals_, to construct a CrsGraph corresponding to
  //    nonlocal rows.  We need, but we have, exact counts of the
  //    number of entries in each nonlocal row.
 
  RCP<crs_graph_type> nonlocalGraph =
      rcp(new crs_graph_type(nonlocalRowMap, numEntPerNonlocalRow()));
  {
    size_type curPos = 0;
    for (auto mapIter = this->nonlocals_.begin();
         mapIter != this->nonlocals_.end();
         ++mapIter, ++curPos) {
      const GO gblRow          = mapIter->first;
      std::vector<GO>& gblCols = mapIter->second;  // by ref just to avoid copy
      const LO numEnt          = static_cast<LO>(numEntPerNonlocalRow[curPos]);
      nonlocalGraph->insertGlobalIndices(gblRow, numEnt, gblCols.data());
    }
  }
  if (verbose_) {
    std::ostringstream os;
    os << *prefix << "Built nonlocal graph" << endl;
    std::cerr << os.str();
  }
  // There's no need to fill-complete the nonlocals graph.
  // We just use it as a temporary container for the Export.
 
  // 4. If the original row Map is one to one, then we can Export
  //    directly from nonlocalGraph into this.  Otherwise, we have
  //    to create a temporary graph with a one-to-one row Map,
  //    Export into that, then Import from the temporary graph into
  //    *this.
 
  auto origRowMap                 = this->getRowMap();
  const bool origRowMapIsOneToOne = origRowMap->isOneToOne();
 
  if (origRowMapIsOneToOne) {
    if (verbose_) {
      std::ostringstream os;
      os << *prefix << "Original row Map is 1-to-1" << endl;
      std::cerr << os.str();
    }
    export_type exportToOrig(nonlocalRowMap, origRowMap);
    this->doExport(*nonlocalGraph, exportToOrig, Tpetra::INSERT);
    // We're done at this point!
  } else {
    if (verbose_) {
      std::ostringstream os;
      os << *prefix << "Original row Map is NOT 1-to-1" << endl;
      std::cerr << os.str();
    }
    // If you ask a Map whether it is one to one, it does some
    // communication and stashes intermediate results for later use
    // by createOneToOne.  Thus, calling createOneToOne doesn't cost
    // much more then the original cost of calling isOneToOne.
    auto oneToOneRowMap = Tpetra::createOneToOne(origRowMap);
    export_type exportToOneToOne(nonlocalRowMap, oneToOneRowMap);
 
    // Create a temporary graph with the one-to-one row Map.
    //
    // TODO (mfh 09 Sep 2016) Estimate the number of entries in each
    // row, to avoid reallocation during the Export operation.
    crs_graph_type oneToOneGraph(oneToOneRowMap, 0);
 
    // Export from graph of nonlocals into the temp one-to-one graph.
    if (verbose_) {
      std::ostringstream os;
      os << *prefix << "Export nonlocal graph" << endl;
      std::cerr << os.str();
    }
    oneToOneGraph.doExport(*nonlocalGraph, exportToOneToOne, Tpetra::INSERT);
 
    // We don't need the graph of nonlocals anymore, so get rid of
    // it, to keep the memory high-water mark down.
    nonlocalGraph = Teuchos::null;
 
    // Import from the one-to-one graph to the original graph.
    import_type importToOrig(oneToOneRowMap, origRowMap);
    if (verbose_) {
      std::ostringstream os;
      os << *prefix << "Import nonlocal graph" << endl;
      std::cerr << os.str();
    }
    this->doImport(oneToOneGraph, importToOrig, Tpetra::INSERT);
  }
 
  // It's safe now to clear out nonlocals_, since we've already
  // committed side effects to *this.  The standard idiom for
  // clearing a Container like std::map, is to swap it with an empty
  // Container and let the swapped Container fall out of scope.
  decltype(this->nonlocals_) newNonlocals;
  std::swap(this->nonlocals_, newNonlocals);
 
  checkInternalState();
  if (verbose_) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    resumeFill(const Teuchos::RCP<Teuchos::ParameterList>& params) {
  clearGlobalConstants();
  if (params != Teuchos::null) this->setParameterList(params);
  // either still sorted/merged or initially sorted/merged
  indicesAreSorted_ = true;
  noRedundancies_   = true;
  fillComplete_     = false;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    fillComplete(const Teuchos::RCP<Teuchos::ParameterList>& params) {
  // If the graph already has domain and range Maps, don't clobber
  // them.  If it doesn't, use the current row Map for both the
  // domain and range Maps.
  //
  // NOTE (mfh 28 Sep 2014): If the graph was constructed without a
  // column Map, and column indices are inserted which are not in
  // the row Map on any process, this will cause troubles.  However,
  // that is not a common case for most applications that we
  // encounter, and checking for it might require more
  // communication.
  Teuchos::RCP<const map_type> domMap = this->getDomainMap();
  if (domMap.is_null()) {
    domMap = this->getRowMap();
  }
  Teuchos::RCP<const map_type> ranMap = this->getRangeMap();
  if (ranMap.is_null()) {
    ranMap = this->getRowMap();
  }
  this->fillComplete(domMap, ranMap, params);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    fillComplete(const Teuchos::RCP<const map_type>& domainMap,
                 const Teuchos::RCP<const map_type>& rangeMap,
                 const Teuchos::RCP<Teuchos::ParameterList>& params) {
  using std::endl;
 
  const char tfecfFuncName[] = "fillComplete: ";
  const bool verbose         = verbose_;
 
  Details::ProfilingRegion regionFC("Tpetra::CrsGraph::fillComplete");
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "fillComplete");
    std::ostringstream os;
    os << *prefix << "Start" << endl;
    std::cerr << os.str();
  }
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!isFillActive() || isFillComplete(), std::runtime_error,
                                        "Graph fill state must be active (isFillActive() "
                                        "must be true) before calling fillComplete().");
 
  const int numProcs = getComm()->getSize();
 
  //
  // Read and set parameters
  //
 
  // Does the caller want to sort remote GIDs (within those owned by
  // the same process) in makeColMap()?
  if (!params.is_null()) {
    if (params->isParameter("sort column map ghost gids")) {
      sortGhostsAssociatedWithEachProcessor_ =
          params->get<bool>("sort column map ghost gids",
                            sortGhostsAssociatedWithEachProcessor_);
    } else if (params->isParameter("Sort column Map ghost GIDs")) {
      sortGhostsAssociatedWithEachProcessor_ =
          params->get<bool>("Sort column Map ghost GIDs",
                            sortGhostsAssociatedWithEachProcessor_);
    }
  }
 
  // If true, the caller promises that no process did nonlocal
  // changes since the last call to fillComplete.
  bool assertNoNonlocalInserts = false;
  if (!params.is_null()) {
    assertNoNonlocalInserts =
        params->get<bool>("No Nonlocal Changes", assertNoNonlocalInserts);
  }
 
  //
  // Allocate indices, if they haven't already been allocated
  //
  if (!indicesAreAllocated()) {
    if (hasColMap()) {
      // We have a column Map, so use local indices.
      allocateIndices(LocalIndices, verbose);
    } else {
      // We don't have a column Map, so use global indices.
      allocateIndices(GlobalIndices, verbose);
    }
  }
 
  //
  // Do global assembly, if requested and if the communicator
  // contains more than one process.
  //
  const bool mayNeedGlobalAssemble = !assertNoNonlocalInserts && numProcs > 1;
  if (mayNeedGlobalAssemble) {
    // This first checks if we need to do global assembly.
    // The check costs a single all-reduce.
    globalAssemble();
  } else {
    const size_t numNonlocals = nonlocals_.size();
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Do not need to call globalAssemble; "
                       "assertNoNonlocalInserts="
         << (assertNoNonlocalInserts ? "true" : "false")
         << "numProcs=" << numProcs
         << ", nonlocals_.size()=" << numNonlocals << endl;
      std::cerr << os.str();
    }
    const int lclNeededGlobalAssemble =
        (numProcs > 1 && numNonlocals != 0) ? 1 : 0;
    if (lclNeededGlobalAssemble != 0 && verbose) {
      std::ostringstream os;
      os << *prefix;
      Details::Impl::verbosePrintMap(
          os, nonlocals_.begin(), nonlocals_.end(),
          nonlocals_.size(), "nonlocals_");
      std::cerr << os.str() << endl;
    }
 
    if (debug_) {
      auto map                    = this->getMap();
      auto comm                   = map.is_null() ? Teuchos::null : map->getComm();
      int gblNeededGlobalAssemble = lclNeededGlobalAssemble;
      if (!comm.is_null()) {
        using Teuchos::REDUCE_MAX;
        using Teuchos::reduceAll;
        reduceAll(*comm, REDUCE_MAX, lclNeededGlobalAssemble,
                  Teuchos::outArg(gblNeededGlobalAssemble));
      }
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(gblNeededGlobalAssemble != 0, std::runtime_error,
                                            "nonlocals_.size()=" << numNonlocals << " != 0 on at "
                                                                                    "least one process in the CrsGraph's communicator.  This "
                                                                                    "means either that you incorrectly set the "
                                                                                    "\"No Nonlocal Changes\" fillComplete parameter to true, "
                                                                                    "or that you inserted invalid entries.  "
                                                                                    "Rerun with the environment variable TPETRA_VERBOSE="
                                                                                    "CrsGraph set to see the entries of nonlocals_ on every "
                                                                                    "MPI process (WARNING: lots of output).");
    } else {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(lclNeededGlobalAssemble != 0, std::runtime_error,
                                            "nonlocals_.size()=" << numNonlocals << " != 0 on the "
                                                                                    "calling process.  This means either that you incorrectly "
                                                                                    "set the \"No Nonlocal Changes\" fillComplete parameter "
                                                                                    "to true, or that you inserted invalid entries.  "
                                                                                    "Rerun with the environment "
                                                                                    "variable TPETRA_VERBOSE=CrsGraph set to see the entries "
                                                                                    "of nonlocals_ on every MPI process (WARNING: lots of "
                                                                                    "output).");
    }
  }
 
  // Set domain and range Map.  This may clear the Import / Export
  // objects if the new Maps differ from any old ones.
  setDomainRangeMaps(domainMap, rangeMap);
 
  // If the graph does not already have a column Map (either from
  // the user constructor calling the version of the constructor
  // that takes a column Map, or from a previous fillComplete call),
  // then create it.
  Teuchos::Array<int> remotePIDs(0);
  const bool mustBuildColMap = !this->hasColMap();
  if (mustBuildColMap) {
    this->makeColMap(remotePIDs);  // resized on output
  }
 
  // Make indices local, if they aren't already.
  // The method doesn't do any work if the indices are already local.
  const std::pair<size_t, std::string> makeIndicesLocalResult =
      this->makeIndicesLocal(verbose);
 
  if (debug_) {
    using Details::gathervPrint;
    using Teuchos::outArg;
    using Teuchos::RCP;
    using Teuchos::REDUCE_MIN;
    using Teuchos::reduceAll;
 
    RCP<const map_type> map = this->getMap();
    RCP<const Teuchos::Comm<int>> comm;
    if (!map.is_null()) {
      comm = map->getComm();
    }
    if (comm.is_null()) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(makeIndicesLocalResult.first != 0, std::runtime_error,
                                            makeIndicesLocalResult.second);
    } else {
      const int lclSuccess = (makeIndicesLocalResult.first == 0);
      int gblSuccess       = 0;  // output argument
      reduceAll(*comm, REDUCE_MIN, lclSuccess, outArg(gblSuccess));
      if (gblSuccess != 1) {
        std::ostringstream os;
        gathervPrint(os, makeIndicesLocalResult.second, *comm);
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
      }
    }
  } else {
    // TODO (mfh 20 Jul 2017) Instead of throwing here, pass along
    // the error state to makeImportExport or
    // computeGlobalConstants, which may do all-reduces and thus may
    // have the opportunity to communicate that error state.
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(makeIndicesLocalResult.first != 0, std::runtime_error,
                                          makeIndicesLocalResult.second);
  }
 
  // If this process has no indices, then CrsGraph considers it
  // already trivially sorted and merged.  Thus, this method need
  // not be called on all processes in the row Map's communicator.
  this->sortAndMergeAllIndices(this->isSorted(), this->isMerged());
 
  // Make Import and Export objects, if they haven't been made
  // already.  If we made a column Map above, reuse information from
  // that process to avoid communiation in the Import setup.
  this->makeImportExport(remotePIDs, mustBuildColMap);
 
  // Create the KokkosSparse::StaticCrsGraph, if it doesn't already exist.
  this->fillLocalGraph(params);
 
  const bool callComputeGlobalConstants = params.get() == nullptr ||
                                          params->get("compute global constants", true);
  if (callComputeGlobalConstants) {
    this->computeGlobalConstants();
  } else {
    this->computeLocalConstants();
  }
  this->fillComplete_ = true;
  this->checkInternalState();
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    expertStaticFillComplete(const Teuchos::RCP<const map_type>& domainMap,
                             const Teuchos::RCP<const map_type>& rangeMap,
                             const Teuchos::RCP<const import_type>& importer,
                             const Teuchos::RCP<const export_type>& exporter,
                             const Teuchos::RCP<Teuchos::ParameterList>& params) {
  const char tfecfFuncName[] = "expertStaticFillComplete: ";
  Tpetra::Details::ProfilingRegion prESFC("Tpetra::CrsGraph::expertStaticFillComplete");
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      domainMap.is_null() || rangeMap.is_null(),
      std::runtime_error, "The input domain Map and range Map must be nonnull.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      isFillComplete() || !hasColMap(), std::runtime_error,
      "You may not "
      "call this method unless the graph has a column Map.");
  auto rowPtrsUnpackedLength = this->getRowPtrsUnpackedDevice().extent(0);
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      getLocalNumRows() > 0 && rowPtrsUnpackedLength == 0,
      std::runtime_error, "The calling process has getLocalNumRows() = " << getLocalNumRows() << " > 0 rows, but the row offsets array has not "
                                                                                                 "been set.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      static_cast<size_t>(rowPtrsUnpackedLength) != getLocalNumRows() + 1,
      std::runtime_error, "The row offsets array has length " << rowPtrsUnpackedLength << " != getLocalNumRows()+1 = " << (getLocalNumRows() + 1) << ".");
 
  // Note: We don't need to do the following things which are normally done in fillComplete:
  // allocateIndices, globalAssemble, makeColMap, makeIndicesLocal, sortAndMergeAllIndices
 
  // Constants from allocateIndices
  //
  // mfh 08 Aug 2014: numAllocForAllRows_ and k_numAllocPerRow_ go
  // away once the graph is allocated.  expertStaticFillComplete
  // either presumes that the graph is allocated, or "allocates" it.
  //
  // FIXME (mfh 08 Aug 2014) The goal for the Kokkos refactor
  // version of CrsGraph is to allocate in the constructor, not
  // lazily on first insert.  That will make both
  // numAllocForAllRows_ and k_numAllocPerRow_ obsolete.
  numAllocForAllRows_  = 0;
  k_numAllocPerRow_    = decltype(k_numAllocPerRow_)();
  indicesAreAllocated_ = true;
 
  // Constants from makeIndicesLocal
  //
  // The graph has a column Map, so its indices had better be local.
  indicesAreLocal_  = true;
  indicesAreGlobal_ = false;
 
  // set domain/range map: may clear the import/export objects
  {
    Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-Maps");
    setDomainRangeMaps(domainMap, rangeMap);
  }
 
  // Presume the user sorted and merged the arrays first
  indicesAreSorted_ = true;
  noRedundancies_   = true;
 
  // makeImportExport won't create a new importer/exporter if I set one here first.
 
  importer_ = Teuchos::null;
  exporter_ = Teuchos::null;
  if (importer != Teuchos::null) {
    Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-mIXcheckI");
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
        !importer->getSourceMap()->isSameAs(*getDomainMap()) ||
            !importer->getTargetMap()->isSameAs(*getColMap()),
        std::invalid_argument, ": importer does not match matrix maps.");
    importer_ = importer;
  }
 
  if (exporter != Teuchos::null) {
    Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-mIXcheckE");
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
        !exporter->getSourceMap()->isSameAs(*getRowMap()) ||
            !exporter->getTargetMap()->isSameAs(*getRangeMap()),
        std::invalid_argument, ": exporter does not match matrix maps.");
    exporter_ = exporter;
  }
 
  {
    Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-mIXmake");
    Teuchos::Array<int> remotePIDs(0);  // unused output argument
    this->makeImportExport(remotePIDs, false);
  }
 
  {
    Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-fLG");
    this->fillLocalGraph(params);
  }
 
  const bool callComputeGlobalConstants = params.get() == nullptr ||
                                          params->get("compute global constants", true);
 
  if (callComputeGlobalConstants) {
    Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-cGC (const)");
    this->computeGlobalConstants();
  } else {
    Tpetra::Details::ProfilingRegion pr("Tpetra ESFC-G-cGC (noconst)");
    this->computeLocalConstants();
  }
 
  fillComplete_ = true;
 
  checkInternalState();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    fillLocalGraph(const Teuchos::RCP<Teuchos::ParameterList>& params) {
  using ::Tpetra::Details::computeOffsetsFromCounts;
  typedef typename local_graph_device_type::row_map_type row_map_type;
  typedef typename row_map_type::non_const_type non_const_row_map_type;
  typedef typename local_graph_device_type::entries_type::non_const_type lclinds_1d_type;
  const char tfecfFuncName[] =
      "fillLocalGraph (called from fillComplete or "
      "expertStaticFillComplete): ";
  const size_t lclNumRows = this->getLocalNumRows();
 
  Details::ProfilingRegion regionFLG("Tpetra::CrsGraph::fillLocalGraph");
 
  // This method's goal is to fill in the two arrays (compressed
  // sparse row format) that define the sparse graph's structure.
 
  bool requestOptimizedStorage = true;
  if (!params.is_null() && !params->get("Optimize Storage", true)) {
    requestOptimizedStorage = false;
  }
 
  // The graph's column indices are currently stored in a 1-D
  // format, with row offsets in rowPtrsUnpacked_host_ and local column indices
  // in k_lclInds1D_.
 
  if (debug_) {
    auto rowPtrsUnpacked = this->getRowPtrsUnpackedHost();
    // The graph's array of row offsets must already be allocated.
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked.extent(0) == 0, std::logic_error,
                                          "rowPtrsUnpacked_host_ has size zero, but shouldn't");
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsUnpacked.extent(0) != lclNumRows + 1, std::logic_error,
                                          "rowPtrsUnpacked_host_.extent(0) = "
                                              << rowPtrsUnpacked.extent(0) << " != (lclNumRows + 1) = "
                                              << (lclNumRows + 1) << ".");
    const size_t numOffsets = rowPtrsUnpacked.extent(0);
    const auto valToCheck   = rowPtrsUnpacked(numOffsets - 1);
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numOffsets != 0 &&
                                              lclIndsUnpacked_wdv.extent(0) != valToCheck,
                                          std::logic_error, "numOffsets=" << numOffsets << " != 0 "
                                                                                           " and lclIndsUnpacked_wdv.extent(0)="
                                                                          << lclIndsUnpacked_wdv.extent(0) << " != rowPtrsUnpacked_host_(" << numOffsets << ")=" << valToCheck << ".");
  }
 
  size_t allocSize = 0;
  try {
    allocSize = this->getLocalAllocationSize();
  } catch (std::logic_error& e) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
                                          "getLocalAllocationSize threw "
                                          "std::logic_error: "
                                              << e.what());
  } catch (std::runtime_error& e) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                          "getLocalAllocationSize threw "
                                          "std::runtime_error: "
                                              << e.what());
  } catch (std::exception& e) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                          "getLocalAllocationSize threw "
                                          "std::exception: "
                                              << e.what());
  } catch (...) {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                          "getLocalAllocationSize threw "
                                          "an exception not a subclass of std::exception.");
  }
 
  if (this->getLocalNumEntries() != allocSize) {
    // Use the nonconst version of row_map_type for ptr_d, because
    // the latter is const and we need to modify ptr_d here.
    non_const_row_map_type ptr_d;
    row_map_type ptr_d_const;
 
    // The graph's current 1-D storage is "unpacked."  This means
    // the row offsets may differ from what the final row offsets
    // should be.  This could happen, for example, if the user set
    // an upper bound on the number of entries in each row, but
    // didn't fill all those entries.
 
    if (debug_) {
      auto rowPtrsUnpacked = this->getRowPtrsUnpackedHost();
      if (rowPtrsUnpacked.extent(0) != 0) {
        const size_t numOffsets =
            static_cast<size_t>(rowPtrsUnpacked.extent(0));
        const auto valToCheck = rowPtrsUnpacked(numOffsets - 1);
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != size_t(lclIndsUnpacked_wdv.extent(0)),
                                              std::logic_error,
                                              "(Unpacked branch) Before allocating "
                                              "or packing, k_rowPtrs_("
                                                  << (numOffsets - 1) << ")="
                                                  << valToCheck << " != lclIndsUnpacked_wdv.extent(0)="
                                                  << lclIndsUnpacked_wdv.extent(0) << ".");
      }
    }
 
    // Pack the row offsets into ptr_d, by doing a sum-scan of the
    // array of valid entry counts per row (k_numRowEntries_).
 
    // Total number of entries in the matrix on the calling
    // process.  We will compute this in the loop below.  It's
    // cheap to compute and useful as a sanity check.
    size_t lclTotalNumEntries = 0;
    {
      // Allocate the packed row offsets array.
      ptr_d =
          non_const_row_map_type("Tpetra::CrsGraph::ptr", lclNumRows + 1);
      ptr_d_const = ptr_d;
 
      // It's ok that k_numRowEntries_ is a host View; the
      // function can handle this.
      typename num_row_entries_type::const_type numRowEnt_h = k_numRowEntries_;
      if (debug_) {
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(size_t(numRowEnt_h.extent(0)) != lclNumRows,
                                              std::logic_error,
                                              "(Unpacked branch) "
                                              "numRowEnt_h.extent(0)="
                                                  << numRowEnt_h.extent(0)
                                                  << " != getLocalNumRows()=" << lclNumRows << "");
      }
 
      lclTotalNumEntries = computeOffsetsFromCounts(ptr_d, numRowEnt_h);
 
      if (debug_) {
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(ptr_d.extent(0)) != lclNumRows + 1,
                                              std::logic_error,
                                              "(Unpacked branch) After allocating "
                                              "ptr_d, ptr_d.extent(0) = "
                                                  << ptr_d.extent(0)
                                                  << " != lclNumRows+1 = " << (lclNumRows + 1) << ".");
        const auto valToCheck =
            ::Tpetra::Details::getEntryOnHost(ptr_d, lclNumRows);
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != lclTotalNumEntries, std::logic_error,
                                              "Tpetra::CrsGraph::fillLocalGraph: In unpacked branch, "
                                              "after filling ptr_d, ptr_d(lclNumRows="
                                                  << lclNumRows
                                                  << ") = " << valToCheck << " != total number of entries "
                                                                             "on the calling process = "
                                                  << lclTotalNumEntries
                                                  << ".");
      }
    }
 
    // Allocate the array of packed column indices.
    lclinds_1d_type ind_d =
        lclinds_1d_type("Tpetra::CrsGraph::lclInd", lclTotalNumEntries);
 
    // k_rowPtrs_ and lclIndsUnpacked_wdv are currently unpacked.  Pack
    // them, using the packed row offsets array ptr_d that we
    // created above.
    //
    // FIXME (mfh 08 Aug 2014) If "Optimize Storage" is false (in
    // CrsMatrix?), we need to keep around the unpacked row
    // offsets and column indices.
 
    // Pack the column indices from unpacked lclIndsUnpacked_wdv into
    // packed ind_d.  We will replace lclIndsUnpacked_wdv below.
    typedef pack_functor<
        typename local_graph_device_type::entries_type::non_const_type,
        typename local_inds_dualv_type::t_dev::const_type,
        row_map_type,
        typename local_graph_device_type::row_map_type>
        inds_packer_type;
    inds_packer_type f(ind_d,
                       lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly),
                       ptr_d, this->getRowPtrsUnpackedDevice());
    {
      typedef typename decltype(ind_d)::execution_space exec_space;
      typedef Kokkos::RangePolicy<exec_space, LocalOrdinal> range_type;
      Kokkos::parallel_for(range_type(0, lclNumRows), f);
    }
 
    if (debug_) {
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(ptr_d.extent(0) == 0, std::logic_error,
                                            "(\"Optimize Storage\"=true branch) After packing, "
                                            "ptr_d.extent(0)=0.");
      if (ptr_d.extent(0) != 0) {
        const size_t numOffsets = static_cast<size_t>(ptr_d.extent(0));
        const auto valToCheck =
            ::Tpetra::Details::getEntryOnHost(ptr_d, numOffsets - 1);
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(valToCheck) != ind_d.extent(0),
                                              std::logic_error,
                                              "(\"Optimize Storage\"=true branch) "
                                              "After packing, ptr_d("
                                                  << (numOffsets - 1) << ")="
                                                  << valToCheck << " != ind_d.extent(0)="
                                                  << ind_d.extent(0) << ".");
      }
    }
    // Build the local graph.
    if (requestOptimizedStorage)
      setRowPtrs(ptr_d_const);
    else
      setRowPtrsPacked(ptr_d_const);
    lclIndsPacked_wdv = local_inds_wdv_type(ind_d);
  } else {  // We don't have to pack, so just set the pointers.
    // Set both packed and unpacked rowptrs to this
    this->setRowPtrs(rowPtrsUnpacked_dev_);
    lclIndsPacked_wdv = lclIndsUnpacked_wdv;
 
    if (debug_) {
      auto rowPtrsPacked_dev  = this->getRowPtrsPackedDevice();
      auto rowPtrsPacked_host = this->getRowPtrsPackedHost();
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(rowPtrsPacked_dev.extent(0) == 0, std::logic_error,
                                            "(\"Optimize Storage\"=false branch) "
                                            "rowPtrsPacked_dev_.extent(0) = 0.");
      if (rowPtrsPacked_dev.extent(0) != 0) {
        const size_t numOffsets =
            static_cast<size_t>(rowPtrsPacked_dev.extent(0));
        const size_t valToCheck =
            rowPtrsPacked_host(numOffsets - 1);
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(valToCheck != size_t(lclIndsPacked_wdv.extent(0)),
                                              std::logic_error,
                                              "(\"Optimize Storage\"=false branch) "
                                              "rowPtrsPacked_dev_("
                                                  << (numOffsets - 1) << ")="
                                                  << valToCheck
                                                  << " != lclIndsPacked_wdv.extent(0)="
                                                  << lclIndsPacked_wdv.extent(0) << ".");
      }
    }
  }
 
  if (debug_) {
    auto rowPtrsPacked_dev  = this->getRowPtrsPackedDevice();
    auto rowPtrsPacked_host = this->getRowPtrsPackedHost();
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(rowPtrsPacked_dev.extent(0)) != lclNumRows + 1,
                                          std::logic_error, "After packing, rowPtrsPacked_dev_.extent(0) = " << rowPtrsPacked_dev.extent(0) << " != lclNumRows+1 = " << (lclNumRows + 1) << ".");
    if (rowPtrsPacked_dev.extent(0) != 0) {
      const size_t numOffsets = static_cast<size_t>(rowPtrsPacked_dev.extent(0));
      const auto valToCheck   = rowPtrsPacked_host(numOffsets - 1);
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<size_t>(valToCheck) != lclIndsPacked_wdv.extent(0),
                                            std::logic_error, "After packing, rowPtrsPacked_dev_(" << (numOffsets - 1) << ") = " << valToCheck << " != lclIndsPacked_wdv.extent(0) = " << lclIndsPacked_wdv.extent(0) << ".");
    }
  }
 
  if (requestOptimizedStorage) {
    // With optimized storage, we don't need to store
    // the array of row entry counts.
 
    // Free graph data structures that are only needed for
    // unpacked 1-D storage.
    k_numRowEntries_ = num_row_entries_type();
 
    // Keep the new 1-D packed allocations.
    lclIndsUnpacked_wdv = lclIndsPacked_wdv;
 
    storageStatus_ = Details::STORAGE_1D_PACKED;
  }
 
  set_need_sync_host_uvm_access();  // make sure kernel setup of indices is fenced before a host access
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    replaceColMap(const Teuchos::RCP<const map_type>& newColMap) {
  // NOTE: This safety check matches the code, but not the documentation of Crsgraph
  //
  // FIXME (mfh 18 Aug 2014) This will break if the calling process
  // has no entries, because in that case, currently it is neither
  // locally nor globally indexed.  This will change once we get rid
  // of lazy allocation (so that the constructor allocates indices
  // and therefore commits to local vs. global).
  const char tfecfFuncName[] = "replaceColMap: ";
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      isLocallyIndexed() || isGloballyIndexed(), std::runtime_error,
      "Requires matching maps and non-static graph.");
  colMap_ = newColMap;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    reindexColumns(const Teuchos::RCP<const map_type>& newColMap,
                   const Teuchos::RCP<const import_type>& newImport,
                   const bool sortIndicesInEachRow) {
  using Teuchos::RCP;
  using Teuchos::REDUCE_MIN;
  using Teuchos::reduceAll;
  typedef GlobalOrdinal GO;
  typedef LocalOrdinal LO;
  using col_inds_type_dev    = typename local_inds_dualv_type::t_dev;
  const char tfecfFuncName[] = "reindexColumns: ";
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      isFillComplete(), std::runtime_error,
      "The graph is fill complete "
      "(isFillComplete() returns true).  You must call resumeFill() before "
      "you may call this method.");
 
  // mfh 19 Aug 2014: This method does NOT redistribute data; it
  // doesn't claim to do the work of an Import or Export.  This
  // means that for all processes, the calling process MUST own all
  // column indices, in both the old column Map (if it exists) and
  // the new column Map.  We check this via an all-reduce.
  //
  // Some processes may be globally indexed, others may be locally
  // indexed, and others (that have no graph entries) may be
  // neither.  This method will NOT change the graph's current
  // state.  If it's locally indexed, it will stay that way, and
  // vice versa.  It would easy to add an option to convert indices
  // from global to local, so as to save a global-to-local
  // conversion pass.  However, we don't do this here.  The intended
  // typical use case is that the graph already has a column Map and
  // is locally indexed, and this is the case for which we optimize.
 
  const LO lclNumRows = static_cast<LO>(this->getLocalNumRows());
 
  // Attempt to convert indices to the new column Map's version of
  // local.  This will fail if on the calling process, the graph has
  // indices that are not on that process in the new column Map.
  // After the local conversion attempt, we will do an all-reduce to
  // see if any processes failed.
 
  // If this is false, then either the graph contains a column index
  // which is invalid in the CURRENT column Map, or the graph is
  // locally indexed but currently has no column Map.  In either
  // case, there is no way to convert the current local indices into
  // global indices, so that we can convert them into the new column
  // Map's local indices.  It's possible for this to be true on some
  // processes but not others, due to replaceColMap.
  bool allCurColIndsValid = true;
  // On the calling process, are all valid current column indices
  // also in the new column Map on the calling process?  In other
  // words, does local reindexing suffice, or should the user have
  // done an Import or Export instead?
  bool localSuffices = true;
 
  {
    // Final arrays for the local indices.  We will allocate exactly
    // one of these ONLY if the graph is locally indexed on the
    // calling process, and ONLY if the graph has one or more entries
    // (is not empty) on the calling process.  In that case, we
    // allocate the first (1-D storage) if the graph has a static
    // profile, else we allocate the second (2-D storage).
    col_inds_type_dev newLclInds1D_dev;
 
    // If indices aren't allocated, that means the calling process
    // owns no entries in the graph.  Thus, there is nothing to
    // convert, and it trivially succeeds locally.
    if (indicesAreAllocated()) {
      if (isLocallyIndexed()) {
        if (hasColMap()) {  // locally indexed, and currently has a column Map
          const map_type& oldColMap = *(getColMap());
 
          // Allocate storage for the new local indices.
          const size_t allocSize = this->getLocalAllocationSize();
          auto oldLclInds1D      = lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly);
          newLclInds1D_dev       = col_inds_type_dev("Tpetra::CrsGraph::lclIndsReindexed",
                                                     allocSize);
          auto oldLclColMap      = oldColMap.getLocalMap();
          auto newLclColMap      = newColMap->getLocalMap();
 
          const auto LO_INVALID = Teuchos::OrdinalTraits<LO>::invalid();
          const auto GO_INVALID = Teuchos::OrdinalTraits<GO>::invalid();
 
          const int NOT_ALL_LOCAL_INDICES_ARE_VALID = 1;
          const int LOCAL_DOES_NOT_SUFFICE          = 2;
          int errorStatus                           = 0;
          Kokkos::parallel_reduce(
              "Tpetra::CrsGraph::reindexColumns",
              Kokkos::RangePolicy<LocalOrdinal, execution_space>(0, allocSize),
              KOKKOS_LAMBDA(const LocalOrdinal k, int& result) {
                const LocalOrdinal oldLclCol = oldLclInds1D(k);
                if (oldLclCol == LO_INVALID) {
                  result &= NOT_ALL_LOCAL_INDICES_ARE_VALID;
                } else {
                  const GO gblCol = oldLclColMap.getGlobalElement(oldLclCol);
                  if (gblCol == GO_INVALID) {
                    result &= LOCAL_DOES_NOT_SUFFICE;
                  } else {
                    const LocalOrdinal newLclCol = newLclColMap.getLocalElement(gblCol);
                    if (newLclCol == LO_INVALID) {
                      result &= NOT_ALL_LOCAL_INDICES_ARE_VALID;
                    } else {
                      newLclInds1D_dev(k) = newLclCol;
                    }
                  }
                }
              },
              Kokkos::LOr<int>(errorStatus));
          allCurColIndsValid = !(errorStatus & NOT_ALL_LOCAL_INDICES_ARE_VALID);
          localSuffices      = !(errorStatus & LOCAL_DOES_NOT_SUFFICE);
        } else {  // locally indexed, but no column Map
          // This case is only possible if replaceColMap() was called
          // with a null argument on the calling process.  It's
          // possible, but it means that this method can't possibly
          // succeed, since we have no way of knowing how to convert
          // the current local indices to global indices.
          allCurColIndsValid = false;
        }
      } else {  // globally indexed
        // If the graph is globally indexed, we don't need to save
        // local indices, but we _do_ need to know whether the current
        // global indices are valid in the new column Map.  We may
        // need to do a getRemoteIndexList call to find this out.
        //
        // In this case, it doesn't matter whether the graph currently
        // has a column Map.  We don't need the old column Map to
        // convert from global indices to the _new_ column Map's local
        // indices.  Furthermore, we can use the same code, whether
        // the graph is static or dynamic profile.
 
        // Test whether the current global indices are in the new
        // column Map on the calling process.
        for (LO lclRow = 0; lclRow < lclNumRows; ++lclRow) {
          const RowInfo rowInfo = this->getRowInfo(lclRow);
          auto oldGblRowView    = this->getGlobalIndsViewHost(rowInfo);
          for (size_t k = 0; k < rowInfo.numEntries; ++k) {
            const GO gblCol = oldGblRowView(k);
            if (!newColMap->isNodeGlobalElement(gblCol)) {
              localSuffices = false;
              break;  // Stop at the first invalid index
            }
          }  // for each entry in the current row
        }    // for each locally owned row
      }      // locally or globally indexed
    }        // whether indices are allocated
 
    // Do an all-reduce to check both possible error conditions.
    int lclSuccess[2];
    lclSuccess[0] = allCurColIndsValid ? 1 : 0;
    lclSuccess[1] = localSuffices ? 1 : 0;
    int gblSuccess[2];
    gblSuccess[0] = 0;
    gblSuccess[1] = 0;
    RCP<const Teuchos::Comm<int>> comm =
        getRowMap().is_null() ? Teuchos::null : getRowMap()->getComm();
    if (!comm.is_null()) {
      reduceAll<int, int>(*comm, REDUCE_MIN, 2, lclSuccess, gblSuccess);
    }
 
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
        gblSuccess[0] == 0, std::runtime_error,
        "It is not possible to continue."
        "  The most likely reason is that the graph is locally indexed, but the "
        "column Map is missing (null) on some processes, due to a previous call "
        "to replaceColMap().");
 
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
        gblSuccess[1] == 0, std::runtime_error,
        "On some process, the graph "
        "contains column indices that are in the old column Map, but not in the "
        "new column Map (on that process).  This method does NOT redistribute "
        "data; it does not claim to do the work of an Import or Export operation."
        "  This means that for all processess, the calling process MUST own all "
        "column indices, in both the old column Map and the new column Map.  In "
        "this case, you will need to do an Import or Export operation to "
        "redistribute data.");
 
    // Commit the results.
    if (isLocallyIndexed()) {
      lclIndsUnpacked_wdv = local_inds_wdv_type(newLclInds1D_dev);
    }
    // end of scope for newLclInds1D_dev
    // sortAndMergeAllIndices needs host access
  }
 
  if (isLocallyIndexed()) {
    // We've reindexed, so we don't know if the indices are sorted.
    //
    // FIXME (mfh 17 Sep 2014) It could make sense to check this,
    // since we're already going through all the indices above.  We
    // could also sort each row in place; that way, we would only
    // have to make one pass over the rows.
    indicesAreSorted_ = false;
    if (sortIndicesInEachRow) {
      // NOTE (mfh 17 Sep 2014) The graph must be locally indexed in
      // order to call this method.
      //
      // FIXME (mfh 17 Sep 2014) This violates the strong exception
      // guarantee.  It would be better to sort the new index arrays
      // before committing them.
      const bool sorted = false;  // need to resort
      const bool merged = true;   // no need to merge, since no dups
      this->sortAndMergeAllIndices(sorted, merged);
    }
  }
  colMap_ = newColMap;
 
  if (newImport.is_null()) {
    // FIXME (mfh 19 Aug 2014) Should use the above all-reduce to
    // check whether the input Import is null on any process.
    //
    // If the domain Map hasn't been set yet, we can't compute a new
    // Import object.  Leave it what it is; it should be null, but
    // it doesn't matter.  If the domain Map _has_ been set, then
    // compute a new Import object if necessary.
    if (!domainMap_.is_null()) {
      if (!domainMap_->isSameAs(*newColMap)) {
        importer_ = Teuchos::rcp(new import_type(domainMap_, newColMap));
      } else {
        importer_ = Teuchos::null;  // don't need an Import
      }
    }
  } else {
    // The caller gave us an Import object.  Assume that it's valid.
    importer_ = newImport;
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    replaceDomainMap(const Teuchos::RCP<const map_type>& newDomainMap) {
  const char prefix[] = "Tpetra::CrsGraph::replaceDomainMap: ";
  TEUCHOS_TEST_FOR_EXCEPTION(
      colMap_.is_null(), std::invalid_argument, prefix << "You may not call "
                                                          "this method unless the graph already has a column Map.");
  TEUCHOS_TEST_FOR_EXCEPTION(
      newDomainMap.is_null(), std::invalid_argument,
      prefix << "The new domain Map must be nonnull.");
 
  // Create a new importer, if needed
  Teuchos::RCP<const import_type> newImporter = Teuchos::null;
  if (newDomainMap != colMap_ && (!newDomainMap->isSameAs(*colMap_))) {
    newImporter = rcp(new import_type(newDomainMap, colMap_));
  }
  this->replaceDomainMapAndImporter(newDomainMap, newImporter);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    replaceDomainMapAndImporter(const Teuchos::RCP<const map_type>& newDomainMap,
                                const Teuchos::RCP<const import_type>& newImporter) {
  const char prefix[] = "Tpetra::CrsGraph::replaceDomainMapAndImporter: ";
  TEUCHOS_TEST_FOR_EXCEPTION(
      colMap_.is_null(), std::invalid_argument, prefix << "You may not call "
                                                          "this method unless the graph already has a column Map.");
  TEUCHOS_TEST_FOR_EXCEPTION(
      newDomainMap.is_null(), std::invalid_argument,
      prefix << "The new domain Map must be nonnull.");
 
  if (debug_) {
    if (newImporter.is_null()) {
      // It's not a good idea to put expensive operations in a macro
      // clause, even if they are side effect - free, because macros
      // don't promise that they won't evaluate their arguments more
      // than once.  It's polite for them to do so, but not required.
      const bool colSameAsDom = colMap_->isSameAs(*newDomainMap);
      TEUCHOS_TEST_FOR_EXCEPTION(!colSameAsDom, std::invalid_argument,
                                 "If the new Import is null, "
                                 "then the new domain Map must be the same as the current column Map.");
    } else {
      const bool colSameAsTgt =
          colMap_->isSameAs(*(newImporter->getTargetMap()));
      const bool newDomSameAsSrc =
          newDomainMap->isSameAs(*(newImporter->getSourceMap()));
      TEUCHOS_TEST_FOR_EXCEPTION(!colSameAsTgt || !newDomSameAsSrc, std::invalid_argument,
                                 "If the "
                                 "new Import is nonnull, then the current column Map must be the same "
                                 "as the new Import's target Map, and the new domain Map must be the "
                                 "same as the new Import's source Map.");
    }
  }
 
  domainMap_ = newDomainMap;
  importer_  = Teuchos::rcp_const_cast<import_type>(newImporter);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    replaceRangeMap(const Teuchos::RCP<const map_type>& newRangeMap) {
  const char prefix[] = "Tpetra::CrsGraph::replaceRangeMap: ";
  TEUCHOS_TEST_FOR_EXCEPTION(
      rowMap_.is_null(), std::invalid_argument, prefix << "You may not call "
                                                          "this method unless the graph already has a row Map.");
  TEUCHOS_TEST_FOR_EXCEPTION(
      newRangeMap.is_null(), std::invalid_argument,
      prefix << "The new range Map must be nonnull.");
 
  // Create a new exporter, if needed
  Teuchos::RCP<const export_type> newExporter = Teuchos::null;
  if (newRangeMap != rowMap_ && (!newRangeMap->isSameAs(*rowMap_))) {
    newExporter = rcp(new export_type(rowMap_, newRangeMap));
  }
  this->replaceRangeMapAndExporter(newRangeMap, newExporter);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    replaceRangeMapAndExporter(const Teuchos::RCP<const map_type>& newRangeMap,
                               const Teuchos::RCP<const export_type>& newExporter) {
  const char prefix[] = "Tpetra::CrsGraph::replaceRangeMapAndExporter: ";
  TEUCHOS_TEST_FOR_EXCEPTION(
      rowMap_.is_null(), std::invalid_argument, prefix << "You may not call "
                                                          "this method unless the graph already has a column Map.");
  TEUCHOS_TEST_FOR_EXCEPTION(
      newRangeMap.is_null(), std::invalid_argument,
      prefix << "The new domain Map must be nonnull.");
 
  if (debug_) {
    if (newExporter.is_null()) {
      // It's not a good idea to put expensive operations in a macro
      // clause, even if they are side effect - free, because macros
      // don't promise that they won't evaluate their arguments more
      // than once.  It's polite for them to do so, but not required.
      const bool rowSameAsRange = rowMap_->isSameAs(*newRangeMap);
      TEUCHOS_TEST_FOR_EXCEPTION(!rowSameAsRange, std::invalid_argument,
                                 "If the new Export is null, "
                                 "then the new range Map must be the same as the current row Map.");
    } else {
      const bool newRangeSameAsTgt =
          newRangeMap->isSameAs(*(newExporter->getTargetMap()));
      const bool rowSameAsSrc =
          rowMap_->isSameAs(*(newExporter->getSourceMap()));
      TEUCHOS_TEST_FOR_EXCEPTION(!rowSameAsSrc || !newRangeSameAsTgt, std::invalid_argument,
                                 "If the "
                                 "new Export is nonnull, then the current row Map must be the same "
                                 "as the new Export's source Map, and the new range Map must be the "
                                 "same as the new Export's target Map.");
    }
  }
 
  rangeMap_ = newRangeMap;
  exporter_ = Teuchos::rcp_const_cast<export_type>(newExporter);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_graph_device_type
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalGraphDevice() const {
  return local_graph_device_type(
      lclIndsPacked_wdv.getDeviceView(Access::ReadWrite),
      this->getRowPtrsPackedDevice());
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_graph_host_type
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalGraphHost() const {
  return local_graph_host_type(
      lclIndsPacked_wdv.getHostView(Access::ReadWrite),
      this->getRowPtrsPackedHost());
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    computeGlobalConstants() {
  using Teuchos::ArrayView;
  using Teuchos::outArg;
  using Teuchos::reduceAll;
  using ::Tpetra::Details::ProfilingRegion;
  typedef global_size_t GST;
 
  ProfilingRegion regionCGC("Tpetra::CrsGraph::computeGlobalConstants");
 
  GST lcl, gbl;
  std::shared_ptr<Details::CommRequest> req;
  if (!this->haveGlobalConstants_) {
    lcl = static_cast<GST>(this->getLocalNumEntries());
    req = Details::iallreduce(lcl, gbl, Teuchos::REDUCE_SUM, *this->getComm());
  }
 
  this->computeLocalConstants();
 
  // Compute global constants from local constants.  Processes that
  // already have local constants still participate in the
  // all-reduces, using their previously computed values.
  if (!this->haveGlobalConstants_) {
    // Promote all the nodeNum* and nodeMaxNum* quantities from
    // size_t to global_size_t, when doing the all-reduces for
    // globalNum* / globalMaxNum* results.
    //
    // FIXME (mfh 07 May 2013) Unfortunately, we either have to do
    // this in two all-reduces (one for the sum and the other for
    // the max), or use a custom MPI_Op that combines the sum and
    // the max.  The latter might even be slower than two
    // all-reduces on modern network hardware.  It would also be a
    // good idea to use nonblocking all-reduces (MPI 3), so that we
    // don't have to wait around for the first one to finish before
    // starting the second one.
    const GST lclMaxNumRowEnt = static_cast<GST>(this->nodeMaxNumRowEntries_);
    auto req2                 = Details::iallreduce(lclMaxNumRowEnt, this->globalMaxNumRowEntries_, Teuchos::REDUCE_MAX, *this->getComm());
 
    req->wait();
    this->globalNumEntries_ = gbl;
 
    req2->wait();
    this->haveGlobalConstants_ = true;
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    computeLocalConstants() {
  using ::Tpetra::Details::ProfilingRegion;
 
  ProfilingRegion regionCLC("Tpetra::CrsGraph::computeLocalConstants");
  if (this->haveLocalConstants_) {
    return;
  }
 
  // Reset local properties
  this->nodeMaxNumRowEntries_ =
      Teuchos::OrdinalTraits<size_t>::invalid();
 
  using LO = local_ordinal_type;
 
  auto ptr            = this->getRowPtrsPackedDevice();
  const LO lclNumRows = ptr.extent(0) == 0 ? static_cast<LO>(0) : (static_cast<LO>(ptr.extent(0)) - static_cast<LO>(1));
 
  const LO lclMaxNumRowEnt =
      ::Tpetra::Details::maxDifference("Tpetra::CrsGraph: nodeMaxNumRowEntries",
                                       ptr, lclNumRows);
  this->nodeMaxNumRowEntries_ = static_cast<size_t>(lclMaxNumRowEnt);
  this->haveLocalConstants_   = true;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
std::pair<size_t, std::string>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    makeIndicesLocal(const bool verbose) {
  using Details::ProfilingRegion;
  using std::endl;
  using Teuchos::arcp;
  using Teuchos::Array;
  typedef LocalOrdinal LO;
  typedef GlobalOrdinal GO;
  typedef device_type DT;
  typedef typename local_graph_device_type::row_map_type::non_const_value_type offset_type;
  typedef typename num_row_entries_type::non_const_value_type num_ent_type;
  const char tfecfFuncName[] = "makeIndicesLocal: ";
  ProfilingRegion regionMakeIndicesLocal("Tpetra::CrsGraph::makeIndicesLocal");
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "makeIndicesLocal");
    std::ostringstream os;
    os << *prefix << "lclNumRows: " << getLocalNumRows() << endl;
    std::cerr << os.str();
  }
 
  // These are somewhat global properties, so it's safe to have
  // exception checks for them, rather than returning an error code.
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::logic_error,
                                        "The graph does not have a "
                                        "column Map yet.  This method should never be called in that case.  "
                                        "Please report this bug to the Tpetra developers.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->getColMap().is_null(), std::logic_error,
                                        "The graph claims "
                                        "that it has a column Map, because hasColMap() returns true.  However, "
                                        "the result of getColMap() is null.  This should never happen.  Please "
                                        "report this bug to the Tpetra developers.");
 
  // Return value 1: The number of column indices (counting
  // duplicates) that could not be converted to local indices,
  // because they were not in the column Map on the calling process.
  size_t lclNumErrs = 0;
  std::ostringstream errStrm;  // for return value 2 (error string)
 
  const LO lclNumRows    = static_cast<LO>(this->getLocalNumRows());
  const map_type& colMap = *(this->getColMap());
 
  if (this->isGloballyIndexed() && lclNumRows != 0) {
    // This is a host-accessible View.
    typename num_row_entries_type::const_type h_numRowEnt =
        this->k_numRowEntries_;
 
    auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
 
    // Allocate space for local indices.
    if (rowPtrsUnpacked_host.extent(0) == 0) {
      errStrm << "Unpacked row pointers (rowPtrsUnpacked_dev_) has length 0.  This should never "
                 "happen here.  Please report this bug to the Tpetra developers."
              << endl;
      // Need to return early.
      return std::make_pair(Tpetra::Details::OrdinalTraits<size_t>::invalid(),
                            errStrm.str());
    }
    const auto numEnt = rowPtrsUnpacked_host(lclNumRows);
 
    // mfh 17 Dec 2016: We don't need initial zero-fill of
    // lclIndsUnpacked_wdv, because we will fill it below anyway.
    // AllowPadding would only help for aligned access (e.g.,
    // for vectorization) if we also were to pad each row to the
    // same alignment, so we'll skip AllowPadding for now.
 
    // using Kokkos::AllowPadding;
    using Kokkos::view_alloc;
    using Kokkos::WithoutInitializing;
 
    // When giving the label as an argument to
    // Kokkos::view_alloc, the label must be a string and not a
    // char*, else the code won't compile.  This is because
    // view_alloc also allows a raw pointer as its first
    // argument.  See
    // https://github.com/kokkos/kokkos/issues/434.  This is a
    // large allocation typically, so the overhead of creating
    // an std::string is minor.
    const std::string label("Tpetra::CrsGraph::lclInd");
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "(Re)allocate lclInd_wdv: old="
         << lclIndsUnpacked_wdv.extent(0) << ", new=" << numEnt << endl;
      std::cerr << os.str();
    }
 
    local_inds_dualv_type lclInds_dualv =
        local_inds_dualv_type(view_alloc(label, WithoutInitializing),
                              numEnt);
    lclIndsUnpacked_wdv = local_inds_wdv_type(lclInds_dualv);
 
    auto lclColMap = colMap.getLocalMap();
    // This is a "device mirror" of the host View h_numRowEnt.
    //
    // NOTE (mfh 27 Sep 2016) Currently, the right way to get a
    // Device instance is to use its default constructor.  See the
    // following Kokkos issue:
    //
    // https://github.com/kokkos/kokkos/issues/442
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Allocate device mirror k_numRowEnt: "
         << h_numRowEnt.extent(0) << endl;
      std::cerr << os.str();
    }
    auto k_numRowEnt =
        Kokkos::create_mirror_view_and_copy(device_type(), h_numRowEnt);
 
    using ::Tpetra::Details::convertColumnIndicesFromGlobalToLocal;
    lclNumErrs =
        convertColumnIndicesFromGlobalToLocal<LO, GO, DT, offset_type, num_ent_type>(
            lclIndsUnpacked_wdv.getDeviceView(Access::OverwriteAll),
            gblInds_wdv.getDeviceView(Access::ReadOnly),
            this->getRowPtrsUnpackedDevice(),
            lclColMap,
            k_numRowEnt);
    if (lclNumErrs != 0) {
      const int myRank = [this]() {
        auto map = this->getMap();
        if (map.is_null()) {
          return 0;
        } else {
          auto comm = map->getComm();
          return comm.is_null() ? 0 : comm->getRank();
        }
      }();
      const bool pluralNumErrs = (lclNumErrs != static_cast<size_t>(1));
      errStrm << "(Process " << myRank << ") When converting column "
                                          "indices from global to local, we encountered "
              << lclNumErrs
              << " ind" << (pluralNumErrs ? "ices" : "ex")
              << " that do" << (pluralNumErrs ? "" : "es")
              << " not live in the column Map on this process." << endl;
    }
 
    // We've converted column indices from global to local, so we
    // can deallocate the global column indices (which we know are
    // in 1-D storage, because the graph has static profile).
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Free gblInds_wdv: "
         << gblInds_wdv.extent(0) << endl;
      std::cerr << os.str();
    }
    gblInds_wdv = global_inds_wdv_type();
  }  // globallyIndexed() && lclNumRows > 0
 
  this->indicesAreLocal_  = true;
  this->indicesAreGlobal_ = false;
  this->checkInternalState();
 
  return std::make_pair(lclNumErrs, errStrm.str());
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    makeColMap(Teuchos::Array<int>& remotePIDs) {
  using Details::ProfilingRegion;
  using std::endl;
  const char tfecfFuncName[] = "makeColMap";
 
  ProfilingRegion regionSortAndMerge("Tpetra::CrsGraph::makeColMap");
  std::unique_ptr<std::string> prefix;
  if (verbose_) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "Start" << endl;
    std::cerr << os.str();
  }
 
  // this->colMap_ should be null at this point, but we accept the
  // future possibility that it might not be (esp. if we decide
  // later to support graph structure changes after first
  // fillComplete, which CrsGraph does not currently (as of 12 Feb
  // 2017) support).
  Teuchos::RCP<const map_type> colMap = this->colMap_;
  const bool sortEachProcsGids =
      this->sortGhostsAssociatedWithEachProcessor_;
 
  // FIXME (mfh 12 Feb 2017) ::Tpetra::Details::makeColMap returns a
  // per-process error code.  If an error does occur on a process,
  // ::Tpetra::Details::makeColMap does NOT promise that all processes will
  // notice that error.  This is the caller's responsibility.  For
  // now, we only propagate (to all processes) and report the error
  // in debug mode.  In the future, we need to add the local/global
  // error handling scheme used in BlockCrsMatrix to this class.
  if (debug_) {
    using Teuchos::outArg;
    using Teuchos::REDUCE_MIN;
    using Teuchos::reduceAll;
 
    std::ostringstream errStrm;
    const int lclErrCode =
        Details::makeColMap(colMap, remotePIDs,
                            getDomainMap(), *this, sortEachProcsGids, &errStrm);
    auto comm = this->getComm();
    if (!comm.is_null()) {
      const int lclSuccess = (lclErrCode == 0) ? 1 : 0;
      int gblSuccess       = 0;  // output argument
      reduceAll<int, int>(*comm, REDUCE_MIN, lclSuccess,
                          outArg(gblSuccess));
      if (gblSuccess != 1) {
        std::ostringstream os;
        Details::gathervPrint(os, errStrm.str(), *comm);
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error,
                                              ": An error happened on at "
                                              "least one process in the CrsGraph's communicator.  "
                                              "Here are all processes' error messages:"
                                                  << std::endl
                                                  << os.str());
      }
    }
  } else {
    (void)Details::makeColMap(colMap, remotePIDs,
                              getDomainMap(), *this, sortEachProcsGids, nullptr);
  }
  // See above.  We want to admit the possibility of makeColMap
  // actually revising an existing column Map, even though that
  // doesn't currently (as of 10 May 2017) happen.
  this->colMap_ = colMap;
 
  checkInternalState();
  if (verbose_) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class execution_space, class LO, class rowptr_type, class colinds_type, class numRowEntries_type>
void prepareSortMergeUnpackedGraph(rowptr_type rowptr, colinds_type colinds, numRowEntries_type numRowEntries) {
  using ATS         = KokkosKernels::ArithTraits<LO>;
  const auto unused = ATS::max();
 
  auto numRows = rowptr.extent(0) - 1;
 
  // make sure that unused entries will get ordered last
  Kokkos::parallel_for(
      "flag_unused_entries", Kokkos::RangePolicy<execution_space, LO>(0, numRows), KOKKOS_LAMBDA(const LO rlid) {
        for (size_t jj = rowptr(rlid) + numRowEntries(rlid); jj < rowptr(rlid + 1); ++jj) {
          colinds(jj) = unused;
        }
      });
}
 
template <class execution_space, class LO, class rowptr_type, class colinds_type, class numRowEntries_type>
void mergeUnpackedGraph(rowptr_type rowptr, colinds_type colinds, numRowEntries_type numRowEntries) {
  auto numRows = rowptr.extent(0) - 1;
 
  // merge
  // We cannot use KokkosSparse::sort_and_merge_matrix since we
  // do not actually want to change the allocations.
 
  Kokkos::parallel_for(
      "merge_entries", Kokkos::RangePolicy<execution_space>(0, numRows), KOKKOS_LAMBDA(const LO rlid) {
        auto rowNNZ = numRowEntries(rlid);
        if (rowNNZ == 0) {
          return;
        }
        auto rowBegin = rowptr(rlid);
        auto pos      = rowBegin;
        for (size_t offset = rowBegin + 1; offset < rowBegin + rowNNZ; ++offset) {
          if ((colinds(offset) != colinds(pos))) {
            colinds(++pos) = colinds(offset);
          }
        }
        numRowEntries(rlid) = pos + 1 - rowBegin;
      });
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    sortAndMergeAllIndices(const bool sorted, const bool merged) {
  using std::endl;
  const char tfecfFuncName[] = "sortAndMergeAllIndices";
 
  std::unique_ptr<std::string> prefix;
  if (verbose_) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "Start: "
       << "sorted=" << (sorted ? "true" : "false")
       << ", merged=" << (merged ? "true" : "false") << endl;
    std::cerr << os.str();
  }
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isGloballyIndexed(), std::logic_error,
                                        "This method may only be called after makeIndicesLocal.");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!merged && this->isStorageOptimized(), std::logic_error,
                                        "The graph is already storage optimized, so we shouldn't be "
                                        "merging any indices.  "
                                        "Please report this bug to the Tpetra developers.");
 
  if (!sorted || !merged) {
    Details::ProfilingRegion regionSortAndMerge("Tpetra::CrsGraph::sortAndMergeAllIndices");
 
    if (storageStatus_ == Details::STORAGE_1D_UNPACKED) {
      // We are sorting & merging the unpacked views.
      // This means that not all entries are actually in use. We need to take k_numRowEntries_ into account.
      auto rowptr  = rowPtrsUnpacked_dev_;
      auto colinds = lclIndsUnpacked_wdv.getDeviceView(Access::ReadWrite);
 
      // Create a device copy of k_numRowEntries_.
      auto k_numRowEntries_d = Kokkos::create_mirror_view_and_copy(execution_space(), k_numRowEntries_);
 
      // set set unused column entries so they get sorted last
      prepareSortMergeUnpackedGraph<execution_space, LocalOrdinal>(rowptr, colinds, k_numRowEntries_d);
 
      if (!sorted) {
        // For this to work correctly, we require that the unused column entries have been filled
        // with indices that get ordered last.
        Import_Util::sortCrsEntries(rowptr, colinds);
        this->indicesAreSorted_ = true;  // we just sorted every row
      }
      if (!merged) {
        mergeUnpackedGraph<execution_space, LocalOrdinal>(rowptr, colinds, k_numRowEntries_d);
        Kokkos::deep_copy(k_numRowEntries_, k_numRowEntries_d);
        this->noRedundancies_ = true;  // we just merged every row
      }
    } else {
      auto rowptr  = rowPtrsPacked_dev_;
      auto colinds = lclIndsPacked_wdv.getDeviceView(Access::ReadWrite);
      if (!sorted && merged) {
        Import_Util::sortCrsEntries(rowptr, colinds);
        this->indicesAreSorted_ = true;  // we just sorted every row
      } else {
        TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error,
                                              "We should never get here."
                                              "Please report this bug to the Tpetra developers.");
      }
    }
  }
 
  if (verbose_) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    makeImportExport(Teuchos::Array<int>& remotePIDs,
                     const bool useRemotePIDs) {
  using Teuchos::ParameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;
  using ::Tpetra::Details::ProfilingRegion;
  const char tfecfFuncName[] = "makeImportExport: ";
  ProfilingRegion regionMIE("Tpetra::CrsGraph::makeImportExport");
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::logic_error,
                                        "This method may not be called unless the graph has a column Map.");
  RCP<ParameterList> params = this->getNonconstParameterList();  // could be null
 
  // Don't do any checks to see if we need to create the Import, if
  // it exists already.
  //
  // FIXME (mfh 25 Mar 2013) This will become incorrect if we
  // change CrsGraph in the future to allow changing the column
  // Map after fillComplete.  For now, the column Map is fixed
  // after the first fillComplete call.
  if (importer_.is_null()) {
    // Create the Import instance if necessary.
    if (domainMap_ != colMap_ && (!domainMap_->isSameAs(*colMap_))) {
      if (params.is_null() || !params->isSublist("Import")) {
        if (useRemotePIDs) {
          importer_ = rcp(new import_type(domainMap_, colMap_, remotePIDs));
        } else {
          importer_ = rcp(new import_type(domainMap_, colMap_));
        }
      } else {
        RCP<ParameterList> importSublist = sublist(params, "Import", true);
        if (useRemotePIDs) {
          RCP<import_type> newImp =
              rcp(new import_type(domainMap_, colMap_, remotePIDs,
                                  importSublist));
          importer_ = newImp;
        } else {
          importer_ = rcp(new import_type(domainMap_, colMap_, importSublist));
        }
      }
    }
  }
 
  // Don't do any checks to see if we need to create the Export, if
  // it exists already.
  if (exporter_.is_null()) {
    // Create the Export instance if necessary.
    if (rangeMap_ != rowMap_ && !rangeMap_->isSameAs(*rowMap_)) {
      if (params.is_null() || !params->isSublist("Export")) {
        exporter_ = rcp(new export_type(rowMap_, rangeMap_));
      } else {
        RCP<ParameterList> exportSublist = sublist(params, "Export", true);
        exporter_                        = rcp(new export_type(rowMap_, rangeMap_, exportSublist));
      }
    }
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
std::string
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    description() const {
  std::ostringstream oss;
  oss << dist_object_type::description();
  if (isFillComplete()) {
    oss << "{status = fill complete"
        << ", global rows = " << getGlobalNumRows()
        << ", global cols = " << getGlobalNumCols()
        << ", global num entries = " << getGlobalNumEntries()
        << "}";
  } else {
    oss << "{status = fill not complete"
        << ", global rows = " << getGlobalNumRows()
        << "}";
  }
  return oss.str();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    describe(Teuchos::FancyOStream& out,
             const Teuchos::EVerbosityLevel verbLevel) const {
  using std::endl;
  using std::setw;
  using Teuchos::ArrayView;
  using Teuchos::Comm;
  using Teuchos::RCP;
  using Teuchos::VERB_DEFAULT;
  using Teuchos::VERB_EXTREME;
  using Teuchos::VERB_HIGH;
  using Teuchos::VERB_LOW;
  using Teuchos::VERB_MEDIUM;
  using Teuchos::VERB_NONE;
 
  Teuchos::EVerbosityLevel vl = verbLevel;
  if (vl == VERB_DEFAULT) vl = VERB_LOW;
  RCP<const Comm<int>> comm = this->getComm();
  const int myImageID       = comm->getRank(),
            numImages       = comm->getSize();
  size_t width              = 1;
  for (size_t dec = 10; dec < getGlobalNumRows(); dec *= 10) {
    ++width;
  }
  width = std::max<size_t>(width, static_cast<size_t>(11)) + 2;
  Teuchos::OSTab tab(out);
  //    none: print nothing
  //     low: print O(1) info from node 0
  //  medium: print O(P) info, num entries per node
  //    high: print O(N) info, num entries per row
  // extreme: print O(NNZ) info: print graph indices
  //
  // for medium and higher, print constituent objects at specified verbLevel
  if (vl != VERB_NONE) {
    if (myImageID == 0) out << this->description() << std::endl;
    // O(1) globals, minus what was already printed by description()
    if (isFillComplete() && myImageID == 0) {
      out << "Global max number of row entries = " << globalMaxNumRowEntries_ << std::endl;
    }
    // constituent objects
    if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
      if (myImageID == 0) out << "\nRow map: " << std::endl;
      rowMap_->describe(out, vl);
      if (colMap_ != Teuchos::null) {
        if (myImageID == 0) out << "\nColumn map: " << std::endl;
        colMap_->describe(out, vl);
      }
      if (domainMap_ != Teuchos::null) {
        if (myImageID == 0) out << "\nDomain map: " << std::endl;
        domainMap_->describe(out, vl);
      }
      if (rangeMap_ != Teuchos::null) {
        if (myImageID == 0) out << "\nRange map: " << std::endl;
        rangeMap_->describe(out, vl);
      }
    }
    // O(P) data
    if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
      for (int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
        if (myImageID == imageCtr) {
          out << "Node ID = " << imageCtr << std::endl
              << "Node number of entries = " << this->getLocalNumEntries() << std::endl
              << "Node max number of entries = " << nodeMaxNumRowEntries_ << std::endl;
          if (!indicesAreAllocated()) {
            out << "Indices are not allocated." << std::endl;
          }
        }
        comm->barrier();
        comm->barrier();
        comm->barrier();
      }
    }
    // O(N) and O(NNZ) data
    if (vl == VERB_HIGH || vl == VERB_EXTREME) {
      for (int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
        if (myImageID == imageCtr) {
          out << std::setw(width) << "Node ID"
              << std::setw(width) << "Global Row"
              << std::setw(width) << "Num Entries";
          if (vl == VERB_EXTREME) {
            out << " Entries";
          }
          out << std::endl;
          const LocalOrdinal lclNumRows =
              static_cast<LocalOrdinal>(this->getLocalNumRows());
          for (LocalOrdinal r = 0; r < lclNumRows; ++r) {
            const RowInfo rowinfo = this->getRowInfo(r);
            GlobalOrdinal gid     = rowMap_->getGlobalElement(r);
            out << std::setw(width) << myImageID
                << std::setw(width) << gid
                << std::setw(width) << rowinfo.numEntries;
            if (vl == VERB_EXTREME) {
              out << " ";
              if (isGloballyIndexed()) {
                auto rowview = gblInds_wdv.getHostView(Access::ReadOnly);
                for (size_t j = 0; j < rowinfo.numEntries; ++j) {
                  GlobalOrdinal colgid = rowview[j + rowinfo.offset1D];
                  out << colgid << " ";
                }
              } else if (isLocallyIndexed()) {
                auto rowview = lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
                for (size_t j = 0; j < rowinfo.numEntries; ++j) {
                  LocalOrdinal collid = rowview[j + rowinfo.offset1D];
                  out << colMap_->getGlobalElement(collid) << " ";
                }
              }
            }
            out << std::endl;
          }
        }
        comm->barrier();
        comm->barrier();
        comm->barrier();
      }
    }
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    checkSizes(const SrcDistObject& /* source */) {
  // It's not clear what kind of compatibility checks on sizes can
  // be performed here.  Epetra_CrsGraph doesn't check any sizes for
  // compatibility.
  return true;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    copyAndPermute(const SrcDistObject& source,
                   const size_t numSameIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                                          buffer_device_type>& permuteToLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                                          buffer_device_type>& permuteFromLIDs,
                   const CombineMode /*CM*/) {
  using std::endl;
  using LO                   = local_ordinal_type;
  using GO                   = global_ordinal_type;
  using this_CRS_type        = CrsGraph<LO, GO, node_type>;
  const char tfecfFuncName[] = "copyAndPermute: ";
  const bool verbose         = verbose_;
 
  if (Details::Behavior::useNewCopyAndPermute()) {
    const row_graph_type& srcRowGraph = dynamic_cast<const row_graph_type&>(source);
    copyAndPermuteNew(srcRowGraph, *this, numSameIDs, permuteToLIDs, permuteFromLIDs, INSERT);
    return;
  }
 
  Details::ProfilingRegion regionCAP("Tpetra::CrsGraph::copyAndPermute");
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "copyAndPermute");
    std::ostringstream os;
    os << *prefix << endl;
    std::cerr << os.str();
  }
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(permuteToLIDs.extent(0) != permuteFromLIDs.extent(0),
                                        std::runtime_error, "permuteToLIDs.extent(0) = " << permuteToLIDs.extent(0) << " != permuteFromLIDs.extent(0) = " << permuteFromLIDs.extent(0) << ".");
 
  // We know from checkSizes that the source object is a
  // row_graph_type, so we don't need to check again.
  const row_graph_type& srcRowGraph =
      dynamic_cast<const row_graph_type&>(source);
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Compute padding" << endl;
    std::cerr << os.str();
  }
  auto padding = computeCrsPadding(srcRowGraph, numSameIDs,
                                   permuteToLIDs, permuteFromLIDs, verbose);
  applyCrsPadding(*padding, verbose);
 
  // If the source object is actually a CrsGraph, we can use view
  // mode instead of copy mode to access the entries in each row,
  // if the graph is not fill complete.
  const this_CRS_type* srcCrsGraph =
      dynamic_cast<const this_CRS_type*>(&source);
 
  const map_type& srcRowMap = *(srcRowGraph.getRowMap());
  const map_type& tgtRowMap = *(getRowMap());
  const bool src_filled     = srcRowGraph.isFillComplete();
  nonconst_global_inds_host_view_type row_copy;
  LO myid = 0;
 
  //
  // "Copy" part of "copy and permute."
  //
  if (src_filled || srcCrsGraph == nullptr) {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "src_filled || srcCrsGraph == nullptr" << endl;
      std::cerr << os.str();
    }
    // If the source graph is fill complete, we can't use view mode,
    // because the data might be stored in a different format not
    // compatible with the expectations of view mode.  Also, if the
    // source graph is not a CrsGraph, we can't use view mode,
    // because RowGraph only provides copy mode access to the data.
    for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
      const GO gid      = srcRowMap.getGlobalElement(myid);
      size_t row_length = srcRowGraph.getNumEntriesInGlobalRow(gid);
      Kokkos::resize(row_copy, row_length);
      size_t check_row_length = 0;
      srcRowGraph.getGlobalRowCopy(gid, row_copy, check_row_length);
      this->insertGlobalIndices(gid, row_length, row_copy.data());
    }
  } else {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "! src_filled && srcCrsGraph != nullptr" << endl;
      std::cerr << os.str();
    }
    for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
      const GO gid = srcRowMap.getGlobalElement(myid);
      global_inds_host_view_type row;
      srcCrsGraph->getGlobalRowView(gid, row);
      this->insertGlobalIndices(gid, row.extent(0), row.data());
    }
  }
 
  //
  // "Permute" part of "copy and permute."
  //
  auto permuteToLIDs_h   = permuteToLIDs.view_host();
  auto permuteFromLIDs_h = permuteFromLIDs.view_host();
 
  if (src_filled || srcCrsGraph == nullptr) {
    for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
      const GO mygid    = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
      const GO srcgid   = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
      size_t row_length = srcRowGraph.getNumEntriesInGlobalRow(srcgid);
      Kokkos::resize(row_copy, row_length);
      size_t check_row_length = 0;
      srcRowGraph.getGlobalRowCopy(srcgid, row_copy, check_row_length);
      this->insertGlobalIndices(mygid, row_length, row_copy.data());
    }
  } else {
    for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
      const GO mygid  = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
      const GO srcgid = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
      global_inds_host_view_type row;
      srcCrsGraph->getGlobalRowView(srcgid, row);
      this->insertGlobalIndices(mygid, row.extent(0), row.data());
    }
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    applyCrsPadding(const padding_type& padding,
                    const bool verbose) {
  using Details::padCrsArrays;
  using Details::ProfilingRegion;
  using std::endl;
  using LO = local_ordinal_type;
  using row_ptrs_type =
      typename local_graph_device_type::row_map_type::non_const_type;
  using range_policy =
      Kokkos::RangePolicy<execution_space, Kokkos::IndexType<LO>>;
  const char tfecfFuncName[] = "applyCrsPadding";
  ProfilingRegion regionCAP("Tpetra::CrsGraph::applyCrsPadding");
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "padding: ";
    padding.print(os);
    os << endl;
    std::cerr << os.str();
  }
  const int myRank = !verbose ? -1 : [&]() {
    auto map = this->getMap();
    if (map.is_null()) {
      return -1;
    }
    auto comm = map->getComm();
    if (comm.is_null()) {
      return -1;
    }
    return comm->getRank();
  }();
 
  // FIXME (mfh 10 Feb 2020) We shouldn't actually reallocate
  // row_ptrs_beg or allocate row_ptrs_end unless the allocation
  // size needs to increase.  That should be the job of
  // padCrsArrays.
 
  // Assume global indexing we don't have any indices yet
  if (!indicesAreAllocated()) {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Call allocateIndices" << endl;
      std::cerr << os.str();
    }
    allocateIndices(GlobalIndices, verbose);
  }
  TEUCHOS_ASSERT(indicesAreAllocated());
 
  // Making copies here because k_rowPtrs_ has a const type. Otherwise, we
  // would use it directly.
 
  auto rowPtrsUnpacked_dev = this->getRowPtrsUnpackedDevice();
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Allocate row_ptrs_beg: "
       << rowPtrsUnpacked_dev.extent(0) << endl;
    std::cerr << os.str();
  }
  using Kokkos::view_alloc;
  using Kokkos::WithoutInitializing;
  row_ptrs_type row_ptrs_beg(
      view_alloc("row_ptrs_beg", WithoutInitializing),
      rowPtrsUnpacked_dev.extent(0));
  // DEEP_COPY REVIEW - DEVICE-TO-DEVICE
  Kokkos::deep_copy(execution_space(), row_ptrs_beg, rowPtrsUnpacked_dev);
 
  const size_t N = row_ptrs_beg.extent(0) == 0 ? size_t(0) : size_t(row_ptrs_beg.extent(0) - 1);
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Allocate row_ptrs_end: " << N << endl;
    std::cerr << os.str();
  }
  row_ptrs_type row_ptrs_end(
      view_alloc("row_ptrs_end", WithoutInitializing), N);
  row_ptrs_type num_row_entries;
 
  const bool refill_num_row_entries = k_numRowEntries_.extent(0) != 0;
 
  execution_space().fence();  // we need above deep_copy to be done
 
  if (refill_num_row_entries) {  // Case 1: Unpacked storage
    // We can't assume correct *this capture until C++17, and it's
    // likely more efficient just to capture what we need anyway.
    num_row_entries =
        row_ptrs_type(view_alloc("num_row_entries", WithoutInitializing), N);
    Kokkos::deep_copy(num_row_entries, this->k_numRowEntries_);
    Kokkos::parallel_for(
        "Fill end row pointers", range_policy(0, N),
        KOKKOS_LAMBDA(const size_t i) {
          row_ptrs_end(i) = row_ptrs_beg(i) + num_row_entries(i);
        });
  } else {
    // FIXME (mfh 10 Feb 2020) Fix padCrsArrays so that if packed
    // storage, we don't need row_ptr_end to be separate allocation;
    // could just have it alias row_ptr_beg+1.
    Kokkos::parallel_for(
        "Fill end row pointers", range_policy(0, N),
        KOKKOS_LAMBDA(const size_t i) {
          row_ptrs_end(i) = row_ptrs_beg(i + 1);
        });
  }
 
  if (isGloballyIndexed()) {
    padCrsArrays(row_ptrs_beg, row_ptrs_end, gblInds_wdv,
                 padding, myRank, verbose);
  } else {
    padCrsArrays(row_ptrs_beg, row_ptrs_end, lclIndsUnpacked_wdv,
                 padding, myRank, verbose);
  }
 
  if (refill_num_row_entries) {
    Kokkos::parallel_for(
        "Fill num entries", range_policy(0, N),
        KOKKOS_LAMBDA(const size_t i) {
          num_row_entries(i) = row_ptrs_end(i) - row_ptrs_beg(i);
        });
    Kokkos::deep_copy(this->k_numRowEntries_, num_row_entries);
  }
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Reassign k_rowPtrs_; old size: "
       << rowPtrsUnpacked_dev.extent(0) << ", new size: "
       << row_ptrs_beg.extent(0) << endl;
    std::cerr << os.str();
    TEUCHOS_ASSERT(rowPtrsUnpacked_dev.extent(0) == row_ptrs_beg.extent(0));
  }
 
  setRowPtrsUnpacked(row_ptrs_beg);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
std::unique_ptr<
    typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    computeCrsPadding(
        const RowGraph<LocalOrdinal, GlobalOrdinal, Node>& source,
        const size_t numSameIDs,
        const Kokkos::DualView<const local_ordinal_type*,
                               buffer_device_type>& permuteToLIDs,
        const Kokkos::DualView<const local_ordinal_type*,
                               buffer_device_type>& permuteFromLIDs,
        const bool verbose) const {
  using LO = local_ordinal_type;
  using std::endl;
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph",
                                "computeCrsPadding(same & permute)");
    std::ostringstream os;
    os << *prefix << "{numSameIDs: " << numSameIDs
       << ", numPermutes: " << permuteFromLIDs.extent(0) << "}"
       << endl;
    std::cerr << os.str();
  }
 
  const int myRank = [&]() {
    auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
    return comm.is_null() ? -1 : comm->getRank();
  }();
  std::unique_ptr<padding_type> padding(
      new padding_type(myRank, numSameIDs,
                       permuteFromLIDs.extent(0)));
 
  computeCrsPaddingForSameIDs(*padding, source,
                              static_cast<LO>(numSameIDs));
  computeCrsPaddingForPermutedIDs(*padding, source, permuteToLIDs,
                                  permuteFromLIDs);
  return padding;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    computeCrsPaddingForSameIDs(
        padding_type& padding,
        const RowGraph<local_ordinal_type, global_ordinal_type,
                       node_type>& source,
        const local_ordinal_type numSameIDs) const {
  using LO = local_ordinal_type;
  using GO = global_ordinal_type;
  using Details::Impl::getRowGraphGlobalRow;
  using std::endl;
  const char tfecfFuncName[] = "computeCrsPaddingForSameIds";
 
  std::unique_ptr<std::string> prefix;
  const bool verbose = verbose_;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "numSameIDs: " << numSameIDs << endl;
    std::cerr << os.str();
  }
 
  if (numSameIDs == 0) {
    return;
  }
 
  const map_type& srcRowMap   = *(source.getRowMap());
  const map_type& tgtRowMap   = *rowMap_;
  using this_CRS_type         = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
  const this_CRS_type* srcCrs = dynamic_cast<const this_CRS_type*>(&source);
  const bool src_is_unique =
      srcCrs == nullptr ? false : srcCrs->isMerged();
  const bool tgt_is_unique = this->isMerged();
 
  std::vector<GO> srcGblColIndsScratch;
  std::vector<GO> tgtGblColIndsScratch;
 
  execute_sync_host_uvm_access();  // protect host UVM access
  for (LO lclRowInd = 0; lclRowInd < numSameIDs; ++lclRowInd) {
    const GO srcGblRowInd = srcRowMap.getGlobalElement(lclRowInd);
    const GO tgtGblRowInd = tgtRowMap.getGlobalElement(lclRowInd);
    auto srcGblColInds    = getRowGraphGlobalRow(
           srcGblColIndsScratch, source, srcGblRowInd);
    auto tgtGblColInds = getRowGraphGlobalRow(
        tgtGblColIndsScratch, *this, tgtGblRowInd);
    padding.update_same(lclRowInd, tgtGblColInds.getRawPtr(),
                        tgtGblColInds.size(), tgt_is_unique,
                        srcGblColInds.getRawPtr(),
                        srcGblColInds.size(), src_is_unique);
  }
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    computeCrsPaddingForPermutedIDs(
        padding_type& padding,
        const RowGraph<local_ordinal_type, global_ordinal_type,
                       node_type>& source,
        const Kokkos::DualView<const local_ordinal_type*,
                               buffer_device_type>& permuteToLIDs,
        const Kokkos::DualView<const local_ordinal_type*,
                               buffer_device_type>& permuteFromLIDs) const {
  using LO = local_ordinal_type;
  using GO = global_ordinal_type;
  using Details::Impl::getRowGraphGlobalRow;
  using std::endl;
  const char tfecfFuncName[] = "computeCrsPaddingForPermutedIds";
 
  std::unique_ptr<std::string> prefix;
  const bool verbose = verbose_;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "permuteToLIDs.extent(0): "
       << permuteToLIDs.extent(0)
       << ", permuteFromLIDs.extent(0): "
       << permuteFromLIDs.extent(0) << endl;
    std::cerr << os.str();
  }
 
  if (permuteToLIDs.extent(0) == 0) {
    return;
  }
 
  const map_type& srcRowMap   = *(source.getRowMap());
  const map_type& tgtRowMap   = *rowMap_;
  using this_CRS_type         = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
  const this_CRS_type* srcCrs = dynamic_cast<const this_CRS_type*>(&source);
  const bool src_is_unique =
      srcCrs == nullptr ? false : srcCrs->isMerged();
  const bool tgt_is_unique = this->isMerged();
 
  TEUCHOS_ASSERT(!permuteToLIDs.need_sync_host());
  auto permuteToLIDs_h = permuteToLIDs.view_host();
  TEUCHOS_ASSERT(!permuteFromLIDs.need_sync_host());
  auto permuteFromLIDs_h = permuteFromLIDs.view_host();
 
  std::vector<GO> srcGblColIndsScratch;
  std::vector<GO> tgtGblColIndsScratch;
  const LO numPermutes = static_cast<LO>(permuteToLIDs_h.extent(0));
 
  execute_sync_host_uvm_access();  // protect host UVM access
  for (LO whichPermute = 0; whichPermute < numPermutes; ++whichPermute) {
    const LO srcLclRowInd = permuteFromLIDs_h[whichPermute];
    const GO srcGblRowInd = srcRowMap.getGlobalElement(srcLclRowInd);
    auto srcGblColInds    = getRowGraphGlobalRow(
           srcGblColIndsScratch, source, srcGblRowInd);
    const LO tgtLclRowInd = permuteToLIDs_h[whichPermute];
    const GO tgtGblRowInd = tgtRowMap.getGlobalElement(tgtLclRowInd);
    auto tgtGblColInds    = getRowGraphGlobalRow(
           tgtGblColIndsScratch, *this, tgtGblRowInd);
    padding.update_permute(whichPermute, tgtLclRowInd,
                           tgtGblColInds.getRawPtr(),
                           tgtGblColInds.size(), tgt_is_unique,
                           srcGblColInds.getRawPtr(),
                           srcGblColInds.size(), src_is_unique);
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
std::unique_ptr<
    typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    computeCrsPaddingForImports(
        const Kokkos::DualView<const local_ordinal_type*,
                               buffer_device_type>& importLIDs,
        Kokkos::DualView<packet_type*, buffer_device_type> imports,
        Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
        const bool verbose) const {
  using Details::Impl::getRowGraphGlobalRow;
  using std::endl;
  using LO                   = local_ordinal_type;
  using GO                   = global_ordinal_type;
  const char tfecfFuncName[] = "computeCrsPaddingForImports";
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "importLIDs.extent(0): "
       << importLIDs.extent(0)
       << ", imports.extent(0): "
       << imports.extent(0)
       << ", numPacketsPerLID.extent(0): "
       << numPacketsPerLID.extent(0) << endl;
    std::cerr << os.str();
  }
 
  const LO numImports = static_cast<LO>(importLIDs.extent(0));
  const int myRank    = [&]() {
    auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
    return comm.is_null() ? -1 : comm->getRank();
  }();
  std::unique_ptr<padding_type> padding(
      new padding_type(myRank, numImports));
 
  if (imports.need_sync_host()) {
    imports.sync_host();
  }
  auto imports_h = imports.view_host();
  if (numPacketsPerLID.need_sync_host()) {
    numPacketsPerLID.sync_host();
  }
  auto numPacketsPerLID_h = numPacketsPerLID.view_host();
 
  TEUCHOS_ASSERT(!importLIDs.need_sync_host());
  auto importLIDs_h = importLIDs.view_host();
 
  const map_type& tgtRowMap = *rowMap_;
  // Always merge source column indices, since isMerged() is
  // per-process state, and we don't know its value on other
  // processes that sent us data.
  constexpr bool src_is_unique = false;
  const bool tgt_is_unique     = isMerged();
 
  std::vector<GO> tgtGblColIndsScratch;
  size_t offset = 0;
  execute_sync_host_uvm_access();  // protect host UVM access
  for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
    // CrsGraph packs just global column indices, while CrsMatrix
    // packs bytes (first the number of entries in the row, then the
    // global column indices, then other stuff like the matrix
    // values in that row).
    const LO origSrcNumEnt =
        static_cast<LO>(numPacketsPerLID_h[whichImport]);
    GO* const srcGblColInds = imports_h.data() + offset;
 
    const LO tgtLclRowInd = importLIDs_h[whichImport];
    const GO tgtGblRowInd =
        tgtRowMap.getGlobalElement(tgtLclRowInd);
    auto tgtGblColInds = getRowGraphGlobalRow(
        tgtGblColIndsScratch, *this, tgtGblRowInd);
    const size_t origTgtNumEnt(tgtGblColInds.size());
 
    padding->update_import(whichImport, tgtLclRowInd,
                           tgtGblColInds.getRawPtr(),
                           origTgtNumEnt, tgt_is_unique,
                           srcGblColInds,
                           origSrcNumEnt, src_is_unique);
    offset += origSrcNumEnt;
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
  return padding;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
std::unique_ptr<
    typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type>
CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    computePaddingForCrsMatrixUnpack(
        const Kokkos::DualView<const local_ordinal_type*,
                               buffer_device_type>& importLIDs,
        Kokkos::DualView<char*, buffer_device_type> imports,
        Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
        const bool verbose) const {
  using Details::PackTraits;
  using Details::Impl::getRowGraphGlobalRow;
  using std::endl;
  using LO                   = local_ordinal_type;
  using GO                   = global_ordinal_type;
  const char tfecfFuncName[] = "computePaddingForCrsMatrixUnpack";
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "importLIDs.extent(0): "
       << importLIDs.extent(0)
       << ", imports.extent(0): "
       << imports.extent(0)
       << ", numPacketsPerLID.extent(0): "
       << numPacketsPerLID.extent(0) << endl;
    std::cerr << os.str();
  }
  const bool extraVerbose =
      verbose && Details::Behavior::verbose("CrsPadding");
 
  const LO numImports = static_cast<LO>(importLIDs.extent(0));
  TEUCHOS_ASSERT(LO(numPacketsPerLID.extent(0)) >= numImports);
  const int myRank = [&]() {
    auto comm = rowMap_.is_null() ? Teuchos::null : rowMap_->getComm();
    return comm.is_null() ? -1 : comm->getRank();
  }();
  std::unique_ptr<padding_type> padding(
      new padding_type(myRank, numImports));
 
  if (imports.need_sync_host()) {
    imports.sync_host();
  }
  auto imports_h = imports.view_host();
  if (numPacketsPerLID.need_sync_host()) {
    numPacketsPerLID.sync_host();
  }
  auto numPacketsPerLID_h = numPacketsPerLID.view_host();
 
  TEUCHOS_ASSERT(!importLIDs.need_sync_host());
  auto importLIDs_h = importLIDs.view_host();
 
  const map_type& tgtRowMap = *rowMap_;
  // Always merge source column indices, since isMerged() is
  // per-process state, and we don't know its value on other
  // processes that sent us data.
  constexpr bool src_is_unique = false;
  const bool tgt_is_unique     = isMerged();
 
  std::vector<GO> srcGblColIndsScratch;
  std::vector<GO> tgtGblColIndsScratch;
  size_t offset = 0;
  execute_sync_host_uvm_access();  // protect host UVM access
  for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
    // CrsGraph packs just global column indices, while CrsMatrix
    // packs bytes (first the number of entries in the row, then the
    // global column indices, then other stuff like the matrix
    // values in that row).
    const size_t numBytes = numPacketsPerLID_h[whichImport];
    if (extraVerbose) {
      std::ostringstream os;
      os << *prefix << "whichImport=" << whichImport
         << ", numImports=" << numImports
         << ", numBytes=" << numBytes << endl;
      std::cerr << os.str();
    }
    if (numBytes == 0) {
      continue;  // special case: no entries to unpack for this row
    }
    LO origSrcNumEnt       = 0;
    const size_t numEntBeg = offset;
    const size_t numEntLen =
        PackTraits<LO>::packValueCount(origSrcNumEnt);
    TEUCHOS_ASSERT(numBytes >= numEntLen);
    TEUCHOS_ASSERT(imports_h.extent(0) >= numEntBeg + numEntLen);
    PackTraits<LO>::unpackValue(origSrcNumEnt,
                                imports_h.data() + numEntBeg);
    if (extraVerbose) {
      std::ostringstream os;
      os << *prefix << "whichImport=" << whichImport
         << ", numImports=" << numImports
         << ", origSrcNumEnt=" << origSrcNumEnt << endl;
      std::cerr << os.str();
    }
    TEUCHOS_ASSERT(origSrcNumEnt >= LO(0));
    TEUCHOS_ASSERT(numBytes >= size_t(numEntLen + origSrcNumEnt * sizeof(GO)));
    const size_t gidsBeg = numEntBeg + numEntLen;
    if (srcGblColIndsScratch.size() < size_t(origSrcNumEnt)) {
      srcGblColIndsScratch.resize(origSrcNumEnt);
    }
    GO* const srcGblColInds = srcGblColIndsScratch.data();
    PackTraits<GO>::unpackArray(srcGblColInds,
                                imports_h.data() + gidsBeg,
                                origSrcNumEnt);
    const LO tgtLclRowInd = importLIDs_h[whichImport];
    const GO tgtGblRowInd =
        tgtRowMap.getGlobalElement(tgtLclRowInd);
    auto tgtGblColInds = getRowGraphGlobalRow(
        tgtGblColIndsScratch, *this, tgtGblRowInd);
    const size_t origNumTgtEnt(tgtGblColInds.size());
 
    if (extraVerbose) {
      std::ostringstream os;
      os << *prefix << "whichImport=" << whichImport
         << ", numImports=" << numImports
         << ": Call padding->update_import" << endl;
      std::cerr << os.str();
    }
    padding->update_import(whichImport, tgtLclRowInd,
                           tgtGblColInds.getRawPtr(),
                           origNumTgtEnt, tgt_is_unique,
                           srcGblColInds,
                           origSrcNumEnt, src_is_unique);
    offset += numBytes;
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
  return padding;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    packAndPrepare(const SrcDistObject& source,
                   const Kokkos::DualView<const local_ordinal_type*,
                                          buffer_device_type>& exportLIDs,
                   Kokkos::DualView<packet_type*,
                                    buffer_device_type>& exports,
                   Kokkos::DualView<size_t*,
                                    buffer_device_type>
                       numPacketsPerLID,
                   size_t& constantNumPackets) {
  using Tpetra::Details::ProfilingRegion;
  using GO = global_ordinal_type;
  using std::endl;
  using crs_graph_type =
      CrsGraph<local_ordinal_type, global_ordinal_type, node_type>;
  const char tfecfFuncName[] = "packAndPrepare: ";
  ProfilingRegion region_papn("Tpetra::CrsGraph::packAndPrepare");
 
  const bool verbose = verbose_;
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "packAndPrepare");
    std::ostringstream os;
    os << *prefix << "Start" << endl;
    std::cerr << os.str();
  }
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(exportLIDs.extent(0) != numPacketsPerLID.extent(0),
                                        std::runtime_error,
                                        "exportLIDs.extent(0) = " << exportLIDs.extent(0)
                                                                  << " != numPacketsPerLID.extent(0) = " << numPacketsPerLID.extent(0)
                                                                  << ".");
  const row_graph_type* srcRowGraphPtr =
      dynamic_cast<const row_graph_type*>(&source);
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(srcRowGraphPtr == nullptr, std::invalid_argument,
                                        "Source of an Export "
                                        "or Import operation to a CrsGraph must be a RowGraph with the same "
                                        "template parameters.");
  // We don't check whether src_graph has had fillComplete called,
  // because it doesn't matter whether the *source* graph has been
  // fillComplete'd. The target graph can not be fillComplete'd yet.
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isFillComplete(), std::runtime_error,
                                        "The target graph of an Import or Export must not be fill complete.");
 
  const crs_graph_type* srcCrsGraphPtr =
      dynamic_cast<const crs_graph_type*>(&source);
 
  if (srcCrsGraphPtr == nullptr) {
    using Teuchos::ArrayView;
    using LO = local_ordinal_type;
 
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Source is a RowGraph but not a CrsGraph"
         << endl;
      std::cerr << os.str();
    }
    // RowGraph::pack serves the "old" DistObject interface.  It
    // takes Teuchos::ArrayView and Teuchos::Array&.  The latter
    // entails deep-copying the exports buffer on output.  RowGraph
    // is a convenience interface when not a CrsGraph, so we accept
    // the performance hit.
    TEUCHOS_ASSERT(!exportLIDs.need_sync_host());
    auto exportLIDs_h = exportLIDs.view_host();
    ArrayView<const LO> exportLIDs_av(exportLIDs_h.data(),
                                      exportLIDs_h.extent(0));
    Teuchos::Array<GO> exports_a;
 
    numPacketsPerLID.clear_sync_state();
    numPacketsPerLID.modify_host();
    auto numPacketsPerLID_h = numPacketsPerLID.view_host();
    ArrayView<size_t> numPacketsPerLID_av(numPacketsPerLID_h.data(),
                                          numPacketsPerLID_h.extent(0));
    srcRowGraphPtr->pack(exportLIDs_av, exports_a, numPacketsPerLID_av,
                         constantNumPackets);
    const size_t newSize = static_cast<size_t>(exports_a.size());
    if (static_cast<size_t>(exports.extent(0)) != newSize) {
      using exports_dv_type = Kokkos::DualView<packet_type*, buffer_device_type>;
      exports               = exports_dv_type("exports", newSize);
    }
    Kokkos::View<const packet_type*, Kokkos::HostSpace,
                 Kokkos::MemoryUnmanaged>
        exports_a_h(exports_a.getRawPtr(), newSize);
    exports.clear_sync_state();
    exports.modify_host();
    // DEEP_COPY REVIEW - NOT TESTED
    Kokkos::deep_copy(exports.view_host(), exports_a_h);
  }
  // packCrsGraphNew requires k_rowPtrsPacked_ to be set
  else if (!getColMap().is_null() &&
           (this->getRowPtrsPackedDevice().extent(0) != 0 ||
            getRowMap()->getLocalNumElements() == 0)) {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "packCrsGraphNew path" << endl;
      std::cerr << os.str();
    }
    using export_pids_type =
        Kokkos::DualView<const int*, buffer_device_type>;
    export_pids_type exportPIDs;  // not filling it; needed for syntax
    using LO = local_ordinal_type;
    using NT = node_type;
    using Tpetra::Details::packCrsGraphNew;
    packCrsGraphNew<LO, GO, NT>(*srcCrsGraphPtr, exportLIDs, exportPIDs,
                                exports, numPacketsPerLID,
                                constantNumPackets, false);
  } else {
    srcCrsGraphPtr->packFillActiveNew(exportLIDs, exports, numPacketsPerLID,
                                      constantNumPackets);
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    pack(const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
         Teuchos::Array<GlobalOrdinal>& exports,
         const Teuchos::ArrayView<size_t>& numPacketsPerLID,
         size_t& constantNumPackets) const {
  auto col_map = this->getColMap();
  // packCrsGraph requires k_rowPtrsPacked to be set
  if (!col_map.is_null() && (this->getRowPtrsPackedDevice().extent(0) != 0 || getRowMap()->getLocalNumElements() == 0)) {
    using Tpetra::Details::packCrsGraph;
    packCrsGraph<LocalOrdinal, GlobalOrdinal, Node>(*this, exports, numPacketsPerLID,
                                                    exportLIDs, constantNumPackets);
  } else {
    this->packFillActive(exportLIDs, exports, numPacketsPerLID,
                         constantNumPackets);
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    packFillActive(const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
                   Teuchos::Array<GlobalOrdinal>& exports,
                   const Teuchos::ArrayView<size_t>& numPacketsPerLID,
                   size_t& constantNumPackets) const {
  using std::endl;
  using LO = LocalOrdinal;
  using GO = GlobalOrdinal;
  using host_execution_space =
      typename Kokkos::View<size_t*, device_type>::
          host_mirror_type::execution_space;
  const char tfecfFuncName[] = "packFillActive: ";
  const bool verbose         = verbose_;
 
  const auto numExportLIDs = exportLIDs.size();
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "allocateIndices");
    std::ostringstream os;
    os << *prefix << "numExportLIDs=" << numExportLIDs << endl;
    std::cerr << os.str();
  }
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numExportLIDs != numPacketsPerLID.size(), std::runtime_error,
                                        "exportLIDs.size() = " << numExportLIDs << " != numPacketsPerLID.size()"
                                                                                   " = "
                                                               << numPacketsPerLID.size() << ".");
 
  const map_type& rowMap          = *(this->getRowMap());
  const map_type* const colMapPtr = this->colMap_.getRawPtr();
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() && colMapPtr == nullptr, std::logic_error,
                                        "This graph claims to be locally indexed, but its column Map is nullptr.  "
                                        "This should never happen.  Please report this bug to the Tpetra "
                                        "developers.");
 
  // We may pack different amounts of data for different rows.
  constantNumPackets = 0;
 
  // mfh 20 Sep 2017: Teuchos::ArrayView isn't thread safe (well,
  // it might be now, but we might as well be safe).
  size_t* const numPacketsPerLID_raw = numPacketsPerLID.getRawPtr();
  const LO* const exportLIDs_raw     = exportLIDs.getRawPtr();
 
  // Count the total number of packets (column indices, in the case
  // of a CrsGraph) to pack.  While doing so, set
  // numPacketsPerLID[i] to the number of entries owned by the
  // calling process in (local) row exportLIDs[i] of the graph, that
  // the caller wants us to send out.
  Kokkos::RangePolicy<host_execution_space, LO> inputRange(0, numExportLIDs);
  size_t totalNumPackets = 0;
  size_t errCount        = 0;
  // lambdas turn what they capture const, so we can't
  // atomic_add(&errCount,1).  Instead, we need a View to modify.
  typedef Kokkos::Device<host_execution_space, Kokkos::HostSpace>
      host_device_type;
  Kokkos::View<size_t, host_device_type> errCountView(&errCount);
  constexpr size_t ONE = 1;
 
  execute_sync_host_uvm_access();  // protect host UVM access
  Kokkos::parallel_reduce(
      "Tpetra::CrsGraph::pack: totalNumPackets",
      inputRange,
      [=, *this](const LO& i, size_t& curTotalNumPackets) {
        const GO gblRow = rowMap.getGlobalElement(exportLIDs_raw[i]);
        if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid()) {
          Kokkos::atomic_add(&errCountView(), ONE);
          numPacketsPerLID_raw[i] = 0;
        } else {
          const size_t numEnt     = this->getNumEntriesInGlobalRow(gblRow);
          numPacketsPerLID_raw[i] = numEnt;
          curTotalNumPackets += numEnt;
        }
      },
      totalNumPackets);
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "totalNumPackets=" << totalNumPackets << endl;
    std::cerr << os.str();
  }
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
                                        "totalNumPackets count encountered "
                                        "one or more errors!  errCount = "
                                            << errCount
                                            << ", totalNumPackets = " << totalNumPackets << ".");
  errCount = 0;
 
  // Allocate space for all the column indices to pack.
  exports.resize(totalNumPackets);
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->supportsRowViews(), std::logic_error,
                                        "this->supportsRowViews() returns false; this should never happen.  "
                                        "Please report this bug to the Tpetra developers.");
 
  // Loop again over the rows to export, and pack rows of indices
  // into the output buffer.
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Pack into exports" << endl;
    std::cerr << os.str();
  }
 
  // Teuchos::ArrayView may not be thread safe, or may not be
  // efficiently thread safe.  Better to use the raw pointer.
  GO* const exports_raw = exports.getRawPtr();
  errCount              = 0;
  Kokkos::parallel_scan("Tpetra::CrsGraph::pack: pack from views",
                        inputRange, [=, &prefix, *this](const LO i, size_t& exportsOffset, const bool final) {
                          const size_t curOffset = exportsOffset;
                          const GO gblRow        = rowMap.getGlobalElement(exportLIDs_raw[i]);
                          const RowInfo rowInfo =
                              this->getRowInfoFromGlobalRowIndex(gblRow);
 
                          using TDO = Tpetra::Details::OrdinalTraits<size_t>;
                          if (rowInfo.localRow == TDO::invalid()) {
                            if (verbose) {
                              std::ostringstream os;
                              os << *prefix << ": INVALID rowInfo: i=" << i
                                 << ", lclRow=" << exportLIDs_raw[i] << endl;
                              std::cerr << os.str();
                            }
                            Kokkos::atomic_add(&errCountView(), ONE);
                          } else if (curOffset + rowInfo.numEntries > totalNumPackets) {
                            if (verbose) {
                              std::ostringstream os;
                              os << *prefix << ": UH OH!  For i=" << i << ", lclRow="
                                 << exportLIDs_raw[i] << ", gblRow=" << gblRow << ", curOffset "
                                                                                  "(= "
                                 << curOffset << ") + numEnt (= " << rowInfo.numEntries
                                 << ") > totalNumPackets (= " << totalNumPackets << ")."
                                 << endl;
                              std::cerr << os.str();
                            }
                            Kokkos::atomic_add(&errCountView(), ONE);
                          } else {
                            const LO numEnt = static_cast<LO>(rowInfo.numEntries);
                            if (this->isLocallyIndexed()) {
                              auto lclColInds = getLocalIndsViewHost(rowInfo);
                              if (final) {
                                for (LO k = 0; k < numEnt; ++k) {
                                  const LO lclColInd = lclColInds(k);
                                  const GO gblColInd = colMapPtr->getGlobalElement(lclColInd);
                                  // Pack it, even if it's wrong.  Let the receiving
                                  // process deal with it.  Otherwise, we'll miss out
                                  // on any correct data.
                                  exports_raw[curOffset + k] = gblColInd;
                                }  // for each entry in the row
                              }    // final pass?
                              exportsOffset = curOffset + numEnt;
                            } else if (this->isGloballyIndexed()) {
                              auto gblColInds = getGlobalIndsViewHost(rowInfo);
                              if (final) {
                                for (LO k = 0; k < numEnt; ++k) {
                                  const GO gblColInd = gblColInds(k);
                                  // Pack it, even if it's wrong.  Let the receiving
                                  // process deal with it.  Otherwise, we'll miss out
                                  // on any correct data.
                                  exports_raw[curOffset + k] = gblColInd;
                                }  // for each entry in the row
                              }    // final pass?
                              exportsOffset = curOffset + numEnt;
                            }
                            // If neither globally nor locally indexed, then the graph
                            // has no entries in this row (or indeed, in any row on this
                            // process) to pack.
                          }
                        });
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
                                        "Packing encountered "
                                        "one or more errors!  errCount = "
                                            << errCount
                                            << ", totalNumPackets = " << totalNumPackets << ".");
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    packFillActiveNew(const Kokkos::DualView<const local_ordinal_type*,
                                             buffer_device_type>& exportLIDs,
                      Kokkos::DualView<packet_type*,
                                       buffer_device_type>& exports,
                      Kokkos::DualView<size_t*,
                                       buffer_device_type>
                          numPacketsPerLID,
                      size_t& constantNumPackets) const {
  using std::endl;
  using LO                   = local_ordinal_type;
  using GO                   = global_ordinal_type;
  using host_execution_space = typename Kokkos::View<size_t*,
                                                     device_type>::host_mirror_type::execution_space;
  using host_device_type =
      Kokkos::Device<host_execution_space, Kokkos::HostSpace>;
  using exports_dv_type =
      Kokkos::DualView<packet_type*, buffer_device_type>;
  const char tfecfFuncName[] = "packFillActiveNew: ";
  const bool verbose         = verbose_;
 
  const auto numExportLIDs = exportLIDs.extent(0);
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "packFillActiveNew");
    std::ostringstream os;
    os << *prefix << "numExportLIDs: " << numExportLIDs
       << ", numPacketsPerLID.extent(0): "
       << numPacketsPerLID.extent(0) << endl;
    std::cerr << os.str();
  }
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(numExportLIDs != numPacketsPerLID.extent(0), std::runtime_error,
                                        "exportLIDs.extent(0) = " << numExportLIDs
                                                                  << " != numPacketsPerLID.extent(0) = "
                                                                  << numPacketsPerLID.extent(0) << ".");
  TEUCHOS_ASSERT(!exportLIDs.need_sync_host());
  auto exportLIDs_h = exportLIDs.view_host();
 
  const map_type& rowMap          = *(this->getRowMap());
  const map_type* const colMapPtr = this->colMap_.getRawPtr();
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(this->isLocallyIndexed() && colMapPtr == nullptr, std::logic_error,
                                        "This graph claims to be locally indexed, but its column Map is nullptr.  "
                                        "This should never happen.  Please report this bug to the Tpetra "
                                        "developers.");
 
  // We may pack different amounts of data for different rows.
  constantNumPackets = 0;
 
  numPacketsPerLID.clear_sync_state();
  numPacketsPerLID.modify_host();
  auto numPacketsPerLID_h = numPacketsPerLID.view_host();
 
  // Count the total number of packets (column indices, in the case
  // of a CrsGraph) to pack.  While doing so, set
  // numPacketsPerLID[i] to the number of entries owned by the
  // calling process in (local) row exportLIDs[i] of the graph, that
  // the caller wants us to send out.
  using range_type = Kokkos::RangePolicy<host_execution_space, LO>;
  range_type inputRange(0, numExportLIDs);
  size_t totalNumPackets = 0;
  size_t errCount        = 0;
  // lambdas turn what they capture const, so we can't
  // atomic_add(&errCount,1).  Instead, we need a View to modify.
  Kokkos::View<size_t, host_device_type> errCountView(&errCount);
  constexpr size_t ONE = 1;
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Compute totalNumPackets" << endl;
    std::cerr << os.str();
  }
 
  execute_sync_host_uvm_access();  // protect host UVM access
  totalNumPackets = 0;
  for (size_t i = 0; i < numExportLIDs; ++i) {
    const LO lclRow = exportLIDs_h[i];
    const GO gblRow = rowMap.getGlobalElement(lclRow);
    if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid()) {
      if (verbose) {
        std::ostringstream os;
        os << *prefix << "For i=" << i << ", lclRow=" << lclRow
           << " not in row Map on this process" << endl;
        std::cerr << os.str();
      }
      Kokkos::atomic_add(&errCountView(), ONE);
      numPacketsPerLID_h(i) = 0;
    } else {
      const size_t numEnt   = this->getNumEntriesInGlobalRow(gblRow);
      numPacketsPerLID_h(i) = numEnt;
      totalNumPackets += numEnt;
    }
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "totalNumPackets: " << totalNumPackets
       << ", errCount: " << errCount << endl;
    std::cerr << os.str();
  }
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(errCount != 0, std::logic_error,
                                        "totalNumPackets count encountered "
                                        "one or more errors! totalNumPackets: "
                                            << totalNumPackets
                                            << ", errCount: " << errCount << ".");
 
  // Allocate space for all the column indices to pack.
  if (size_t(exports.extent(0)) < totalNumPackets) {
    // FIXME (mfh 09 Apr 2019) Create without initializing.
    exports = exports_dv_type("exports", totalNumPackets);
  }
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->supportsRowViews(), std::logic_error,
                                        "this->supportsRowViews() returns false; this should never happen.  "
                                        "Please report this bug to the Tpetra developers.");
 
  // Loop again over the rows to export, and pack rows of indices
  // into the output buffer.
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Pack into exports buffer" << endl;
    std::cerr << os.str();
  }
 
  exports.clear_sync_state();
  exports.modify_host();
  auto exports_h = exports.view_host();
 
  errCount = 0;
 
  // The following parallel_scan needs const host access to lclIndsUnpacked_wdv
  // (if locally indexed) or gblInds_wdv (if globally indexed).
  if (isLocallyIndexed())
    lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
  else if (isGloballyIndexed())
    gblInds_wdv.getHostView(Access::ReadOnly);
 
  Details::disableWDVTracking();
  Kokkos::parallel_scan("Tpetra::CrsGraph::packFillActiveNew: Pack exports",
                        inputRange, [=, &prefix, *this](const LO i, size_t& exportsOffset, const bool final) {
                          const size_t curOffset = exportsOffset;
                          const LO lclRow        = exportLIDs_h(i);
                          const GO gblRow        = rowMap.getGlobalElement(lclRow);
                          if (gblRow == Details::OrdinalTraits<GO>::invalid()) {
                            if (verbose) {
                              std::ostringstream os;
                              os << *prefix << "For i=" << i << ", lclRow=" << lclRow
                                 << " not in row Map on this process" << endl;
                              std::cerr << os.str();
                            }
                            Kokkos::atomic_add(&errCountView(), ONE);
                            return;
                          }
 
                          const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex(gblRow);
                          if (rowInfo.localRow == Details::OrdinalTraits<size_t>::invalid()) {
                            if (verbose) {
                              std::ostringstream os;
                              os << *prefix << "For i=" << i << ", lclRow=" << lclRow
                                 << ", gblRow=" << gblRow << ": invalid rowInfo"
                                 << endl;
                              std::cerr << os.str();
                            }
                            Kokkos::atomic_add(&errCountView(), ONE);
                            return;
                          }
 
                          if (curOffset + rowInfo.numEntries > totalNumPackets) {
                            if (verbose) {
                              std::ostringstream os;
                              os << *prefix << "For i=" << i << ", lclRow=" << lclRow
                                 << ", gblRow=" << gblRow << ", curOffset (= "
                                 << curOffset << ") + numEnt (= " << rowInfo.numEntries
                                 << ") > totalNumPackets (= " << totalNumPackets
                                 << ")." << endl;
                              std::cerr << os.str();
                            }
                            Kokkos::atomic_add(&errCountView(), ONE);
                            return;
                          }
 
                          const LO numEnt = static_cast<LO>(rowInfo.numEntries);
                          if (this->isLocallyIndexed()) {
                            auto lclColInds = getLocalIndsViewHost(rowInfo);
                            if (final) {
                              for (LO k = 0; k < numEnt; ++k) {
                                const LO lclColInd = lclColInds(k);
                                const GO gblColInd = colMapPtr->getGlobalElement(lclColInd);
                                // Pack it, even if it's wrong.  Let the receiving
                                // process deal with it.  Otherwise, we'll miss out
                                // on any correct data.
                                exports_h(curOffset + k) = gblColInd;
                              }  // for each entry in the row
                            }    // final pass?
                            exportsOffset = curOffset + numEnt;
                          } else if (this->isGloballyIndexed()) {
                            auto gblColInds = getGlobalIndsViewHost(rowInfo);
                            if (final) {
                              for (LO k = 0; k < numEnt; ++k) {
                                const GO gblColInd = gblColInds(k);
                                // Pack it, even if it's wrong.  Let the receiving
                                // process deal with it.  Otherwise, we'll miss out
                                // on any correct data.
                                exports_h(curOffset + k) = gblColInd;
                              }  // for each entry in the row
                            }    // final pass?
                            exportsOffset = curOffset + numEnt;
                          }
                          // If neither globally nor locally indexed, then the graph
                          // has no entries in this row (or indeed, in any row on this
                          // process) to pack.
                        });
  Details::enableWDVTracking();
 
  // TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
  //   (errCount != 0, std::logic_error, "Packing encountered "
  //    "one or more errors!  errCount = " << errCount
  //    << ", totalNumPackets = " << totalNumPackets << ".");
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "errCount=" << errCount << "; Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    unpackAndCombine(const Kokkos::DualView<const local_ordinal_type*,
                                            buffer_device_type>& importLIDs,
                     Kokkos::DualView<packet_type*,
                                      buffer_device_type>
                         imports,
                     Kokkos::DualView<size_t*,
                                      buffer_device_type>
                         numPacketsPerLID,
                     const size_t /* constantNumPackets */,
                     const CombineMode /* combineMode */) {
  using Details::ProfilingRegion;
  using std::endl;
  using LO                   = local_ordinal_type;
  using GO                   = global_ordinal_type;
  const char tfecfFuncName[] = "unpackAndCombine";
 
  ProfilingRegion regionCGC("Tpetra::CrsGraph::unpackAndCombine");
  const bool verbose = verbose_;
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", tfecfFuncName);
    std::ostringstream os;
    os << *prefix << "Start" << endl;
    std::cerr << os.str();
  }
  {
    auto padding = computeCrsPaddingForImports(
        importLIDs, imports, numPacketsPerLID, verbose);
    applyCrsPadding(*padding, verbose);
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Done computing & applying padding" << endl;
      std::cerr << os.str();
    }
  }
 
  // FIXME (mfh 02 Apr 2012) REPLACE combine mode has a perfectly
  // reasonable meaning, whether or not the matrix is fill complete.
  // It's just more work to implement.
 
  // We are not checking the value of the CombineMode input
  // argument.  For CrsGraph, we only support import/export
  // operations if fillComplete has not yet been called.  Any
  // incoming column-indices are inserted into the target graph. In
  // this context, CombineMode values of ADD vs INSERT are
  // equivalent. What is the meaning of REPLACE for CrsGraph? If a
  // duplicate column-index is inserted, it will be compressed out
  // when fillComplete is called.
  //
  // Note: I think REPLACE means that an existing row is replaced by
  // the imported row, i.e., the existing indices are cleared. CGB,
  // 6/17/2010
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(importLIDs.extent(0) != numPacketsPerLID.extent(0),
                                        std::runtime_error, ": importLIDs.extent(0) = " << importLIDs.extent(0) << " != numPacketsPerLID.extent(0) = " << numPacketsPerLID.extent(0) << ".");
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isFillComplete(), std::runtime_error,
                                        ": Import or Export operations are not allowed on a target "
                                        "CrsGraph that is fillComplete.");
 
  const size_t numImportLIDs(importLIDs.extent(0));
  if (numPacketsPerLID.need_sync_host()) {
    numPacketsPerLID.sync_host();
  }
  auto numPacketsPerLID_h = numPacketsPerLID.view_host();
  if (imports.need_sync_host()) {
    imports.sync_host();
  }
  auto imports_h = imports.view_host();
  TEUCHOS_ASSERT(!importLIDs.need_sync_host());
  auto importLIDs_h = importLIDs.view_host();
 
  // If we're inserting in local indices, let's pre-allocate
  Teuchos::Array<LO> lclColInds;
  if (isLocallyIndexed()) {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Preallocate local indices scratch" << endl;
      std::cerr << os.str();
    }
    size_t maxNumInserts = 0;
    for (size_t i = 0; i < numImportLIDs; ++i) {
      maxNumInserts = std::max(maxNumInserts, numPacketsPerLID_h[i]);
    }
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "Local indices scratch size: "
         << maxNumInserts << endl;
      std::cerr << os.str();
    }
    lclColInds.resize(maxNumInserts);
  } else {
    if (verbose) {
      std::ostringstream os;
      os << *prefix;
      if (isGloballyIndexed()) {
        os << "Graph is globally indexed";
      } else {
        os << "Graph is neither locally nor globally indexed";
      }
      os << endl;
      std::cerr << os.str();
    }
  }
 
  TEUCHOS_ASSERT(!rowMap_.is_null());
  const map_type& rowMap = *rowMap_;
 
  try {
    size_t importsOffset = 0;
    for (size_t i = 0; i < numImportLIDs; ++i) {
      if (verbose) {
        std::ostringstream os;
        os << *prefix << "i=" << i << ", numImportLIDs="
           << numImportLIDs << endl;
        std::cerr << os.str();
      }
      // We can only unpack into owned rows, since we only have
      // local row indices.
      const LO lclRow = importLIDs_h[i];
      const GO gblRow = rowMap.getGlobalElement(lclRow);
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(gblRow == Teuchos::OrdinalTraits<GO>::invalid(),
                                            std::logic_error, "importLIDs[i=" << i << "]=" << lclRow << " is not in the row Map on the calling "
                                                                                                        "process.");
      const LO numEnt            = numPacketsPerLID_h[i];
      const GO* const gblColInds = (numEnt == 0) ? nullptr : imports_h.data() + importsOffset;
      if (!isLocallyIndexed()) {
        insertGlobalIndicesFiltered(lclRow, gblColInds, numEnt);
      } else {
        // FIXME (mfh 09 Feb 2020) Now would be a good time to do
        // column Map filtering.
        for (LO j = 0; j < numEnt; j++) {
          lclColInds[j] = colMap_->getLocalElement(gblColInds[j]);
        }
        insertLocalIndices(lclRow, numEnt, lclColInds.data());
      }
      importsOffset += numEnt;
    }
  } catch (std::exception& e) {
    TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
                               "Tpetra::CrsGraph::unpackAndCombine: Insert loop threw an "
                               "exception: "
                                   << endl
                                   << e.what());
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    removeEmptyProcessesInPlace(const Teuchos::RCP<const map_type>& newMap) {
  using Teuchos::Comm;
  using Teuchos::null;
  using Teuchos::ParameterList;
  using Teuchos::RCP;
 
  // We'll set all the state "transactionally," so that this method
  // satisfies the strong exception guarantee.  This object's state
  // won't be modified until the end of this method.
  RCP<const map_type> rowMap, domainMap, rangeMap, colMap;
  RCP<import_type> importer;
  RCP<export_type> exporter;
 
  rowMap = newMap;
  RCP<const Comm<int>> newComm =
      (newMap.is_null()) ? null : newMap->getComm();
 
  if (!domainMap_.is_null()) {
    if (domainMap_.getRawPtr() == rowMap_.getRawPtr()) {
      // Common case: original domain and row Maps are identical.
      // In that case, we need only replace the original domain Map
      // with the new Map.  This ensures that the new domain and row
      // Maps _stay_ identical.
      domainMap = newMap;
    } else {
      domainMap = domainMap_->replaceCommWithSubset(newComm);
    }
  }
  if (!rangeMap_.is_null()) {
    if (rangeMap_.getRawPtr() == rowMap_.getRawPtr()) {
      // Common case: original range and row Maps are identical.  In
      // that case, we need only replace the original range Map with
      // the new Map.  This ensures that the new range and row Maps
      // _stay_ identical.
      rangeMap = newMap;
    } else {
      rangeMap = rangeMap_->replaceCommWithSubset(newComm);
    }
  }
  if (!colMap_.is_null()) {
    colMap = colMap_->replaceCommWithSubset(newComm);
  }
 
  // (Re)create the Export and / or Import if necessary.
  if (!newComm.is_null()) {
    RCP<ParameterList> params = this->getNonconstParameterList();  // could be null
    //
    // The operations below are collective on the new communicator.
    //
    // (Re)create the Export object if necessary.  If I haven't
    // called fillComplete yet, I don't have a rangeMap, so I must
    // first check if the _original_ rangeMap is not null.  Ditto
    // for the Import object and the domain Map.
    if (!rangeMap_.is_null() &&
        rangeMap != rowMap &&
        !rangeMap->isSameAs(*rowMap)) {
      if (params.is_null() || !params->isSublist("Export")) {
        exporter = rcp(new export_type(rowMap, rangeMap));
      } else {
        RCP<ParameterList> exportSublist = sublist(params, "Export", true);
        exporter                         = rcp(new export_type(rowMap, rangeMap, exportSublist));
      }
    }
    // (Re)create the Import object if necessary.
    if (!domainMap_.is_null() &&
        domainMap != colMap &&
        !domainMap->isSameAs(*colMap)) {
      if (params.is_null() || !params->isSublist("Import")) {
        importer = rcp(new import_type(domainMap, colMap));
      } else {
        RCP<ParameterList> importSublist = sublist(params, "Import", true);
        importer                         = rcp(new import_type(domainMap, colMap, importSublist));
      }
    }
  }  // if newComm is not null
 
  // Defer side effects until the end.  If no destructors throw
  // exceptions (they shouldn't anyway), then this method satisfies
  // the strong exception guarantee.
  exporter_ = exporter;
  importer_ = importer;
  rowMap_   = rowMap;
  // mfh 31 Mar 2013: DistObject's map_ is the row Map of a CrsGraph
  // or CrsMatrix.  CrsGraph keeps a redundant pointer (rowMap_) to
  // the same object.  We might want to get rid of this redundant
  // pointer sometime, but for now, we'll leave it alone and just
  // set map_ to the same object.
  this->map_ = rowMap;
  domainMap_ = domainMap;
  rangeMap_  = rangeMap;
  colMap_    = colMap;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalDiagOffsets(const Kokkos::View<size_t*, device_type, Kokkos::MemoryUnmanaged>& offsets) const {
  using std::endl;
  using LO                   = LocalOrdinal;
  using GO                   = GlobalOrdinal;
  const char tfecfFuncName[] = "getLocalDiagOffsets: ";
  const bool verbose         = verbose_;
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "getLocalDiagOffsets");
    std::ostringstream os;
    os << *prefix << "offsets.extent(0)=" << offsets.extent(0)
       << endl;
    std::cerr << os.str();
  }
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error, "The graph must have a column Map.");
  const LO lclNumRows = static_cast<LO>(this->getLocalNumRows());
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(static_cast<LO>(offsets.extent(0)) < lclNumRows,
                                        std::invalid_argument, "offsets.extent(0) = " << offsets.extent(0) << " < getLocalNumRows() = " << lclNumRows << ".");
 
  const map_type& rowMap = *(this->getRowMap());
  const map_type& colMap = *(this->getColMap());
 
  // We only use these in debug mode, but since debug mode is a
  // run-time option, they need to exist here.  That's why we create
  // the vector with explicit size zero, to avoid overhead if debug
  // mode is off.
  bool allRowMapDiagEntriesInColMap = true;
  bool allDiagEntriesFound          = true;
  bool allOffsetsCorrect            = true;
  bool noOtherWeirdness             = true;
  using wrong_offsets_type          = std::vector<std::pair<LO, size_t>>;
  wrong_offsets_type wrongOffsets(0);
 
  // mfh 12 Mar 2016: LocalMap works on (CUDA) device.  It has just
  // the subset of Map functionality that we need below.
  auto lclRowMap = rowMap.getLocalMap();
  auto lclColMap = colMap.getLocalMap();
 
  // FIXME (mfh 16 Dec 2015) It's easy to thread-parallelize this
  // setup, at least on the host.  For CUDA, we have to use LocalMap
  // (that comes from each of the two Maps).
 
  const bool sorted = this->isSorted();
  if (isFillComplete()) {
    auto lclGraph = this->getLocalGraphDevice();
    ::Tpetra::Details::getGraphDiagOffsets(offsets, lclRowMap, lclColMap,
                                           lclGraph.row_map,
                                           lclGraph.entries, sorted);
  } else {
    // NOTE (mfh 22 Feb 2017): We have to run this code on host,
    // since the graph is not fill complete.  The previous version
    // of this code assumed UVM; this version does not.
    auto offsets_h = Kokkos::create_mirror_view(offsets);
 
    for (LO lclRowInd = 0; lclRowInd < lclNumRows; ++lclRowInd) {
      // Find the diagonal entry.  Since the row Map and column Map
      // may differ, we have to compare global row and column
      // indices, not local.
      const GO gblRowInd = lclRowMap.getGlobalElement(lclRowInd);
      const GO gblColInd = gblRowInd;
      const LO lclColInd = lclColMap.getLocalElement(gblColInd);
 
      if (lclColInd == Tpetra::Details::OrdinalTraits<LO>::invalid()) {
        allRowMapDiagEntriesInColMap = false;
        offsets_h(lclRowInd)         = Tpetra::Details::OrdinalTraits<size_t>::invalid();
      } else {
        const RowInfo rowInfo = this->getRowInfo(lclRowInd);
        if (static_cast<LO>(rowInfo.localRow) == lclRowInd &&
            rowInfo.numEntries > 0) {
          auto colInds      = this->getLocalIndsViewHost(rowInfo);
          const size_t hint = 0;  // not needed for this algorithm
          const size_t offset =
              KokkosSparse::findRelOffset(colInds, rowInfo.numEntries,
                                          lclColInd, hint, sorted);
          offsets_h(lclRowInd) = offset;
 
          if (debug_) {
            // Now that we have what we think is an offset, make sure
            // that it really does point to the diagonal entry.  Offsets
            // are _relative_ to each row, not absolute (for the whole
            // (local) graph).
            typename local_inds_dualv_type::t_host::const_type lclColInds;
            try {
              lclColInds = this->getLocalIndsViewHost(rowInfo);
            } catch (...) {
              noOtherWeirdness = false;
            }
            // Don't continue with error checking if the above failed.
            if (noOtherWeirdness) {
              const size_t numEnt = lclColInds.extent(0);
              if (offset >= numEnt) {
                // Offsets are relative to each row, so this means that
                // the offset is out of bounds.
                allOffsetsCorrect = false;
                wrongOffsets.push_back(std::make_pair(lclRowInd, offset));
              } else {
                const LO actualLclColInd = lclColInds(offset);
                const GO actualGblColInd = lclColMap.getGlobalElement(actualLclColInd);
                if (actualGblColInd != gblColInd) {
                  allOffsetsCorrect = false;
                  wrongOffsets.push_back(std::make_pair(lclRowInd, offset));
                }
              }
            }
          }       // debug_
        } else {  // either row is empty, or something went wrong w/ getRowInfo()
          offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid();
          allDiagEntriesFound  = false;
        }
      }  // whether lclColInd is a valid local column index
    }    // for each local row
    // DEEP_COPY REVIEW - NOT TESTED
    Kokkos::deep_copy(offsets, offsets_h);
  }  // whether the graph is fill complete
 
  if (verbose && wrongOffsets.size() != 0) {
    std::ostringstream os;
    os << *prefix << "Wrong offsets: [";
    for (size_t k = 0; k < wrongOffsets.size(); ++k) {
      os << "(" << wrongOffsets[k].first << ","
         << wrongOffsets[k].second << ")";
      if (k + 1 < wrongOffsets.size()) {
        os << ", ";
      }
    }
    os << "]" << endl;
    std::cerr << os.str();
  }
 
  if (debug_) {
    using std::endl;
    using Teuchos::reduceAll;
    Teuchos::RCP<const Teuchos::Comm<int>> comm = this->getComm();
    const bool localSuccess =
        allRowMapDiagEntriesInColMap && allDiagEntriesFound && allOffsetsCorrect;
    const int numResults = 5;
    int lclResults[5];
    lclResults[0] = allRowMapDiagEntriesInColMap ? 1 : 0;
    lclResults[1] = allDiagEntriesFound ? 1 : 0;
    lclResults[2] = allOffsetsCorrect ? 1 : 0;
    lclResults[3] = noOtherWeirdness ? 1 : 0;
    // min-all-reduce will compute least rank of all the processes
    // that didn't succeed.
    lclResults[4] = !localSuccess ? comm->getRank() : comm->getSize();
 
    int gblResults[5];
    gblResults[0] = 0;
    gblResults[1] = 0;
    gblResults[2] = 0;
    gblResults[3] = 0;
    gblResults[4] = 0;
    reduceAll<int, int>(*comm, Teuchos::REDUCE_MIN,
                        numResults, lclResults, gblResults);
 
    if (gblResults[0] != 1 || gblResults[1] != 1 || gblResults[2] != 1 || gblResults[3] != 1) {
      std::ostringstream os;  // build error message
      os << "Issue(s) that we noticed (on Process " << gblResults[4] << ", "
                                                                        "possibly among others): "
         << endl;
      if (gblResults[0] == 0) {
        os << "  - The column Map does not contain at least one diagonal entry "
              "of the graph."
           << endl;
      }
      if (gblResults[1] == 0) {
        os << "  - On one or more processes, some row does not contain a "
              "diagonal entry."
           << endl;
      }
      if (gblResults[2] == 0) {
        os << "  - On one or more processes, some offsets are incorrect."
           << endl;
      }
      if (gblResults[3] == 0) {
        os << "  - One or more processes had some other error."
           << endl;
      }
      TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::runtime_error, os.str());
    }
  }  // debug_
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalOffRankOffsets(offset_device_view_type& offsets) const {
  using std::endl;
  const char tfecfFuncName[] = "getLocalOffRankOffsets: ";
  const bool verbose         = verbose_;
 
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "getLocalOffRankOffsets");
    std::ostringstream os;
    os << *prefix << "offsets.extent(0)=" << offsets.extent(0)
       << endl;
    std::cerr << os.str();
  }
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!hasColMap(), std::runtime_error, "The graph must have a column Map.");
  // Instead of throwing, we could also copy the rowPtr to k_offRankOffsets_.
 
  const size_t lclNumRows = this->getLocalNumRows();
 
  if (haveLocalOffRankOffsets_ && k_offRankOffsets_.extent(0) == lclNumRows + 1) {
    offsets = k_offRankOffsets_;
    return;
  }
  haveLocalOffRankOffsets_ = false;
 
  const map_type& colMap = *(this->getColMap());
  const map_type& domMap = *(this->getDomainMap());
 
  // mfh 12 Mar 2016: LocalMap works on (CUDA) device.  It has just
  // the subset of Map functionality that we need below.
  auto lclColMap = colMap.getLocalMap();
  auto lclDomMap = domMap.getLocalMap();
 
  // FIXME (mfh 16 Dec 2015) It's easy to thread-parallelize this
  // setup, at least on the host.  For CUDA, we have to use LocalMap
  // (that comes from each of the two Maps).
 
  TEUCHOS_ASSERT(this->isSorted());
  if (isFillComplete()) {
    k_offRankOffsets_ = offset_device_view_type(Kokkos::ViewAllocateWithoutInitializing("offRankOffset"), lclNumRows + 1);
    auto lclGraph     = this->getLocalGraphDevice();
    ::Tpetra::Details::getGraphOffRankOffsets(k_offRankOffsets_,
                                              lclColMap, lclDomMap,
                                              lclGraph);
    offsets                  = k_offRankOffsets_;
    haveLocalOffRankOffsets_ = true;
  } else {
    TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(true, std::logic_error, "Can't get off-rank offsets for non-fill-complete graph");
  }
}
 
namespace {  // (anonymous)
 
// mfh 21 Jan 2016: This is useful for getLocalDiagOffsets (see
// below).  The point is to avoid the deep copy between the input
// Teuchos::ArrayRCP and the internally used Kokkos::View.  We
// can't use UVM to avoid the deep copy with CUDA, because the
// ArrayRCP is a host pointer, while the input to the graph's
// getLocalDiagOffsets method is a device pointer.  Assigning a
// host pointer to a device pointer is incorrect unless the host
// pointer points to host pinned memory.  The goal is to get rid
// of the Teuchos::ArrayRCP overload anyway, so we accept the deep
// copy for backwards compatibility.
//
// We have to use template magic because
// "staticGraph_->getLocalDiagOffsets(offsetsHosts)" won't compile
// if device_type::memory_space is not Kokkos::HostSpace (as is
// the case with CUDA).
 
template <class DeviceType,
          const bool memSpaceIsHostSpace =
              std::is_same<typename DeviceType::memory_space,
                           Kokkos::HostSpace>::value>
struct HelpGetLocalDiagOffsets {};
 
template <class DeviceType>
struct HelpGetLocalDiagOffsets<DeviceType, true> {
  typedef DeviceType device_type;
  typedef Kokkos::View<size_t*, Kokkos::HostSpace,
                       Kokkos::MemoryUnmanaged>
      device_offsets_type;
  typedef Kokkos::View<size_t*, Kokkos::HostSpace,
                       Kokkos::MemoryUnmanaged>
      host_offsets_type;
 
  static device_offsets_type
  getDeviceOffsets(const host_offsets_type& hostOffsets) {
    // Host and device are the same; no need to allocate a
    // temporary device View.
    return hostOffsets;
  }
 
  static void
  copyBackIfNeeded(const host_offsets_type& /* hostOffsets */,
                   const device_offsets_type& /* deviceOffsets */) { /* copy back not needed; host and device are the same */
  }
};
 
template <class DeviceType>
struct HelpGetLocalDiagOffsets<DeviceType, false> {
  typedef DeviceType device_type;
  // We have to do a deep copy, since host memory space != device
  // memory space.  Thus, the device View is managed (we need to
  // allocate a temporary device View).
  typedef Kokkos::View<size_t*, device_type> device_offsets_type;
  typedef Kokkos::View<size_t*, Kokkos::HostSpace,
                       Kokkos::MemoryUnmanaged>
      host_offsets_type;
 
  static device_offsets_type
  getDeviceOffsets(const host_offsets_type& hostOffsets) {
    // Host memory space != device memory space, so we must
    // allocate a temporary device View for the graph.
    return device_offsets_type("offsets", hostOffsets.extent(0));
  }
 
  static void
  copyBackIfNeeded(const host_offsets_type& hostOffsets,
                   const device_offsets_type& deviceOffsets) {
    // DEEP_COPY REVIEW - NOT TESTED
    Kokkos::deep_copy(hostOffsets, deviceOffsets);
  }
};
}  // namespace
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    getLocalDiagOffsets(Teuchos::ArrayRCP<size_t>& offsets) const {
  typedef LocalOrdinal LO;
  const char tfecfFuncName[] = "getLocalDiagOffsets: ";
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!this->hasColMap(), std::runtime_error,
                                        "The graph does not yet have a column Map.");
  const LO myNumRows = static_cast<LO>(this->getLocalNumRows());
  if (static_cast<LO>(offsets.size()) != myNumRows) {
    // NOTE (mfh 21 Jan 2016) This means that the method does not
    // satisfy the strong exception guarantee (no side effects
    // unless successful).
    offsets.resize(myNumRows);
  }
 
  // mfh 21 Jan 2016: This method unfortunately takes a
  // Teuchos::ArrayRCP, which is host memory.  The graph wants a
  // device pointer.  We can't access host memory from the device;
  // that's the wrong direction for UVM.  (It's the right direction
  // for inefficient host pinned memory, but we don't want to use
  // that here.)  Thus, if device memory space != host memory space,
  // we allocate and use a temporary device View to get the offsets.
  // If the two spaces are equal, the template magic makes the deep
  // copy go away.
  typedef HelpGetLocalDiagOffsets<device_type> helper_type;
  typedef typename helper_type::host_offsets_type host_offsets_type;
  // Unmanaged host View that views the output array.
  host_offsets_type hostOffsets(offsets.getRawPtr(), myNumRows);
  // Allocate temp device View if host != device, else reuse host array.
  auto deviceOffsets = helper_type::getDeviceOffsets(hostOffsets);
  // NOT recursion; this calls the overload that takes a device View.
  this->getLocalDiagOffsets(deviceOffsets);
  helper_type::copyBackIfNeeded(hostOffsets, deviceOffsets);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    supportsRowViews() const {
  return true;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    transferAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
                            const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node>& rowTransfer,
                            const Teuchos::RCP<const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node>>& domainTransfer,
                            const Teuchos::RCP<const map_type>& domainMap,
                            const Teuchos::RCP<const map_type>& rangeMap,
                            const Teuchos::RCP<Teuchos::ParameterList>& params) const {
  using std::string;
  using Teuchos::ArrayRCP;
  using Teuchos::ArrayView;
  using Teuchos::Comm;
  using Teuchos::ParameterList;
  using Teuchos::rcp;
  using Teuchos::RCP;
  using Tpetra::Details::getArrayViewFromDualView;
  using Tpetra::Details::packCrsGraphWithOwningPIDs;
  using Tpetra::Details::unpackAndCombineIntoCrsArrays;
  using Tpetra::Details::unpackAndCombineWithOwningPIDsCount;
 
  using LO            = LocalOrdinal;
  using GO            = GlobalOrdinal;
  using NT            = node_type;
  using this_CRS_type = CrsGraph<LO, GO, NT>;
  using ivector_type  = Vector<int, LO, GO, NT>;
 
  const char* prefix = "Tpetra::CrsGraph::transferAndFillComplete: ";
 
  auto MM = rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC Pack-1"));
 
  // Make sure that the input argument rowTransfer is either an
  // Import or an Export.  Import and Export are the only two
  // subclasses of Transfer that we defined, but users might
  // (unwisely, for now at least) decide to implement their own
  // subclasses.  Exclude this possibility.
  const import_type* xferAsImport = dynamic_cast<const import_type*>(&rowTransfer);
  const export_type* xferAsExport = dynamic_cast<const export_type*>(&rowTransfer);
  TEUCHOS_TEST_FOR_EXCEPTION(
      xferAsImport == nullptr && xferAsExport == nullptr, std::invalid_argument,
      prefix << "The 'rowTransfer' input argument must be either an Import or "
                "an Export, and its template parameters must match the corresponding "
                "template parameters of the CrsGraph.");
 
  // Make sure that the input argument domainTransfer is either an
  // Import or an Export.  Import and Export are the only two
  // subclasses of Transfer that we defined, but users might
  // (unwisely, for now at least) decide to implement their own
  // subclasses.  Exclude this possibility.
  Teuchos::RCP<const import_type> xferDomainAsImport =
      Teuchos::rcp_dynamic_cast<const import_type>(domainTransfer);
  Teuchos::RCP<const export_type> xferDomainAsExport =
      Teuchos::rcp_dynamic_cast<const export_type>(domainTransfer);
 
  if (!domainTransfer.is_null()) {
    TEUCHOS_TEST_FOR_EXCEPTION(
        (xferDomainAsImport.is_null() && xferDomainAsExport.is_null()), std::invalid_argument,
        prefix << "The 'domainTransfer' input argument must be either an "
                  "Import or an Export, and its template parameters must match the "
                  "corresponding template parameters of the CrsGraph.");
 
    TEUCHOS_TEST_FOR_EXCEPTION(
        (xferAsImport != nullptr || !xferDomainAsImport.is_null()) &&
            ((xferAsImport != nullptr && xferDomainAsImport.is_null()) ||
             (xferAsImport == nullptr && !xferDomainAsImport.is_null())),
        std::invalid_argument,
        prefix << "The 'rowTransfer' and 'domainTransfer' input arguments "
                  "must be of the same type (either Import or Export).");
 
    TEUCHOS_TEST_FOR_EXCEPTION(
        (xferAsExport != nullptr || !xferDomainAsExport.is_null()) &&
            ((xferAsExport != nullptr && xferDomainAsExport.is_null()) ||
             (xferAsExport == nullptr && !xferDomainAsExport.is_null())),
        std::invalid_argument,
        prefix << "The 'rowTransfer' and 'domainTransfer' input arguments "
                  "must be of the same type (either Import or Export).");
 
  }  // domainTransfer != null
 
  // FIXME (mfh 15 May 2014) Wouldn't communication still be needed,
  // if the source Map is not distributed but the target Map is?
  const bool communication_needed = rowTransfer.getSourceMap()->isDistributed();
 
  //
  // Get the caller's parameters
  //
 
  bool reverseMode  = false;       // Are we in reverse mode?
  bool restrictComm = false;       // Do we need to restrict the communicator?
  RCP<ParameterList> graphparams;  // parameters for the destination graph
  if (!params.is_null()) {
    reverseMode  = params->get("Reverse Mode", reverseMode);
    restrictComm = params->get("Restrict Communicator", restrictComm);
    graphparams  = sublist(params, "CrsGraph");
  }
 
  // Get the new domain and range Maps.  We need some of them for error
  // checking, now that we have the reverseMode parameter.
  RCP<const map_type> MyRowMap = reverseMode ? rowTransfer.getSourceMap() : rowTransfer.getTargetMap();
  RCP<const map_type> MyColMap;  // create this below
  RCP<const map_type> MyDomainMap   = !domainMap.is_null() ? domainMap : getDomainMap();
  RCP<const map_type> MyRangeMap    = !rangeMap.is_null() ? rangeMap : getRangeMap();
  RCP<const map_type> BaseRowMap    = MyRowMap;
  RCP<const map_type> BaseDomainMap = MyDomainMap;
 
  // If the user gave us a nonnull destGraph, then check whether it's
  // "pristine."  That means that it has no entries.
  //
  // FIXME (mfh 15 May 2014) If this is not true on all processes,
  // then this exception test may hang.  It would be better to
  // forward an error flag to the next communication phase.
  if (!destGraph.is_null()) {
    // FIXME (mfh 15 May 2014): The Epetra idiom for checking
    // whether a graph or matrix has no entries on the calling
    // process, is that it is neither locally nor globally indexed.
    // This may change eventually with the Kokkos refactor version
    // of Tpetra, so it would be better just to check the quantity
    // of interest directly.  Note that with the Kokkos refactor
    // version of Tpetra, asking for the total number of entries in
    // a graph or matrix that is not fill complete might require
    // computation (kernel launch), since it is not thread scalable
    // to update a count every time an entry is inserted.
    const bool NewFlag =
        !destGraph->isLocallyIndexed() && !destGraph->isGloballyIndexed();
    TEUCHOS_TEST_FOR_EXCEPTION(!NewFlag, std::invalid_argument,
                               prefix << "The input argument 'destGraph' is only allowed to be nonnull, "
                                         "if its graph is empty (neither locally nor globally indexed).");
 
    // FIXME (mfh 15 May 2014) At some point, we want to change
    // graphs and matrices so that their DistObject Map
    // (this->getMap()) may differ from their row Map.  This will
    // make redistribution for 2-D distributions more efficient.  I
    // hesitate to change this check, because I'm not sure how much
    // the code here depends on getMap() and getRowMap() being the
    // same.
    TEUCHOS_TEST_FOR_EXCEPTION(
        !destGraph->getRowMap()->isSameAs(*MyRowMap), std::invalid_argument,
        prefix << "The (row) Map of the input argument 'destGraph' is not the "
                  "same as the (row) Map specified by the input argument 'rowTransfer'.");
 
    TEUCHOS_TEST_FOR_EXCEPTION(
        !destGraph->checkSizes(*this), std::invalid_argument,
        prefix << "You provided a nonnull destination graph, but checkSizes() "
                  "indicates that it is not a legal legal target for redistribution from "
                  "the source graph (*this).  This may mean that they do not have the "
                  "same dimensions.");
  }
 
  // If forward mode (the default), then *this's (row) Map must be
  // the same as the source Map of the Transfer.  If reverse mode,
  // then *this's (row) Map must be the same as the target Map of
  // the Transfer.
  //
  // FIXME (mfh 15 May 2014) At some point, we want to change graphs
  // and matrices so that their DistObject Map (this->getMap()) may
  // differ from their row Map.  This will make redistribution for
  // 2-D distributions more efficient.  I hesitate to change this
  // check, because I'm not sure how much the code here depends on
  // getMap() and getRowMap() being the same.
  TEUCHOS_TEST_FOR_EXCEPTION(
      !(reverseMode || getRowMap()->isSameAs(*rowTransfer.getSourceMap())),
      std::invalid_argument, prefix << "rowTransfer->getSourceMap() must match this->getRowMap() in forward mode.");
 
  TEUCHOS_TEST_FOR_EXCEPTION(
      !(!reverseMode || getRowMap()->isSameAs(*rowTransfer.getTargetMap())),
      std::invalid_argument, prefix << "rowTransfer->getTargetMap() must match this->getRowMap() in reverse mode.");
 
  // checks for domainTransfer
  TEUCHOS_TEST_FOR_EXCEPTION(
      !xferDomainAsImport.is_null() && !xferDomainAsImport->getTargetMap()->isSameAs(*domainMap),
      std::invalid_argument,
      prefix << "The target map of the 'domainTransfer' input argument must be "
                "the same as the rebalanced domain map 'domainMap'");
 
  TEUCHOS_TEST_FOR_EXCEPTION(
      !xferDomainAsExport.is_null() && !xferDomainAsExport->getSourceMap()->isSameAs(*domainMap),
      std::invalid_argument,
      prefix << "The source map of the 'domainTransfer' input argument must be "
                "the same as the rebalanced domain map 'domainMap'");
 
  // The basic algorithm here is:
  //
  // 1. Call the moral equivalent of "Distor.do" to handle the import.
  // 2. Copy all the Imported and Copy/Permuted data into the raw
  //    CrsGraph pointers, still using GIDs.
  // 3. Call an optimized version of MakeColMap that avoids the
  //    Directory lookups (since the importer knows who owns all the
  //    GIDs) AND reindexes to LIDs.
  // 4. Call expertStaticFillComplete()
 
  // Get information from the Importer
  const size_t NumSameIDs             = rowTransfer.getNumSameIDs();
  ArrayView<const LO> ExportLIDs      = reverseMode ? rowTransfer.getRemoteLIDs() : rowTransfer.getExportLIDs();
  ArrayView<const LO> RemoteLIDs      = reverseMode ? rowTransfer.getExportLIDs() : rowTransfer.getRemoteLIDs();
  ArrayView<const LO> PermuteToLIDs   = reverseMode ? rowTransfer.getPermuteFromLIDs() : rowTransfer.getPermuteToLIDs();
  ArrayView<const LO> PermuteFromLIDs = reverseMode ? rowTransfer.getPermuteToLIDs() : rowTransfer.getPermuteFromLIDs();
  Distributor& Distor                 = rowTransfer.getDistributor();
 
  // Owning PIDs
  Teuchos::Array<int> SourcePids;
  Teuchos::Array<int> TargetPids;
  int MyPID = getComm()->getRank();
 
  // Temp variables for sub-communicators
  RCP<const map_type> ReducedRowMap, ReducedColMap,
      ReducedDomainMap, ReducedRangeMap;
  RCP<const Comm<int>> ReducedComm;
 
  // If the user gave us a null destGraph, then construct the new
  // destination graph.  We will replace its column Map later.
  if (destGraph.is_null()) {
    destGraph = rcp(new this_CRS_type(MyRowMap, 0, graphparams));
  }
 
  /***************************************************/
  /***** 1) First communicator restriction phase ****/
  /***************************************************/
  if (restrictComm) {
    ReducedRowMap = MyRowMap->removeEmptyProcesses();
    ReducedComm   = ReducedRowMap.is_null() ? Teuchos::null : ReducedRowMap->getComm();
    destGraph->removeEmptyProcessesInPlace(ReducedRowMap);
 
    ReducedDomainMap = MyRowMap.getRawPtr() == MyDomainMap.getRawPtr() ? ReducedRowMap : MyDomainMap->replaceCommWithSubset(ReducedComm);
    ReducedRangeMap  = MyRowMap.getRawPtr() == MyRangeMap.getRawPtr() ? ReducedRowMap : MyRangeMap->replaceCommWithSubset(ReducedComm);
 
    // Reset the "my" maps
    MyRowMap    = ReducedRowMap;
    MyDomainMap = ReducedDomainMap;
    MyRangeMap  = ReducedRangeMap;
 
    // Update my PID, if we've restricted the communicator
    if (!ReducedComm.is_null()) {
      MyPID = ReducedComm->getRank();
    } else {
      MyPID = -2;  // For debugging
    }
  } else {
    ReducedComm = MyRowMap->getComm();
  }
 
  /***************************************************/
  /***** 2) From Tpera::DistObject::doTransfer() ****/
  /***************************************************/
  MM = Teuchos::null;
  MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC ImportSetup"));
  // Get the owning PIDs
  RCP<const import_type> MyImporter = getImporter();
 
  // check whether domain maps of source graph and base domain map is the same
  bool bSameDomainMap = BaseDomainMap->isSameAs(*getDomainMap());
 
  if (!restrictComm && !MyImporter.is_null() && bSameDomainMap) {
    // Same domain map as source graph
    //
    // NOTE: This won't work for restrictComm (because the Import
    // doesn't know the restricted PIDs), though writing an
    // optimized version for that case would be easy (Import an
    // IntVector of the new PIDs).  Might want to add this later.
    Import_Util::getPids(*MyImporter, SourcePids, false);
  } else if (restrictComm && !MyImporter.is_null() && bSameDomainMap) {
    // Same domain map as source graph (restricted communicator)
    // We need one import from the domain to the column map
    ivector_type SourceDomain_pids(getDomainMap(), true);
    ivector_type SourceCol_pids(getColMap());
    // SourceDomain_pids contains the restricted pids
    SourceDomain_pids.putScalar(MyPID);
 
    SourceCol_pids.doImport(SourceDomain_pids, *MyImporter, INSERT);
    SourcePids.resize(getColMap()->getLocalNumElements());
    SourceCol_pids.get1dCopy(SourcePids());
  } else if (MyImporter.is_null() && bSameDomainMap) {
    // Graph has no off-process entries
    SourcePids.resize(getColMap()->getLocalNumElements());
    SourcePids.assign(getColMap()->getLocalNumElements(), MyPID);
  } else if (!MyImporter.is_null() &&
             !domainTransfer.is_null()) {
    // general implementation for rectangular matrices with
    // domain map different than SourceGraph domain map.
    // User has to provide a DomainTransfer object. We need
    // to communications (import/export)
 
    // TargetDomain_pids lives on the rebalanced new domain map
    ivector_type TargetDomain_pids(domainMap);
    TargetDomain_pids.putScalar(MyPID);
 
    // SourceDomain_pids lives on the non-rebalanced old domain map
    ivector_type SourceDomain_pids(getDomainMap());
 
    // SourceCol_pids lives on the non-rebalanced old column map
    ivector_type SourceCol_pids(getColMap());
 
    if (!reverseMode && !xferDomainAsImport.is_null()) {
      SourceDomain_pids.doExport(TargetDomain_pids, *xferDomainAsImport, INSERT);
    } else if (reverseMode && !xferDomainAsExport.is_null()) {
      SourceDomain_pids.doExport(TargetDomain_pids, *xferDomainAsExport, INSERT);
    } else if (!reverseMode && !xferDomainAsExport.is_null()) {
      SourceDomain_pids.doImport(TargetDomain_pids, *xferDomainAsExport, INSERT);
    } else if (reverseMode && !xferDomainAsImport.is_null()) {
      SourceDomain_pids.doImport(TargetDomain_pids, *xferDomainAsImport, INSERT);
    } else {
      TEUCHOS_TEST_FOR_EXCEPTION(
          true, std::logic_error,
          prefix << "Should never get here!  Please report this bug to a Tpetra developer.");
    }
    SourceCol_pids.doImport(SourceDomain_pids, *MyImporter, INSERT);
    SourcePids.resize(getColMap()->getLocalNumElements());
    SourceCol_pids.get1dCopy(SourcePids());
  } else if (BaseDomainMap->isSameAs(*BaseRowMap) &&
             getDomainMap()->isSameAs(*getRowMap())) {
    // We can use the rowTransfer + SourceGraph's Import to find out who owns what.
    ivector_type TargetRow_pids(domainMap);
    ivector_type SourceRow_pids(getRowMap());
    ivector_type SourceCol_pids(getColMap());
 
    TargetRow_pids.putScalar(MyPID);
    if (!reverseMode && xferAsImport != nullptr) {
      SourceRow_pids.doExport(TargetRow_pids, *xferAsImport, INSERT);
    } else if (reverseMode && xferAsExport != nullptr) {
      SourceRow_pids.doExport(TargetRow_pids, *xferAsExport, INSERT);
    } else if (!reverseMode && xferAsExport != nullptr) {
      SourceRow_pids.doImport(TargetRow_pids, *xferAsExport, INSERT);
    } else if (reverseMode && xferAsImport != nullptr) {
      SourceRow_pids.doImport(TargetRow_pids, *xferAsImport, INSERT);
    } else {
      TEUCHOS_TEST_FOR_EXCEPTION(
          true, std::logic_error,
          prefix << "Should never get here!  Please report this bug to a Tpetra developer.");
    }
    SourceCol_pids.doImport(SourceRow_pids, *MyImporter, INSERT);
    SourcePids.resize(getColMap()->getLocalNumElements());
    SourceCol_pids.get1dCopy(SourcePids());
  } else {
    TEUCHOS_TEST_FOR_EXCEPTION(
        true, std::invalid_argument,
        prefix << "This method only allows either domainMap == getDomainMap(), "
                  "or (domainMap == rowTransfer.getTargetMap() and getDomainMap() == getRowMap()).");
  }
 
  // Tpetra-specific stuff
  size_t constantNumPackets = destGraph->constantNumberOfPackets();
  if (constantNumPackets == 0) {
    destGraph->reallocArraysForNumPacketsPerLid(ExportLIDs.size(),
                                                RemoteLIDs.size());
  } else {
    // There are a constant number of packets per element.  We
    // already know (from the number of "remote" (incoming)
    // elements) how many incoming elements we expect, so we can
    // resize the buffer accordingly.
    const size_t rbufLen = RemoteLIDs.size() * constantNumPackets;
    destGraph->reallocImportsIfNeeded(rbufLen, false, nullptr);
  }
 
  {
    // packAndPrepare* methods modify numExportPacketsPerLID_.
    destGraph->numExportPacketsPerLID_.modify_host();
    Teuchos::ArrayView<size_t> numExportPacketsPerLID =
        getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
 
    // Pack & Prepare w/ owning PIDs
    packCrsGraphWithOwningPIDs(*this, destGraph->exports_,
                               numExportPacketsPerLID, ExportLIDs,
                               SourcePids, constantNumPackets);
  }
 
  // Do the exchange of remote data.
  MM = Teuchos::null;
  MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC Transfer"));
 
  if (communication_needed) {
    if (reverseMode) {
      if (constantNumPackets == 0) {  // variable number of packets per LID
        // Make sure that host has the latest version, since we're
        // using the version on host.  If host has the latest
        // version, syncing to host does nothing.
        destGraph->numExportPacketsPerLID_.sync_host();
        Teuchos::ArrayView<const size_t> numExportPacketsPerLID =
            getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
        destGraph->numImportPacketsPerLID_.sync_host();
        Teuchos::ArrayView<size_t> numImportPacketsPerLID =
            getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
 
        Distor.doReversePostsAndWaits(destGraph->numExportPacketsPerLID_.view_host(), 1,
                                      destGraph->numImportPacketsPerLID_.view_host());
        size_t totalImportPackets = 0;
        for (Array_size_type i = 0; i < numImportPacketsPerLID.size(); ++i) {
          totalImportPackets += numImportPacketsPerLID[i];
        }
 
        // Reallocation MUST go before setting the modified flag,
        // because it may clear out the flags.
        destGraph->reallocImportsIfNeeded(totalImportPackets, false, nullptr);
        destGraph->imports_.modify_host();
        auto hostImports = destGraph->imports_.view_host();
        // This is a legacy host pack/unpack path, so use the host
        // version of exports_.
        destGraph->exports_.sync_host();
        auto hostExports = destGraph->exports_.view_host();
        Distor.doReversePostsAndWaits(hostExports,
                                      numExportPacketsPerLID,
                                      hostImports,
                                      numImportPacketsPerLID);
      } else {  // constant number of packets per LI
        destGraph->imports_.modify_host();
        auto hostImports = destGraph->imports_.view_host();
        // This is a legacy host pack/unpack path, so use the host
        // version of exports_.
        destGraph->exports_.sync_host();
        auto hostExports = destGraph->exports_.view_host();
        Distor.doReversePostsAndWaits(hostExports,
                                      constantNumPackets,
                                      hostImports);
      }
    } else {                          // forward mode (the default)
      if (constantNumPackets == 0) {  // variable number of packets per LID
        // Make sure that host has the latest version, since we're
        // using the version on host.  If host has the latest
        // version, syncing to host does nothing.
        destGraph->numExportPacketsPerLID_.sync_host();
        destGraph->numImportPacketsPerLID_.sync_host();
        Distor.doPostsAndWaits(destGraph->numExportPacketsPerLID_.view_host(), 1,
                               destGraph->numImportPacketsPerLID_.view_host());
 
        Teuchos::ArrayView<const size_t> numImportPacketsPerLID =
            getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
        size_t totalImportPackets = 0;
        for (Array_size_type i = 0; i < numImportPacketsPerLID.size(); ++i) {
          totalImportPackets += numImportPacketsPerLID[i];
        }
 
        // Reallocation MUST go before setting the modified flag,
        // because it may clear out the flags.
        destGraph->reallocImportsIfNeeded(totalImportPackets, false, nullptr);
        destGraph->imports_.modify_host();
        auto hostImports = destGraph->imports_.view_host();
        // This is a legacy host pack/unpack path, so use the host
        // version of exports_.
        destGraph->exports_.sync_host();
        auto hostExports = destGraph->exports_.view_host();
        Teuchos::ArrayView<const size_t> numExportPacketsPerLID =
            getArrayViewFromDualView(destGraph->numExportPacketsPerLID_);
        Distor.doPostsAndWaits(hostExports, numExportPacketsPerLID, hostImports, numImportPacketsPerLID);
      } else {  // constant number of packets per LID
        destGraph->imports_.modify_host();
        auto hostImports = destGraph->imports_.view_host();
        // This is a legacy host pack/unpack path, so use the host
        // version of exports_.
        destGraph->exports_.sync_host();
        auto hostExports = destGraph->exports_.view_host();
        Distor.doPostsAndWaits(hostExports, constantNumPackets, hostImports);
      }
    }
  }
 
  /*********************************************************************/
  /**** 3) Copy all of the Same/Permute/Remote data into CSR_arrays ****/
  /*********************************************************************/
 
  MM = Teuchos::null;
  MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC Unpack-1"));
 
  // Backwards compatibility measure.  We'll use this again below.
  destGraph->numImportPacketsPerLID_.sync_host();
  Teuchos::ArrayView<const size_t> numImportPacketsPerLID =
      getArrayViewFromDualView(destGraph->numImportPacketsPerLID_);
  destGraph->imports_.sync_host();
  Teuchos::ArrayView<const packet_type> hostImports =
      getArrayViewFromDualView(destGraph->imports_);
  size_t mynnz =
      unpackAndCombineWithOwningPIDsCount(*this, RemoteLIDs, hostImports,
                                          numImportPacketsPerLID,
                                          constantNumPackets, INSERT,
                                          NumSameIDs, PermuteToLIDs, PermuteFromLIDs);
  size_t N = BaseRowMap->getLocalNumElements();
 
  // Allocations
  ArrayRCP<size_t> CSR_rowptr(N + 1);
  ArrayRCP<GO> CSR_colind_GID;
  ArrayRCP<LO> CSR_colind_LID;
  CSR_colind_GID.resize(mynnz);
 
  // If LO and GO are the same, we can reuse memory when
  // converting the column indices from global to local indices.
  if (typeid(LO) == typeid(GO)) {
    CSR_colind_LID = Teuchos::arcp_reinterpret_cast<LO>(CSR_colind_GID);
  } else {
    CSR_colind_LID.resize(mynnz);
  }
 
  // FIXME (mfh 15 May 2014) Why can't we abstract this out as an
  // unpackAndCombine method on a "CrsArrays" object?  This passing
  // in a huge list of arrays is icky.  Can't we have a bit of an
  // abstraction?  Implementing a concrete DistObject subclass only
  // takes five methods.
  unpackAndCombineIntoCrsArrays(*this, RemoteLIDs, hostImports,
                                numImportPacketsPerLID, constantNumPackets,
                                INSERT, NumSameIDs, PermuteToLIDs,
                                PermuteFromLIDs, N, mynnz, MyPID,
                                CSR_rowptr(), CSR_colind_GID(),
                                SourcePids(), TargetPids);
 
  /**************************************************************/
  /**** 4) Call Optimized MakeColMap w/ no Directory Lookups ****/
  /**************************************************************/
  MM = Teuchos::null;
  MM = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC Unpack-2"));
  // Call an optimized version of makeColMap that avoids the
  // Directory lookups (since the Import object knows who owns all
  // the GIDs).
  Teuchos::Array<int> RemotePids;
  Import_Util::lowCommunicationMakeColMapAndReindex(CSR_rowptr(),
                                                    CSR_colind_LID(),
                                                    CSR_colind_GID(),
                                                    BaseDomainMap,
                                                    TargetPids, RemotePids,
                                                    MyColMap,
                                                    params);
 
  /*******************************************************/
  /**** 4) Second communicator restriction phase      ****/
  /*******************************************************/
  if (restrictComm) {
    ReducedColMap = (MyRowMap.getRawPtr() == MyColMap.getRawPtr()) ? ReducedRowMap : MyColMap->replaceCommWithSubset(ReducedComm);
    MyColMap      = ReducedColMap;  // Reset the "my" maps
  }
 
  // Replace the col map
  destGraph->replaceColMap(MyColMap);
 
  // Short circuit if the processor is no longer in the communicator
  //
  // NOTE: Epetra replaces modifies all "removed" processes so they
  // have a dummy (serial) Map that doesn't touch the original
  // communicator.  Duplicating that here might be a good idea.
  if (ReducedComm.is_null()) {
    return;
  }
 
  /***************************************************/
  /**** 5) Sort                                   ****/
  /***************************************************/
  if ((!reverseMode && xferAsImport != nullptr) ||
      (reverseMode && xferAsExport != nullptr)) {
    Import_Util::sortCrsEntries(CSR_rowptr(),
                                CSR_colind_LID());
  } else if ((!reverseMode && xferAsExport != nullptr) ||
             (reverseMode && xferAsImport != nullptr)) {
    Import_Util::sortAndMergeCrsEntries(CSR_rowptr(),
                                        CSR_colind_LID());
    if (CSR_rowptr[N] != mynnz) {
      CSR_colind_LID.resize(CSR_rowptr[N]);
    }
  } else {
    TEUCHOS_TEST_FOR_EXCEPTION(
        true, std::logic_error,
        prefix << "Should never get here!  Please report this bug to a Tpetra developer.");
  }
  /***************************************************/
  /**** 6) Reset the colmap and the arrays        ****/
  /***************************************************/
 
  // Call constructor for the new graph (restricted as needed)
  //
  destGraph->setAllIndices(CSR_rowptr, CSR_colind_LID);
 
  /***************************************************/
  /**** 7) Build Importer & Call ESFC             ****/
  /***************************************************/
  // Pre-build the importer using the existing PIDs
  Teuchos::ParameterList esfc_params;
  MM                        = Teuchos::null;
  MM                        = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC CreateImporter"));
  RCP<import_type> MyImport = rcp(new import_type(MyDomainMap, MyColMap, RemotePids));
  MM                        = Teuchos::null;
  MM                        = Teuchos::rcp(new Tpetra::Details::ProfilingRegion("Tpetra CrsGraph TAFC ESFC"));
 
  if (!params.is_null())
    esfc_params.set("compute global constants", params->get("compute global constants", true));
 
  destGraph->expertStaticFillComplete(MyDomainMap, MyRangeMap,
                                      MyImport, Teuchos::null, rcp(&esfc_params, false));
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    importAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
                          const import_type& importer,
                          const Teuchos::RCP<const map_type>& domainMap,
                          const Teuchos::RCP<const map_type>& rangeMap,
                          const Teuchos::RCP<Teuchos::ParameterList>& params) const {
  transferAndFillComplete(destGraph, importer, Teuchos::null, domainMap, rangeMap, params);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    importAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
                          const import_type& rowImporter,
                          const import_type& domainImporter,
                          const Teuchos::RCP<const map_type>& domainMap,
                          const Teuchos::RCP<const map_type>& rangeMap,
                          const Teuchos::RCP<Teuchos::ParameterList>& params) const {
  transferAndFillComplete(destGraph, rowImporter, Teuchos::rcpFromRef(domainImporter), domainMap, rangeMap, params);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    exportAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
                          const export_type& exporter,
                          const Teuchos::RCP<const map_type>& domainMap,
                          const Teuchos::RCP<const map_type>& rangeMap,
                          const Teuchos::RCP<Teuchos::ParameterList>& params) const {
  transferAndFillComplete(destGraph, exporter, Teuchos::null, domainMap, rangeMap, params);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    exportAndFillComplete(Teuchos::RCP<CrsGraph<LocalOrdinal, GlobalOrdinal, Node>>& destGraph,
                          const export_type& rowExporter,
                          const export_type& domainExporter,
                          const Teuchos::RCP<const map_type>& domainMap,
                          const Teuchos::RCP<const map_type>& rangeMap,
                          const Teuchos::RCP<Teuchos::ParameterList>& params) const {
  transferAndFillComplete(destGraph, rowExporter, Teuchos::rcpFromRef(domainExporter), domainMap, rangeMap, params);
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    swap(CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& graph) {
  std::swap(graph.need_sync_host_uvm_access, this->need_sync_host_uvm_access);
 
  std::swap(graph.rowMap_, this->rowMap_);
  std::swap(graph.colMap_, this->colMap_);
  std::swap(graph.rangeMap_, this->rangeMap_);
  std::swap(graph.domainMap_, this->domainMap_);
 
  std::swap(graph.importer_, this->importer_);
  std::swap(graph.exporter_, this->exporter_);
 
  std::swap(graph.nodeMaxNumRowEntries_, this->nodeMaxNumRowEntries_);
 
  std::swap(graph.globalNumEntries_, this->globalNumEntries_);
  std::swap(graph.globalMaxNumRowEntries_, this->globalMaxNumRowEntries_);
 
  std::swap(graph.numAllocForAllRows_, this->numAllocForAllRows_);
 
  std::swap(graph.rowPtrsPacked_dev_, this->rowPtrsPacked_dev_);
  std::swap(graph.rowPtrsPacked_host_, this->rowPtrsPacked_host_);
 
  std::swap(graph.rowPtrsUnpacked_dev_, this->rowPtrsUnpacked_dev_);
  std::swap(graph.rowPtrsUnpacked_host_, this->rowPtrsUnpacked_host_);
  std::swap(graph.packedUnpackedRowPtrsMatch_, this->packedUnpackedRowPtrsMatch_);
 
  std::swap(graph.k_offRankOffsets_, this->k_offRankOffsets_);
 
  std::swap(graph.lclIndsUnpacked_wdv, this->lclIndsUnpacked_wdv);
  std::swap(graph.gblInds_wdv, this->gblInds_wdv);
  std::swap(graph.lclIndsPacked_wdv, this->lclIndsPacked_wdv);
 
  std::swap(graph.storageStatus_, this->storageStatus_);
 
  std::swap(graph.indicesAreAllocated_, this->indicesAreAllocated_);
  std::swap(graph.indicesAreLocal_, this->indicesAreLocal_);
  std::swap(graph.indicesAreGlobal_, this->indicesAreGlobal_);
  std::swap(graph.fillComplete_, this->fillComplete_);
  std::swap(graph.indicesAreSorted_, this->indicesAreSorted_);
  std::swap(graph.noRedundancies_, this->noRedundancies_);
  std::swap(graph.haveLocalConstants_, this->haveLocalConstants_);
  std::swap(graph.haveGlobalConstants_, this->haveGlobalConstants_);
  std::swap(graph.haveLocalOffRankOffsets_, this->haveLocalOffRankOffsets_);
 
  std::swap(graph.sortGhostsAssociatedWithEachProcessor_, this->sortGhostsAssociatedWithEachProcessor_);
 
  std::swap(graph.k_numAllocPerRow_, this->k_numAllocPerRow_);  // View
  std::swap(graph.k_numRowEntries_, this->k_numRowEntries_);    // View
  std::swap(graph.nonlocals_, this->nonlocals_);                // std::map
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
bool CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
    isIdenticalTo(const CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& graph) const {
  auto compare_nonlocals = [&](const nonlocals_type& m1, const nonlocals_type& m2) {
    bool output = true;
    output      = m1.size() == m2.size() ? output : false;
    for (auto& it_m : m1) {
      size_t key = it_m.first;
      output     = m2.find(key) != m2.end() ? output : false;
      if (output) {
        auto v1 = m1.find(key)->second;
        auto v2 = m2.find(key)->second;
        std::sort(v1.begin(), v1.end());
        std::sort(v2.begin(), v2.end());
 
        output = v1.size() == v2.size() ? output : false;
        for (size_t i = 0; output && i < v1.size(); i++) {
          output = v1[i] == v2[i] ? output : false;
        }
      }
    }
    return output;
  };
 
  bool output = true;
 
  output = this->rowMap_->isSameAs(*(graph.rowMap_)) ? output : false;
  output = this->colMap_->isSameAs(*(graph.colMap_)) ? output : false;
  output = this->rangeMap_->isSameAs(*(graph.rangeMap_)) ? output : false;
  output = this->domainMap_->isSameAs(*(graph.domainMap_)) ? output : false;
 
  output = this->nodeMaxNumRowEntries_ == graph.nodeMaxNumRowEntries_ ? output : false;
 
  output = this->globalNumEntries_ == graph.globalNumEntries_ ? output : false;
  output = this->globalMaxNumRowEntries_ == graph.globalMaxNumRowEntries_ ? output : false;
 
  output = this->numAllocForAllRows_ == graph.numAllocForAllRows_ ? output : false;
 
  output = this->storageStatus_ == graph.storageStatus_ ? output : false;  // EStorageStatus is an enum
 
  output = this->indicesAreAllocated_ == graph.indicesAreAllocated_ ? output : false;
  output = this->indicesAreLocal_ == graph.indicesAreLocal_ ? output : false;
  output = this->indicesAreGlobal_ == graph.indicesAreGlobal_ ? output : false;
  output = this->fillComplete_ == graph.fillComplete_ ? output : false;
  output = this->indicesAreSorted_ == graph.indicesAreSorted_ ? output : false;
  output = this->noRedundancies_ == graph.noRedundancies_ ? output : false;
  output = this->haveLocalConstants_ == graph.haveLocalConstants_ ? output : false;
  output = this->haveGlobalConstants_ == graph.haveGlobalConstants_ ? output : false;
  output = this->haveLocalOffRankOffsets_ == graph.haveLocalOffRankOffsets_ ? output : false;
  output = this->sortGhostsAssociatedWithEachProcessor_ == graph.sortGhostsAssociatedWithEachProcessor_ ? output : false;
 
  // Compare nonlocals_ -- std::map<GlobalOrdinal, std::vector<GlobalOrdinal> >
  // nonlocals_ isa std::map<GO, std::vector<GO> >
  output = compare_nonlocals(this->nonlocals_, graph.nonlocals_) ? output : false;
 
  // Compare k_numAllocPerRow_ isa Kokkos::View::host_mirror_type
  // - since this is a host_mirror_type type, it should be in host memory already
  output = this->k_numAllocPerRow_.extent(0) == graph.k_numAllocPerRow_.extent(0) ? output : false;
  if (output && this->k_numAllocPerRow_.extent(0) > 0) {
    for (size_t i = 0; output && i < this->k_numAllocPerRow_.extent(0); i++)
      output = this->k_numAllocPerRow_(i) == graph.k_numAllocPerRow_(i) ? output : false;
  }
 
  // Compare k_numRowEntries_ isa Kokkos::View::host_mirror_type
  // - since this is a host_mirror_type type, it should be in host memory already
  output = this->k_numRowEntries_.extent(0) == graph.k_numRowEntries_.extent(0) ? output : false;
  if (output && this->k_numRowEntries_.extent(0) > 0) {
    for (size_t i = 0; output && i < this->k_numRowEntries_.extent(0); i++)
      output = this->k_numRowEntries_(i) == graph.k_numRowEntries_(i) ? output : false;
  }
 
  // Compare this->k_rowPtrs_ isa Kokkos::View<LocalOrdinal*, ...>
  {
    auto rowPtrsThis  = this->getRowPtrsUnpackedHost();
    auto rowPtrsGraph = graph.getRowPtrsUnpackedHost();
    output            = rowPtrsThis.extent(0) == rowPtrsGraph.extent(0) ? output : false;
    for (size_t i = 0; output && i < rowPtrsThis.extent(0); i++)
      output = rowPtrsThis(i) == rowPtrsGraph(i) ? output : false;
  }
 
  // Compare lclIndsUnpacked_wdv isa Kokkos::View<LocalOrdinal*, ...>
  output = this->lclIndsUnpacked_wdv.extent(0) == graph.lclIndsUnpacked_wdv.extent(0) ? output : false;
  if (output && this->lclIndsUnpacked_wdv.extent(0) > 0) {
    auto indThis  = this->lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
    auto indGraph = graph.lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
    for (size_t i = 0; output && i < indThis.extent(0); i++)
      output = indThis(i) == indGraph(i) ? output : false;
  }
 
  // Compare gblInds_wdv isa Kokkos::View<GlobalOrdinal*, ...>
  output = this->gblInds_wdv.extent(0) == graph.gblInds_wdv.extent(0) ? output : false;
  if (output && this->gblInds_wdv.extent(0) > 0) {
    auto indtThis  = this->gblInds_wdv.getHostView(Access::ReadOnly);
    auto indtGraph = graph.gblInds_wdv.getHostView(Access::ReadOnly);
    for (size_t i = 0; output && i < indtThis.extent(0); i++)
      output = indtThis(i) == indtGraph(i) ? output : false;
  }
 
  // Check lclGraph_ isa
  // KokkosSparse::StaticCrsGraph<LocalOrdinal, Kokkos::LayoutLeft, execution_space>
  // KokkosSparse::StaticCrsGraph has 3 data members in it:
  //   Kokkos::View<size_type*, ...> row_map
  //           (local_graph_device_type::row_map_type)
  //   Kokkos::View<data_type*, ...> entries
  //           (local_graph_device_type::entries_type)
  //   Kokkos::View<size_type*, ...> row_block_offsets
  //           (local_graph_device_type::row_block_type)
  // There is currently no KokkosSparse::StaticCrsGraph comparison function
  // that's built-in, so we will just compare
  // the three data items here. This can be replaced if Kokkos ever
  // puts in its own comparison routine.
  local_graph_host_type thisLclGraph  = this->getLocalGraphHost();
  local_graph_host_type graphLclGraph = graph.getLocalGraphHost();
 
  output = thisLclGraph.row_map.extent(0) == graphLclGraph.row_map.extent(0)
               ? output
               : false;
  if (output && thisLclGraph.row_map.extent(0) > 0) {
    auto lclGraph_rowmap_host_this  = thisLclGraph.row_map;
    auto lclGraph_rowmap_host_graph = graphLclGraph.row_map;
    for (size_t i = 0; output && i < lclGraph_rowmap_host_this.extent(0); i++)
      output = lclGraph_rowmap_host_this(i) == lclGraph_rowmap_host_graph(i)
                   ? output
                   : false;
  }
 
  output = thisLclGraph.entries.extent(0) == graphLclGraph.entries.extent(0)
               ? output
               : false;
  if (output && thisLclGraph.entries.extent(0) > 0) {
    auto lclGraph_entries_host_this  = thisLclGraph.entries;
    auto lclGraph_entries_host_graph = graphLclGraph.entries;
    for (size_t i = 0; output && i < lclGraph_entries_host_this.extent(0); i++)
      output = lclGraph_entries_host_this(i) == lclGraph_entries_host_graph(i)
                   ? output
                   : false;
  }
 
  output =
      thisLclGraph.row_block_offsets.extent(0) ==
              graphLclGraph.row_block_offsets.extent(0)
          ? output
          : false;
  if (output && thisLclGraph.row_block_offsets.extent(0) > 0) {
    auto lclGraph_rbo_host_this  = thisLclGraph.row_block_offsets;
    auto lclGraph_rbo_host_graph = graphLclGraph.row_block_offsets;
    for (size_t i = 0; output && i < lclGraph_rbo_host_this.extent(0); i++)
      output = lclGraph_rbo_host_this(i) == lclGraph_rbo_host_graph(i)
                   ? output
                   : false;
  }
 
  // For Importer and Exporter, we don't need to explicitly check them since
  // they will be consistent with the maps.
  // Note: importer_  isa Teuchos::RCP<const import_type>
  //       exporter_  isa Teuchos::RCP<const export_type>
 
  return output;
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::insertGlobalIndicesDevice(
    const CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& srcCrsGraph,
    CrsGraph<LocalOrdinal, GlobalOrdinal, Node>& tgtCrsGraph,
    const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteToLIDs,
    const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteFromLIDs,
    LocalOrdinal loopEnd) {
  using crs_graph_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
  using LO             = LocalOrdinal;
  using GO             = GlobalOrdinal;
  typedef typename crs_graph_type::global_inds_device_view_type::non_const_value_type global_inds_device_value_t;
  typedef typename Node::execution_space exec_space;
  typedef Kokkos::RangePolicy<exec_space, LO> range_type;
 
  const LocalOrdinal LINV  = Teuchos::OrdinalTraits<LocalOrdinal>::invalid();
  const GlobalOrdinal GINV = Teuchos::OrdinalTraits<GlobalOrdinal>::invalid();
 
  using local_map_type          = typename crs_graph_type::map_type::local_map_type;
  local_map_type srcRowMapLocal = srcCrsGraph.getRowMap()->getLocalMap();
  local_map_type srcColMapLocal = srcCrsGraph.getColMap()->getLocalMap();
  local_map_type tgtRowMapLocal = tgtCrsGraph.getRowMap()->getLocalMap();
 
  auto tgtLocalRowPtrsDevice = tgtCrsGraph.getRowPtrsUnpackedDevice();
  auto tgtGlobalColInds      = tgtCrsGraph.gblInds_wdv.getDeviceView(Access::ReadWrite);
  auto srcLocalRowPtrsDevice = srcCrsGraph.getLocalRowPtrsDevice();
  auto srcLocalColIndsDevice = srcCrsGraph.lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly);
 
  typename crs_graph_type::num_row_entries_type::non_const_type h_numRowEnt = tgtCrsGraph.k_numRowEntries_;
 
  auto k_numRowEnt = Kokkos::create_mirror_view_and_copy(device_type(), h_numRowEnt);
 
  const bool sorted = false;
 
  bool hasMap            = permuteFromLIDs.extent(0) > 0;
  auto permuteToLIDs_d   = permuteToLIDs.view_device();
  auto permuteFromLIDs_d = permuteFromLIDs.view_device();
 
#ifdef CRSGRAPH_INNER_ABORT
#undef CRSGRAPH_INNER_ABORT
#endif
 
#ifdef KOKKOS_ENABLE_SYCL
#define CRSGRAPH_INNER_ABORT(lin)                                                      \
  do {                                                                                 \
    sycl::ext::oneapi::experimental::printf("ERROR: Tpetra_CrsGraph_def.hpp:%d", lin); \
    Kokkos::abort("error");                                                            \
  } while (0)
#else
#define CRSGRAPH_INNER_ABORT(lin)                     \
  do {                                                \
    printf("ERROR: Tpetra_CrsGraph_def.hpp:%d", lin); \
    Kokkos::abort("error");                           \
  } while (0)
#endif
 
  Kokkos::parallel_for(
      "Tpetra_CrsGraph::copyAndPermuteNew",
      range_type(0, loopEnd),
      KOKKOS_LAMBDA(const LO sourceLID) {
        auto srcLid = sourceLID;
        auto tgtLid = sourceLID;
        if (hasMap) {
          srcLid = permuteFromLIDs_d(srcLid);
          tgtLid = permuteToLIDs_d(tgtLid);
        }
        auto srcGid = srcRowMapLocal.getGlobalElement(srcLid);
        if (srcGid == GINV) CRSGRAPH_INNER_ABORT(__LINE__);
        auto tgtGid      = tgtRowMapLocal.getGlobalElement(tgtLid);
        auto tgtLocalRow = tgtRowMapLocal.getLocalElement(tgtGid);
        if (tgtLocalRow == LINV) CRSGRAPH_INNER_ABORT(__LINE__);
        if (tgtLocalRow != tgtLid) CRSGRAPH_INNER_ABORT(__LINE__);
        auto tgtNumEntries = k_numRowEnt(tgtLocalRow);
 
        // FIXME no auto use
        auto start     = srcLocalRowPtrsDevice(srcLid);
        auto end       = srcLocalRowPtrsDevice(srcLid + 1);
        auto rowLength = (end - start);
 
        auto tstart = tgtLocalRowPtrsDevice(tgtLocalRow);
        auto tend   = tstart + tgtNumEntries;
        auto tend1  = tgtLocalRowPtrsDevice(tgtLocalRow + 1);
 
        const size_t num_avail = (tend1 < tend) ? size_t(0) : tend1 - tend;
        size_t num_inserted    = 0;
 
        global_inds_device_value_t* tgtGlobalColIndsPtr = tgtGlobalColInds.data();
 
        size_t hint = 0;
        for (size_t j = 0; j < rowLength; j++) {
          auto ci = srcLocalColIndsDevice(start + j);
          GO gi   = srcColMapLocal.getGlobalElement(ci);
          if (gi == GINV) CRSGRAPH_INNER_ABORT(__LINE__);
          auto numInTgtRow = (tend - tstart);
 
          const size_t offset = KokkosSparse::findRelOffset(
              tgtGlobalColIndsPtr + tstart, numInTgtRow, gi, hint, sorted);
 
          if (offset == numInTgtRow) {
            if (num_inserted >= num_avail) {  // not enough room
              Kokkos::abort("num_avail");
            }
            tgtGlobalColIndsPtr[tstart + offset] = gi;
            ++tend;
            hint = offset + 1;
            ++num_inserted;
          }
        }
        k_numRowEnt(tgtLocalRow) += num_inserted;
        return size_t(0);
      });
  Kokkos::deep_copy(tgtCrsGraph.k_numRowEntries_, k_numRowEnt);
  tgtCrsGraph.setLocallyModified();
}
 
template <class LocalOrdinal, class GlobalOrdinal, class Node>
void CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::copyAndPermuteNew(
    const row_graph_type& srcRowGraph,
    row_graph_type& tgtRowGraph,
    const size_t numSameIDs,
    const Kokkos::DualView<const local_ordinal_type*,
                           buffer_device_type>& permuteToLIDs,
    const Kokkos::DualView<const local_ordinal_type*,
                           buffer_device_type>& permuteFromLIDs,
    const CombineMode CM) {
  using std::endl;
  using LO                   = local_ordinal_type;
  using GO                   = global_ordinal_type;
  const char tfecfFuncName[] = "copyAndPermuteNew: ";
  const bool verbose         = verbose_;
 
  Details::ProfilingRegion regionCAP("Tpetra::CrsGraph::copyAndPermuteNew");
  std::unique_ptr<std::string> prefix;
  if (verbose) {
    prefix = this->createPrefix("CrsGraph", "copyAndPermuteNew");
    std::ostringstream os;
    os << *prefix << endl;
    std::cerr << os.str();
  }
 
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      permuteToLIDs.extent(0) != permuteFromLIDs.extent(0),
      std::runtime_error,
      "permuteToLIDs.extent(0) = " << permuteToLIDs.extent(0) << " != permuteFromLIDs.extent(0) = " << permuteFromLIDs.extent(0) << ".");
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Compute padding" << endl;
    std::cerr << os.str();
  }
 
  using crs_graph_type                 = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
  const crs_graph_type* srcCrsGraphPtr = dynamic_cast<const crs_graph_type*>(&srcRowGraph);
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      !srcCrsGraphPtr, std::runtime_error, "error srcGraph type= " << typeid(srcRowGraph).name());
  const crs_graph_type& srcCrsGraph = *srcCrsGraphPtr;
 
  crs_graph_type* tgtCrsGraphPtr = dynamic_cast<crs_graph_type*>(&tgtRowGraph);
  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
      !tgtCrsGraphPtr, std::runtime_error, "error tgtGraph type= " << typeid(tgtRowGraph).name());
 
  crs_graph_type& tgtCrsGraph = *tgtCrsGraphPtr;
  auto padding                = tgtCrsGraph.computeCrsPadding(
                     srcRowGraph, numSameIDs, permuteToLIDs, permuteFromLIDs, verbose);
  tgtCrsGraph.applyCrsPadding(*padding, verbose);
 
  const map_type& srcRowMap = *(srcRowGraph.getRowMap());
  const map_type& tgtRowMap = *(tgtRowGraph.getRowMap());
  const bool src_filled     = srcRowGraph.isFillComplete();
  nonconst_global_inds_host_view_type row_copy;
  LO myid = 0;
 
  //
  // "Copy" part of "copy and permute."
  //
  LO numSameIDs_as_LID = static_cast<LO>(numSameIDs);
 
  if (src_filled || srcCrsGraphPtr == nullptr) {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "src_filled || srcCrsGraph == nullptr" << endl;
      std::cerr << os.str();
    }
    // If the source graph is fill complete, we can't use view mode,
    // because the data might be stored in a different format not
    // compatible with the expectations of view mode.  Also, if the
    // source graph is not a CrsGraph, we can't use view mode,
    // because RowGraph only provides copy mode access to the data.
    Kokkos::DualView<const local_ordinal_type*, buffer_device_type> noPermute;
    insertGlobalIndicesDevice(srcCrsGraph, tgtCrsGraph,
                              noPermute, noPermute,
                              numSameIDs_as_LID);
  } else {
    if (verbose) {
      std::ostringstream os;
      os << *prefix << "! src_filled && srcCrsGraph != nullptr" << endl;
      std::cerr << os.str();
    }
    for (size_t i = 0; i < numSameIDs; ++i, ++myid) {
      const GO gid = srcRowMap.getGlobalElement(myid);
      global_inds_host_view_type row;
      srcCrsGraph.getGlobalRowView(gid, row);
      tgtCrsGraph.insertGlobalIndices(gid, row.extent(0), row.data());
    }
  }
 
  //
  // "Permute" part of "copy and permute."
  //
  auto permuteToLIDs_h   = permuteToLIDs.view_host();
  auto permuteFromLIDs_h = permuteFromLIDs.view_host();
  auto permuteToLIDs_d   = permuteToLIDs.view_device();
  auto permuteFromLIDs_d = permuteFromLIDs.view_device();
 
  if (src_filled || srcCrsGraphPtr == nullptr) {
    insertGlobalIndicesDevice(
        srcCrsGraph,
        tgtCrsGraph,
        permuteToLIDs,
        permuteFromLIDs,  // note reversed arg order, tgt, then src
        static_cast<LO>(permuteToLIDs_h.extent(0)));
  } else {
    for (LO i = 0; i < static_cast<LO>(permuteToLIDs_h.extent(0)); ++i) {
      const GO mygid  = tgtRowMap.getGlobalElement(permuteToLIDs_h[i]);
      const GO srcgid = srcRowMap.getGlobalElement(permuteFromLIDs_h[i]);
      global_inds_host_view_type row;
      srcCrsGraph.getGlobalRowView(srcgid, row);
      tgtCrsGraph.insertGlobalIndices(mygid, row.extent(0), row.data());
    }
  }
 
  if (verbose) {
    std::ostringstream os;
    os << *prefix << "Done" << endl;
    std::cerr << os.str();
  }
}
 
}  // namespace Tpetra
 
//
// Explicit instantiation macros
//
// Must be expanded from within the Tpetra namespace!
//
 
#define TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE)                                       \
  template <>                                                                                                \
  Teuchos::RCP<CrsGraph<LO, GO, NODE>>                                                                       \
  importAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph,               \
                                const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type,                     \
                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,                    \
                                             CrsGraph<LO, GO, NODE>::node_type>& importer,                   \
                                const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type,     \
                                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,    \
                                                             CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
                                const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type,     \
                                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,    \
                                                             CrsGraph<LO, GO, NODE>::node_type>>& rangeMap,  \
                                const Teuchos::RCP<Teuchos::ParameterList>& params);
 
#define TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE)                                   \
  template <>                                                                                                \
  Teuchos::RCP<CrsGraph<LO, GO, NODE>>                                                                       \
  importAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph,               \
                                const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type,                     \
                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,                    \
                                             CrsGraph<LO, GO, NODE>::node_type>& rowImporter,                \
                                const Import<CrsGraph<LO, GO, NODE>::local_ordinal_type,                     \
                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,                    \
                                             CrsGraph<LO, GO, NODE>::node_type>& domainImporter,             \
                                const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type,     \
                                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,    \
                                                             CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
                                const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type,     \
                                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,    \
                                                             CrsGraph<LO, GO, NODE>::node_type>>& rangeMap,  \
                                const Teuchos::RCP<Teuchos::ParameterList>& params);
 
#define TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE)                                       \
  template <>                                                                                                \
  Teuchos::RCP<CrsGraph<LO, GO, NODE>>                                                                       \
  exportAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph,               \
                                const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type,                     \
                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,                    \
                                             CrsGraph<LO, GO, NODE>::node_type>& exporter,                   \
                                const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type,     \
                                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,    \
                                                             CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
                                const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type,     \
                                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,    \
                                                             CrsGraph<LO, GO, NODE>::node_type>>& rangeMap,  \
                                const Teuchos::RCP<Teuchos::ParameterList>& params);
 
#define TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE)                                   \
  template <>                                                                                                \
  Teuchos::RCP<CrsGraph<LO, GO, NODE>>                                                                       \
  exportAndFillCompleteCrsGraph(const Teuchos::RCP<const CrsGraph<LO, GO, NODE>>& sourceGraph,               \
                                const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type,                     \
                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,                    \
                                             CrsGraph<LO, GO, NODE>::node_type>& rowExporter,                \
                                const Export<CrsGraph<LO, GO, NODE>::local_ordinal_type,                     \
                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,                    \
                                             CrsGraph<LO, GO, NODE>::node_type>& domainExporter,             \
                                const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type,     \
                                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,    \
                                                             CrsGraph<LO, GO, NODE>::node_type>>& domainMap, \
                                const Teuchos::RCP<const Map<CrsGraph<LO, GO, NODE>::local_ordinal_type,     \
                                                             CrsGraph<LO, GO, NODE>::global_ordinal_type,    \
                                                             CrsGraph<LO, GO, NODE>::node_type>>& rangeMap,  \
                                const Teuchos::RCP<Teuchos::ParameterList>& params);
 
#define TPETRA_CRSGRAPH_INSTANT(LO, GO, NODE)                        \
  template class CrsGraph<LO, GO, NODE>;                             \
  TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE)     \
  TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT(LO, GO, NODE)     \
  TPETRA_CRSGRAPH_IMPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE) \
  TPETRA_CRSGRAPH_EXPORT_AND_FILL_COMPLETE_INSTANT_TWO(LO, GO, NODE)
 
#endif  // TPETRA_CRSGRAPH_DEF_HPP