10#ifndef TPETRA_DISTRIBUTOR_HPP
11#define TPETRA_DISTRIBUTOR_HPP
13#include "Tpetra_Details_DistributorActor.hpp"
17#include "Teuchos_as.hpp"
18#include "Teuchos_Describable.hpp"
19#include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
20#include "Teuchos_VerboseObject.hpp"
23#include "KokkosCompat_View.hpp"
24#include "Kokkos_Core.hpp"
25#include "Kokkos_TeuchosCommAdapters.hpp"
32#if defined(TPETRA_ENABLE_DEPRECATED_CODE)
105 public Teuchos::ParameterListAcceptorDefaultBase {
118 explicit Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
131 Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
132 const Teuchos::RCP<Teuchos::FancyOStream>&
out);
147 Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
148 const Teuchos::RCP<Teuchos::ParameterList>&
plist);
166 Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
167 const Teuchos::RCP<Teuchos::FancyOStream>&
out,
168 const Teuchos::RCP<Teuchos::ParameterList>&
plist);
260 template <
class Ordinal>
311 Teuchos::ArrayView<const int>
getProcsTo()
const;
336 return plan_.howInitialized();
395 template <
class ExpView,
class ImpView>
396 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
423 template <
class ExpView,
class ImpView>
424 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
454 template <
class ExpView,
class ImpView>
455 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
478 template <
class ExpView,
class ImpView>
479 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
489 template <
class ExpView,
class ImpView>
490 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
499 template <
class ExpView,
class ImpView>
500 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
510 template <
class ExpView,
class ImpView>
511 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
520 template <
class ExpView,
class ImpView>
521 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
557 const Teuchos::EVerbosityLevel
verbLevel =
558 Teuchos::Describable::verbLevel_default)
const;
569 Details::DistributorActor actor_;
575 static bool getVerbose();
581 std::unique_ptr<std::string>
585 bool verbose_ = getVerbose();
592 mutable Teuchos::RCP<Distributor> reverseDistributor_;
606 template <
class Ordinal>
607 void computeSends(
const Teuchos::ArrayView<const Ordinal>& remoteGIDs,
613 void createReverseDistributor()
const;
620 localDescribeToString(
const Teuchos::EVerbosityLevel
vl)
const;
623template <
class ExpView,
class ImpView>
624typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
629 actor_.doPostsAndWaits(plan_, exports,
numPackets, imports);
632template <
class ExpView,
class ImpView>
633typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
642template <
class ExpView,
class ImpView>
643typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
648 actor_.doPosts(plan_, exports,
numPackets, imports);
651template <
class ExpView,
class ImpView>
652typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
661template <
class ExpView,
class ImpView>
662typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
671template <
class ExpView,
class ImpView>
672typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
683template <
class ExpView,
class ImpView>
684typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
691 !plan_.getIndicesTo().is_null(), std::runtime_error,
692 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
693 "reverse communication when original data are blocked by process.");
694 if (reverseDistributor_.is_null()) {
695 createReverseDistributor();
697 reverseDistributor_->doPosts(exports,
numPackets, imports);
700template <
class ExpView,
class ImpView>
701typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
709 !plan_.getIndicesTo().is_null(), std::runtime_error,
710 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
711 "reverse communication when original data are blocked by process.");
712 if (reverseDistributor_.is_null()) {
713 createReverseDistributor();
719template <
class OrdinalType>
721 computeSends(
const Teuchos::ArrayView<const OrdinalType>&
importGIDs,
733 using Teuchos::ArrayView;
735 const char errPrefix[] =
"Tpetra::Distributor::computeSends: ";
737 " Please report this bug to the Tpetra developers.";
739 const int myRank = plan_.getComm()->getRank();
755 Distributor tempPlan(plan_.getComm());
758 const size_t numExportsAsSizeT =
759 tempPlan.createFromSends(importProcIDs);
760 const size_type numExports =
761 static_cast<size_type
>(numExportsAsSizeT);
762 TEUCHOS_TEST_FOR_EXCEPTION(numExports < 0, std::logic_error, errPrefix <<
"tempPlan.createFromSends() returned numExports=" << numExportsAsSizeT <<
" as a size_t, which overflows to " << numExports <<
" when cast to " << Teuchos::TypeNameTraits<size_type>::name() <<
"." << suffix);
763 TEUCHOS_TEST_FOR_EXCEPTION(size_type(tempPlan.getTotalReceiveLength()) != numExports,
764 std::logic_error, errPrefix <<
"tempPlan.getTotalReceiveLength()=" << tempPlan.getTotalReceiveLength() <<
" != numExports=" << numExports <<
"." << suffix);
766 if (numExports > 0) {
767 exportGIDs.resize(numExports);
768 exportProcIDs.resize(numExports);
779 static_assert(
sizeof(size_t) >=
sizeof(OrdinalType),
780 "Tpetra::Distributor::computeSends: "
781 "sizeof(size_t) < sizeof(OrdinalType).");
783 TEUCHOS_TEST_FOR_EXCEPTION(tempPlan.getTotalReceiveLength() <
size_t(numExports),
785 errPrefix <<
"tempPlan.getTotalReceiveLength()="
786 << tempPlan.getTotalReceiveLength() <<
" < numExports="
787 << numExports <<
"." << suffix);
789 Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs(
"exportObjs", tempPlan.getTotalReceiveLength() * 2);
790 tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
793 for (size_type i = 0; i < numExports; ++i) {
794 exportGIDs[i] =
static_cast<OrdinalType
>(exportObjs[2 * i]);
795 exportProcIDs[i] =
static_cast<int>(exportObjs[2 * i + 1]);
799 plan_ = tempPlan.plan_;
802template <
class OrdinalType>
804 createFromRecvs(
const Teuchos::ArrayView<const OrdinalType>& remoteGIDs,
805 const Teuchos::ArrayView<const int>& remoteProcIDs,
806 Teuchos::Array<OrdinalType>& exportGIDs,
807 Teuchos::Array<int>& exportProcIDs) {
809 const char errPrefix[] =
"Tpetra::Distributor::createFromRecvs: ";
810 const int myRank = plan_.getComm()->getRank();
812 std::unique_ptr<std::string> prefix;
814 prefix = createPrefix(
"createFromRecvs");
815 std::ostringstream os;
816 os << *prefix <<
"Start" << endl;
817 std::cerr << os.str();
822 using Teuchos::outArg;
823 using Teuchos::REDUCE_MAX;
824 using Teuchos::reduceAll;
828 (remoteGIDs.size() != remoteProcIDs.size()) ? myRank : -1;
830 reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
831 TEUCHOS_TEST_FOR_EXCEPTION(maxErrProc != -1, std::runtime_error, errPrefix <<
"Lists "
832 "of remote IDs and remote process IDs must have the same "
833 "size on all participating processes. Maximum process ID "
835 << maxErrProc <<
".");
839 TEUCHOS_TEST_FOR_EXCEPTION(remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
840 errPrefix <<
"On Process " << myRank <<
": "
842 << remoteGIDs.size() <<
" != remoteProcIDs.size()=" << remoteProcIDs.size() <<
".");
845 computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
847 plan_.createFromRecvs(remoteProcIDs);
850 std::ostringstream os;
851 os << *prefix <<
"Done" << endl;
852 std::cerr << os.str();
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Stand-alone utility functions and macros.
Struct that holds views of the contents of a CrsMatrix.
static bool debug()
Whether Tpetra is in debug mode.
Sets up and executes a communication plan for a Tpetra DistObject.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
std::string description() const
Return a one-line description of this object.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
Teuchos::Array< std::string > distributorSendTypes()
Valid string values for Distributor's "Send type" parameter.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Namespace Tpetra contains the class and methods constituting the Tpetra library.