Tpetra_Details_DistributorPlan.hpp

Go to the documentation of this file.

// @HEADER
// *****************************************************************************
//          Tpetra: Templated Linear Algebra Services Package
//
// Copyright 2008 NTESS and the Tpetra contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER
 
#ifndef TPETRA_DETAILS_DISTRIBUTOR_PLAN_HPP
#define TPETRA_DETAILS_DISTRIBUTOR_PLAN_HPP
 
#include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
#include "Teuchos_Array.hpp"
#include "Teuchos_Comm.hpp"
#include "Teuchos_RCP.hpp"
#include "TpetraCore_config.h"
 
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
#include <mpi_advance.h>
#endif
 
namespace Tpetra {
namespace Details {
 
enum EDistributorSendType {
  DISTRIBUTOR_ISEND,    // Use MPI_Isend (Teuchos::isend)
  DISTRIBUTOR_SEND,     // Use MPI_Send (Teuchos::send)
  DISTRIBUTOR_ALLTOALL  // Use MPI_Alltoall
#if defined(HAVE_TPETRA_MPI)
  ,
  DISTRIBUTOR_IALLTOFEWV
#endif
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  ,
  DISTRIBUTOR_MPIADVANCE_ALLTOALL,
  DISTRIBUTOR_MPIADVANCE_NBRALLTOALLV
#endif
};
 
std::string
DistributorSendTypeEnumToString(EDistributorSendType sendType);
 
EDistributorSendType
DistributorSendTypeStringToEnum(const std::string_view s);
 
Teuchos::Array<std::string> distributorSendTypes();
 
Teuchos::Array<EDistributorSendType> distributorSendTypeEnums();
 
Teuchos::Array<std::string> distributorSendTypes();
 
const std::string& validSendTypeOrThrow(const std::string& s);
 
enum EDistributorHowInitialized {
  DISTRIBUTOR_NOT_INITIALIZED,                           // Not initialized yet
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS,          // By createFromSends
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_RECVS,          // By createFromRecvs
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS_N_RECVS,  // By createFromSendsAndRecvs
  DISTRIBUTOR_INITIALIZED_BY_REVERSE,                    // By createReverseDistributor
  DISTRIBUTOR_INITIALIZED_BY_COPY,                       // By copy constructor
};
 
std::string
DistributorHowInitializedEnumToString(EDistributorHowInitialized how);
 
class DistributorPlan : public Teuchos::ParameterListAcceptorDefaultBase {
  static constexpr int DEFAULT_MPI_TAG = 0;
 
 public:
  using IndexView     = std::vector<size_t>;
  using SubViewLimits = std::pair<IndexView, IndexView>;
 
  DistributorPlan(Teuchos::RCP<const Teuchos::Comm<int>> comm);
  DistributorPlan(const DistributorPlan& otherPlan);
 
  size_t createFromSends(const Teuchos::ArrayView<const int>& exportProcIDs);
  void createFromRecvs(const Teuchos::ArrayView<const int>& remoteProcIDs);
  void createFromSendsAndRecvs(const Teuchos::ArrayView<const int>& exportProcIDs,
                               const Teuchos::ArrayView<const int>& remoteProcIDs);
 
  void setParameterList(const Teuchos::RCP<Teuchos::ParameterList>& plist);
 
  Teuchos::RCP<DistributorPlan> getReversePlan() const;
 
  Teuchos::RCP<const Teuchos::Comm<int>> getComm() const { return comm_; }
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  Teuchos::RCP<MPIX_Comm*> getMPIXComm() const { return mpixComm_; }
#endif
  EDistributorSendType getSendType() const { return sendType_; }
  size_t getNumReceives() const { return numReceives_; }
  size_t getNumSends() const { return numSendsToOtherProcs_; }
  bool hasSelfMessage() const { return sendMessageToSelf_; }
  size_t getMaxSendLength() const { return maxSendLength_; }
  size_t getTotalReceiveLength() const { return totalReceiveLength_; }
  Teuchos::ArrayView<const int> getProcsFrom() const { return procsFrom_; }
  Teuchos::ArrayView<const int> getProcsTo() const { return procIdsToSendTo_; }
  Teuchos::ArrayView<const size_t> getLengthsFrom() const { return lengthsFrom_; }
  Teuchos::ArrayView<const size_t> getLengthsTo() const { return lengthsTo_; }
  Teuchos::ArrayView<const size_t> getStartsTo() const { return startsTo_; }
  Teuchos::ArrayView<const size_t> getIndicesTo() const { return indicesTo_; }
  Details::EDistributorHowInitialized howInitialized() const { return howInitialized_; }
 
  SubViewLimits getImportViewLimits(size_t numPackets) const;
  SubViewLimits getImportViewLimits(const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) const;
  SubViewLimits getExportViewLimits(size_t numPackets) const;
  SubViewLimits getExportViewLimits(const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID) const;
 
#if defined(HAVE_TPETRA_MPI)
  const std::vector<int> getRoots() const {
    return roots_;
  }
#endif
 private:
  // after the plan has been created we have the info we need to initialize the MPI advance communicator
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  void initializeMpiAdvance();
#endif
 
#if defined(HAVE_TPETRA_MPI)
  void maybeInitializeRoots();
#endif
 
  Teuchos::RCP<const Teuchos::ParameterList> getValidParameters() const;
 
  void createReversePlan() const;
 
  void computeReceives();
 
  Teuchos::RCP<const Teuchos::Comm<int>> comm_;
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  Teuchos::RCP<MPIX_Comm*> mpixComm_;
#endif
 
  Details::EDistributorHowInitialized howInitialized_;
  mutable Teuchos::RCP<DistributorPlan> reversePlan_;
 
 
  EDistributorSendType sendType_;
 
  bool sendMessageToSelf_;
  size_t numSendsToOtherProcs_;
  Teuchos::Array<int> procIdsToSendTo_;
 
  Teuchos::Array<size_t> startsTo_;
 
  Teuchos::Array<size_t> lengthsTo_;
 
  size_t maxSendLength_;
 
  Teuchos::Array<size_t> indicesTo_;
 
  size_t numReceives_;
 
  size_t totalReceiveLength_;
 
  Teuchos::Array<size_t> lengthsFrom_;
 
  Teuchos::Array<int> procsFrom_;
 
  Teuchos::Array<size_t> startsFrom_;
 
  Teuchos::Array<size_t> indicesFrom_;
 
#if defined(HAVE_TPETRA_MPI)
  std::vector<int> roots_;
 
#endif
};
 
}  // namespace Details
}  // namespace Tpetra
 
#endif