Teuchos_DefaultMpiComm_decl.hpp

// @HEADER
// *****************************************************************************
//                    Teuchos: Common Tools Package
//
// Copyright 2004 NTESS and the Teuchos contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER
 
#ifndef TEUCHOS_DEFAULTMPICOMM_DECL_HPP
#define TEUCHOS_DEFAULTMPICOMM_DECL_HPP
 
#include "Teuchos_Comm.hpp"
 
namespace Teuchos {

template<class OrdinalType>
class MpiCommStatus : public CommStatus<OrdinalType> {
public:
  MpiCommStatus (MPI_Status status);

  virtual ~MpiCommStatus();

  OrdinalType getSourceRank ();

  OrdinalType getTag ();

  OrdinalType getError ();
 
private:
  MpiCommStatus ();

  MPI_Status status_;
};

template<class OrdinalType>
RCP<MpiCommStatus<OrdinalType> >
mpiCommStatus (MPI_Status rawMpiStatus);

template<class OrdinalType>
class MpiCommRequestBase : public CommRequest<OrdinalType> {
public:
  MpiCommRequestBase ();

  MpiCommRequestBase (MPI_Request rawMpiRequest);

  MPI_Request releaseRawMpiRequest();

  bool isNull() const;
 
  bool isReady();

  RCP<CommStatus<OrdinalType> > wait ();

  RCP<CommStatus<OrdinalType> > cancel ();

  virtual ~MpiCommRequestBase ();
 
private:
  MPI_Request rawMpiRequest_;
};
 

template<class OrdinalType>
class MpiCommRequest : public MpiCommRequestBase<OrdinalType> {
public:
  MpiCommRequest ();

  MpiCommRequest (MPI_Request rawMpiRequest,
                  const ArrayView<char>::size_type numBytesInMessage);

  ArrayView<char>::size_type numBytes () const;

  virtual ~MpiCommRequest ();
 
private:
  ArrayView<char>::size_type numBytes_;
};

template<class OrdinalType>
RCP<MpiCommRequest<OrdinalType> >
mpiCommRequest (MPI_Request rawMpiRequest,
                const ArrayView<char>::size_type numBytes);

template<typename Ordinal>
class MpiComm : public Comm<Ordinal> {
public:


  explicit MpiComm (MPI_Comm rawMpiComm);

  MpiComm (const RCP<const OpaqueWrapper<MPI_Comm> >& rawMpiComm);

  MpiComm (const RCP<const OpaqueWrapper<MPI_Comm> >& rawMpiComm,
           const int defaultTag);

  MpiComm (const MpiComm<Ordinal>& other);

  RCP<const OpaqueWrapper<MPI_Comm> > getRawMpiComm () const {
    return rawMpiComm_;
  }

  void setErrorHandler (const RCP<const OpaqueWrapper<MPI_Errhandler> >& errHandler);



  virtual int getRank() const;

  virtual int getSize() const;

  virtual void barrier() const;

  virtual void
  alltoAll(const Ordinal sendBytes, const char sendBuffer[],
           const Ordinal recvBytes, char recvBuffer[]) const;

  virtual void broadcast(
    const int rootRank, const Ordinal bytes, char buffer[]
    ) const;

  virtual void
  gather (const Ordinal sendBytes, const char sendBuffer[],
          const Ordinal recvBytes, char recvBuffer[],
          const int root) const;
  virtual void gatherAll(
    const Ordinal sendBytes, const char sendBuffer[]
    ,const Ordinal recvBytes, char recvBuffer[]
    ) const;
  virtual void reduceAll(
    const ValueTypeReductionOp<Ordinal,char> &reductOp
    ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
    ) const;
  virtual void scan(
    const ValueTypeReductionOp<Ordinal,char> &reductOp
    ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
    ) const;
  virtual void send(
    const Ordinal bytes, const char sendBuffer[], const int destRank
    ) const;
  virtual void
  send (const Ordinal bytes,
        const char sendBuffer[],
        const int destRank,
        const int tag) const;
  virtual void ssend(
    const Ordinal bytes, const char sendBuffer[], const int destRank
    ) const;
  virtual void
  ssend (const Ordinal bytes,
         const char sendBuffer[],
         const int destRank,
         const int tag) const;
  virtual int receive(
    const int sourceRank, const Ordinal bytes, char recvBuffer[]
    ) const;
  virtual void readySend(
    const ArrayView<const char> &sendBuffer,
    const int destRank
    ) const;
  virtual void
  readySend (const Ordinal bytes,
             const char sendBuffer[],
             const int destRank,
             const int tag) const;
  virtual RCP<CommRequest<Ordinal> > isend(
    const ArrayView<const char> &sendBuffer,
    const int destRank
    ) const;
  virtual RCP<CommRequest<Ordinal> >
  isend (const ArrayView<const char> &sendBuffer,
         const int destRank,
         const int tag) const;
  virtual RCP<CommRequest<Ordinal> > ireceive(
    const ArrayView<char> &Buffer,
    const int sourceRank
    ) const;
  virtual RCP<CommRequest<Ordinal> >
  ireceive (const ArrayView<char> &Buffer,
            const int sourceRank,
            const int tag) const;
  virtual void waitAll(
    const ArrayView<RCP<CommRequest<Ordinal> > > &requests
    ) const;
  virtual void
  waitAll (const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
           const ArrayView<RCP<CommStatus<Ordinal> > >& statuses) const;
  virtual RCP<CommStatus<Ordinal> >
  wait (const Ptr<RCP<CommRequest<Ordinal> > >& request) const;
  virtual RCP< Comm<Ordinal> > duplicate() const;
  virtual RCP< Comm<Ordinal> > split(const int color, const int key) const;
  virtual RCP< Comm<Ordinal> > createSubcommunicator(
    const ArrayView<const int>& ranks) const;



  std::string description() const;

 
  // These should be private but the PGI compiler requires them be public
 
  static int const minTag_ = 26000; // These came from Teuchos::MpiComm???
  static int const maxTag_ = 26099; // ""

  int getTag () const { return tag_; }
 
  int incrementTag() {
    ++tag_;
    // Wrap at MPI_TAG_UB, NOT std::numeric_limits<int>::max(). The tag is
    // passed straight to MPI_Isend/Irecv, and MPI rejects any tag above
    // MPI_TAG_UB with MPI_ERR_TAG. MPI_TAG_UB is far below INT_MAX on real
    // implementations (e.g. 2^23-1 = 8388607 on OpenMPI), so wrapping only
    // at INT_MAX let the tag climb out of the valid range: the Tpetra
    // Distributor calls this once per communication round, and a long run
    // (~8.4M rounds) eventually aborts mid-solve with MPI_ERR_TAG. tagUb_
    // is the queried MPI_TAG_UB (see setupMembersFromComm); wrapping back
    // to minTag_ keeps the tag valid and matches the documented intent of
    // "loop around back to a small value".
    if (tag_ >= tagUb_)
      tag_ = minTag_;
    return tag_;
  }
 
private:

  void setupMembersFromComm();
  static int tagCounter_;

  RCP<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;

  int rank_;

  int size_;

  int tag_;

  int tagUb_ = 32767;

  RCP<const OpaqueWrapper<MPI_Errhandler> > customErrorHandler_;
 
  void assertRank(const int rank, const std::string &rankName) const;
 
  // Not defined and not to be called!
  MpiComm();
 
#ifdef TEUCHOS_MPI_COMM_DUMP
public:
  static bool show_dump;
#endif // TEUCHOS_MPI_COMM_DUMP
 
};
 

template<typename Ordinal>
RCP<MpiComm<Ordinal> >
createMpiComm(
  const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
  );
 

template<typename Ordinal>
RCP<MpiComm<Ordinal> >
createMpiComm(
              const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm,
              const int defaultTag
  );
 

template<typename Ordinal>
MPI_Comm
getRawMpiComm(const Comm<Ordinal> &comm);
 
}
 
#endif