Panzer_BlockedTpetraLinearObjFactory.hpp

Go to the documentation of this file.

// @HEADER
// *****************************************************************************
//           Panzer: A partial differential equation assembly
//       engine for strongly coupled complex multiphysics systems
//
// Copyright 2011 NTESS and the Panzer contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER
 
#ifndef __Panzer_BlockedTpetraLinearObjFactory_hpp__
#define __Panzer_BlockedTpetraLinearObjFactory_hpp__
 
#include <map>
 
// Tpetra includes
#include "Tpetra_Map.hpp"
#include "Tpetra_CrsGraph.hpp"
#include "Tpetra_FECrsGraph.hpp"
#include "Tpetra_FECrsMatrix.hpp"
#include "Tpetra_Import.hpp"
#include "Tpetra_Export.hpp"
 
#include "PanzerDiscFE_config.hpp"
#include "Panzer_GlobalIndexer.hpp"
#include "Panzer_GlobalIndexer_Utilities.hpp"
#include "Panzer_LinearObjFactory.hpp"
#include "Panzer_TpetraLinearObjContainer.hpp"
#include "Panzer_BlockedTpetraLinearObjContainer.hpp"
#include "Panzer_BlockedDOFManager.hpp"
#include "Panzer_CloneableEvaluator.hpp"
#include "Panzer_HashUtils.hpp" // for pair_hash
 
#include "Panzer_GatherOrientation.hpp"
#include "Panzer_GatherSolution_BlockedTpetra.hpp"
#include "Panzer_GatherSolution_Tpetra.hpp"
#include "Panzer_GatherTangent_BlockedTpetra.hpp"
#include "Panzer_ScatterResidual_BlockedTpetra.hpp"
#include "Panzer_ScatterDirichletResidual_BlockedTpetra.hpp"
#include "Panzer_ThyraObjFactory.hpp"
 
#include "Thyra_BlockedLinearOpBase.hpp"
#include "Thyra_ProductVectorBase.hpp"
 
#include "Teuchos_RCP.hpp"
#include "Teuchos_DefaultMpiComm.hpp"
#include "Teuchos_OpaqueWrapper.hpp"
 
namespace panzer {
 
template <typename Traits,typename ScalarT,typename LocalOrdinalT,typename GlobalOrdinalT,typename NodeT=panzer::TpetraNodeType>
class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
                                    , public ThyraObjFactory<double> {
public:
   typedef BlockedTpetraLinearObjContainer<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> BTLOC;
   typedef Tpetra::Vector<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> VectorType;
   typedef Tpetra::CrsMatrix<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> CrsMatrixType;
   typedef Tpetra::Operator<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> OperatorType;
   typedef Tpetra::CrsGraph<LocalOrdinalT,GlobalOrdinalT,NodeT> CrsGraphType;
   typedef Tpetra::Map<LocalOrdinalT,GlobalOrdinalT,NodeT> MapType;
   typedef Tpetra::Import<LocalOrdinalT,GlobalOrdinalT,NodeT> ImportType;
   typedef Tpetra::Export<LocalOrdinalT,GlobalOrdinalT,NodeT> ExportType;
 
   typedef Tpetra::FECrsGraph<LocalOrdinalT,GlobalOrdinalT,NodeT> FECrsGraphType;
   typedef Tpetra::FECrsMatrix<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> FECrsMatrixType;
 
   typedef Thyra::TpetraVector<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> ThyraVector;
   typedef Thyra::TpetraLinearOp<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> ThyraLinearOp;
 
 
   BlockedTpetraLinearObjFactory(const Teuchos::RCP<const Teuchos::MpiComm<int> > & comm,
                                 const Teuchos::RCP<const BlockedDOFManager> & gidProvider,
                                 bool useFEAssembly = false);
 
   BlockedTpetraLinearObjFactory(const Teuchos::RCP<const Teuchos::MpiComm<int> > & comm,
                                 const std::vector<Teuchos::RCP<const panzer::GlobalIndexer>> & gidProviders,
                                 bool useFEAssembly = false);
 
   BlockedTpetraLinearObjFactory(const Teuchos::RCP<const Teuchos::MpiComm<int> > & comm,
                                 const Teuchos::RCP<const GlobalIndexer> & rowProvider,
                                 const Teuchos::RCP<const GlobalIndexer> & colProvider,
                                 bool useFEAssembly = false);
 
   virtual ~BlockedTpetraLinearObjFactory();
 
/*************** Linear object factory methods *******************/
 
   virtual void readVector(const std::string & /* identifier */, LinearObjContainer & /* loc */, int /* id */) const
   { TEUCHOS_ASSERT(false); }
 
   virtual void writeVector(const std::string & /* identifier */, const LinearObjContainer & /* loc */, int /* id */) const
   { TEUCHOS_ASSERT(false); }
 
   virtual Teuchos::RCP<LinearObjContainer> buildLinearObjContainer() const;
 
   virtual Teuchos::RCP<LinearObjContainer> buildPrimitiveLinearObjContainer() const
   { return buildLinearObjContainer(); }
 
   virtual Teuchos::RCP<LinearObjContainer> buildGhostedLinearObjContainer() const;
 
   virtual Teuchos::RCP<LinearObjContainer> buildPrimitiveGhostedLinearObjContainer() const
   { return buildGhostedLinearObjContainer(); }
 
   virtual void globalToGhostContainer(const LinearObjContainer & container,
                                       LinearObjContainer & ghostContainer,int) const;
   virtual void ghostToGlobalContainer(const LinearObjContainer & ghostContainer,
                                       LinearObjContainer & container,int) const;
 
   virtual void adjustForDirichletConditions(const LinearObjContainer & localBCRows,
                                             const LinearObjContainer & globalBCRows,
                                             LinearObjContainer & ghostedObjs,
                                             bool zeroVectorRows=false, bool adjustX = false) const;
 
   virtual void applyDirichletBCs(const LinearObjContainer & counter,
                                  LinearObjContainer & result) const;
 
   virtual Teuchos::RCP<ReadOnlyVector_GlobalEvaluationData> buildReadOnlyDomainContainer() const;
 
#ifdef PANZER_HAVE_EPETRA_STACK
   virtual Teuchos::RCP<WriteVector_GlobalEvaluationData> buildWriteDomainContainer() const;
#endif
 
   Teuchos::MpiComm<int> getComm() const;
 
   template <typename EvalT>
   Teuchos::RCP<panzer::CloneableEvaluator> buildScatter() const
   {
     if(!hasColProvider_)
       return Teuchos::rcp(new ScatterResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_));
 
     // Non-square: the columns carry the derivative components, so the scatter
     // needs the column indexers to place them.
     return Teuchos::rcp(new ScatterResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_,colGidProviders_));
   }
 
   template <typename EvalT>
   Teuchos::RCP<panzer::CloneableEvaluator > buildGather() const
   { return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
 
   template <typename EvalT>
   Teuchos::RCP<panzer::CloneableEvaluator > buildGatherTangent() const
   { return Teuchos::rcp(new GatherTangent_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
 
   template <typename EvalT>
   Teuchos::RCP<panzer::CloneableEvaluator > buildGatherDomain() const
   {
     if(!hasColProvider_)
       return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_));
 
     // A blocked column side gathers into a product vector; a flat one is a
     // single vector, which only the non-blocked gather knows how to fill.
     if(colBlockedDOFManager_!=Teuchos::null)
       return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(colBlockedDOFManager_));
 
     return Teuchos::rcp(new GatherSolution_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(colGidProviders_[0]));
   }
 
   template <typename EvalT>
   Teuchos::RCP<panzer::CloneableEvaluator > buildGatherOrientation() const
   { return Teuchos::rcp(new GatherOrientation<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT>(nc2c_vector(blockedDOFManager_->getFieldDOFManagers()))); }
 
   template <typename EvalT>
   Teuchos::RCP<panzer::CloneableEvaluator> buildScatterDirichlet() const
   {
     if(!hasColProvider_)
       return Teuchos::rcp(new ScatterDirichletResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_));
 
     return Teuchos::rcp(new ScatterDirichletResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_,colGidProviders_));
   }
 
/*************** Generic helper functions for container setup *******************/
 
   void initializeContainer(int,LinearObjContainer & loc) const;
 
   void initializeContainer(int mem,BTLOC & loc) const;
 
   void initializeGhostedContainer(int,LinearObjContainer & loc) const;
 
   void initializeGhostedContainer(int mem,BTLOC & loc) const;
 
/*************** Thyra based methods *******************/
 
   Teuchos::RCP<const Thyra::VectorSpaceBase<ScalarT> > getThyraDomainSpace() const;
 
   Teuchos::RCP<const Thyra::VectorSpaceBase<ScalarT> > getThyraRangeSpace() const;
 
   Teuchos::RCP<const Thyra::VectorSpaceBase<ScalarT> > getThyraDomainSpace(int blk) const;
 
   Teuchos::RCP<const Thyra::VectorSpaceBase<ScalarT> > getThyraRangeSpace(int blk) const;
 
   Teuchos::RCP<Thyra::VectorBase<ScalarT> > getThyraDomainVector() const;
 
   Teuchos::RCP<Thyra::VectorBase<ScalarT> > getThyraRangeVector() const;
 
   Teuchos::RCP<Thyra::LinearOpBase<ScalarT> > getThyraMatrix() const;
 
   // and now the ghosted versions
 
   Teuchos::RCP<const Thyra::VectorSpaceBase<ScalarT> > getGhostedThyraDomainSpace() const;
 
   Teuchos::RCP<const Thyra::VectorSpaceBase<ScalarT> > getGhostedThyraRangeSpace() const;
 
   Teuchos::RCP<Thyra::VectorBase<ScalarT> > getGhostedThyraDomainVector() const;
 
   Teuchos::RCP<Thyra::VectorBase<ScalarT> > getGhostedThyraRangeVector() const;
 
   Teuchos::RCP<Thyra::BlockedLinearOpBase<ScalarT> > getGhostedThyraMatrix() const;
 
/*************** Tpetra based methods *******************/
 
   virtual Teuchos::RCP<const MapType> getMap(int i) const;
 
   virtual Teuchos::RCP<const MapType> getGhostedMap(int i) const;
 
   virtual Teuchos::RCP<const CrsGraphType> getGraph(int i,int j) const;
 
   virtual Teuchos::RCP<const CrsGraphType> getGhostedGraph(int i,int j) const;
 
   virtual Teuchos::RCP<const ImportType> getGhostedImport(int i) const;
 
   virtual Teuchos::RCP<const ExportType> getGhostedExport(int j) const;
 
   virtual Teuchos::RCP<const MapType> getColMap(int i) const;
   virtual Teuchos::RCP<const MapType> getGhostedColMap(int i) const;
   virtual Teuchos::RCP<const ImportType> getGhostedColImport(int i) const;
   virtual Teuchos::RCP<const ExportType> getGhostedColExport(int i) const;
 
   Teuchos::RCP<CrsMatrixType> getTpetraMatrix(int i,int j) const;
   Teuchos::RCP<CrsMatrixType> getGhostedTpetraMatrix(int i,int j) const;
 
/*************** FE (finite-element) construction functions *******************/
 
   bool useFEAssembly() const { return useFEAssembly_; }
 
   Teuchos::RCP<FECrsGraphType> getFEGraph(int i,int j) const;
 
   Teuchos::RCP<FECrsMatrixType> getFEMatrix(int i,int j) const;
 
   Teuchos::RCP<VectorType> getTpetraDomainVector(int i) const;
   Teuchos::RCP<VectorType> getGhostedTpetraDomainVector(int i) const;
 
   Teuchos::RCP<VectorType> getTpetraRangeVector(int i) const;
   Teuchos::RCP<VectorType> getGhostedTpetraRangeVector(int i) const;
 
   int getBlockRowCount() const;
 
   int getBlockColCount() const;
 
   void addExcludedPair(int rowBlock,int colBlock);
 
   void addExcludedPairs(const std::vector<std::pair<int,int> > & exPairs);
 
   virtual void beginFill(LinearObjContainer & loc) const;
 
   virtual void beginFill(LinearObjContainer & ghostContainer,
                          const LinearObjContainer & container) const;
   virtual void endFill(LinearObjContainer & loc) const;
 
   virtual void endFill(LinearObjContainer & ghostContainer,
                        const LinearObjContainer & container) const;
 
   Teuchos::RCP<const panzer::BlockedDOFManager> getGlobalIndexer() const
   { return blockedDOFManager_; }
 
   Teuchos::RCP<const panzer::GlobalIndexer> getDomainGlobalIndexer() const
   { return hasColProvider_ ? colBlockProvider_ : blockProvider_; }
 
   Teuchos::RCP<const panzer::GlobalIndexer> getRangeGlobalIndexer() const
   { return blockProvider_; }
 
protected:
/*************** Generic methods/members *******************/
 
   // Get the global indexer associated with a particular block
   Teuchos::RCP<const GlobalIndexer> getGlobalIndexer(int i) const;
 
   // Get the column global indexer associated with a particular block
   Teuchos::RCP<const GlobalIndexer> getColGlobalIndexer(int i) const;
 
   static void splitIntoBlocks(const Teuchos::RCP<const GlobalIndexer> & ugi,
                               Teuchos::RCP<const BlockedDOFManager> & blocked,
                               std::vector<Teuchos::RCP<const GlobalIndexer> > & blocks);
 
   void makeRoomForBlocks(std::size_t blockCnt,std::size_t colBlockCnt=0);
 
   Teuchos::RCP<const GlobalIndexer> blockProvider_;
   Teuchos::RCP<const BlockedDOFManager> blockedDOFManager_;
   std::vector<Teuchos::RCP<const GlobalIndexer> > gidProviders_;
 
   bool hasColProvider_;
   Teuchos::RCP<const GlobalIndexer> colBlockProvider_;
   Teuchos::RCP<const BlockedDOFManager> colBlockedDOFManager_;
   std::vector<Teuchos::RCP<const GlobalIndexer> > colGidProviders_;
 
   // which block entries are ignored
  std::unordered_set<std::pair<int,int>,panzer::pair_hash> excludedPairs_;
 
/*************** Thyra based methods/members *******************/
 
   void ghostToGlobalThyraVector(const Teuchos::RCP<const Thyra::VectorBase<ScalarT> > & in,
                                 const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out,bool col) const;
   void ghostToGlobalThyraMatrix(const Thyra::LinearOpBase<ScalarT> & in,Thyra::LinearOpBase<ScalarT> & out) const;
   void globalToGhostThyraVector(const Teuchos::RCP<const Thyra::VectorBase<ScalarT> > & in,
                                 const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out,bool col) const;
 
   mutable Teuchos::RCP<Thyra::ProductVectorSpaceBase<ScalarT> > rangeSpace_;
   mutable Teuchos::RCP<Thyra::ProductVectorSpaceBase<ScalarT> > ghostedRangeSpace_;
 
   mutable Teuchos::RCP<const Thyra::VectorSpaceBase<ScalarT> > domainSpace_;
   mutable Teuchos::RCP<const Thyra::VectorSpaceBase<ScalarT> > ghostedDomainSpace_;
 
/*************** Tpetra based methods/members *******************/
 
   void adjustForDirichletConditions(const VectorType & local_bcs,
                                     const VectorType & global_bcs,
                                     const Teuchos::Ptr<VectorType> & f,
                                     const Teuchos::Ptr<CrsMatrixType> & A,
                                     bool zeroVectorRows) const;
 
   void ghostToGlobalTpetraVector(int i,const VectorType & in,VectorType & out,bool col) const;
   void ghostToGlobalTpetraMatrix(int blockRow,const CrsMatrixType & in,CrsMatrixType & out) const;
   void globalToGhostTpetraVector(int i,const VectorType & in,VectorType & out,bool col) const;
 
   // get the map from the matrix
   virtual Teuchos::RCP<const MapType> buildTpetraMap(int i) const;
   virtual Teuchos::RCP<const MapType> buildTpetraGhostedMap(int i) const;
   virtual Teuchos::RCP<const MapType> buildColTpetraMap(int i) const;
   virtual Teuchos::RCP<const MapType> buildColTpetraGhostedMap(int i) const;
 
   // get the graph of the crs matrix
   virtual Teuchos::RCP<const CrsGraphType> buildTpetraGraph(int i,int j) const;
 
   // build the FE graph for block (i,j) (owned+shared/owned unified via Tpetra::FECrsGraph)
   virtual Teuchos::RCP<FECrsGraphType> buildFEGraph(int i,int j) const;
 
 
 public:
   virtual Teuchos::RCP<const CrsGraphType> buildTpetraGhostedGraph(int i,int j) const;
 protected:
 
   // storage for Tpetra graphs and maps
   Teuchos::RCP<const Teuchos::MpiComm<int> > comm_;
   mutable std::vector<Teuchos::RCP<const MapType> > maps_;
   mutable std::vector<Teuchos::RCP<const MapType> > ghostedMaps_;
   mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> graphs_ ;
   mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> ghostedGraphs_;
 
   mutable std::vector<Teuchos::RCP<const ImportType> > importers_;
   mutable std::vector<Teuchos::RCP<const ExportType> > exporters_;
 
   // Only populated when hasColProvider_; otherwise the column accessors
   // forward to the row-side objects above.
   mutable std::vector<Teuchos::RCP<const MapType> > colMaps_;
   mutable std::vector<Teuchos::RCP<const MapType> > ghostedColMaps_;
   mutable std::vector<Teuchos::RCP<const ImportType> > colImporters_;
   mutable std::vector<Teuchos::RCP<const ExportType> > colExporters_;
 
/*************** FE based members *******************/
 
   bool useFEAssembly_;
 
   mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<FECrsGraphType>,panzer::pair_hash> feGraphs_;
 
   mutable std::unordered_map<std::pair<int,int>,const FECrsMatrixType *,panzer::pair_hash> feAssemblyOpenOn_;
};
 
}
 
#endif