18#ifndef AMESOS2_MUMPS_DEF_HPP
19#define AMESOS2_MUMPS_DEF_HPP
21#include <Teuchos_Tuple.hpp>
22#include <Teuchos_ParameterList.hpp>
23#include <Teuchos_StandardParameterEntryValidators.hpp>
26#include <Teuchos_DefaultMpiComm.hpp>
38 template <
class Matrix,
class Vector>
39 MUMPS<Matrix,Vector>::MUMPS(
40 Teuchos::RCP<const Matrix> A,
41 Teuchos::RCP<Vector> X,
42 Teuchos::RCP<const Vector> B )
43 : SolverCore<Amesos2::MUMPS,Matrix,Vector>(A, X, B)
45 , schur_out_ptr(nullptr)
46 , only_forward_solve(false)
47 , only_backward_solve(false)
48 , is_contiguous_(true)
51 typedef FunctionMap<MUMPS,scalar_type> function_map;
53 MUMPS_MATRIX_LOAD =
false;
55 MUMPS_MATRIX_LOAD_PREORDERING =
false;
59 using Teuchos::MpiComm;
62 using Teuchos::rcp_dynamic_cast;
65 mumps_par.comm_fortran = -987654;
66 RCP<const Comm<int> > matComm = this->matrixA_->getComm();
68 TEUCHOS_TEST_FOR_EXCEPTION(
69 matComm.is_null(), std::logic_error,
"Amesos2::MUMPS:: matComm is null");
70 RCP<const MpiComm<int> > matMpiComm =
71 rcp_dynamic_cast<const MpiComm<int> >(matComm);
73 TEUCHOS_TEST_FOR_EXCEPTION(
75 std::logic_error,
"Amesos2::MUMPS:: matMPIComm is null");
76 TEUCHOS_TEST_FOR_EXCEPTION(
77 matMpiComm->getRawMpiComm().is_null(),
78 std::logic_error,
"Amesos2::MUMPS:: rawMPIComm is null");
79 MPI_Comm rawMpiComm = (* (matMpiComm->getRawMpiComm()) )();
80 mumps_par.comm_fortran = (int) MPI_Comm_c2f(rawMpiComm);
87 function_map::mumps_c(&(mumps_par));
90 mumps_par.n = this->globalNumCols_;
93 mumps_par.icntl[0] = -1;
94 mumps_par.icntl[1] = -1;
95 mumps_par.icntl[2] = -1;
96 mumps_par.icntl[3] = 1;
97 mumps_par.icntl[4] = 0;
98 mumps_par.icntl[5] = 7;
99 mumps_par.icntl[6] = 7;
100 mumps_par.icntl[7] = 7;
101 mumps_par.icntl[8] = 1;
102 mumps_par.icntl[9] = 0;
103 mumps_par.icntl[10] = 0;
104 mumps_par.icntl[11] = 0;
105 mumps_par.icntl[12] = 0;
106 mumps_par.icntl[13] = 20;
107 mumps_par.icntl[17] = 0;
108 mumps_par.icntl[18] = 0;
109 mumps_par.icntl[19] = 0;
110 mumps_par.icntl[20] = 0;
111 mumps_par.icntl[21] = 0;
112 mumps_par.icntl[22] = 0;
113 mumps_par.icntl[23] = 0;
114 mumps_par.icntl[24] = 0;
115 mumps_par.icntl[25] = 0;
116 mumps_par.icntl[27] = 1;
117 mumps_par.icntl[28] = 0;
118 mumps_par.icntl[29] = 0;
119 mumps_par.icntl[30] = 0;
120 mumps_par.icntl[31] = 0;
121 mumps_par.icntl[32] = 0;
124 template <
class Matrix,
class Vector>
125 MUMPS<Matrix,Vector>::~MUMPS( )
128 typedef FunctionMap<MUMPS,scalar_type> function_map;
130 if(MUMPS_STRUCT ==
true)
137 if (this->rank_ < this->nprocs_) {
138 function_map::mumps_c(&(mumps_par));
143 template<
class Matrix,
class Vector>
149 #ifdef HAVE_AMESOS2_TIMERS
150 Teuchos::TimeMonitor preOrderTimer(this->timers_.preOrderTime_);
156 template <
class Matrix,
class Vector>
160 #ifdef HAVE_AMESOS2_TIMERS
161 Teuchos::TimeMonitor symFactTime( this->timers_.symFactTime_ );
165 if ( this->globalNumRows_ > 0 ) {
168 function_map::mumps_c(&(mumps_par));
175 template <
class Matrix,
class Vector>
179 #ifdef HAVE_AMESOS2_TIMERS
180 Teuchos::TimeMonitor numFactTimer(this->timers_.numFactTime_);
184 if ( this->globalNumRows_ > 0 ) {
186 function_map::mumps_c(&(mumps_par));
189 if (schur_size > 0 && schur_out_ptr !=
nullptr) {
193 for (
size_t i = 0; i < schur_size; i++) {
194 for (
size_t j = 0; j < schur_size; j++) {
195 schur_out_ptr[j+i*schur_size] = schur_out(i+j*schur_size);
204 template <
class Matrix,
class Vector>
212 const global_size_type ld_rhs = this->root_ ? X->getGlobalLength() : 0;
213 const size_t nrhs = X->getGlobalNumVectors();
214 const size_t val_store_size = Teuchos::as<size_t>(ld_rhs * nrhs);
216 const bool initialize_data =
true;
217 const bool do_not_initialize_data =
false;
219 #ifdef HAVE_AMESOS2_TIMERS
220 Teuchos::TimeMonitor mvConvTimer(this->timers_.vecConvTime_);
223 Util::get_1d_copy_helper_kokkos_view<MultiVecAdapter<Vector>,
224 host_mv_view>::do_get(initialize_data, B, bvals_,
225 Teuchos::as<size_t>(ld_rhs),
227 this->rowIndexBase_);
228 Util::get_1d_copy_helper_kokkos_view<MultiVecAdapter<Vector>,
229 host_mv_view>::do_get(do_not_initialize_data, X, xvals_,
230 Teuchos::as<size_t>(ld_rhs),
232 this->rowIndexBase_);
235 Kokkos::deep_copy(xvals_, bvals_);
238 if ( this->globalNumRows_ > 0 ) {
239 #ifdef HAVE_AMESOS2_TIMERS
240 Teuchos::TimeMonitor solveTimer(this->timers_.solveTime_);
243 mumps_par.nrhs = nrhs;
244 mumps_par.lrhs = mumps_par.n;
251 mumps_par.rhs = xvals_.data();
254 if (only_forward_solve || only_backward_solve) {
256 if (only_forward_solve) {
257 mumps_par.icntl[25] = 1;
259 mumps_par.icntl[25] = 2;
263 Kokkos::resize(schur_rhs, schur_size*nrhs);
264 size_t n = this->globalNumRows_;
265 size_t n2 = schur_size;
267 for (
size_t i = 0; i < n2; i++) {
268 for (
size_t j = 0; j < nrhs; j++) schur_rhs[i+j*schur_size] = xvals_(n1+i, j);
270 mumps_par.redrhs = schur_rhs.data();
273 mumps_par.icntl[25] = 0;
276 function_map::mumps_c(&(mumps_par));
280 #ifdef HAVE_AMESOS2_TIMERS
281 Teuchos::TimeMonitor redistTimer2(this->timers_.vecRedistTime_);
283 if (only_forward_solve) {
285 size_t n = this->globalNumRows_;
286 size_t n2 = schur_size;
288 for (
size_t i = 0; i < n2; i++) {
289 for (
size_t j = 0; j < nrhs; j++) xvals_(n1+i, j) = schur_rhs[i+j*schur_size];
293 Util::put_1d_data_helper_kokkos_view<MultiVecAdapter<Vector>,
294 host_mv_view>::do_put(X, xvals_,
295 Teuchos::as<size_t>(ld_rhs),
297 this->rowIndexBase_);
300 MUMPS_MATRIX_LOAD_PREORDERING =
false;
305 template <
class Matrix,
class Vector>
310 return( this->globalNumRows_ == this->globalNumCols_ );
314 template <
class Matrix,
class Vector>
319 using Teuchos::getIntegralValue;
320 using Teuchos::ParameterEntryValidator;
322 RCP<const Teuchos::ParameterList> valid_params = getValidParameters_impl();
324 if(parameterList->isParameter(
"ICNTL(1)")){
325 mumps_par.icntl[0] = parameterList->get<
int>(
"ICNTL(1)", -1);
327 if(parameterList->isParameter(
"ICNTL(2)")){
328 mumps_par.icntl[1] = parameterList->get<
int>(
"ICNTL(2)", -1);
330 if(parameterList->isParameter(
"ICNTL(3)")){
331 mumps_par.icntl[2] = parameterList->get<
int>(
"ICNTL(3)", -1);
333 if(parameterList->isParameter(
"ICNTL(4)")){
334 mumps_par.icntl[3] = parameterList->get<
int>(
"ICNTL(4)", 1);
336 if(parameterList->isParameter(
"ICNTL(6)")){
337 mumps_par.icntl[5] = parameterList->get<
int>(
"ICNTL(6)", 0);
339 if(parameterList->isParameter(
"ICNTL(7)")){
340 mumps_par.icntl[6] = parameterList->get<
int>(
"ICNTL(7)", 7);
342 if(parameterList->isParameter(
"ICNTL(9)")){
343 mumps_par.icntl[8] = parameterList->get<
int>(
"ICNTL(9)", 1);
345 if(parameterList->isParameter(
"ICNTL(11)")){
346 mumps_par.icntl[10] = parameterList->get<
int>(
"ICNTL(11)", 0);
348 if(parameterList->isParameter(
"ICNTL(14)")){
349 mumps_par.icntl[13] = parameterList->get<
int>(
"ICNTL(14)", 20);
352 if(parameterList->isParameter(
"PartialFacto")) {
353 int partial_facto = parameterList->get<
int>(
"PartialFacto");
354 if (partial_facto == 1 || partial_facto == 2) {
355 mumps_par.icntl[18] = 1;
358 if(parameterList->isParameter(
"SchurPart")) {
360 auto schur_part_ptr = parameterList->get<
const local_ordinal_type*>(
"SchurPart");
362 for (global_size_type i=0; i<this->globalNumCols_; i++) {
363 if (schur_part_ptr[i] == 1) schur_size ++;
366 Kokkos::resize(schur_part, schur_size);
367 Kokkos::resize(schur_out, schur_size*schur_size);
369 for (global_size_type i=0; i<this->globalNumCols_; i++) {
370 if (schur_part_ptr[i] == 1) {
371 schur_part(schur_size) = i+1;
375 mumps_par.size_schur = schur_size;
376 mumps_par.listvar_schur = schur_part.data();
377 mumps_par.schur = schur_out.data();
379 if(parameterList->isParameter(
"SchurOut")) {
381 schur_out_ptr = parameterList->get<scalar_type*>(
"SchurOut");
383 if(parameterList->isParameter(
"OnlyForwardSolve")) {
384 only_forward_solve = parameterList->get<
bool>(
"OnlyForwardSolve");
386 if(parameterList->isParameter(
"OnlyBackwardSolve")) {
387 only_backward_solve = parameterList->get<
bool>(
"OnlyBackwardSolve");
390 if(parameterList->isParameter(
"verbose")){
391 bool verbose = parameterList->get<
bool>(
"verbose");
392 if (verbose) mumps_par.icntl[3] = 2;
394 if( parameterList->isParameter(
"IsContiguous") ){
395 is_contiguous_ = parameterList->get<
bool>(
"IsContiguous");
400 template <
class Matrix,
class Vector>
401 Teuchos::RCP<const Teuchos::ParameterList>
404 using Teuchos::ParameterList;
406 static Teuchos::RCP<const Teuchos::ParameterList> valid_params;
408 if( is_null(valid_params) ){
409 Teuchos::RCP<Teuchos::ParameterList> pl = Teuchos::parameterList();
411 pl->set(
"ICNTL(1)", -1,
"Output stream for error messages." );
412 pl->set(
"ICNTL(2)", -1,
"Output stream for diagnostic." );
413 pl->set(
"ICNTL(3)", -1,
"Output stream for global information." );
414 pl->set(
"ICNTL(4)", 1,
"Level of printing." );
415 pl->set(
"ICNTL(6)", 0,
"Permutes the matrix to a zero-free diagonal" );
416 pl->set(
"ICNTL(9)", 1,
"Transpose solve, if not 1" );
417 pl->set(
"ICNTL(11)", 0,
"Computes statistics for error analysis" );
418 pl->set(
"ICNTL(14)", 20,
"Percentage increase in the estimated working space" );
421 scalar_type *dummy_scalar_ptr;
422 const local_ordinal_type *dummy_ordinal_ptr;
423 pl->set(
"PartialFacto", 0,
424 "Perform partial factorization to extract dense Schur complement (0: no, 1: form + factor Schur, 2: ony form");
425 pl->set(
"SchurPart", dummy_ordinal_ptr,
426 "Specify rows/columns belonging to Schur complement for partial factorization");
427 pl->set(
"SchurOut", dummy_scalar_ptr,
428 "Store output Schur complement from partial factorization");
429 pl->set(
"OnlyForwardSolve",
false,
430 "Perform only the forward substitution");
431 pl->set(
"OnlyBackwardSolve",
false,
432 "Perform only the backward substitution");
434 pl->set(
"verbose",
false,
"Whether verbose");
435 pl->set(
"IsContiguous",
true,
"Whether GIDs contiguous");
443 template <
class Matrix,
class Vector>
447 #ifdef HAVE_AMESOS2_TIMERS
448 Teuchos::TimeMonitor convTimer(this->timers_.mtxConvTime_);
450 if(MUMPS_MATRIX_LOAD ==
false || current_phase==NUMFACT)
453 if( !MUMPS_MATRIX_LOAD && this->root_ ) {
454 Kokkos::resize(host_nzvals_view_, this->globalNumNonZeros_);
455 Kokkos::resize(host_rows_view_, this->globalNumNonZeros_);
456 Kokkos::resize(host_col_ptr_view_, this->globalNumRows_ + 1);
459 #ifdef HAVE_AMESOS2_TIMERS
460 Teuchos::TimeMonitor mtxRedistTimer( this->timers_.mtxRedistTime_ );
463 local_ordinal_type nnz_ret = 0;
466 ::do_get(this->matrixA_.ptr(), host_nzvals_view_, host_rows_view_, host_col_ptr_view_, nnz_ret,
469 this->rowIndexBase_);
471 Teuchos::broadcast(*(this->getComm()), 0, &nnz_ret);
472 TEUCHOS_TEST_FOR_EXCEPTION( nnz_ret != Teuchos::as<local_ordinal_type>(this->globalNumNonZeros_),
473 std::runtime_error,
"Did not get the expected number of non-zero vals");
483 if (current_phase==PREORDERING){
484 MUMPS_MATRIX_LOAD_PREORDERING =
true;
488 MUMPS_MATRIX_LOAD =
true;
492 template <
class Matrix,
class Vector>
495 const Teuchos::EVerbosityLevel verbLevel)
const
497 out <<
" MUMPS current parameters:" << std::endl;
498 out <<
" > ICNTL(1) = " << mumps_par.icntl[0] << std::endl;
499 out <<
" > ICNTL(2) = " << mumps_par.icntl[1] << std::endl;
500 out <<
" > ICNTL(3) = " << mumps_par.icntl[2] << std::endl;
501 out <<
" > ICNTL(4) = " << mumps_par.icntl[3] << std::endl;
502 out <<
" > ICNTL(6) = " << mumps_par.icntl[5] << std::endl;
503 out <<
" > ICNTL(7) = " << mumps_par.icntl[6] << std::endl;
504 out <<
" > ICNTL(9) = " << mumps_par.icntl[8] << std::endl;
505 out <<
" > ICNTL(11) = " << mumps_par.icntl[10] << std::endl;
506 out <<
" > ICNTL(14) = " << mumps_par.icntl[13] << std::endl;
507 out <<
" > IsContiguous = " << (is_contiguous_ ?
"YES" :
"NO") << std::endl;
511 template <
class Matrix,
class Vector>
515 if ( !MUMPS_STRUCT ) {
517 mumps_par.n = this->globalNumCols_;
518 mumps_par.nz = this->globalNumNonZeros_;
519 mumps_par.a = (mumps_type*)malloc(mumps_par.nz *
sizeof(mumps_type));
520 mumps_par.irn = (MUMPS_INT*)malloc(mumps_par.nz *
sizeof(MUMPS_INT));
521 mumps_par.jcn = (MUMPS_INT*)malloc(mumps_par.nz *
sizeof(MUMPS_INT));
523 if((mumps_par.a == NULL) || (mumps_par.irn == NULL)
524 || (mumps_par.jcn == NULL)) {
529 local_ordinal_type tri_count = 0;
530 local_ordinal_type i,j;
531 local_ordinal_type max_local_ordinal = 0;
533 for(i = 0; i < (local_ordinal_type)this->globalNumCols_; i++) {
534 for( j = host_col_ptr_view_(i); j < host_col_ptr_view_(i+1)-1; j++) {
535 mumps_par.jcn[tri_count] = (MUMPS_INT)i+1;
536 mumps_par.irn[tri_count] = (MUMPS_INT)host_rows_view_(j)+1;
537 mumps_par.a[tri_count] = host_nzvals_view_(j);
542 j = host_col_ptr_view_(i+1)-1;
543 mumps_par.jcn[tri_count] = (MUMPS_INT)i+1;
544 mumps_par.irn[tri_count] = (MUMPS_INT)host_rows_view_(j)+1;
545 mumps_par.a[tri_count] = host_nzvals_view_(j);
549 if(host_rows_view_(j) > max_local_ordinal) {
550 max_local_ordinal = host_rows_view_(j);
553 TEUCHOS_TEST_FOR_EXCEPTION(std::numeric_limits<MUMPS_INT>::max() <= max_local_ordinal,
555 "Matrix index larger than MUMPS_INT");
560 template<
class Matrix,
class Vector>
562 MUMPS<Matrix,Vector>::MUMPS_ERROR()
const
566 bool Wrong = ((mumps_par.info[0] != 0) || (mumps_par.infog[0] != 0)) && (this->rank_ < this->nprocs_);
568 if (this->rank_==0) {
569 std::cerr <<
"Amesos2_Mumps : ERROR" << std::endl;
570 if ( this->status_.getNumSolve() > 0) {
571 std::cerr <<
" Last Phase : SOLVE" << std::endl;
572 }
else if( this->status_.numericFactorizationDone() ){
573 std::cerr <<
" Last Phase : NUMFACT" << std::endl;
574 }
else if( this->status_.symbolicFactorizationDone() ){
575 std::cerr <<
" Last Phase : SYMBFACT" << std::endl;
576 }
else if( this->status_.preOrderingDone() ){
577 std::cerr <<
" Last Phase : PREORDERING" << std::endl;
579 std::cerr <<
" Last Phase : CLEAN" << std::endl;
581 std::cerr <<
"Amesos2_Mumps : INFOG(1) = " << mumps_par.infog[0] << std::endl;
582 std::cerr <<
"Amesos2_Mumps : INFOG(2) = " << mumps_par.infog[1] << std::endl;
584 if (mumps_par.info[0] != 0 && Wrong) {
585 std::cerr <<
"Amesos2_Mumps : On process " << this->matrixA_->getComm()->getRank()
586 <<
", INFO(1) = " << mumps_par.info[0] << std::endl;
587 std::cerr <<
"Amesos2_Mumps : On process " << this->matrixA_->getComm()->getRank()
588 <<
", INFO(2) = " << mumps_par.info[1] << std::endl;
594 int WrongInt = Wrong;
595 RCP<const Comm<int> > matComm = this->matrixA_->getComm();
596 Teuchos::broadcast<int,int>(*matComm,0,1,&WrongInt);
597 TEUCHOS_TEST_FOR_EXCEPTION(WrongInt>0,
604 template<
class Matrix,
class Vector>
605 const char* MUMPS<Matrix,Vector>::name =
"MUMPS";
Amesos2 MUMPS declarations.
@ ROOTED
Definition Amesos2_TypeDecl.hpp:93
@ CONTIGUOUS_AND_ROOTED
Definition Amesos2_TypeDecl.hpp:94
@ ARBITRARY
Definition Amesos2_TypeDecl.hpp:109
Amesos2 interface to the MUMPS package.
Definition Amesos2_MUMPS_decl.hpp:51
A Matrix adapter interface for Amesos2.
Definition Amesos2_MatrixAdapter_decl.hpp:42
EPhase
Used to indicate a phase in the direct solution.
Definition Amesos2_TypeDecl.hpp:31
Passes functions to TPL functions based on type.
Definition Amesos2_FunctionMap.hpp:43
A templated MultiVector class adapter for Amesos2.
Definition Amesos2_MultiVecAdapter_decl.hpp:142
A generic helper class for getting a CCS representation of a Matrix.
Definition Amesos2_Util.hpp:589