Zoltan2
Loading...
Searching...
No Matches
Zoltan2_AlltoAll.hpp
Go to the documentation of this file.
1// @HEADER
2// *****************************************************************************
3// Zoltan2: A package of combinatorial algorithms for scientific computing
4//
5// Copyright 2012 NTESS and the Zoltan2 contributors.
6// SPDX-License-Identifier: BSD-3-Clause
7// *****************************************************************************
8// @HEADER
9
14#ifndef _ZOLTAN2_ALLTOALL_HPP_
15#define _ZOLTAN2_ALLTOALL_HPP_
16
17#include <Zoltan2_Standards.hpp>
19
20#include <vector>
21#include <climits>
22
23namespace Zoltan2
24{
25
26extern void AlltoAllCount(const Comm<int> &comm, const Environment &env,
27 const ArrayView<const int> &sendCount, const ArrayView<int> &recvCount);
28
56template <typename T>
57void AlltoAllv(const Comm<int> &comm,
58 const Environment &env,
59 const ArrayView<const T> &sendBuf,
60 const ArrayView<const int> &sendCount,
61 ArrayRCP<T> &recvBuf, // output, allocated here
62 const ArrayView<int> &recvCount // output
63)
64{
65 int nprocs = comm.getSize();
66 int rank = comm.getRank();
67
68 try{
69 Zoltan2::AlltoAllCount(comm, env, sendCount, recvCount);
70 }
72
73 // Allocate the receive buffer.
74 size_t totalrecv = 0;
75 int maxMsg = 0;
76 int nrecvranks = 0;
77 for(int i = 0; i < nprocs; i++) {
78 if (recvCount[i] > 0) {
79 totalrecv += recvCount[i];
80 nrecvranks++;
81 if (recvCount[i] > maxMsg) maxMsg = recvCount[i];
82 }
83 }
84
85
86 T *rbuf = NULL;
87 if (totalrecv) rbuf = new T[totalrecv];
88
89 if (nprocs > 1) {
90
91 RCP<CommRequest<int> > *requests = new RCP<CommRequest<int> > [nrecvranks];
92
93 // Error checking for memory and message size.
94 int OK[2] = {1,1};
95 // OK[0] -- true/false indicating whether each message size
96 // fits in an int (for MPI).
97 // OK[1] -- true/false indicating whether memory allocs are OK
98 int gOK[2]; // For global reduce of OK.
99
100 if (size_t(maxMsg) * sizeof(T) > INT_MAX && nprocs > 1) OK[0] = false;
101 if (totalrecv && !rbuf) OK[1] = 0;
102 if (!requests) OK[1] = 0;
103
104 // Post receives
105
106 size_t offset = 0;
107 size_t myrecvoffset = 0;
108 size_t mysendoffset = 0;
109
110 if (OK[0] && OK[1]) {
111 int rcnt = 0;
112 for (int i = 0; i < nprocs; i++) {
113 if (i != rank && recvCount[i]) {
114 try {
115 requests[rcnt++] = Teuchos::ireceive<int,T>(comm,
116 Teuchos::arcp(&rbuf[offset],0,recvCount[i],false),
117 i);
118 }
120 }
121 else if (i == rank) {
122 myrecvoffset = offset;
123 }
124 offset += recvCount[i];
125 }
126 }
127
128 // Use barrier for error checking
129 Teuchos::reduceAll<int>(comm, Teuchos::REDUCE_MIN, 2, OK, gOK);
130 if (!gOK[0] || !gOK[1]) {
131 delete [] rbuf;
132 delete [] requests;
133 if (!gOK[0])
134 throw std::runtime_error("Max single message length exceeded");
135 else
136 throw std::bad_alloc();
137 }
138
139 // Send data; can use readySend since receives are posted.
140 offset = 0;
141 for (int i = 0; i < nprocs; i++) {
142 if (i != rank && sendCount[i]) {
143 try {
144 Teuchos::readySend<int,T>(comm,
145 Teuchos::arrayView(&sendBuf[offset],sendCount[i]),
146 i);
147 }
149 }
150 else if (i == rank) {
151 mysendoffset = offset;
152 }
153 offset += sendCount[i];
154 }
155
156 // Copy local data
157 for (int j = 0; j < sendCount[rank]; j++)
158 rbuf[myrecvoffset++] = sendBuf[mysendoffset++];
159
160 // Wait for messages to return.
161 try {
162 Teuchos::waitAll<int>(comm, Teuchos::arrayView(requests, nrecvranks));
163 }
165
166 delete [] requests;
167 }
168 else { // nprocs == 1; no communication needed
169
170 if (totalrecv && !rbuf)
171 throw std::bad_alloc();
172
173 for (int j = 0; j < sendCount[0]; j++)
174 rbuf[j] = sendBuf[j];
175 }
176
177 if (totalrecv)
178 recvBuf = ArrayRCP<T>(rbuf, 0, totalrecv, true);
179 else
180 recvBuf = Teuchos::null;
181}
182
183/* \brief Specialization for std::string.
184
185 For string of char. Number of chars in a string limited to SCHAR_MAX.
186 Send as chars: 1 char for length of string, then chars in string,
187 1 char for length of next string, and so on.
188 \todo error checking
189 */
190template <>
191void AlltoAllv(const Comm<int> &comm,
192 const Environment &env,
193 const ArrayView<const std::string> &sendBuf,
194 const ArrayView<const int> &sendCount,
195 ArrayRCP<std::string> &recvBuf,
196 const ArrayView<int> &recvCount);
197
198/* \brief Specialization for unsigned long long
199 */
200#ifdef HAVE_TPETRA_INT_LONG_LONG
201template <>
202void AlltoAllv(const Comm<int> &comm,
203 const Environment &env,
204 const ArrayView<const unsigned long long> &sendBuf,
205 const ArrayView<const int> &sendCount,
206 ArrayRCP<unsigned long long> &recvBuf,
207 const ArrayView<int> &recvCount);
208#endif
209
210/* \brief Specialization for unsigned short
211 */
212template <>
213void AlltoAllv(const Comm<int> &comm,
214 const Environment &env,
215 const ArrayView<const unsigned short> &sendBuf,
216 const ArrayView<const int> &sendCount,
217 ArrayRCP<unsigned short> &recvBuf,
218 const ArrayView<int> &recvCount);
219
220/* \brief For data type unsigned char (no Teuchos::DirectSerializationTraits)
221 */
222template <>
223void AlltoAllv(const Comm<int> &comm,
224 const Environment &env,
225 const ArrayView<const unsigned char> &sendBuf,
226 const ArrayView<const int> &sendCount,
227 ArrayRCP<unsigned char> &recvBuf,
228 const ArrayView<int> &recvCount);
229
230} // namespace Z2
231#endif
Defines the Environment class.
#define Z2_FORWARD_EXCEPTIONS
Forward an exception back through call stack.
#define Z2_THROW_OUTSIDE_ERROR(env)
Throw an error returned from outside the Zoltan2 library.
Gathering definitions used in software development.
The user parameters, debug, timing and memory profiling output objects, and error checking methods.
Created by mbenlioglu on Aug 31, 2020.
void AlltoAllCount(const Comm< int > &comm, const Environment &env, const ArrayView< const int > &sendCount, const ArrayView< int > &recvCount)
Each process sends a value to every process, an all-to-all.
void AlltoAllv(const Comm< int > &comm, const Environment &env, const ArrayView< const std::string > &sendBuf, const ArrayView< const int > &sendCount, ArrayRCP< std::string > &recvBuf, const ArrayView< int > &recvCount)