Zoltan2
Loading...
Searching...
No Matches
Zoltan2_findUniqueGids.hpp
Go to the documentation of this file.
1// @HEADER
2// *****************************************************************************
3// Zoltan2: A package of combinatorial algorithms for scientific computing
4//
5// Copyright 2012 NTESS and the Zoltan2 contributors.
6// SPDX-License-Identifier: BSD-3-Clause
7// *****************************************************************************
8// @HEADER
9
15#ifndef _ZOLTAN2_FINDUNIQUEGIDS_HPP_
16#define _ZOLTAN2_FINDUNIQUEGIDS_HPP_
17
18#include <Zoltan2_Standards.hpp>
19#include <vector>
20
21#include <Tpetra_MultiVector.hpp>
22#include <Tpetra_Vector.hpp>
23
24#include <Zoltan2_TPLTraits.hpp>
25
26#include <zoltan_dd.h>
27#include <zoltan_dd_const.h>
28
29namespace Zoltan2
30{
31
32template <typename gno_t>
34 size_t num_keys,
35 int num_gid,
36 ZOLTAN_ID_PTR ddkeys,
37 char *ddnewgids,
38 MPI_Comm mpicomm
39)
40{
41 int num_lid = 0; // Local IDs not needed
42 int debug_level = 0;
43 int num_user = sizeof(gno_t);
44
45 Zoltan_DD_Struct *dd = NULL;
46 Zoltan_DD_Create(&dd, mpicomm, num_gid, num_lid, num_user, num_keys,
47 debug_level);
48
49 ZOLTAN_ID_PTR ddnotneeded = NULL; // Local IDs not needed
50 Zoltan_DD_Update(dd, ddkeys, ddnotneeded, ddnewgids, NULL, int(num_keys));
51
53 // Insert unique GIDs for DD entries in User data here.
54
55 // Get value of first gid on this rank
56 typedef long long mpi_t;
57 mpi_t nDDEntries = (mpi_t)(dd->nodecnt);
58 mpi_t firstIdx;
59 MPI_Scan(&nDDEntries, &firstIdx, 1, MPI_LONG_LONG, MPI_SUM, mpicomm);
60 firstIdx -= nDDEntries; // do not include this rank's entries in prefix sum
61
62 // Loop over all directory entries, updating their userdata with updated gid
63 DD_NodeIdx cnt = 0;
64
65 for (DD_NodeIdx i = 0; i < dd->nodelistlen; i++) {
66 DD_Node *ptr = &(dd->nodelist[i]);
67 if (!(ptr->free)) {
68 char *userchar = (char*)(ptr->gid + (dd->gid_length + dd->lid_length));
69 gno_t *newgid = (gno_t*) userchar;
70 *newgid = gno_t(firstIdx + cnt);
71 cnt++;
72 }
73 }
74
76 // Retrieve the global numbers and put in the result gids vector
77 Zoltan_DD_Find(dd, ddkeys, ddnotneeded, ddnewgids, NULL, int(num_keys), NULL);
78
79 Zoltan_DD_Destroy(&dd);
80
81 mpi_t nUnique = 0;
82 MPI_Allreduce(&nDDEntries, &nUnique, 1, MPI_LONG_LONG, MPI_SUM, mpicomm);
83
84 return size_t(nUnique);
85}
86
88template <typename lno_t, typename gno_t>
90 Tpetra::MultiVector<gno_t, lno_t, gno_t> &keys,
91 Tpetra::Vector<gno_t, lno_t, gno_t> &gids
92)
93{
94 // Input: Tpetra MultiVector of keys; key length = numVectors()
95 // May contain duplicate keys within a processor.
96 // May contain duplicate keys across processors.
97 // Input: Empty Tpetra Vector with same map for holding the results
98 // Output: Filled gids vector, containing unique global numbers for
99 // each unique key. Global numbers are in range [0,#UniqueKeys).
100
101 size_t num_keys = keys.getLocalLength();
102 size_t num_entries = keys.getNumVectors();
103
104#ifdef HAVE_ZOLTAN2_MPI
105 MPI_Comm mpicomm = Teuchos::getRawMpiComm(*(keys.getMap()->getComm()));
106#else
107 // Zoltan's siMPI will be used here
108 {
109 int flag;
110 MPI_Initialized(&flag);
111 if (!flag) {
112 int narg = 0;
113 char **argv = NULL;
114 MPI_Init(&narg, &argv);
115 }
116 }
117 MPI_Comm mpicomm = MPI_COMM_WORLD; // Will get MPI_COMM_WORLD from siMPI
118#endif
119
120 int num_gid = TPL_Traits<ZOLTAN_ID_PTR,gno_t>::NUM_ID * num_entries;
121 int num_user = sizeof(gno_t);
122
123 // Buffer the keys for Zoltan_DD
124 Teuchos::ArrayRCP<const gno_t> *tmpKeyVecs =
125 new Teuchos::ArrayRCP<const gno_t>[num_entries];
126 for (size_t v = 0; v < num_entries; v++) tmpKeyVecs[v] = keys.getData(v);
127
128 ZOLTAN_ID_PTR ddkeys = new ZOLTAN_ID_TYPE[num_gid * num_keys];
129 size_t idx = 0;
130 for (size_t i = 0; i < num_keys; i++) {
131 for (size_t v = 0; v < num_entries; v++) {
132 ZOLTAN_ID_PTR ddkey = &(ddkeys[idx]);
133 TPL_Traits<ZOLTAN_ID_PTR,gno_t>::ASSIGN(ddkey, tmpKeyVecs[v][i]);
135 }
136 }
137 delete [] tmpKeyVecs;
138
139 // Allocate memory for the result
140 char *ddnewgids = new char[num_user * num_keys];
141
142 // Compute the new GIDs
143 size_t nUnique = findUniqueGidsCommon<gno_t>(num_keys, num_gid,
144 ddkeys, ddnewgids, mpicomm);
145
146 // Copy the result into the output vector
147 gno_t *result = (gno_t *)ddnewgids;
148 for (size_t i = 0; i < num_keys; i++)
149 gids.replaceLocalValue(i, result[i]);
150
151 // Clean up
152 delete [] ddkeys;
153 delete [] ddnewgids;
154
155 return nUnique;
156}
157
159template <typename key_t, typename gno_t>
161 std::vector<key_t> &keys,
162 std::vector<gno_t> &gids,
163 const Teuchos::Comm<int> &comm
164)
165{
166 // Input: Vector of keys; key length = key_t.size()
167 // Each key must have the same size. std::array<gno_t, N> is
168 // an example of a good key_t.
169 // May contain duplicate keys within a processor.
170 // May contain duplicate keys across processors.
171 // Input: Empty vector for holding the results
172 // Output: Filled gids vector, containing unique global numbers for
173 // each unique key. Global numbers are in range [0,#UniqueKeys).
174 //
175 // Note: This code uses the Zoltan Distributed Directory to assign the
176 // unique global numbers. Right now, it hacks into the Zoltan_DD
177 // data structures. If we like this approach, we can add some
178 // elegance to the Zoltan_DD, allowing operations internal to the
179 // directory.
180
181 size_t num_keys = keys.size();
182 key_t dummy;
183 size_t num_entries = dummy.size();
184
185#ifdef HAVE_ZOLTAN2_MPI
186 MPI_Comm mpicomm = Teuchos::getRawMpiComm(comm);
187#else
188 // Zoltan's siMPI will be used here
189 {
190 int flag;
191 MPI_Initialized(&flag);
192 if (!flag) {
193 int narg = 0;
194 char **argv = NULL;
195 MPI_Init(&narg, &argv);
196 }
197 }
198 MPI_Comm mpicomm = MPI_COMM_WORLD; // Will get MPI_COMM_WORLD from siMPI
199#endif
200
201 int num_gid = TPL_Traits<ZOLTAN_ID_PTR,gno_t>::NUM_ID * num_entries;
202 int num_user = sizeof(gno_t);
203
204 // Buffer the keys for Zoltan_DD
205 ZOLTAN_ID_PTR ddkeys = new ZOLTAN_ID_TYPE[num_gid * num_keys];
206 size_t idx = 0;
207 for (size_t i = 0; i < num_keys; i++) {
208 for (size_t v = 0; v < num_entries; v++) {
209 ZOLTAN_ID_PTR ddkey = &(ddkeys[idx]);
212 }
213 }
214
215 // Allocate memory for the result
216 char *ddnewgids = new char[num_user * num_keys];
217
218 // Compute the new GIDs
219 size_t nUnique = findUniqueGidsCommon<gno_t>(num_keys, num_gid,
220 ddkeys, ddnewgids, mpicomm);
221
222 // Copy the result into the output vector
223 gno_t *result = (gno_t *)ddnewgids;
224 for (size_t i = 0; i < num_keys; i++)
225 gids[i] = result[i];
226
227 // Clean up
228 delete [] ddkeys;
229 delete [] ddnewgids;
230
231 return nUnique;
232}
233
234
235} // namespace Zoltan2
236#endif
Gathering definitions used in software development.
Traits class to handle conversions between gno_t/lno_t and TPL data types (e.g., ParMETIS's idx_t,...
map_t::global_ordinal_type gno_t
Created by mbenlioglu on Aug 31, 2020.
size_t findUniqueGids(Tpetra::MultiVector< gno_t, lno_t, gno_t > &keys, Tpetra::Vector< gno_t, lno_t, gno_t > &gids)
size_t findUniqueGidsCommon(size_t num_keys, int num_gid, ZOLTAN_ID_PTR ddkeys, char *ddnewgids, MPI_Comm mpicomm)
static void ASSIGN(first_t &a, second_t b)