1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278 | // *****************************************************************************
/*!
\file src/Inciter/Sorter.hpp
\copyright 2012-2015 J. Bakosi,
2016-2018 Los Alamos National Security, LLC.,
2019-2021 Triad National Security, LLC.,
2022-2025 J. Bakosi
All rights reserved. See the LICENSE file for details.
\brief Mesh sorter for global distributed mesh reordering
\details Mesh sorter is Charm++ chare array and is used to do global
distributed mesh node reordering that yields consecutive unique global node
IDs with increasing PE IDs in asynchronous distributed-memory parallel
fashion.
*/
// *****************************************************************************
#ifndef Sorter_h
#define Sorter_h
#include <vector><--- Include file: not found. Please note: Cppcheck does not need standard library headers to get proper results.
#include <map><--- Include file:
#include <unordered_map><--- Include file: not found. Please note: Cppcheck does not need standard library headers to get proper results.
#include "TaggedTuple.hpp"
#include "Callback.hpp"
#include "UnsMesh.hpp"<--- Include file: "UnsMesh.hpp" not found.
#include "UnsMesh.hpp"<--- Include file: "UnsMesh.hpp" not found.
#include "NoWarning/transporter.decl.h"<--- Include file: "NoWarning/transporter.decl.h" not found.
#include "NoWarning/sorter.decl.h"<--- Include file: "NoWarning/sorter.decl.h" not found.
namespace inciter {
//! Mesh sorter for global distributed mesh node reordering
class Sorter : public CBase_Sorter {
#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunused-parameter"
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
#elif defined(STRICT_GNUC)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#elif defined(__INTEL_COMPILER)
#pragma warning( push )
#pragma warning( disable: 1478 )
#endif
// Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
// charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
Sorter_SDAG_CODE
#if defined(__clang__)
#pragma clang diagnostic pop
#elif defined(STRICT_GNUC)
#pragma GCC diagnostic pop
#elif defined(__INTEL_COMPILER)
#pragma warning( pop )
#endif
public:
//! Constructor
explicit Sorter( std::size_t meshid,
const CProxy_Transporter& transporter,
const tk::CProxy_MeshWriter& meshwriter,
const tk::SorterCallback& cbs,
const std::vector< CProxy_Discretization >& discretization,
const CProxy_RieCG& riecg,
const CProxy_LaxCG& laxcg,
const CProxy_ZalCG& zalcg,
const CProxy_KozCG& kozcg,
const CProxy_ChoCG& chocg,
const CProxy_LohCG& lohcg,
const tk::CProxy_ConjugateGradients& cgpre,
const tk::CProxy_ConjugateGradients& cgmom,
CkCallback reorderRefiner,
const std::vector< std::size_t >& ginpoel,
const tk::UnsMesh::CoordMap& coordmap,
const tk::UnsMesh::Chunk& el,
const std::map< int, std::vector< std::size_t > >& bface,
const std::vector< std::size_t >& triinpoel,
const std::map< int, std::vector< std::size_t > >& bnode,
int nchare );
#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wundefined-func-template"
#endif
//! Migrate constructor
// cppcheck-suppress uninitMemberVarPrivate
explicit Sorter( CkMigrateMessage* m ) : CBase_Sorter( m ) {}
#if defined(__clang__)
#pragma clang diagnostic pop
#endif
//! Configure Charm++ reduction types
static void registerReducers();
//! Setup chare mesh boundary node communication map
void setup( std::size_t npoin );
//! \brief Incoming query for a list mesh nodes for which this chare
//! compiles communication maps
void query( int fromch, const std::unordered_set< std::size_t >& bnd );
//! Report receipt of boundary node lists
void recvquery();
//! Respond to boundary node list queries
void response();
//! Receive boundary node communication map for our mesh chunk
void bnd( int fromch,
const std::map< int, std::unordered_set< std::size_t > >& nodeCommMap );
//! Receive receipt of boundary node communication map
void recvbnd();
//! Start reordering (if user enabled it)
void start();
//! \brief Receive number of uniquely assigned global mesh node IDs from
//! chares with lower indices
void offset( int c, std::size_t u );
//! Request new global node IDs for old node IDs
void request( int c, const std::unordered_set< std::size_t >& nd );
//! Receive lower bound of node IDs our PE operates on after reordering
void lower( std::size_t low );
//! Receive new (reordered) global node IDs and coordinates
void neworder( const std::unordered_map< std::size_t,
std::tuple< std::size_t, tk::UnsMesh::Coord > >& nodes );
//! Create worker chare array elements on this PE
void createWorkers();
//! Update mesh data we hold for whoever calls this function
void mesh( std::vector< std::size_t >& ginpoel,
tk::UnsMesh::CoordMap& coordmap,
std::vector< std::size_t >& triinpoel,
std::map< int, std::vector< std::size_t > >& bnode );
/** @name Charm++ pack/unpack serializer member functions */
///@{
//! \brief Pack/Unpack serialize member function
//! \param[in,out] p Charm++'s PUP::er serializer object reference
void pup( PUP::er &p ) override {
p | m_meshid;
p | m_host;
p | m_meshwriter;
p | m_cbs;
p | m_discretization;
p | m_riecg;
p | m_laxcg;
p | m_kozcg;
p | m_chocg;
p | m_lohcg;
p | m_cgpre;
p | m_cgmom;
p | m_reorderRefiner;
p | m_ginpoel;
p | m_coordmap;
p | m_el;
p | m_nbnd;
p | m_bface;
p | m_triinpoel;
p | m_bnode;
p | m_nchare;
p | m_nodeset;
p | m_noffset;
p | m_nodech;
p | m_chnode;
p | m_nodeCommMap;
p | m_reordcomm;
p | m_start;
p | m_newnodes;
p | m_newcoordmap;
p | m_reqnodes;
p | m_lower;
p | m_upper;
}
//! \brief Pack/Unpack serialize operator|
//! \param[in,out] p Charm++'s PUP::er serializer object reference
//! \param[in,out] s Sorter object reference
friend void operator|( PUP::er& p, Sorter& s ) { s.pup(p); }
//@}
private:
//! Mesh ID
std::size_t m_meshid;
//! Host proxy
CProxy_Transporter m_host;
//! MeshWriter proxy
tk::CProxy_MeshWriter m_meshwriter;
//! Charm++ callbacks associated to compile-time tags for sorter
tk::SorterCallback m_cbs;
//! Discretization proxy for all meshes
std::vector< CProxy_Discretization > m_discretization;
//! Discretization scheme proxy
CProxy_RieCG m_riecg;
//! Discretization scheme proxy
CProxy_LaxCG m_laxcg;
//! Discretization scheme proxy
CProxy_ZalCG m_zalcg;
//! Discretization scheme proxy
CProxy_KozCG m_kozcg;
//! Discretization scheme proxy
CProxy_ChoCG m_chocg;
//! Discretization scheme proxy
CProxy_LohCG m_lohcg;
//! Conjugate Gradients Charm++ proxy for pressure solve
tk::CProxy_ConjugateGradients m_cgpre;
//! Conjugate Gradients Charm++ proxy for momentum solve
tk::CProxy_ConjugateGradients m_cgmom;
//! Callback to use to send reordered mesh to Refiner
CkCallback m_reorderRefiner;
//! Tetrtahedron element connectivity of our chunk of the mesh (global ids)
std::vector< std::size_t > m_ginpoel;
//! Coordinates associated to global node IDs of our mesh chunk
tk::UnsMesh::CoordMap m_coordmap;
//! Elements of the mesh chunk we operate on
tk::UnsMesh::Chunk m_el;
//! Counter for number of chares contributing to chare boundary nodes
std::size_t m_nbnd;
//! List of boundary faces associated to side-set IDs
std::map< int, std::vector< std::size_t > > m_bface;
//! Boundary face-node connectivity
std::vector< std::size_t > m_triinpoel;
//! List of boundary nodes associated to side-set IDs
std::map< int, std::vector< std::size_t > > m_bnode;
//! Total number of sorter chares
int m_nchare;
//! Unique global node IDs chares on our PE will contribute to
std::set< std::size_t > m_nodeset;
//! \brief Counter for the number of chares from which this chare has
//! received node reordering offsets from
int m_noffset;
//! Node->chare map used to build boundary node communication maps
std::unordered_map< std::size_t, std::vector< int > > m_nodech;
//! Chare->node map used to build boundary node communication maps
std::unordered_map< int, std::unordered_set< std::size_t > > m_chnode;
//! Node lists associated to chare IDs
std::map< int, std::unordered_set< std::size_t > > m_nodeCommMap;
//! \brief Communication map used for distributed mesh node reordering
//! \details This map associates the list of global mesh point
//! indices to fellow chare IDs from which this chare receives new node
//! IDs during reordering. Only data that will be received from chares
//! with a lower index are stored, thus this is an asymmetric
//! communication map.
std::unordered_map< int, std::unordered_set< std::size_t > > m_reordcomm;
//! \brief Starting global mesh node ID for node reordering on this chare
//! during mesh node reordering
std::size_t m_start;
//! Map associating new node IDs (value) to old node IDs (key)
std::unordered_map< std::size_t, std::size_t > m_newnodes;
//! Coordinates associated to global (new) node IDs during reordering
tk::UnsMesh::CoordMap m_newcoordmap;
//! Queue of requested node IDs from chares
std::vector< std::pair< int, std::unordered_set<std::size_t> > > m_reqnodes;
//! Lower bound of node IDs this chare contributes to in a linear system
std::size_t m_lower;
//! Upper bound of node IDs this chare contributes to in a linear system
std::size_t m_upper;
//! Start preparing for mesh node reordering in parallel
void mask();
//! Reorder global mesh node IDs
void reorder();
//! Associate new node IDs to old ones and return them to the requestor(s)
void prepare();
//! Compute final result of reordering
void finish();
//! Create Discretization chare array elements on this PE
void createDiscWorkers();
};
} // inciter::
#endif // Sorter_h
|