Line data Source code
1 : // *****************************************************************************
2 : /*!
3 : \file src/Inciter/Sorter.hpp
4 : \copyright 2012-2015 J. Bakosi,
5 : 2016-2018 Los Alamos National Security, LLC.,
6 : 2019-2021 Triad National Security, LLC.,
7 : 2022-2025 J. Bakosi
8 : All rights reserved. See the LICENSE file for details.
9 : \brief Mesh sorter for global distributed mesh reordering
10 : \details Mesh sorter is Charm++ chare array and is used to do global
11 : distributed mesh node reordering that yields consecutive unique global node
12 : IDs with increasing PE IDs in asynchronous distributed-memory parallel
13 : fashion.
14 : */
15 : // *****************************************************************************
16 : #ifndef Sorter_h
17 : #define Sorter_h
18 :
19 : #include <vector>
20 : #include <map>
21 : #include <unordered_map>
22 :
23 : #include "TaggedTuple.hpp"
24 : #include "Callback.hpp"
25 : #include "UnsMesh.hpp"
26 : #include "UnsMesh.hpp"
27 :
28 : #include "NoWarning/transporter.decl.h"
29 : #include "NoWarning/sorter.decl.h"
30 :
31 : namespace inciter {
32 :
33 : //! Mesh sorter for global distributed mesh node reordering
34 : class Sorter : public CBase_Sorter {
35 :
36 : #if defined(__clang__)
37 : #pragma clang diagnostic push
38 : #pragma clang diagnostic ignored "-Wunused-parameter"
39 : #pragma clang diagnostic ignored "-Wdeprecated-declarations"
40 : #elif defined(STRICT_GNUC)
41 : #pragma GCC diagnostic push
42 : #pragma GCC diagnostic ignored "-Wunused-parameter"
43 : #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
44 : #elif defined(__INTEL_COMPILER)
45 : #pragma warning( push )
46 : #pragma warning( disable: 1478 )
47 : #endif
48 : // Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
49 : // charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
50 : Sorter_SDAG_CODE
51 : #if defined(__clang__)
52 : #pragma clang diagnostic pop
53 : #elif defined(STRICT_GNUC)
54 : #pragma GCC diagnostic pop
55 : #elif defined(__INTEL_COMPILER)
56 : #pragma warning( pop )
57 : #endif
58 :
59 : public:
60 : //! Constructor
61 : explicit Sorter( std::size_t meshid,
62 : const CProxy_Transporter& transporter,
63 : const tk::CProxy_MeshWriter& meshwriter,
64 : const tk::SorterCallback& cbs,
65 : const std::vector< CProxy_Discretization >& discretization,
66 : const CProxy_RieCG& riecg,
67 : const CProxy_LaxCG& laxcg,
68 : const CProxy_ZalCG& zalcg,
69 : const CProxy_KozCG& kozcg,
70 : const CProxy_ChoCG& chocg,
71 : const CProxy_LohCG& lohcg,
72 : const tk::CProxy_ConjugateGradients& cgpre,
73 : const tk::CProxy_ConjugateGradients& cgmom,
74 : CkCallback reorderRefiner,
75 : const std::vector< std::size_t >& ginpoel,
76 : const tk::UnsMesh::CoordMap& coordmap,
77 : const tk::UnsMesh::Chunk& el,
78 : const std::map< int, std::vector< std::size_t > >& bface,
79 : const std::vector< std::size_t >& triinpoel,
80 : const std::map< int, std::vector< std::size_t > >& bnode,
81 : int nchare );
82 :
83 : #if defined(__clang__)
84 : #pragma clang diagnostic push
85 : #pragma clang diagnostic ignored "-Wundefined-func-template"
86 : #endif
87 : //! Migrate constructor
88 : // cppcheck-suppress uninitMemberVarPrivate
89 4495 : explicit Sorter( CkMigrateMessage* m ) : CBase_Sorter( m ) {}
90 : #if defined(__clang__)
91 : #pragma clang diagnostic pop
92 : #endif
93 :
94 : //! Configure Charm++ reduction types
95 : static void registerReducers();
96 :
97 : //! Setup chare mesh boundary node communication map
98 : void setup( std::size_t npoin );
99 :
100 : //! \brief Incoming query for a list mesh nodes for which this chare
101 : //! compiles communication maps
102 : void query( int fromch, const std::unordered_set< std::size_t >& bnd );
103 : //! Report receipt of boundary node lists
104 : void recvquery();
105 : //! Respond to boundary node list queries
106 : void response();
107 : //! Receive boundary node communication map for our mesh chunk
108 : void bnd( int fromch,
109 : const std::map< int, std::unordered_set< std::size_t > >& nodeCommMap );
110 : //! Receive receipt of boundary node communication map
111 : void recvbnd();
112 : //! Start reordering (if user enabled it)
113 : void start();
114 :
115 : //! \brief Receive number of uniquely assigned global mesh node IDs from
116 : //! chares with lower indices
117 : void offset( int c, std::size_t u );
118 :
119 : //! Request new global node IDs for old node IDs
120 : void request( int c, const std::unordered_set< std::size_t >& nd );
121 :
122 : //! Receive lower bound of node IDs our PE operates on after reordering
123 : void lower( std::size_t low );
124 :
125 : //! Receive new (reordered) global node IDs and coordinates
126 : void neworder( const std::unordered_map< std::size_t,
127 : std::tuple< std::size_t, tk::UnsMesh::Coord > >& nodes );
128 :
129 : //! Create worker chare array elements on this PE
130 : void createWorkers();
131 :
132 : //! Update mesh data we hold for whoever calls this function
133 : void mesh( std::vector< std::size_t >& ginpoel,
134 : tk::UnsMesh::CoordMap& coordmap,
135 : std::vector< std::size_t >& triinpoel,
136 : std::map< int, std::vector< std::size_t > >& bnode );
137 :
138 : /** @name Charm++ pack/unpack serializer member functions */
139 : ///@{
140 : //! \brief Pack/Unpack serialize member function
141 : //! \param[in,out] p Charm++'s PUP::er serializer object reference
142 14279 : void pup( PUP::er &p ) override {
143 14279 : p | m_meshid;
144 : p | m_host;
145 : p | m_meshwriter;
146 : p | m_cbs;
147 14279 : p | m_discretization;
148 : p | m_riecg;
149 : p | m_laxcg;
150 : p | m_kozcg;
151 : p | m_chocg;
152 : p | m_lohcg;
153 : p | m_cgpre;
154 : p | m_cgmom;
155 14279 : p | m_reorderRefiner;
156 14279 : p | m_ginpoel;
157 14279 : p | m_coordmap;
158 14279 : p | m_el;
159 14279 : p | m_nbnd;
160 14279 : p | m_bface;
161 14279 : p | m_triinpoel;
162 14279 : p | m_bnode;
163 14279 : p | m_nchare;
164 14279 : p | m_nodeset;
165 14279 : p | m_noffset;
166 14279 : p | m_nodech;
167 14279 : p | m_chnode;
168 14279 : p | m_nodeCommMap;
169 14279 : p | m_reordcomm;
170 14279 : p | m_start;
171 14279 : p | m_newnodes;
172 14279 : p | m_newcoordmap;
173 14279 : p | m_reqnodes;
174 14279 : p | m_lower;
175 14279 : p | m_upper;
176 14279 : }
177 : //! \brief Pack/Unpack serialize operator|
178 : //! \param[in,out] p Charm++'s PUP::er serializer object reference
179 : //! \param[in,out] s Sorter object reference
180 : friend void operator|( PUP::er& p, Sorter& s ) { s.pup(p); }
181 : //@}
182 :
183 : private:
184 : //! Mesh ID
185 : std::size_t m_meshid;
186 : //! Host proxy
187 : CProxy_Transporter m_host;
188 : //! MeshWriter proxy
189 : tk::CProxy_MeshWriter m_meshwriter;
190 : //! Charm++ callbacks associated to compile-time tags for sorter
191 : tk::SorterCallback m_cbs;
192 : //! Discretization proxy for all meshes
193 : std::vector< CProxy_Discretization > m_discretization;
194 : //! Discretization scheme proxy
195 : CProxy_RieCG m_riecg;
196 : //! Discretization scheme proxy
197 : CProxy_LaxCG m_laxcg;
198 : //! Discretization scheme proxy
199 : CProxy_ZalCG m_zalcg;
200 : //! Discretization scheme proxy
201 : CProxy_KozCG m_kozcg;
202 : //! Discretization scheme proxy
203 : CProxy_ChoCG m_chocg;
204 : //! Discretization scheme proxy
205 : CProxy_LohCG m_lohcg;
206 : //! Conjugate Gradients Charm++ proxy for pressure solve
207 : tk::CProxy_ConjugateGradients m_cgpre;
208 : //! Conjugate Gradients Charm++ proxy for momentum solve
209 : tk::CProxy_ConjugateGradients m_cgmom;
210 : //! Callback to use to send reordered mesh to Refiner
211 : CkCallback m_reorderRefiner;
212 : //! Tetrtahedron element connectivity of our chunk of the mesh (global ids)
213 : std::vector< std::size_t > m_ginpoel;
214 : //! Coordinates associated to global node IDs of our mesh chunk
215 : tk::UnsMesh::CoordMap m_coordmap;
216 : //! Elements of the mesh chunk we operate on
217 : tk::UnsMesh::Chunk m_el;
218 : //! Counter for number of chares contributing to chare boundary nodes
219 : std::size_t m_nbnd;
220 : //! List of boundary faces associated to side-set IDs
221 : std::map< int, std::vector< std::size_t > > m_bface;
222 : //! Boundary face-node connectivity
223 : std::vector< std::size_t > m_triinpoel;
224 : //! List of boundary nodes associated to side-set IDs
225 : std::map< int, std::vector< std::size_t > > m_bnode;
226 : //! Total number of sorter chares
227 : int m_nchare;
228 : //! Unique global node IDs chares on our PE will contribute to
229 : std::set< std::size_t > m_nodeset;
230 : //! \brief Counter for the number of chares from which this chare has
231 : //! received node reordering offsets from
232 : int m_noffset;
233 : //! Node->chare map used to build boundary node communication maps
234 : std::unordered_map< std::size_t, std::vector< int > > m_nodech;
235 : //! Chare->node map used to build boundary node communication maps
236 : std::unordered_map< int, std::unordered_set< std::size_t > > m_chnode;
237 : //! Node lists associated to chare IDs
238 : std::map< int, std::unordered_set< std::size_t > > m_nodeCommMap;
239 : //! \brief Communication map used for distributed mesh node reordering
240 : //! \details This map associates the list of global mesh point
241 : //! indices to fellow chare IDs from which this chare receives new node
242 : //! IDs during reordering. Only data that will be received from chares
243 : //! with a lower index are stored, thus this is an asymmetric
244 : //! communication map.
245 : std::unordered_map< int, std::unordered_set< std::size_t > > m_reordcomm;
246 : //! \brief Starting global mesh node ID for node reordering on this chare
247 : //! during mesh node reordering
248 : std::size_t m_start;
249 : //! Map associating new node IDs (value) to old node IDs (key)
250 : std::unordered_map< std::size_t, std::size_t > m_newnodes;
251 : //! Coordinates associated to global (new) node IDs during reordering
252 : tk::UnsMesh::CoordMap m_newcoordmap;
253 : //! Queue of requested node IDs from chares
254 : std::vector< std::pair< int, std::unordered_set<std::size_t> > > m_reqnodes;
255 : //! Lower bound of node IDs this chare contributes to in a linear system
256 : std::size_t m_lower;
257 : //! Upper bound of node IDs this chare contributes to in a linear system
258 : std::size_t m_upper;
259 :
260 : //! Start preparing for mesh node reordering in parallel
261 : void mask();
262 :
263 : //! Reorder global mesh node IDs
264 : void reorder();
265 :
266 : //! Associate new node IDs to old ones and return them to the requestor(s)
267 : void prepare();
268 :
269 : //! Compute final result of reordering
270 : void finish();
271 :
272 : //! Create Discretization chare array elements on this PE
273 : void createDiscWorkers();
274 : };
275 :
276 : } // inciter::
277 :
278 : #endif // Sorter_h
|
