Line data Source code
1 : // *****************************************************************************
2 : /*!
3 : \file src/Inciter/Partitioner.hpp
4 : \copyright 2012-2015 J. Bakosi,
5 : 2016-2018 Los Alamos National Security, LLC.,
6 : 2019-2021 Triad National Security, LLC.,
7 : 2022-2025 J. Bakosi
8 : All rights reserved. See the LICENSE file for details.
9 : \brief Charm++ chare partitioner nodegroup used to perform mesh
10 : partitioning
11 : \details Charm++ chare partitioner nodegroup used to perform mesh read and
12 : partitioning, one worker per compute node.
13 : */
14 : // *****************************************************************************
15 : #ifndef Partitioner_h
16 : #define Partitioner_h
17 :
18 : #include <array>
19 : #include <stddef.h>
20 :
21 : #include "ContainerUtil.hpp"
22 : #include "DerivedData.hpp"
23 : #include "UnsMesh.hpp"
24 : #include "Sorter.hpp"
25 : #include "Refiner.hpp"
26 : #include "Callback.hpp"
27 :
28 : #include "NoWarning/partitioner.decl.h"
29 :
30 : namespace inciter {
31 :
32 : //! Partitioner Charm++ chare nodegroup class
33 : //! \details Instantiations of Partitioner comprise a processor aware Charm++
34 : //! chare node group. When instantiated, a new object is created on each
35 : //! compute node and not more (as opposed to individual chares or chare array
36 : //! object elements). See also the Charm++ interface file partitioner.ci.
37 : class Partitioner : public CBase_Partitioner {
38 :
39 : private:
40 : //! \brief Mesh data used for categorizing mesh chunks assigned to chares
41 : //! after mesh partitioning and before mesh distribution across chares
42 : using MeshData =
43 : std::tuple<
44 : // Tetrahedron (domain element) connectivity
45 : std::vector< std::size_t >,
46 : // Boundary face connectivity for each side set
47 : std::unordered_map< int, std::vector< std::size_t > >,
48 : // Boundary node lists for each side set
49 : std::unordered_map< int, std::vector< std::size_t > > >;
50 :
51 : public:
52 : //! Constructor
53 : Partitioner( std::size_t meshid,
54 : const std::string& filename,
55 : const tk::PartitionerCallback& cbp,
56 : const tk::RefinerCallback& cbr,
57 : const tk::SorterCallback& cbs,
58 : const CProxy_Transporter& host,
59 : const CProxy_Refiner& refiner,
60 : const CProxy_Sorter& sorter,
61 : const tk::CProxy_MeshWriter& meshwriter,
62 : const std::vector< CProxy_Discretization >& discretization,
63 : const CProxy_RieCG& riecg,
64 : const CProxy_LaxCG& laxcg,
65 : const CProxy_ZalCG& zalcg,
66 : const CProxy_KozCG& kozcg,
67 : const CProxy_ChoCG& chocg,
68 : const CProxy_LohCG& lohcg,
69 : const tk::CProxy_ConjugateGradients& cgpre,
70 : const tk::CProxy_ConjugateGradients& cgmom,
71 : const std::map< int, std::vector< std::size_t > >& bface,
72 : const std::map< int, std::vector< std::size_t > >& faces,
73 : const std::map< int, std::vector< std::size_t > >& bnode );
74 :
75 : #if defined(__clang__)
76 : #pragma clang diagnostic push
77 : #pragma clang diagnostic ignored "-Wundefined-func-template"
78 : #endif
79 : //! Migrate constructor
80 : // cppcheck-suppress uninitMemberVar
81 30 : explicit Partitioner( CkMigrateMessage* m ) : CBase_Partitioner( m ) {}
82 : #if defined(__clang__)
83 : #pragma clang diagnostic pop
84 : #endif
85 :
86 : //! Configure Charm++ reduction types
87 : static void registerReducers();
88 :
89 : //! Aggregate mesh graph for mesh nodes owned
90 : void query( int fromnode,
91 : const std::unordered_map< std::size_t,
92 : std::unordered_set< std::size_t > >& psup );
93 : //! Receive receipt of list of points surrounding points to query
94 : void recvquery();
95 : //! Respond to graph queries
96 : void response();
97 : //! Receive mesh graphs for our mesh chunk
98 : void psup( int fromnode, const std::unordered_map< std::size_t,
99 : std::unordered_set< std::size_t > >& graph );
100 : //! Receive receipt of mesh nodes graphs
101 : void recvpsup();
102 : // Compute total load across distributed mesh
103 : void load();
104 :
105 : //! Reduction target to aggregate mesh partition assignments
106 : void parts( CkReductionMsg* msg );
107 :
108 : //! Partition the computational mesh into a number of chares
109 : void partition( int nchare );
110 :
111 : //! Receive mesh associated to chares we own after refinement
112 : void addMesh( int fromnode,
113 : const std::unordered_map< int,
114 : std::tuple<
115 : std::vector< std::size_t >,
116 : tk::UnsMesh::CoordMap,
117 : std::unordered_map< int, std::vector< std::size_t > >,
118 : std::unordered_map< int, std::vector< std::size_t > >
119 : > >& chmesh );
120 :
121 : //! Acknowledge received mesh after initial mesh refinement
122 : void recvMesh();
123 :
124 : //! Optionally start refining the mesh
125 : void refine();
126 :
127 : /** @name Charm++ pack/unpack serializer member functions */
128 : ///@{
129 : //! \brief Pack/Unpack serialize member function
130 : //! \param[in,out] p Charm++'s PUP::er serializer object reference
131 : //! \note This is a Charm++ nodegroup, pup() is thus only for
132 : //! checkpoint/restart.
133 398 : void pup( PUP::er &p ) override {
134 398 : p | m_meshid;
135 398 : p | m_npsup;
136 : p | m_cbp;
137 : p | m_cbr;
138 : p | m_cbs;
139 : p | m_host;
140 : p | m_refiner;
141 : p | m_sorter;
142 : p | m_meshwriter;
143 398 : p | m_discretization;
144 : p | m_riecg;
145 : p | m_laxcg;
146 : p | m_zalcg;
147 : p | m_kozcg;
148 : p | m_chocg;
149 : p | m_lohcg;
150 : p | m_cgpre;
151 : p | m_cgmom;
152 398 : p | m_ginpoel;
153 398 : p | m_graph;
154 398 : p | m_graphnode;
155 : p | m_coord;
156 398 : p | m_inpoel;
157 398 : p | m_lid;
158 398 : p | m_ndist;
159 398 : p | m_nchare;
160 398 : p | m_nface;
161 398 : p | m_linnodes;
162 398 : p | m_chinpoel;
163 398 : p | m_chcoordmap;
164 398 : p | m_chbface;
165 398 : p | m_chtriinpoel;
166 398 : p | m_chbnode;
167 398 : p | m_bface;
168 398 : p | m_triinpoel;
169 398 : p | m_bnode;
170 398 : }
171 : //! \brief Pack/Unpack serialize operator|
172 : //! \param[in,out] p Charm++'s PUP::er serializer object reference
173 : //! \param[in,out] i Partitioner object reference
174 : friend void operator|( PUP::er& p, Partitioner& i ) { i.pup(p); }
175 : //@}
176 :
177 : private:
178 : //! Mesh ID
179 : std::size_t m_meshid;
180 : //! Counter for number of nodes contributing to mesn graphs
181 : std::size_t m_npsup;
182 : //! Charm++ callbacks associated to compile-time tags for partitioner
183 : tk::PartitionerCallback m_cbp;
184 : //! Charm++ callbacks associated to compile-time tags for refiner
185 : tk::RefinerCallback m_cbr;
186 : //! Charm++ callbacks associated to compile-time tags for sorter
187 : tk::SorterCallback m_cbs;
188 : //! Host proxy
189 : CProxy_Transporter m_host;
190 : //! Mesh refiner proxy
191 : CProxy_Refiner m_refiner;
192 : //! Mesh sorter proxy
193 : CProxy_Sorter m_sorter;
194 : //! Mesh writer proxy
195 : tk::CProxy_MeshWriter m_meshwriter;
196 : //! Discretization proxy for all meshes
197 : std::vector< CProxy_Discretization > m_discretization;
198 : //! Discretization scheme proxy
199 : CProxy_RieCG m_riecg;
200 : //! Discretization scheme proxy
201 : CProxy_LaxCG m_laxcg;
202 : //! Discretization scheme proxy
203 : CProxy_ZalCG m_zalcg;
204 : //! Discretization scheme proxy
205 : CProxy_KozCG m_kozcg;
206 : //! Discretization scheme proxy
207 : CProxy_ChoCG m_chocg;
208 : //! Discretization scheme proxy
209 : CProxy_LohCG m_lohcg;
210 : //! Conjugate Gradients Charm++ proxy for pressure solve
211 : tk::CProxy_ConjugateGradients m_cgpre;
212 : //! Conjugate Gradients Charm++ proxy for momentum solve
213 : tk::CProxy_ConjugateGradients m_cgmom;
214 : //! Element connectivity of this compute node's mesh chunk (global ids)
215 : std::vector< std::size_t > m_ginpoel;
216 : //! Aggregated mesh graph of owned nodes if graph-based partitioner is used
217 : std::unordered_map< std::size_t, std::vector< std::size_t > > m_graph;
218 : //! Graph->node map used to aggregate mesh graph
219 : //! \details Key: global mesh node id, value: 0: list of compute nodes
220 : //! contributing partial graph (points surround points) to the global mesh
221 : //! node id in key, 1: list of global mesh node ids surrounding node in key
222 : std::unordered_map< std::size_t,
223 : std::tuple< std::vector< int >,
224 : std::unordered_set< std::size_t > > > m_graphnode;
225 : //! Coordinates of mesh nodes of this compute node's mesh chunk
226 : tk::UnsMesh::Coords m_coord;
227 : //! \brief Element connectivity with local node IDs of this compute node's
228 : //! mesh chunk
229 : std::vector< std::size_t > m_inpoel;
230 : //! Global->local node IDs of elements of this compute node's mesh chunk
231 : //! \details Key: global node id, value: local node id
232 : std::unordered_map< std::size_t, std::size_t > m_lid;
233 : //! Counter during mesh distribution
234 : std::size_t m_ndist;
235 : //! Total number of chares across all compute nodes
236 : int m_nchare;
237 : //! Counters (for each chare owned) for assigning face ids in parallel
238 : std::unordered_map< int, std::size_t > m_nface;
239 : //! \brief Map associating new node IDs (as in producing contiguous-row-id
240 : //! linear system contributions) as map-values to old node IDs (as in
241 : //! file) as map-keys
242 : std::unordered_map< std::size_t, std::size_t > m_linnodes;
243 : //! Mesh connectivity using global node IDs associated to chares owned
244 : std::unordered_map< int, std::vector< std::size_t > > m_chinpoel;
245 : //! Coordinates associated to global node IDs of our mesh chunk for chares
246 : std::unordered_map< int, tk::UnsMesh::CoordMap > m_chcoordmap;
247 : //! Side set id + boundary face id for each chare
248 : std::unordered_map< int,
249 : std::map< int, std::vector< std::size_t > > > m_chbface;
250 : //! Boundary face connectivity for each chare
251 : std::map< int, std::vector< std::size_t > > m_chtriinpoel;
252 : //! Side set id + boundary nodes for each chare
253 : std::unordered_map< int,
254 : std::map< int, std::vector< std::size_t > > > m_chbnode;
255 : //! Boundary face IDs associated associated to side set IDs
256 : std::map< int, std::vector< std::size_t > > m_bface;
257 : //! Boundary face-node connectivity
258 : std::vector< std::size_t > m_triinpoel;
259 : //! List of boundary nodes associated to side-set IDs
260 : std::map< int, std::vector< std::size_t > > m_bnode;
261 :
262 : //! Categorize mesh elements (given by their gobal node IDs) by target
263 : std::unordered_map< int, MeshData >
264 : categorize( const std::vector< std::size_t >& che ) const;
265 :
266 : //! Extract coordinates associated to global nodes of a mesh chunk
267 : tk::UnsMesh::CoordMap coordmap( const std::vector< std::size_t >& inpoel );
268 :
269 : //! Distribute mesh to target compute nodes after mesh partitioning
270 : void distribute( std::unordered_map< int, MeshData >&& mesh );
271 :
272 : //! Compute chare (partition) distribution across compute nodes
273 : std::array< int, 2 > distribution( int npart ) const;
274 :
275 : //! Return nodegroup id for chare id
276 : int node( int id ) const;
277 :
278 : //! Continue after partitioning finished
279 : void partitioned( std::vector< std::size_t >&& che );
280 : };
281 :
282 : } // inciter::
283 :
284 : #endif // Partitioner_h
|
