Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Inciter/Transporter.hpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.,
7 : : 2022-2025 J. Bakosi
8 : : All rights reserved. See the LICENSE file for details.
9 : : \brief Transporter drives the time integration of transport equations
10 : : \details Transporter drives the time integration of transport equations.
11 : : */
12 : : // *****************************************************************************
13 : : #pragma once
14 : :
15 : : #include <map>
16 : : #include <vector>
17 : : #include <unordered_map>
18 : : #include <unordered_set>
19 : :
20 : : #include "Timer.hpp"
21 : : #include "Types.hpp"
22 : : #include "Partitioner.hpp"
23 : : #include "Progress.hpp"
24 : : #include "ContainerUtil.hpp"
25 : : #include "ConjugateGradients.hpp"
26 : :
27 : : namespace inciter {
28 : :
29 : : //! Indices for progress report on mesh preparation
30 : : enum ProgMesh{ PART=0, DIST, REFINE, BND, COMM, MASK, REORD };
31 : : //! Prefixes for progress report on mesh preparation
32 : : static const std::array< std::string, 7 >
33 : : ProgMeshPrefix = {{ "p", "d", "r", "b", "c", "m", "r" }},
34 : : ProgMeshLegend = {{ "partition", "distribute", "refine", "bnd", "comm",
35 : : "mask", "reorder" }};
36 : :
37 : : //! Indices for progress report on workers preparation
38 : : enum ProgWork{ CREATE=0 };
39 : : //! Prefixes for progress report on workers preparation
40 : : static const std::array< std::string, 1 >
41 : : ProgWorkPrefix = {{ "c" }},
42 : : ProgWorkLegend = {{ "create" }};
43 : :
44 : : //! Transporter drives the time integration of transport equations
45 : : class Transporter : public CBase_Transporter {
46 : :
47 : : public:
48 : : #if defined(__clang__)
49 : : #pragma clang diagnostic push
50 : : #pragma clang diagnostic ignored "-Wunused-parameter"
51 : : #pragma clang diagnostic ignored "-Wdeprecated-declarations"
52 : : #elif defined(STRICT_GNUC)
53 : : #pragma GCC diagnostic push
54 : : #pragma GCC diagnostic ignored "-Wunused-parameter"
55 : : #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
56 : : #elif defined(__INTEL_COMPILER)
57 : : #pragma warning( push )
58 : : #pragma warning( disable: 1478 )
59 : : #endif
60 : : // Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
61 : : // charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
62 : : Transporter_SDAG_CODE
63 : : #if defined(__clang__)
64 : : #pragma clang diagnostic pop
65 : : #elif defined(STRICT_GNUC)
66 : : #pragma GCC diagnostic pop
67 : : #elif defined(__INTEL_COMPILER)
68 : : #pragma warning( pop )
69 : : #endif
70 : :
71 : : //! Constructor
72 : : explicit Transporter();
73 : :
74 : : //! Migrate constructor: returning from a checkpoint
75 : : explicit Transporter( CkMigrateMessage* m );
76 : :
77 : : //! Reduction target: the mesh has been read from file on all PEs
78 : : void load( std::size_t meshid, std::size_t nelem );
79 : :
80 : : //! Reduction target: a mesh has been partitioned
81 : : void partitioned( std::size_t meshid );
82 : :
83 : : //! \brief Reduction target: all Solver (PEs) have computed the number of
84 : : //! chares they will recieve contributions from during linear solution
85 : : void partition();
86 : :
87 : : //! \brief Reduction target: all compute nodes have distrbuted their mesh
88 : : //! after partitioning
89 : : void distributed( std::size_t meshid );
90 : :
91 : : //! Reduction target: all Refiner chares have queried their boundary edges
92 : : void queriedRef( std::size_t meshid );
93 : : //! Reduction target: all Refiner chares have setup their boundary edges
94 : : void respondedRef( std::size_t meshid );
95 : :
96 : : //! Reduction target: all compute nodes have created the mesh refiners
97 : : void refinserted( std::size_t meshid, std::size_t error );
98 : :
99 : : //! Reduction target: all Discretization chares have been inserted
100 : : void discinserted( std::size_t meshid );
101 : :
102 : : //! Reduction target: all Discretization constructors have been called
103 : : void disccreated( std::size_t summeshid, std::size_t npoin );
104 : :
105 : : //! \brief Reduction target: all worker (derived discretization) chares have
106 : : //! been inserted
107 : : void workinserted( std::size_t meshid );
108 : :
109 : : //! \brief Reduction target: all Refiner chares have received a round
110 : : //! of edges, and have run their compatibility algorithm
111 : : void compatibility( std::size_t meshid );
112 : :
113 : : //! \brief Reduction target: all Refiner chares have matched/corrected
114 : : //! the tagging of chare-boundary edges, all chares are ready to perform
115 : : //! refinement.
116 : : void matched( std::size_t summeshid, std::size_t nextra, std::size_t nref,
117 : : std::size_t nderef, std::size_t sumrefmode );
118 : :
119 : : //! Compute surface integral across the whole problem and perform leak-test
120 : : void bndint( tk::real sx, tk::real sy, tk::real sz, tk::real cb,
121 : : tk::real summeshid );
122 : :
123 : : //! Reduction target: all chares have refined their mesh
124 : : void refined( std::size_t summeshid, std::size_t nelem, std::size_t npoin );
125 : :
126 : : //! \brief Reduction target: all worker chares have resized their own data
127 : : //! after AMR or ALE
128 : : void resized( std::size_t meshid );
129 : :
130 : : //! Reduction target: all Sorter chares have queried their boundary edges
131 : : void queried( std::size_t meshid );
132 : : //! Reduction target: all Sorter chares have setup their boundary edges
133 : : void responded( std::size_t meshid );
134 : :
135 : : //! Reduction target: all Partitioners have queried their mesh graphs
136 : : void queriedPart( std::size_t meshid );
137 : : //! Reduction target: all Partitioners have responded with their mesh graphs
138 : : void respondedPart( std::size_t meshid );
139 : :
140 : : //! Non-reduction target for receiving progress report on partitioning mesh
141 [ + - ]: 38 : void pepartitioned() { m_progMesh.inc< PART >( tk::Print() ); }
142 : : //! Non-reduction target for receiving progress report on distributing mesh
143 [ + - ]: 38 : void pedistributed() { m_progMesh.inc< DIST >( tk::Print() ); }
144 : : //! Non-reduction target for receiving progress report on node ID comm map
145 [ + - ]: 70 : void chcomm() { m_progMesh.inc< COMM >( tk::Print() ); }
146 : : //! Non-reduction target for receiving progress report on node ID mask
147 [ + - ]: 9 : void chmask() { m_progMesh.inc< MASK >( tk::Print() ); }
148 : : //! Non-reduction target for receiving progress report on reordering mesh
149 [ + - ]: 9 : void chreordered() { m_progMesh.inc< REORD >( tk::Print() ); }
150 : : //! Non-reduction target for receiving progress report on creating workers
151 [ + - ]: 70 : void chcreated() { m_progWork.inc< CREATE >( tk::Print() ); }
152 : :
153 : : //! Reduction target summing total mesh volume
154 : : void totalvol( tk::real v, tk::real initial, tk::real summeshid );
155 : :
156 : : //! \brief Reduction target yielding the minimum mesh statistics across
157 : : //! all workers
158 : : void minstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
159 : : tk::real d4, tk::real d5, tk::real rmeshid );
160 : :
161 : : //! \brief Reduction target yielding the maximum mesh statistics across
162 : : //! all workers
163 : : void maxstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
164 : : tk::real d4, tk::real d5, tk::real rmeshid );
165 : :
166 : : //! \brief Reduction target yielding the sum of mesh statistics across
167 : : //! all workers
168 : : void sumstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
169 : : tk::real d4, tk::real d5, tk::real d6, tk::real d7,
170 : : tk::real d8, tk::real summeshid );
171 : :
172 : : //! \brief Reduction target yielding PDF of mesh statistics across all
173 : : //! workers
174 : : void pdfstat( CkReductionMsg* msg );
175 : :
176 : : //! Reduction target computing total volume of IC box
177 : : void boxvol( tk::real v, tk::real summeshid );
178 : :
179 : : //! Reduction target collecting diagnostics from density-based solvers
180 : : void rhodiagnostics( CkReductionMsg* msg );
181 : :
182 : : //! Reduction target collecting diagnostics from pressure-based solvers
183 : : void prediagnostics( CkReductionMsg* msg );
184 : :
185 : : //! \brief Reduction target collecting diagnostics from artifical
186 : : //! compressibility solvers
187 : : void acdiagnostics( CkReductionMsg* msg );
188 : :
189 : : //! \brief Reduction target optionally collecting integrals
190 : : void integrals( CkReductionMsg* msg );
191 : :
192 : : //! Resume execution from checkpoint/restart files
193 : : void resume();
194 : :
195 : : //! Save checkpoint/restart files
196 : : void checkpoint( std::size_t finished, std::size_t meshid );
197 : :
198 : : //! Normal finish of time stepping
199 : : void finish( std::size_t meshid = 0 );
200 : :
201 : : /** @name Charm++ pack/unpack serializer member functions */
202 : : ///@{
203 : : //! \brief Pack/Unpack serialize member function
204 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
205 : : //! \note This is a Charm++ mainchare, pup() is thus only for
206 : : //! checkpoint/restart.
207 : 163 : void pup( PUP::er &p ) override {
208 : 163 : p | m_input;
209 : 163 : p | m_nchare;
210 : 163 : p | m_meshid;
211 : 163 : p | m_ncit;
212 : 163 : p | m_nload;
213 : 163 : p | m_npart;
214 : 163 : p | m_nstat;
215 : 163 : p | m_ndisc;
216 : 163 : p | m_nchk;
217 : 163 : p | m_ncom;
218 : 163 : p | m_nt0refit;
219 : 163 : p | m_ndtrefit;
220 : 163 : p | m_noutrefit;
221 : 163 : p | m_noutderefit;
222 : 163 : p | m_discretization;
223 : 163 : p | m_riecg;
224 : 163 : p | m_laxcg;
225 : 163 : p | m_zalcg;
226 : 163 : p | m_kozcg;
227 : 163 : p | m_chocg;
228 : 163 : p | m_lohcg;
229 : 163 : p | m_cgpre;
230 : 163 : p | m_cgmom;
231 : 163 : p | m_partitioner;
232 : 163 : p | m_refiner;
233 : 163 : p | m_meshwriter;
234 : 163 : p | m_sorter;
235 : 163 : p | m_nelem;
236 : 163 : p | m_npoin;
237 : : // returning from checkpoint
238 [ + + ]: 163 : if (p.isUnpacking()) m_finished.resize( m_nchare.size(), 0 );
239 : 163 : p | m_meshvol;
240 : 163 : p | m_minstat;
241 : 163 : p | m_maxstat;
242 : 163 : p | m_avgstat;
243 : 163 : p | m_timer;
244 : 163 : }
245 : : //! \brief Pack/Unpack serialize operator|
246 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
247 : : //! \param[in,out] t Transporter object reference
248 : : friend void operator|( PUP::er& p, Transporter& t ) { t.pup(p); }
249 : : //@}
250 : :
251 : : private:
252 : : //! List of mesh files to be used for potentially multiple solvers
253 : : std::vector< std::string > m_input;
254 : : //! Number of worker chares (one per mesh)
255 : : std::vector< int > m_nchare;
256 : : //! Sum of mesh ids (across all chares, key) for each meshid (value)
257 : : std::unordered_map< std::size_t, std::size_t > m_meshid;
258 : : //! Number of mesh ref corr iter (one per mesh)
259 : : std::vector< std::size_t > m_ncit;
260 : : //! Number of meshes loaded
261 : : std::size_t m_nload;
262 : : //! Number of meshes partitioned
263 : : std::size_t m_npart;
264 : : //! Number of mesh statistics computed
265 : : std::size_t m_nstat;
266 : : //! Number of Discretization arrays created
267 : : std::size_t m_ndisc;
268 : : //! Number of worker arrays checkpointed
269 : : std::size_t m_nchk;
270 : : //! Number of worker arrays have finished setting up their comm maps
271 : : std::size_t m_ncom;
272 : : //! Number of t0ref mesh ref iters (one per mesh)
273 : : std::vector< std::size_t > m_nt0refit;
274 : : //! Number of dtref mesh ref iters (one per mesh)
275 : : std::vector< std::size_t > m_ndtrefit;
276 : : //! Number of outref mesh ref iters (one per mesh)
277 : : std::vector< std::size_t > m_noutrefit;
278 : : //! Number of outderef mesh ref iters (one per mesh)
279 : : std::vector< std::size_t > m_noutderefit;
280 : : //! Discretization proxies (one per mesh)
281 : : std::vector< CProxy_Discretization > m_discretization;
282 : : //! Discretization scheme proxies (one per mesh)
283 : : std::vector< CProxy_RieCG > m_riecg;
284 : : //! Discretization scheme proxies (one per mesh)
285 : : std::vector< CProxy_LaxCG > m_laxcg;
286 : : //! Discretization scheme proxies (one per mesh)
287 : : std::vector< CProxy_ZalCG > m_zalcg;
288 : : //! Discretization scheme proxies (one per mesh)
289 : : std::vector< CProxy_KozCG > m_kozcg;
290 : : //! Discretization scheme proxies (one per mesh)
291 : : std::vector< CProxy_ChoCG > m_chocg;
292 : : //! Discretization scheme proxies (one per mesh)
293 : : std::vector< CProxy_LohCG > m_lohcg;
294 : : //! Conjugate gradients solver proxies for pressure solve (one per mesh)
295 : : std::vector< tk::CProxy_ConjugateGradients > m_cgpre;
296 : : //! Conjugate gradients solver proxies for momentum solve (one per mesh)
297 : : std::vector< tk::CProxy_ConjugateGradients > m_cgmom;
298 : : //! Partitioner nodegroup proxies (one per mesh)
299 : : std::vector< CProxy_Partitioner > m_partitioner;
300 : : //! Mesh refiner array proxies (one per mesh)
301 : : std::vector< CProxy_Refiner > m_refiner;
302 : : //! Mesh writer nodegroup proxies (one per mesh)
303 : : std::vector< tk::CProxy_MeshWriter > m_meshwriter;
304 : : //! Mesh sorter array proxy (one per mesh)
305 : : std::vector< CProxy_Sorter > m_sorter;
306 : : //!< Number of mesh elements (per mesh)
307 : : std::vector< std::size_t > m_nelem;
308 : : //!< Number of mesh points (per mesh)
309 : : std::vector< std::size_t > m_npoin;
310 : : //!< Nonzero if finished with timestepping (one per mesh)
311 : : std::vector< std::size_t > m_finished;
312 : : //! Total mesh volume (one per mesh)
313 : : std::vector< tk::real > m_meshvol;
314 : : //! Minimum mesh statistics (one per mesh)
315 : : std::vector< std::array< tk::real, 6 > > m_minstat;
316 : : //! Maximum mesh statistics (one per mesh)
317 : : std::vector< std::array< tk::real, 6 > > m_maxstat;
318 : : //! Average mesh statistics (one per mesh)
319 : : std::vector< std::array< tk::real, 6 > > m_avgstat;
320 : : //! Timer tags
321 : : enum class TimerTag { MESH_READ=0, MESH_PART, MESH_DIST };
322 : : //! Timers
323 : : std::map< TimerTag, std::pair< tk::Timer, tk::real > > m_timer;
324 : : //! Progress object for preparing mesh
325 : : tk::Progress< 7 > m_progMesh;
326 : : //! Progress object for preparing workers
327 : : tk::Progress< 1 > m_progWork;
328 : :
329 : : //! Create mesh partitioner and boundary condition object group
330 : : void createPartitioner();
331 : :
332 : : //! Configure and write diagnostics file header
333 : : void diagHeader();
334 : :
335 : : //! Configure and write integrals file header
336 : : void integralsHeader();
337 : :
338 : : //! Print out time integration header to screen
339 : : void inthead( const tk::Print& print );
340 : :
341 : : //! Echo diagnostics on mesh statistics
342 : : void stat();
343 : :
344 : : //! Verify boundary condition (BC) side sets used exist in mesh file
345 : : bool matchBCs( std::map< int, std::vector< std::size_t > >& bnd );
346 : :
347 : : //! Print out mesh statistics
348 : : void meshstat( const std::string& header ) const;
349 : : };
350 : :
351 : : } // inciter::
|
