Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Inciter/Transporter.hpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.,
7 : : 2022-2025 J. Bakosi
8 : : All rights reserved. See the LICENSE file for details.
9 : : \brief Transporter drives the time integration of transport equations
10 : : \details Transporter drives the time integration of transport equations.
11 : : */
12 : : // *****************************************************************************
13 : : #pragma once
14 : :
15 : : #include <map>
16 : : #include <vector>
17 : : #include <unordered_map>
18 : : #include <unordered_set>
19 : :
20 : : #include "Timer.hpp"
21 : : #include "Types.hpp"
22 : : #include "Partitioner.hpp"
23 : : #include "Progress.hpp"
24 : : #include "ContainerUtil.hpp"
25 : : #include "ConjugateGradients.hpp"
26 : :
27 : : namespace inciter {
28 : :
29 : : //! Indices for progress report on mesh preparation
30 : : enum ProgMesh{ PART=0, DIST, REFINE, BND, COMM, MASK, REORD };
31 : : //! Prefixes for progress report on mesh preparation
32 : : static const std::array< std::string, 7 >
33 : : ProgMeshPrefix = {{ "p", "d", "r", "b", "c", "m", "r" }},
34 : : ProgMeshLegend = {{ "partition", "distribute", "refine", "bnd", "comm",
35 : : "mask", "reorder" }};
36 : :
37 : : //! Indices for progress report on workers preparation
38 : : enum ProgWork{ CREATE=0 };
39 : : //! Prefixes for progress report on workers preparation
40 : : static const std::array< std::string, 1 >
41 : : ProgWorkPrefix = {{ "c" }},
42 : : ProgWorkLegend = {{ "create" }};
43 : :
44 : : //! Transporter drives the time integration of transport equations
45 : : class Transporter : public CBase_Transporter {
46 : :
47 : : public:
48 : : #if defined(__clang__)
49 : : #pragma clang diagnostic push
50 : : #pragma clang diagnostic ignored "-Wunused-parameter"
51 : : #pragma clang diagnostic ignored "-Wdeprecated-declarations"
52 : : #elif defined(STRICT_GNUC)
53 : : #pragma GCC diagnostic push
54 : : #pragma GCC diagnostic ignored "-Wunused-parameter"
55 : : #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
56 : : #elif defined(__INTEL_COMPILER)
57 : : #pragma warning( push )
58 : : #pragma warning( disable: 1478 )
59 : : #endif
60 : : // Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
61 : : // charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
62 : : Transporter_SDAG_CODE
63 : : #if defined(__clang__)
64 : : #pragma clang diagnostic pop
65 : : #elif defined(STRICT_GNUC)
66 : : #pragma GCC diagnostic pop
67 : : #elif defined(__INTEL_COMPILER)
68 : : #pragma warning( pop )
69 : : #endif
70 : :
71 : : //! Constructor
72 : : explicit Transporter();
73 : :
74 : : //! Migrate constructor: returning from a checkpoint
75 : : explicit Transporter( CkMigrateMessage* m );
76 : :
77 : : //! Reduction target: the mesh has been read from file on all PEs
78 : : void load( std::size_t meshid, std::size_t nelem );
79 : :
80 : : //! Reduction target: all meshes have been partitioned
81 : : void partitioned();
82 : :
83 : : //! \brief Reduction target: all Solver (PEs) have computed the number of
84 : : //! chares they will recieve contributions from during linear solution
85 : : void partition();
86 : :
87 : : //! \brief Reduction target: all compute nodes have distrbuted their mesh
88 : : //! after partitioning
89 : : void distributed( std::size_t meshid );
90 : :
91 : : //! Reduction target: all Refiner chares have queried their boundary edges
92 : : void queriedRef( std::size_t meshid );
93 : : //! Reduction target: all Refiner chares have setup their boundary edges
94 : : void respondedRef( std::size_t meshid );
95 : :
96 : : //! Reduction target: all compute nodes have created the mesh refiners
97 : : void refinserted( std::size_t meshid, std::size_t error );
98 : :
99 : : //! Reduction target: all Discretization chares have been inserted
100 : : void discinserted( std::size_t meshid );
101 : :
102 : : //! Reduction target: all Discretization constructors have been called
103 : : void disccreated( std::size_t summeshid, std::size_t npoin );
104 : :
105 : : //! \brief Reduction target: all worker (derived discretization) chares have
106 : : //! been inserted
107 : : void workinserted( std::size_t meshid );
108 : :
109 : : //! \brief Reduction target: all Refiner chares have received a round
110 : : //! of edges, and have run their compatibility algorithm
111 : : void compatibility( std::size_t meshid );
112 : :
113 : : //! \brief Reduction target: all Refiner chares have matched/corrected
114 : : //! the tagging of chare-boundary edges, all chares are ready to perform
115 : : //! refinement.
116 : : void matched( std::size_t summeshid, std::size_t nextra, std::size_t nref,
117 : : std::size_t nderef, std::size_t sumrefmode );
118 : :
119 : : //! Compute surface integral across the whole problem and perform leak-test
120 : : void bndint( tk::real sx, tk::real sy, tk::real sz, tk::real cb,
121 : : tk::real summeshid );
122 : :
123 : : //! Reduction target: all chares have refined their mesh
124 : : void refined( std::size_t summeshid, std::size_t nelem, std::size_t npoin );
125 : :
126 : : //! \brief Reduction target: all worker chares have resized their own data
127 : : //! after AMR or ALE
128 : : void resized( std::size_t meshid );
129 : :
130 : : //! Reduction target: all Sorter chares have queried their boundary edges
131 : : void queried( std::size_t meshid );
132 : : //! Reduction target: all Sorter chares have setup their boundary edges
133 : : void responded( std::size_t meshid );
134 : :
135 : : //! Reduction target: all Partitioners have queried their mesh graphs
136 : : void queriedPart( std::size_t meshid );
137 : : //! Reduction target: all Partitioners have responded with their mesh graphs
138 : : void respondedPart( std::size_t meshid );
139 : :
140 : : //! Non-reduction target for receiving progress report on partitioning mesh
141 [ + - ]: 46 : void pepartitioned() { m_progMesh.inc< PART >( tk::Print() ); }
142 : : //! Non-reduction target for receiving progress report on distributing mesh
143 [ + - ]: 46 : void pedistributed() { m_progMesh.inc< DIST >( tk::Print() ); }
144 : : //! Non-reduction target for receiving progress report on node ID comm map
145 [ + - ]: 86 : void chcomm() { m_progMesh.inc< COMM >( tk::Print() ); }
146 : : //! Non-reduction target for receiving progress report on node ID mask
147 [ + - ]: 9 : void chmask() { m_progMesh.inc< MASK >( tk::Print() ); }
148 : : //! Non-reduction target for receiving progress report on reordering mesh
149 [ + - ]: 9 : void chreordered() { m_progMesh.inc< REORD >( tk::Print() ); }
150 : : //! Non-reduction target for receiving progress report on creating workers
151 [ + - ]: 86 : void chcreated() { m_progWork.inc< CREATE >( tk::Print() ); }
152 : :
153 : : //! Reduction target summing total mesh volume
154 : : void totalvol( tk::real v, tk::real initial, tk::real summeshid );
155 : :
156 : : //! \brief Reduction target yielding the minimum mesh statistics across
157 : : //! all workers
158 : : void minstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
159 : : tk::real d4, tk::real d5, tk::real rmeshid );
160 : :
161 : : //! \brief Reduction target yielding the maximum mesh statistics across
162 : : //! all workers
163 : : void maxstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
164 : : tk::real d4, tk::real d5, tk::real rmeshid );
165 : :
166 : : //! \brief Reduction target yielding the sum of mesh statistics across
167 : : //! all workers
168 : : void sumstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
169 : : tk::real d4, tk::real d5, tk::real d6, tk::real d7,
170 : : tk::real d8, tk::real summeshid );
171 : :
172 : : //! \brief Reduction target yielding PDF of mesh statistics across all
173 : : //! workers
174 : : void pdfstat( CkReductionMsg* msg );
175 : :
176 : : //! Reduction target computing the minimum dt for coupled problems
177 : : void transfer_dt( tk::real dt );
178 : :
179 : : //! Reduction target computing total volume of IC box
180 : : void boxvol( tk::real v, tk::real summeshid );
181 : :
182 : : //! Reduction target collecting diagnostics from density-based solvers
183 : : void rhodiagnostics( CkReductionMsg* msg );
184 : :
185 : : //! Reduction target collecting diagnostics from pressure-based solvers
186 : : void prediagnostics( CkReductionMsg* msg );
187 : :
188 : : //! \brief Reduction target collecting diagnostics from artificial
189 : : //! compressibility solvers
190 : : void acdiagnostics( CkReductionMsg* msg );
191 : :
192 : : //! \brief Reduction target optionally collecting integrals
193 : : void integrals( CkReductionMsg* msg );
194 : :
195 : : //! Resume execution from checkpoint/restart files
196 : : void resume();
197 : :
198 : : //! Save checkpoint/restart files
199 : : void checkpoint( std::size_t finished, std::size_t meshid );
200 : :
201 : : //! Normal finish of time stepping
202 : : void finish( std::size_t meshid = 0 );
203 : :
204 : : /** @name Charm++ pack/unpack serializer member functions */
205 : : ///@{
206 : : //! \brief Pack/Unpack serialize member function
207 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
208 : 171 : void pup( PUP::er &p ) override {
209 : 171 : p | m_input;
210 : 171 : p | m_nchare;
211 : 171 : p | m_meshid;
212 : 171 : p | m_ncit;
213 : 171 : p | m_ndt;
214 : 171 : p | m_mindt;
215 : 171 : p | m_nload;
216 : 171 : p | m_npart;
217 : 171 : p | m_nstat;
218 : 171 : p | m_ndisc;
219 : 171 : p | m_nchk;
220 : 171 : p | m_ncom;
221 : 171 : p | m_nt0refit;
222 : 171 : p | m_ndtrefit;
223 : 171 : p | m_noutrefit;
224 : 171 : p | m_noutderefit;
225 : 171 : p | m_discretization;
226 : 171 : p | m_riecg;
227 : 171 : p | m_laxcg;
228 : 171 : p | m_zalcg;
229 : 171 : p | m_kozcg;
230 : 171 : p | m_chocg;
231 : 171 : p | m_lohcg;
232 : 171 : p | m_cgpre;
233 : 171 : p | m_cgmom;
234 : 171 : p | m_partitioner;
235 : 171 : p | m_refiner;
236 : 171 : p | m_meshwriter;
237 : 171 : p | m_sorter;
238 : 171 : p | m_nelem;
239 : 171 : p | m_npoin;
240 : : // returning from checkpoint
241 [ + + ]: 171 : if (p.isUnpacking()) m_finished.resize( m_nchare.size(), 0 );
242 : 171 : p | m_meshvol;
243 : 171 : p | m_minstat;
244 : 171 : p | m_maxstat;
245 : 171 : p | m_avgstat;
246 : 171 : p | m_timer;
247 : 171 : }
248 : : //! \brief Pack/Unpack serialize operator|
249 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
250 : : //! \param[in,out] t Transporter object reference
251 : : friend void operator|( PUP::er& p, Transporter& t ) { t.pup(p); }
252 : : //@}
253 : :
254 : : private:
255 : : //! List of mesh files to be used for potentially multiple solvers
256 : : std::vector< std::string > m_input;
257 : : //! Number of worker chares (one per mesh)
258 : : std::vector< int > m_nchare;
259 : : //! Sum of mesh ids (across all chares, key) for each meshid (value)
260 : : std::unordered_map< std::size_t, std::size_t > m_meshid;
261 : : //! Number of mesh ref corr iter (one per mesh)
262 : : std::vector< std::size_t > m_ncit;
263 : : //! Number of meshes that have contributed to dt calculation
264 : : std::size_t m_ndt;
265 : : //! Minimum dt across meshes for coupled problems
266 : : tk::real m_mindt;
267 : : //! Number of meshes loaded
268 : : std::size_t m_nload;
269 : : //! Number of meshes partitioned
270 : : std::size_t m_npart;
271 : : //! Number of mesh statistics computed
272 : : std::size_t m_nstat;
273 : : //! Number of Discretization arrays created
274 : : std::size_t m_ndisc;
275 : : //! Number of worker arrays checkpointed
276 : : std::size_t m_nchk;
277 : : //! Number of worker arrays have finished setting up their comm maps
278 : : std::size_t m_ncom;
279 : : //! Number of t0ref mesh ref iters (one per mesh)
280 : : std::vector< std::size_t > m_nt0refit;
281 : : //! Number of dtref mesh ref iters (one per mesh)
282 : : std::vector< std::size_t > m_ndtrefit;
283 : : //! Number of outref mesh ref iters (one per mesh)
284 : : std::vector< std::size_t > m_noutrefit;
285 : : //! Number of outderef mesh ref iters (one per mesh)
286 : : std::vector< std::size_t > m_noutderefit;
287 : : //! Discretization proxies (one per mesh)
288 : : std::vector< CProxy_Discretization > m_discretization;
289 : : //! Discretization scheme proxies (one per mesh)
290 : : std::vector< CProxy_RieCG > m_riecg;
291 : : //! Discretization scheme proxies (one per mesh)
292 : : std::vector< CProxy_LaxCG > m_laxcg;
293 : : //! Discretization scheme proxies (one per mesh)
294 : : std::vector< CProxy_ZalCG > m_zalcg;
295 : : //! Discretization scheme proxies (one per mesh)
296 : : std::vector< CProxy_KozCG > m_kozcg;
297 : : //! Discretization scheme proxies (one per mesh)
298 : : std::vector< CProxy_ChoCG > m_chocg;
299 : : //! Discretization scheme proxies (one per mesh)
300 : : std::vector< CProxy_LohCG > m_lohcg;
301 : : //! Conjugate gradients solver proxies for pressure solve (one per mesh)
302 : : std::vector< tk::CProxy_ConjugateGradients > m_cgpre;
303 : : //! Conjugate gradients solver proxies for momentum solve (one per mesh)
304 : : std::vector< tk::CProxy_ConjugateGradients > m_cgmom;
305 : : //! Partitioner nodegroup proxies (one per mesh)
306 : : std::vector< CProxy_Partitioner > m_partitioner;
307 : : //! Mesh refiner array proxies (one per mesh)
308 : : std::vector< CProxy_Refiner > m_refiner;
309 : : //! Mesh writer nodegroup proxies (one per mesh)
310 : : std::vector< tk::CProxy_MeshWriter > m_meshwriter;
311 : : //! Mesh sorter array proxy (one per mesh)
312 : : std::vector< CProxy_Sorter > m_sorter;
313 : : //!< Number of mesh elements (per mesh)
314 : : std::vector< std::size_t > m_nelem;
315 : : //!< Number of mesh points (per mesh)
316 : : std::vector< std::size_t > m_npoin;
317 : : //!< Nonzero if finished with timestepping (one per mesh)
318 : : std::vector< std::size_t > m_finished;
319 : : //! Total mesh volume (one per mesh)
320 : : std::vector< tk::real > m_meshvol;
321 : : //! Minimum mesh statistics (one per mesh)
322 : : std::vector< std::array< tk::real, 6 > > m_minstat;
323 : : //! Maximum mesh statistics (one per mesh)
324 : : std::vector< std::array< tk::real, 6 > > m_maxstat;
325 : : //! Average mesh statistics (one per mesh)
326 : : std::vector< std::array< tk::real, 6 > > m_avgstat;
327 : : //! Timer tags
328 : : enum class TimerTag { MESH_READ=0, MESH_PART };
329 : : //! Timers
330 : : std::map< TimerTag, std::pair< tk::Timer, tk::real > > m_timer;
331 : : //! Progress object for preparing mesh
332 : : tk::Progress< 7 > m_progMesh;
333 : : //! Progress object for preparing workers
334 : : tk::Progress< 1 > m_progWork;
335 : :
336 : : //! Create mesh partitioner and boundary condition object group
337 : : void createPartitioner();
338 : :
339 : : //! Configure and write diagnostics file header
340 : : void diagHeader();
341 : :
342 : : //! Configure and write integrals file header
343 : : void integralsHeader();
344 : :
345 : : //! Print out time integration header to screen
346 : : void inthead( const tk::Print& print );
347 : :
348 : : //! Echo diagnostics on mesh statistics
349 : : void stat();
350 : :
351 : : //! Verify that side sets referred to in the control file exist in mesh file
352 : : bool matchsets( std::map< int, std::vector< std::size_t > >& bnd,
353 : : const std::string& filenamne );
354 : :
355 : : //! Verify that side sets referred to in the control file exist in mesh file
356 : : bool matchsets_multi( std::map< int, std::vector< std::size_t > >& bnd,
357 : : const std::string& filenamne,
358 : : std::size_t meshid );
359 : :
360 : : //! Print out mesh statistics
361 : : void meshstat( const std::string& header ) const;
362 : : };
363 : :
364 : : } // inciter::
|
