Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Inciter/Transporter.hpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.,
7 : : 2022-2024 J. Bakosi
8 : : All rights reserved. See the LICENSE file for details.
9 : : \brief Transporter drives the time integration of transport equations
10 : : \details Transporter drives the time integration of transport equations.
11 : : */
12 : : // *****************************************************************************
13 : : #pragma once
14 : :
15 : : #include <map>
16 : : #include <vector>
17 : : #include <unordered_map>
18 : : #include <unordered_set>
19 : :
20 : : #include "Timer.hpp"
21 : : #include "Types.hpp"
22 : : #include "Partitioner.hpp"
23 : : #include "Progress.hpp"
24 : : #include "ContainerUtil.hpp"
25 : : #include "ConjugateGradients.hpp"
26 : :
27 : : namespace inciter {
28 : :
29 : : //! Indices for progress report on mesh preparation
30 : : enum ProgMesh{ PART=0, DIST, REFINE, BND, COMM, MASK, REORD };
31 : : //! Prefixes for progress report on mesh preparation
32 : : static const std::array< std::string, 7 >
33 : : ProgMeshPrefix = {{ "p", "d", "r", "b", "c", "m", "r" }},
34 : : ProgMeshLegend = {{ "partition", "distribute", "refine", "bnd", "comm",
35 : : "mask", "reorder" }};
36 : :
37 : : //! Indices for progress report on workers preparation
38 : : enum ProgWork{ CREATE=0 };
39 : : //! Prefixes for progress report on workers preparation
40 : : static const std::array< std::string, 1 >
41 : : ProgWorkPrefix = {{ "c" }},
42 : : ProgWorkLegend = {{ "create" }};
43 : :
44 : : //! Transporter drives the time integration of transport equations
45 : : class Transporter : public CBase_Transporter {
46 : :
47 : : public:
48 : : #if defined(__clang__)
49 : : #pragma clang diagnostic push
50 : : #pragma clang diagnostic ignored "-Wunused-parameter"
51 : : #pragma clang diagnostic ignored "-Wdeprecated-declarations"
52 : : #elif defined(STRICT_GNUC)
53 : : #pragma GCC diagnostic push
54 : : #pragma GCC diagnostic ignored "-Wunused-parameter"
55 : : #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
56 : : #elif defined(__INTEL_COMPILER)
57 : : #pragma warning( push )
58 : : #pragma warning( disable: 1478 )
59 : : #endif
60 : : // Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
61 : : // charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
62 : : Transporter_SDAG_CODE
63 : : #if defined(__clang__)
64 : : #pragma clang diagnostic pop
65 : : #elif defined(STRICT_GNUC)
66 : : #pragma GCC diagnostic pop
67 : : #elif defined(__INTEL_COMPILER)
68 : : #pragma warning( pop )
69 : : #endif
70 : :
71 : : //! Constructor
72 : : explicit Transporter();
73 : :
74 : : //! Migrate constructor: returning from a checkpoint
75 : : explicit Transporter( CkMigrateMessage* m );
76 : :
77 : : //! Reduction target: the mesh has been read from file on all PEs
78 : : void load( std::size_t meshid, std::size_t nelem );
79 : :
80 : : //! Reduction target: a mesh has been partitioned
81 : : void partitioned( std::size_t meshid );
82 : :
83 : : //! \brief Reduction target: all Solver (PEs) have computed the number of
84 : : //! chares they will recieve contributions from during linear solution
85 : : void partition();
86 : :
87 : : //! \brief Reduction target: all compute nodes have distrbuted their mesh
88 : : //! after partitioning
89 : : void distributed( std::size_t meshid );
90 : :
91 : : //! Reduction target: all Refiner chares have queried their boundary edges
92 : : void queriedRef( std::size_t meshid );
93 : : //! Reduction target: all Refiner chares have setup their boundary edges
94 : : void respondedRef( std::size_t meshid );
95 : :
96 : : //! Reduction target: all compute nodes have created the mesh refiners
97 : : void refinserted( std::size_t meshid, std::size_t error );
98 : :
99 : : //! Reduction target: all Discretization chares have been inserted
100 : : void discinserted( std::size_t meshid );
101 : :
102 : : //! Reduction target: all Discretization constructors have been called
103 : : void disccreated( std::size_t summeshid, std::size_t npoin );
104 : :
105 : : //! \brief Reduction target: all worker (derived discretization) chares have
106 : : //! been inserted
107 : : void workinserted( std::size_t meshid );
108 : :
109 : : //! \brief Reduction target: all Refiner chares have received a round
110 : : //! of edges, and have run their compatibility algorithm
111 : : void compatibility( std::size_t meshid );
112 : :
113 : : //! \brief Reduction target: all Refiner chares have matched/corrected
114 : : //! the tagging of chare-boundary edges, all chares are ready to perform
115 : : //! refinement.
116 : : void matched( std::size_t summeshid, std::size_t nextra, std::size_t nref,
117 : : std::size_t nderef, std::size_t sumrefmode );
118 : :
119 : : //! Compute surface integral across the whole problem and perform leak-test
120 : : void bndint( tk::real sx, tk::real sy, tk::real sz, tk::real cb,
121 : : tk::real summeshid );
122 : :
123 : : //! Reduction target: all chares have refined their mesh
124 : : void refined( std::size_t summeshid, std::size_t nelem, std::size_t npoin );
125 : :
126 : : //! \brief Reduction target: all worker chares have resized their own data
127 : : //! after AMR or ALE
128 : : void resized( std::size_t meshid );
129 : :
130 : : //! Reduction target: all Sorter chares have queried their boundary edges
131 : : void queried( std::size_t meshid );
132 : : //! Reduction target: all Sorter chares have setup their boundary edges
133 : : void responded( std::size_t meshid );
134 : :
135 : : //! Non-reduction target for receiving progress report on partitioning mesh
136 [ + - ]: 38 : void pepartitioned() { m_progMesh.inc< PART >( tk::Print() ); }
137 : : //! Non-reduction target for receiving progress report on distributing mesh
138 [ + - ]: 38 : void pedistributed() { m_progMesh.inc< DIST >( tk::Print() ); }
139 : : //! Non-reduction target for receiving progress report on node ID comm map
140 [ + - ]: 70 : void chcomm() { m_progMesh.inc< COMM >( tk::Print() ); }
141 : : //! Non-reduction target for receiving progress report on node ID mask
142 [ + - ]: 9 : void chmask() { m_progMesh.inc< MASK >( tk::Print() ); }
143 : : //! Non-reduction target for receiving progress report on reordering mesh
144 [ + - ]: 9 : void chreordered() { m_progMesh.inc< REORD >( tk::Print() ); }
145 : : //! Non-reduction target for receiving progress report on creating workers
146 [ + - ]: 70 : void chcreated() { m_progWork.inc< CREATE >( tk::Print() ); }
147 : :
148 : : //! Reduction target summing total mesh volume
149 : : void totalvol( tk::real v, tk::real initial, tk::real summeshid );
150 : :
151 : : //! \brief Reduction target yielding the minimum mesh statistics across
152 : : //! all workers
153 : : void minstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
154 : : tk::real d4, tk::real d5, tk::real rmeshid );
155 : :
156 : : //! \brief Reduction target yielding the maximum mesh statistics across
157 : : //! all workers
158 : : void maxstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
159 : : tk::real d4, tk::real d5, tk::real rmeshid );
160 : :
161 : : //! \brief Reduction target yielding the sum of mesh statistics across
162 : : //! all workers
163 : : void sumstat( tk::real d0, tk::real d1, tk::real d2, tk::real d3,
164 : : tk::real d4, tk::real d5, tk::real d6, tk::real d7,
165 : : tk::real d8, tk::real summeshid );
166 : :
167 : : //! \brief Reduction target yielding PDF of mesh statistics across all
168 : : //! workers
169 : : void pdfstat( CkReductionMsg* msg );
170 : :
171 : : //! Reduction target computing total volume of IC box
172 : : void boxvol( tk::real v, tk::real summeshid );
173 : :
174 : : //! Reduction target collecting diagnostics from density-based solvers
175 : : void rhodiagnostics( CkReductionMsg* msg );
176 : :
177 : : //! Reduction target collecting diagnostics from pressure-based solvers
178 : : void prediagnostics( CkReductionMsg* msg );
179 : :
180 : : //! \brief Reduction target collecting diagnostics from artifical
181 : : //! compressibility solvers
182 : : void acdiagnostics( CkReductionMsg* msg );
183 : :
184 : : //! \brief Reduction target optionally collecting integrals
185 : : void integrals( CkReductionMsg* msg );
186 : :
187 : : //! Resume execution from checkpoint/restart files
188 : : void resume();
189 : :
190 : : //! Save checkpoint/restart files
191 : : void checkpoint( std::size_t finished, std::size_t meshid );
192 : :
193 : : //! Normal finish of time stepping
194 : : void finish( std::size_t meshid = 0 );
195 : :
196 : : /** @name Charm++ pack/unpack serializer member functions */
197 : : ///@{
198 : : //! \brief Pack/Unpack serialize member function
199 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
200 : : //! \note This is a Charm++ mainchare, pup() is thus only for
201 : : //! checkpoint/restart.
202 : 154 : void pup( PUP::er &p ) override {
203 : 154 : p | m_input;
204 : 154 : p | m_nchare;
205 : 154 : p | m_meshid;
206 : 154 : p | m_ncit;
207 : 154 : p | m_nload;
208 : 154 : p | m_npart;
209 : 154 : p | m_nstat;
210 : 154 : p | m_ndisc;
211 : 154 : p | m_nchk;
212 : 154 : p | m_ncom;
213 : 154 : p | m_nt0refit;
214 : 154 : p | m_ndtrefit;
215 : 154 : p | m_noutrefit;
216 : 154 : p | m_noutderefit;
217 : 154 : p | m_discretization;
218 : 154 : p | m_riecg;
219 : 154 : p | m_laxcg;
220 : 154 : p | m_zalcg;
221 : 154 : p | m_kozcg;
222 : 154 : p | m_chocg;
223 : 154 : p | m_lohcg;
224 : 154 : p | m_cgpre;
225 : 154 : p | m_cgmom;
226 : 154 : p | m_partitioner;
227 : 154 : p | m_refiner;
228 : 154 : p | m_meshwriter;
229 : 154 : p | m_sorter;
230 : 154 : p | m_nelem;
231 : 154 : p | m_npoin;
232 : : // returning from checkpoint
233 [ + + ]: 154 : if (p.isUnpacking()) m_finished.resize( m_nchare.size(), 0 );
234 : 154 : p | m_meshvol;
235 : 154 : p | m_minstat;
236 : 154 : p | m_maxstat;
237 : 154 : p | m_avgstat;
238 : 154 : p | m_timer;
239 : 154 : }
240 : : //! \brief Pack/Unpack serialize operator|
241 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
242 : : //! \param[in,out] t Transporter object reference
243 : : friend void operator|( PUP::er& p, Transporter& t ) { t.pup(p); }
244 : : //@}
245 : :
246 : : private:
247 : : //! List of mesh files to be used for potentially multiple solvers
248 : : std::vector< std::string > m_input;
249 : : //! Number of worker chares (one per mesh)
250 : : std::vector< int > m_nchare;
251 : : //! Sum of mesh ids (across all chares, key) for each meshid (value)
252 : : std::unordered_map< std::size_t, std::size_t > m_meshid;
253 : : //! Number of mesh ref corr iter (one per mesh)
254 : : std::vector< std::size_t > m_ncit;
255 : : //! Number of meshes loaded
256 : : std::size_t m_nload;
257 : : //! Number of meshes partitioned
258 : : std::size_t m_npart;
259 : : //! Number of mesh statistics computed
260 : : std::size_t m_nstat;
261 : : //! Number of Discretization arrays created
262 : : std::size_t m_ndisc;
263 : : //! Number of worker arrays checkpointed
264 : : std::size_t m_nchk;
265 : : //! Number of worker arrays have finished setting up their comm maps
266 : : std::size_t m_ncom;
267 : : //! Number of t0ref mesh ref iters (one per mesh)
268 : : std::vector< std::size_t > m_nt0refit;
269 : : //! Number of dtref mesh ref iters (one per mesh)
270 : : std::vector< std::size_t > m_ndtrefit;
271 : : //! Number of outref mesh ref iters (one per mesh)
272 : : std::vector< std::size_t > m_noutrefit;
273 : : //! Number of outderef mesh ref iters (one per mesh)
274 : : std::vector< std::size_t > m_noutderefit;
275 : : //! Discretization proxies (one per mesh)
276 : : std::vector< CProxy_Discretization > m_discretization;
277 : : //! Discretization scheme proxies (one per mesh)
278 : : std::vector< CProxy_RieCG > m_riecg;
279 : : //! Discretization scheme proxies (one per mesh)
280 : : std::vector< CProxy_LaxCG > m_laxcg;
281 : : //! Discretization scheme proxies (one per mesh)
282 : : std::vector< CProxy_ZalCG > m_zalcg;
283 : : //! Discretization scheme proxies (one per mesh)
284 : : std::vector< CProxy_KozCG > m_kozcg;
285 : : //! Discretization scheme proxies (one per mesh)
286 : : std::vector< CProxy_ChoCG > m_chocg;
287 : : //! Discretization scheme proxies (one per mesh)
288 : : std::vector< CProxy_LohCG > m_lohcg;
289 : : //! Conjugate gradients solver proxies for pressure solve (one per mesh)
290 : : std::vector< tk::CProxy_ConjugateGradients > m_cgpre;
291 : : //! Conjugate gradients solver proxies for momentum solve (one per mesh)
292 : : std::vector< tk::CProxy_ConjugateGradients > m_cgmom;
293 : : //! Partitioner nodegroup proxies (one per mesh)
294 : : std::vector< CProxy_Partitioner > m_partitioner;
295 : : //! Mesh refiner array proxies (one per mesh)
296 : : std::vector< CProxy_Refiner > m_refiner;
297 : : //! Mesh writer nodegroup proxies (one per mesh)
298 : : std::vector< tk::CProxy_MeshWriter > m_meshwriter;
299 : : //! Mesh sorter array proxy (one per mesh)
300 : : std::vector< CProxy_Sorter > m_sorter;
301 : : //!< Number of mesh elements (per mesh)
302 : : std::vector< std::size_t > m_nelem;
303 : : //!< Number of mesh points (per mesh)
304 : : std::vector< std::size_t > m_npoin;
305 : : //!< Nonzero if finished with timestepping (one per mesh)
306 : : std::vector< std::size_t > m_finished;
307 : : //! Total mesh volume (one per mesh)
308 : : std::vector< tk::real > m_meshvol;
309 : : //! Minimum mesh statistics (one per mesh)
310 : : std::vector< std::array< tk::real, 6 > > m_minstat;
311 : : //! Maximum mesh statistics (one per mesh)
312 : : std::vector< std::array< tk::real, 6 > > m_maxstat;
313 : : //! Average mesh statistics (one per mesh)
314 : : std::vector< std::array< tk::real, 6 > > m_avgstat;
315 : : //! Timer tags
316 : : enum class TimerTag { MESH_READ=0, MESH_PART, MESH_DIST };
317 : : //! Timers
318 : : std::map< TimerTag, std::pair< tk::Timer, tk::real > > m_timer;
319 : : //! Progress object for preparing mesh
320 : : tk::Progress< 7 > m_progMesh;
321 : : //! Progress object for preparing workers
322 : : tk::Progress< 1 > m_progWork;
323 : :
324 : : //! Create mesh partitioner and boundary condition object group
325 : : void createPartitioner();
326 : :
327 : : //! Configure and write diagnostics file header
328 : : void diagHeader();
329 : :
330 : : //! Configure and write integrals file header
331 : : void integralsHeader();
332 : :
333 : : //! Print out time integration header to screen
334 : : void inthead( const tk::Print& print );
335 : :
336 : : //! Echo diagnostics on mesh statistics
337 : : void stat();
338 : :
339 : : //! Verify boundary condition (BC) side sets used exist in mesh file
340 : : bool matchBCs( std::map< int, std::vector< std::size_t > >& bnd );
341 : :
342 : : //! Print out mesh statistics
343 : : void meshstat( const std::string& header ) const;
344 : : };
345 : :
346 : : } // inciter::
|
