Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/IO/MeshWriter.cpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.,
7 : : 2022-2025 J. Bakosi
8 : : All rights reserved. See the LICENSE file for details.
9 : : \brief Charm++ group for outputing mesh data to file
10 : : \details Charm++ group definition used to output data associated to
11 : : unstructured meshes to file(s). Charm++ chares (work units) send mesh and
12 : : field data associated to mesh entities to the MeshWriter class defined here
13 : : to write the data to file(s).
14 : : */
15 : : // *****************************************************************************
16 : :
17 : : #include "XystBuildConfig.hpp"
18 : : #include "MeshWriter.hpp"
19 : : #include "Reorder.hpp"
20 : : #include "ExodusIIMeshWriter.hpp"
21 : :
22 : : using tk::MeshWriter;
23 : :
24 : : void
25 : 770 : MeshWriter::nchare( std::size_t meshid, int n )
26 : : // *****************************************************************************
27 : : // Set the total number of chares
28 : : //! \param[in] meshid Mesh whose number of chares to set
29 : : //! \param[in] n Total number of chares across the whole problem (for a mesh)
30 : : // *****************************************************************************
31 : : {
32 : : Assert( meshid < m_nchare.size(), "Size mismatch" );
33 : 770 : m_nchare[ meshid ] = n;
34 : 770 : }
35 : :
36 : : void
37 : 3645 : MeshWriter::write(
38 : : std::size_t meshid,
39 : : bool meshoutput,
40 : : bool fieldoutput,
41 : : uint64_t itr,
42 : : uint64_t itf,
43 : : tk::real time,
44 : : int chareid,
45 : : const std::string& basefilename,
46 : : const std::vector< std::size_t >& inpoel,
47 : : const UnsMesh::Coords& coord,
48 : : const std::map< int, std::vector< std::size_t > >& bface,
49 : : const std::map< int, std::vector< std::size_t > >& bnode,
50 : : const std::vector< std::size_t >& triinpoel,
51 : : const std::vector< std::string >& elemfieldnames,
52 : : const std::vector< std::string >& nodefieldnames,
53 : : const std::vector< std::string >& elemsurfnames,
54 : : const std::vector< std::string >& nodesurfnames,
55 : : const std::vector< std::vector< tk::real > >& elemfields,
56 : : const std::vector< std::vector< tk::real > >& nodefields,
57 : : const std::vector< std::vector< tk::real > >& elemsurfs,
58 : : const std::vector< std::vector< tk::real > >& nodesurfs,
59 : : const std::set< int >& outsets,
60 : : CkCallback c )
61 : : // *****************************************************************************
62 : : // Output unstructured mesh into file
63 : : //! \param[in] meshid Mesh Id
64 : : //! \param[in] meshoutput True if mesh is to be written
65 : : //! \param[in] fieldoutput True if field data is to be written
66 : : //! \param[in] itr Iteration count since a new mesh. New mesh in this context
67 : : //! means that either the mesh is moved and/or its topology has changed.
68 : : //! \param[in] itf Field output iteration count
69 : : //! \param[in] time Physical time this at this field output dump
70 : : //! \param[in] chareid The chare id the write-to-file request is coming from
71 : : //! \param[in] basefilename String to use as the base of the filename
72 : : //! \param[in] inpoel Mesh connectivity for the mesh chunk to be written with
73 : : //! local ids
74 : : //! \param[in] coord Node coordinates of the mesh chunk to be written
75 : : //! \param[in] bface Map of boundary-face lists mapped to corresponding side set
76 : : //! ids for this mesh chunk
77 : : //! \param[in] bnode Map of boundary-node lists mapped to corresponding side set
78 : : //! ids for this mesh chunk with local ids
79 : : //! \param[in] triinpoel Interconnectivity of points and boundary-face in this
80 : : //! mesh chunk with local ids
81 : : //! \param[in] elemfieldnames Names of element fields to be output to file
82 : : //! \param[in] nodefieldnames Names of node fields to be output to file
83 : : //! \param[in] elemsurfnames Names of eleme surface fields to be output to file
84 : : //! \param[in] nodesurfnames Names of node surface fields to be output to file
85 : : //! \param[in] elemfields Field data in mesh elements to output to file
86 : : //! \param[in] nodefields Field data in mesh nodes to output to file
87 : : //! \param[in] elemsurfs Surface field data in mesh elements to output to file
88 : : //! \param[in] nodesurfs Surface field data in mesh nodes to output to file
89 : : //! \param[in] outsets Unique set of surface side set ids along which to save
90 : : //! solution field variables
91 : : //! \param[in] c Function to continue with after the write
92 : : // *****************************************************************************
93 : : {
94 [ - + ]: 3645 : if (m_benchmark) { c.send(); return; }
95 : :
96 : : // Generate filenames for volume and surface field output
97 : 3645 : auto vf = filename( basefilename, meshid, itr, chareid );
98 : :
99 [ + + ]: 3645 : if (meshoutput) {
100 : : // Write volume mesh and field names
101 [ + - ]: 922 : ExodusIIMeshWriter ev( vf, ExoWriter::CREATE );
102 : : // Write chare mesh (do not write side sets in parallel)
103 [ + + ]: 922 : if (m_nchare[meshid] == 1) {
104 [ + - ][ + - ]: 84 : ev.writeMesh( tk::UnsMesh( inpoel, coord, bnode ) );
105 : : } else {
106 [ + - ]: 838 : ev.writeMesh< 4 >( inpoel, coord );
107 : : }
108 [ + - ]: 922 : ev.writeElemVarNames( elemfieldnames );
109 : : Assert( nodefieldnames.size() == nodefields.size(), "Size mismatch" );
110 [ + - ]: 922 : ev.writeNodeVarNames( nodefieldnames );
111 : :
112 : : // Write surface meshes and surface variable field names
113 [ + + ]: 971 : for (auto s : outsets) {
114 [ + - ]: 49 : auto sf = filename( basefilename, meshid, itr, chareid, s );
115 [ + - ]: 49 : ExodusIIMeshWriter es( sf, ExoWriter::CREATE );
116 : : auto b = bface.find(s);
117 [ + + ]: 49 : if (b == end(bface)) {
118 : : // If a side set does not exist on a chare, write out a
119 : : // connectivity for a single triangle with its node coordinates of
120 : : // zero. This is so the paraview series reader can load side sets
121 : : // distributed across multiple files. See also
122 : : // https://www.paraview.org/Wiki/Restarted_Simulation_Readers.
123 [ + - ][ + - ]: 6 : es.writeMesh< 3 >( std::vector< std::size_t >{1,2,3},
124 [ + - ][ + - ]: 12 : UnsMesh::Coords{{ {{0,0,0}}, {{0,0,0}}, {{0,0,0}} }} );
[ + - ][ - - ]
[ - - ]
125 [ + - ]: 6 : es.writeElemVarNames( elemsurfnames );
126 [ + - ]: 6 : es.writeNodeVarNames( nodesurfnames );
127 : : continue;
128 : : }
129 : : std::vector< std::size_t > nodes;
130 [ + + ][ + + ]: 8887 : for (auto f : b->second) {
131 : : nodes.push_back( triinpoel[f*3+0] );
132 : : nodes.push_back( triinpoel[f*3+1] );
133 : : nodes.push_back( triinpoel[f*3+2] );
134 : : }
135 [ + - ]: 43 : auto [inp,gid,lid] = tk::global2local( nodes );
136 [ + - ]: 43 : tk::unique( nodes );
137 : : auto nnode = nodes.size();
138 : : UnsMesh::Coords scoord;
139 [ + - ]: 43 : scoord[0].resize( nnode );
140 [ + - ]: 43 : scoord[1].resize( nnode );
141 [ + - ]: 43 : scoord[2].resize( nnode );
142 : : std::size_t j = 0;
143 [ + + ]: 5599 : for (auto i : nodes) {
144 : 5556 : scoord[0][j] = coord[0][i];
145 : 5556 : scoord[1][j] = coord[1][i];
146 : 5556 : scoord[2][j] = coord[2][i];
147 : 5556 : ++j;
148 : : }
149 [ + - ]: 43 : es.writeMesh< 3 >( inp, scoord );
150 [ + - ]: 43 : es.writeElemVarNames( elemsurfnames );
151 [ + - ]: 43 : es.writeNodeVarNames( nodesurfnames );
152 : 92 : }
153 : 922 : }
154 : :
155 [ + - ]: 3645 : if (fieldoutput) {
156 : : // Write volume variable fields
157 [ + - ]: 3645 : ExodusIIMeshWriter ev( vf, ExoWriter::OPEN );
158 [ + - ]: 3645 : ev.writeTimeStamp( itf, time );
159 : : // Write volume element variable fields
160 : : int varid = 0;
161 [ + - ][ + + ]: 3939 : for (const auto& v : elemfields) ev.writeElemScalar( itf, ++varid, v );
162 : : // Write volume node variable fields
163 : : varid = 0;
164 [ + - ][ + + ]: 39131 : for (const auto& v : nodefields) ev.writeNodeScalar( itf, ++varid, v );
165 : :
166 : : // Write surface node variable fields
167 : : std::size_t j = 0;
168 : : std::size_t k = 0;
169 : 3645 : auto nvar = static_cast< int >( nodesurfnames.size() ) ;
170 : 3645 : auto nevar = static_cast< int >( elemsurfnames.size() ) ;
171 [ + + ]: 3831 : for (auto s : outsets) {
172 [ + - ]: 186 : auto sf = filename( basefilename, meshid, itr, chareid, s );
173 [ + - ]: 186 : ExodusIIMeshWriter es( sf, ExoWriter::OPEN );
174 [ + - ]: 186 : es.writeTimeStamp( itf, time );
175 [ + + ]: 186 : if (bface.find(s) == end(bface)) {
176 : : // If a side set does not exist on a chare, write out a
177 : : // a node field for a single triangle with zeros. This is so the
178 : : // paraview series reader can load side sets distributed across
179 : : // multiple files. See also
180 : : // https://www.paraview.org/Wiki/Restarted_Simulation_Readers.
181 [ + - ][ + - ]: 156 : for (int i=1; i<=nvar; ++i) es.writeNodeScalar( itf, i, {0,0,0} );
[ + + ]
182 [ - - ][ - - ]: 24 : for (int i=1; i<=nevar; ++i) es.writeElemScalar( itf, i, {0} );
[ - + ]
183 : : continue;
184 : 24 : }
185 [ + + ]: 1099 : for (int i=1; i<=nvar; ++i)
186 [ + - ]: 937 : es.writeNodeScalar( itf, i, nodesurfs[j++] );
187 [ - + ]: 162 : for (int i=1; i<=nevar; ++i)
188 [ - - ]: 0 : es.writeElemScalar( itf, i, elemsurfs[k++] );
189 : 186 : }
190 : 3645 : }
191 : :
192 [ + - ]: 3645 : c.send();
193 : : }
194 : :
195 : : std::string
196 : 3880 : MeshWriter::filename( const std::string& basefilename,
197 : : std::size_t meshid,
198 : : uint64_t itr,
199 : : int chareid,
200 : : int surfid ) const
201 : : // *****************************************************************************
202 : : // Compute filename
203 : : //! \param[in] basefilename String use as the base filename.
204 : : //! \param[in] meshid Mesh Id
205 : : //! \param[in] itr Iteration count since a new mesh. New mesh in this context
206 : : //! means that either the mesh is moved and/or its topology has changed.
207 : : //! \param[in] chareid The chare id the write-to-file request is coming from
208 : : //! \param[in] surfid Surface ID if computing a surface filename
209 : : //! \details We use a file naming convention for large field output data that
210 : : //! allows ParaView to glue multiple files into a single simulation output by
211 : : //! only loading a single file. The base filename is followed by ".e-s.",
212 : : //! which probably stands for Exodus Sequence, followed by 3 integers:
213 : : //! (1) {RS}: counts the number of "restart dumps", but we use this for
214 : : //! counting the number of outputs with a different mesh, e.g., due to
215 : : //! mesh refinement, thus if this first number is new the mesh is new
216 : : //! compared to the previous (first) number afer ".e-s.",
217 : : //! (2) {NP}: total number of partitions (workers, chares), this is more than
218 : : //! the number of PEs with nonzero virtualization (overdecomposition), and
219 : : //! (3) {RANK}: worker (chare) id.
220 : : //! Thus {RANK} does spatial partitioning, while {RS} partitions in time, but
221 : : //! a single {RS} id may contain multiple time steps, which equals to the
222 : : //! number of time steps at which field output is saved without refining the
223 : : //! mesh.
224 : : //! \return Filename computed
225 : : //! \see https://www.paraview.org/Wiki/Restarted_Simulation_Readers
226 : : // *****************************************************************************
227 : : {
228 [ + + ][ + - ]: 8230 : return basefilename + (surfid ? "-surf." + std::to_string(surfid) : "")
[ + - ][ + + ]
[ - - ]
229 [ - + ][ - - ]: 11640 : + (m_nchare.size() > 1 ? '.' + std::to_string(meshid) : "")
[ + - ][ + - ]
[ - + ][ - - ]
230 : 3880 : + ".e-s"
231 [ + - ][ + - ]: 11640 : + '.' + std::to_string( itr ) // iteration count with new mesh
[ + - ]
232 [ + - ][ + - ]: 11640 : + '.' + std::to_string( m_nchare[meshid] ) // total number of workers
233 [ + - ][ + - ]: 11640 : + '.' + std::to_string( chareid ); // new file per worker
234 : : }
235 : :
236 : : #include "NoWarning/meshwriter.def.h"
|
