6 #include <boost/date_time/gregorian/greg_date.hpp> 7 #include <boost/property_tree/xml_parser.hpp> 8 #include <boost/property_tree/json_parser.hpp> 9 #include <boost/filesystem.hpp> 33 using std::ostringstream;
37 Hdf5Saver::Hdf5Saver(
string filename,
const ptree& pt_config,
int frequency,
RunMode run_mode,
int start_timestep)
38 : m_filename(filename), m_frequency(frequency),
39 m_current_step(start_timestep - 1), m_timestep(start_timestep),
45 if (exists(system_complete(
string(filename)))) {
48 H5File file(m_filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT, H5P_DEFAULT);
49 DataSet dataset = DataSet(file.openDataSet(
"amt_timesteps"));
51 dataset.read(data, PredType::NATIVE_UINT);
55 m_save_count = data[0];
60 H5File file(m_filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
62 this->saveConfigs(file, pt_config);
63 this->saveTimestepMetadata(file, 0, 0,
true);
66 }
catch (FileIException error) {
73 if (m_frequency != 0 && m_current_step % m_frequency == 0) {
74 this->saveTimestep(sim);
82 m_timestep = timestep;
84 this->saveTimestep(sim);
88 void Hdf5Saver::saveTimestep(
const Simulator& sim) {
91 H5File file(m_filename.c_str(), H5F_ACC_RDWR);
93 if (m_current_step == 0) {
94 this->savePersonTIData(file, sim);
97 ss <<
"/Timestep_" << std::setfill(
'0') << std::setw(6) << m_timestep;
98 Group group(file.createGroup(ss.str()));
101 if (sim.m_rng !=
nullptr) {
102 saveRngState(group, sim);
104 this->saveCalendar(group, sim);
105 this->savePersonTDData(group, sim);
107 this->saveTravellers(group, sim);
109 this->saveClusters(group,
"household_clusters", sim.m_households);
110 this->saveClusters(group,
"school_clusters", sim.m_school_clusters);
111 this->saveClusters(group,
"work_clusters", sim.m_work_clusters);
112 this->saveClusters(group,
"primary_community_clusters", sim.m_primary_community);
113 this->saveClusters(group,
"secondary_community_clusters", sim.m_secondary_community);
116 this->saveTimestepMetadata(file, m_save_count, m_current_step);
117 m_timestep += m_frequency;
119 }
catch (GroupIException& error) {
122 }
catch (AttributeIException& error) {
125 }
catch (FileIException& error) {
126 std::cout <<
"Trying to open file: " << m_filename <<
" but failed." << std::endl;
129 }
catch (DataSetIException& error) {
132 }
catch (DataSpaceIException& error) {
133 std::cout <<
"Error while interacting with a dataspace." << std::endl;
136 }
catch (Exception& error) {
137 std::cout <<
"Unknown exception?" << std::endl;
143 void Hdf5Saver::saveClusters(Group& group,
string dataset_name,
const vector<Cluster>& clusters)
const {
144 auto getAmtIds = [&]() {
145 unsigned int amt = 0;
146 for (
unsigned int i = 0; i < clusters.size(); i++) amt += clusters.at(i).getSize();
149 unsigned int amtIds = getAmtIds();
151 const unsigned int ndims_clusters = 1;
152 hsize_t dims_clusters[ndims_clusters] {amtIds};
153 DataSpace dataspace_clusters = DataSpace(ndims_clusters, dims_clusters);
154 DataSet dataset_clusters = DataSet(
155 group.createDataSet(H5std_string(dataset_name), PredType::NATIVE_UINT, dataspace_clusters));
157 auto cluster_data = make_unique<vector<unsigned int>>(amtIds);
158 unsigned int index = 0;
159 for (
unsigned int i = 0; i < clusters.size(); i++) {
160 for (
unsigned int j = 0; j < clusters.at(i).getSize(); j++) {
161 (*cluster_data)[index++] = clusters.at(i).m_members.at(j).first->m_id;
164 dataset_clusters.write(cluster_data->data(), PredType::NATIVE_UINT);
165 dataspace_clusters.close();
166 dataset_clusters.close();
170 void Hdf5Saver::savePersonTIData(H5File& file,
const Simulator& sim)
const {
171 hsize_t dims[1] {sim.getPopulation().get()->m_original.size()};
172 DataSpace dataspace = DataSpace(1, dims);
175 DataSet dataset = DataSet(file.createDataSet(
"person_time_independent", type_person_TI, dataspace));
178 unsigned int person_index = 0;
179 hsize_t chunk_dims[1] = {40000};
180 while (person_index < dims[0]) {
181 hsize_t selected_dims[1];
182 if (person_index + chunk_dims[0] < dims[0]) {
183 selected_dims[0] = chunk_dims[0];
185 selected_dims[0] = dims[0] - person_index;
188 PersonTIDataType personData[selected_dims[0]];
189 for (
unsigned int j = 0; j < selected_dims[0]; j++) {
190 #define setAttributePerson(attr_lhs, attr_rhs) personData[j].attr_lhs = sim.getPopulation().get()->m_original.at(person_index).attr_rhs 201 personData[j].m_time_infectiousness =
202 sim.getPopulation().get()->m_original.at(person_index).m_health.getEndInfectiousness() -
203 sim.getPopulation().get()->m_original.at(person_index).m_health.getStartInfectiousness();
204 personData[j].m_time_symptomatic =
205 sim.getPopulation().get()->m_original.at(person_index).m_health.getEndSymptomatic() -
206 sim.getPopulation().get()->m_original.at(person_index).m_health.getStartSymptomatic();
211 DataSpace filespace = DataSpace(dataset.getSpace());
212 hsize_t offset[1] {person_index - selected_dims[0]};
213 filespace.selectHyperslab(H5S_SELECT_SET, selected_dims, offset);
216 DataSpace memspace = DataSpace(1, selected_dims, NULL);
219 dataset.write(personData, type_person_TI, memspace, filespace);
221 #undef setAttributePerson 225 void Hdf5Saver::savePersonTDData(Group& group,
const Simulator& sim)
const {
226 hsize_t dims[1] {sim.getPopulation().get()->m_original.size()};
228 const unsigned int CHUNK_SIZE = 10000;
231 DataSpace dataspace = DataSpace(1, dims);
232 DSetCreatPropList plist = DSetCreatPropList();
233 hsize_t chunk_dims[1] = {CHUNK_SIZE};
234 plist.setChunk(1, chunk_dims);
237 if (dims[0] > CHUNK_SIZE) {
238 dataset = DataSet(group.createDataSet(
"person_time_dependent", type_person_TD, dataspace, plist));
240 dataset = DataSet(group.createDataSet(
"person_time_dependent", type_person_TD, dataspace));
244 const std::vector<PersonType>& population = sim.getPopulation()->m_original;
248 unsigned int person_index = 0;
249 while (person_index < dims[0]) {
250 hsize_t selected_dims[1];
251 if (person_index + chunk_dims[0] < dims[0]) {
252 selected_dims[0] = chunk_dims[0];
254 selected_dims[0] = dims[0] - person_index;
257 PersonTDDataType person_data[selected_dims[0]];
258 for (
unsigned int j = 0; j < selected_dims[0]; j++) {
259 const PersonType& person = population[person_index];
260 person_data[j].m_participant = person.m_is_participant;
261 person_data[j].m_health_status = (
unsigned int) person.m_health.getHealthStatus();
262 person_data[j].m_disease_counter = (
unsigned int) person.m_health.getDiseaseCounter();
263 person_data[j].m_on_vacation = person.m_is_on_vacation;
268 DataSpace filespace = DataSpace(dataset.getSpace());
269 hsize_t offset[1] = {person_index - selected_dims[0]};
270 filespace.selectHyperslab(H5S_SELECT_SET, selected_dims, offset);
273 DataSpace memspace = DataSpace(1, selected_dims, NULL);
276 dataset.write(person_data, type_person_TD, memspace, filespace);
286 void Hdf5Saver::saveTravellers(Group& group,
const Simulator& sim)
const {
292 const Agenda& travellers = sim.m_planner.getAgenda();
293 hsize_t dims[1] {sim.m_planner.size()};
296 DataSpace dataspace = DataSpace(1, dims);
297 DataSet dataset = DataSet(group.createDataSet(
"travellers", type_traveller, dataspace));
298 auto traveller_data = make_unique<std::vector<TravellerDataType>>(dims[0]);
302 vector<Simulator::PersonType*> travellers_seq;
303 for (
auto&& day : sim.m_planner.getAgenda()) {
304 for (
auto&& traveller : *(day)) {
305 travellers_seq.push_back(traveller->getNewPerson());
309 unsigned int current_index = 0;
310 unsigned int list_index = 0;
312 for (
auto&& day : travellers) {
313 const Block& current_day = *(day);
314 for (
auto&& person: current_day) {
315 TravellerDataType traveller;
317 string home_sim_name = person->getHomeSimulatorId();
318 string dest_sim_name = person->getDestinationSimulatorId();
320 traveller.m_days_left = list_index;
321 traveller.m_home_sim_name = home_sim_name.c_str();
322 traveller.m_dest_sim_name = dest_sim_name.c_str();
323 traveller.m_home_sim_index = person->getHomeSimulatorIndex();
324 traveller.m_dest_sim_index = sim.m_population->m_original.size() +
325 (std::find(travellers_seq.begin(), travellers_seq.end(),
326 person->getNewPerson()) - travellers_seq.begin());
328 PersonType original_person = person->getHomePerson();
329 #define setAttributeTraveller(attr_lhs, attr_rhs) traveller.attr_lhs = original_person.attr_rhs 340 traveller.m_time_infectiousness = original_person.m_health.getEndInfectiousness() -
341 original_person.m_health.getStartInfectiousness();
342 traveller.m_time_symptomatic = original_person.m_health.getEndSymptomatic() -
343 original_person.m_health.getStartSymptomatic();
345 PersonType current_person = *person->getNewPerson();
346 traveller.m_participant = current_person.m_is_participant;
347 traveller.m_health_status = (
unsigned int) current_person.m_health.getHealthStatus();;
348 traveller.m_disease_counter = (
unsigned int) current_person.m_health.getDiseaseCounter();;
349 traveller.m_new_id = current_person.m_id;
350 traveller.m_new_household_id = current_person.m_household_id;
351 traveller.m_new_school_id = current_person.m_school_id;
352 traveller.m_new_work_id = current_person.m_work_id;
353 traveller.m_new_prim_comm_id = current_person.m_primary_community_id;
354 traveller.m_new_sec_comm_id = current_person.m_secondary_community_id;
356 (*traveller_data)[current_index++] = traveller;
360 #undef setAttributeTraveller 362 if (sim.m_planner.size() != 0)
369 void Hdf5Saver::saveTimestepMetadata(H5File& file,
unsigned int total_amt,
unsigned int current,
bool create)
const {
371 if (create ==
true) {
373 DataSpace dataspace = DataSpace(1, dims);
374 dataset_amt = file.createDataSet(
"amt_timesteps", PredType::NATIVE_UINT, dataspace);
376 dataset_amt = file.openDataSet(
"amt_timesteps");
378 unsigned int amt_timesteps[1] {total_amt};
379 dataset_amt.write(amt_timesteps, PredType::NATIVE_UINT);
383 DataSet dataset_last;
384 if (create ==
true) {
386 DataSpace dataspace = DataSpace(1, dims);
387 dataset_last = file.createDataSet(
"last_timestep", PredType::NATIVE_UINT, dataspace);
389 dataset_last = file.openDataSet(
"last_timestep");
391 unsigned int last_timestep[1] {current};
392 dataset_last.write(last_timestep, PredType::NATIVE_UINT);
393 dataset_last.close();
397 void Hdf5Saver::saveRngState(Group& group,
const Simulator& sim)
const {
399 DataSpace dataspace = DataSpace(1, dims);
400 DataSet dataset = DataSet(group.createDataSet(
"randomgen", StrType(0, H5T_VARIABLE), dataspace));
404 string cppString = ss.str();
405 const char* rng_state[1] {cppString.c_str()};
407 dataset.write(rng_state, StrType(0, H5T_VARIABLE));
413 void Hdf5Saver::saveCalendar(Group& group,
const Simulator& sim)
const {
416 DataSpace dataspace = DataSpace(1, dims);
417 DataSet dataset = DataSet(group.createDataSet(
"calendar", typeCalendar, dataspace));
420 ss << sim.m_calendar->getYear() <<
"-" << sim.m_calendar->getMonth() <<
"-" << sim.m_calendar->getDay();
421 string save_date = ss.str();
423 CalendarDataType calendar[1];
424 calendar[0].m_day = sim.m_calendar->getSimulationDay();
425 calendar[0].m_date = save_date.c_str();
426 dataset.write(calendar, typeCalendar);
433 void Hdf5Saver::saveConfigs(H5File& file,
const ptree& pt_config)
const {
435 Group group(file.createGroup(
"/Configuration"));
436 DataSpace dataspace = DataSpace(1, dims);
439 DataSet dataset = DataSet(group.createDataSet(H5std_string(
"configuration"), type_conf_data, dataspace));
440 ConfigDataType configData[1];
442 auto getStringXmlPtree = [](
const ptree xml) {
444 boost::property_tree::xml_parser::write_xml(oss, xml);
449 auto getPtreeXmlFile = [](
string filename) {
451 if (!is_regular_file(filepath))
452 throw std::runtime_error(
string(__func__) +
"> File " + filepath.string() +
" not present/regular.");
454 xml_parser::read_xml(filepath.string(), tree, boost::property_tree::xml_parser::trim_whitespace);
458 string content_config = getStringXmlPtree(pt_config);
459 configData[0].m_config_content = content_config.c_str();
460 string content_disease = getStringXmlPtree(getPtreeXmlFile(pt_config.get<
string>(
"run.disease.config")));
461 configData[0].m_disease_content = content_disease.c_str();
462 string content_age = getStringXmlPtree(getPtreeXmlFile(pt_config.get<
string>(
"run.age_contact_matrix_file")));
463 configData[0].m_age_contact_content = content_age.c_str();
467 string filename = pt_config.get<
string>(
"run.holidays");
469 if (!is_regular_file(filepath))
470 throw std::runtime_error(
string(__func__) +
"> File " + filepath.string() +
" not present/regular.");
471 json_parser::read_json(filepath.string(), json_tree);
474 json_parser::write_json(oss, json_tree);
475 string content_holidays = oss.str();
476 configData[0].m_holidays_content = content_holidays.c_str();
479 dataset.write(configData, type_conf_data);
Person< BehaviourPolicy, BeliefPolicy > PersonType
static CompType getCompType()
list< unique_ptr< Block >> Agenda
Interface for install directory queries.
Header file for the Calendar class.
Time Dependent Person DataType.
static boost::filesystem::path getDataDir()
Utility method: get path to the directory for data files.
Utilities for the project.
virtual void update(const Simulator &sim)
Update function which is called by the subject.
Header file for the Person class.
static CompType getCompType()
vector< unique_ptr< T >> Block
Header file for the core Population class.
static CompType getCompType()
Header file for the Saver class for the checkpointing functionality.
#define setAttributePerson(attr_lhs, attr_rhs)
static CompType getCompType()
#define setAttributeTraveller(attr_lhs, attr_rhs)
Header for the core Cluster class.
static CompType getCompType()
void forceSave(const Simulator &sim, int timestep=-1)
Forces a save to the hdf5 file, with an optional timestep argument which specifies a new timestep sav...