Stride Reference Manual  1.0
Hdf5Saver.cpp
Go to the documentation of this file.
1 
6 #include <boost/date_time/gregorian/greg_date.hpp>
7 #include <boost/property_tree/xml_parser.hpp>
8 #include <boost/property_tree/json_parser.hpp>
9 #include <boost/filesystem.hpp>
10 #include <iostream>
11 #include <vector>
12 #include "Hdf5Saver.h"
13 #include "util/InstallDirs.h"
14 #include "calendar/Calendar.h"
15 #include "pop/Population.h"
16 #include "pop/Person.h"
17 #include "pop/Traveller.h"
18 #include "core/Cluster.h"
19 #include "util/etc.h"
20 #include "util/SimplePlanner.h"
26 
27 #include <vector>
28 
29 using namespace H5;
30 using namespace stride::util;
31 using namespace boost::filesystem;
32 using std::string;
33 using std::ostringstream;
34 
35 namespace stride {
36 
37 Hdf5Saver::Hdf5Saver(string filename, const ptree& pt_config, int frequency, RunMode run_mode, int start_timestep)
38  : m_filename(filename), m_frequency(frequency),
39  m_current_step(start_timestep - 1), m_timestep(start_timestep),
40  m_save_count(0) {
41 
42  // Check if the simulator is run in extend mode and not from timestep 0
43  if (start_timestep != 0 && run_mode == RunMode::Extend) {
44  // If the hdf5 file already exists, append the data, otherwise still run the whole constructor
45  if (exists(system_complete(string(filename)))) {
46 
47  // Adjust the amount of saved timesteps
48  H5File file(m_filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT, H5P_DEFAULT);
49  DataSet dataset = DataSet(file.openDataSet("amt_timesteps"));
50  unsigned int data[1];
51  dataset.read(data, PredType::NATIVE_UINT);
52  dataset.close();
53  file.close();
54 
55  m_save_count = data[0];
56  return;
57  }
58  }
59  try {
60  H5File file(m_filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
61 
62  this->saveConfigs(file, pt_config);
63  this->saveTimestepMetadata(file, 0, 0, true);
64 
65  file.close();
66  } catch (FileIException error) {
67  error.printError();
68  }
69 }
70 
71 void Hdf5Saver::update(const Simulator& sim) {
72  m_current_step++;
73  if (m_frequency != 0 && m_current_step % m_frequency == 0) {
74  this->saveTimestep(sim);
75  }
76 }
77 
78 void Hdf5Saver::forceSave(const Simulator& sim, int timestep) {
79  m_current_step++;
80 
81  if (timestep != -1) {
82  m_timestep = timestep;
83  }
84  this->saveTimestep(sim);
85 }
86 
87 
88 void Hdf5Saver::saveTimestep(const Simulator& sim) {
89  try {
90  m_save_count++;
91  H5File file(m_filename.c_str(), H5F_ACC_RDWR);
92 
93  if (m_current_step == 0) {
94  this->savePersonTIData(file, sim);
95  }
96  stringstream ss;
97  ss << "/Timestep_" << std::setfill('0') << std::setw(6) << m_timestep;
98  Group group(file.createGroup(ss.str()));
99 
100 
101  if (sim.m_rng != nullptr) {
102  saveRngState(group, sim);
103  }
104  this->saveCalendar(group, sim);
105  this->savePersonTDData(group, sim);
106 
107  this->saveTravellers(group, sim);
108 
109  this->saveClusters(group, "household_clusters", sim.m_households);
110  this->saveClusters(group, "school_clusters", sim.m_school_clusters);
111  this->saveClusters(group, "work_clusters", sim.m_work_clusters);
112  this->saveClusters(group, "primary_community_clusters", sim.m_primary_community);
113  this->saveClusters(group, "secondary_community_clusters", sim.m_secondary_community);
114 
115 
116  this->saveTimestepMetadata(file, m_save_count, m_current_step);
117  m_timestep += m_frequency;
118  file.close();
119  } catch (GroupIException& error) {
120  error.printError();
121  return;
122  } catch (AttributeIException& error) {
123  error.printError();
124  return;
125  } catch (FileIException& error) {
126  std::cout << "Trying to open file: " << m_filename << " but failed." << std::endl;
127  error.printError();
128  return;
129  } catch (DataSetIException& error) {
130  error.printError();
131  return;
132  } catch (DataSpaceIException& error) {
133  std::cout << "Error while interacting with a dataspace." << std::endl;
134  error.printError();
135  return;
136  } catch (Exception& error) {
137  std::cout << "Unknown exception?" << std::endl;
138  error.printError();
139  }
140  return;
141 }
142 
143 void Hdf5Saver::saveClusters(Group& group, string dataset_name, const vector<Cluster>& clusters) const {
144  auto getAmtIds = [&]() {
145  unsigned int amt = 0;
146  for (unsigned int i = 0; i < clusters.size(); i++) amt += clusters.at(i).getSize();
147  return amt;
148  };
149  unsigned int amtIds = getAmtIds();
150 
151  const unsigned int ndims_clusters = 1;
152  hsize_t dims_clusters[ndims_clusters] {amtIds};
153  DataSpace dataspace_clusters = DataSpace(ndims_clusters, dims_clusters);
154  DataSet dataset_clusters = DataSet(
155  group.createDataSet(H5std_string(dataset_name), PredType::NATIVE_UINT, dataspace_clusters));
156 
157  auto cluster_data = make_unique<vector<unsigned int>>(amtIds);
158  unsigned int index = 0;
159  for (unsigned int i = 0; i < clusters.size(); i++) {
160  for (unsigned int j = 0; j < clusters.at(i).getSize(); j++) {
161  (*cluster_data)[index++] = clusters.at(i).m_members.at(j).first->m_id;
162  }
163  }
164  dataset_clusters.write(cluster_data->data(), PredType::NATIVE_UINT);
165  dataspace_clusters.close();
166  dataset_clusters.close();
167 }
168 
169 
170 void Hdf5Saver::savePersonTIData(H5File& file, const Simulator& sim) const {
171  hsize_t dims[1] {sim.getPopulation().get()->m_original.size()};
172  DataSpace dataspace = DataSpace(1, dims);
173 
174  CompType type_person_TI = PersonTIDataType::getCompType();
175  DataSet dataset = DataSet(file.createDataSet("person_time_independent", type_person_TI, dataspace));
176 
177  // Persons are saved per chunk
178  unsigned int person_index = 0;
179  hsize_t chunk_dims[1] = {40000};
180  while (person_index < dims[0]) {
181  hsize_t selected_dims[1];
182  if (person_index + chunk_dims[0] < dims[0]) {
183  selected_dims[0] = chunk_dims[0];
184  } else {
185  selected_dims[0] = dims[0] - person_index;
186  }
187 
188  PersonTIDataType personData[selected_dims[0]];
189  for (unsigned int j = 0; j < selected_dims[0]; j++) {
190  #define setAttributePerson(attr_lhs, attr_rhs) personData[j].attr_lhs = sim.getPopulation().get()->m_original.at(person_index).attr_rhs
191  setAttributePerson(m_id, m_id);
192  setAttributePerson(m_age, m_age);
193  setAttributePerson(m_gender, m_gender);
194  setAttributePerson(m_household_id, m_household_id);
195  setAttributePerson(m_school_id, m_school_id);
196  setAttributePerson(m_work_id, m_work_id);
197  setAttributePerson(m_prim_comm_id, m_primary_community_id);
198  setAttributePerson(m_sec_comm_id, m_secondary_community_id);
199  setAttributePerson(m_start_infectiousness, m_health.getStartInfectiousness());
200  setAttributePerson(m_start_symptomatic, m_health.getStartSymptomatic());
201  personData[j].m_time_infectiousness =
202  sim.getPopulation().get()->m_original.at(person_index).m_health.getEndInfectiousness() -
203  sim.getPopulation().get()->m_original.at(person_index).m_health.getStartInfectiousness();
204  personData[j].m_time_symptomatic =
205  sim.getPopulation().get()->m_original.at(person_index).m_health.getEndSymptomatic() -
206  sim.getPopulation().get()->m_original.at(person_index).m_health.getStartSymptomatic();
207  person_index++;
208  }
209 
210  // Select a hyperslab in the dataset.
211  DataSpace filespace = DataSpace(dataset.getSpace());
212  hsize_t offset[1] {person_index - selected_dims[0]};
213  filespace.selectHyperslab(H5S_SELECT_SET, selected_dims, offset);
214 
215  // Define memory space.
216  DataSpace memspace = DataSpace(1, selected_dims, NULL);
217 
218  // Write data to the selected portion of the dataset.
219  dataset.write(personData, type_person_TI, memspace, filespace);
220  }
221  #undef setAttributePerson
222 }
223 
224 
225 void Hdf5Saver::savePersonTDData(Group& group, const Simulator& sim) const {
226  hsize_t dims[1] {sim.getPopulation().get()->m_original.size()};
227  CompType type_person_TD = PersonTDDataType::getCompType();
228  const unsigned int CHUNK_SIZE = 10000;
229 
230  // Dataspace can fit all persons but is chunked in parts of 10000 persons
231  DataSpace dataspace = DataSpace(1, dims);
232  DSetCreatPropList plist = DSetCreatPropList();
233  hsize_t chunk_dims[1] = {CHUNK_SIZE};
234  plist.setChunk(1, chunk_dims);
235 
236  DataSet dataset;
237  if (dims[0] > CHUNK_SIZE) {
238  dataset = DataSet(group.createDataSet("person_time_dependent", type_person_TD, dataspace, plist));
239  } else {
240  dataset = DataSet(group.createDataSet("person_time_dependent", type_person_TD, dataspace));
241  }
242 
243  using PersonType = Simulator::PersonType;
244  const std::vector<PersonType>& population = sim.getPopulation()->m_original;
245 
246 
247  // Persons are saved per chunk
248  unsigned int person_index = 0;
249  while (person_index < dims[0]) {
250  hsize_t selected_dims[1];
251  if (person_index + chunk_dims[0] < dims[0]) {
252  selected_dims[0] = chunk_dims[0];
253  } else {
254  selected_dims[0] = dims[0] - person_index;
255  }
256 
257  PersonTDDataType person_data[selected_dims[0]];
258  for (unsigned int j = 0; j < selected_dims[0]; j++) {
259  const PersonType& person = population[person_index];
260  person_data[j].m_participant = person.m_is_participant;
261  person_data[j].m_health_status = (unsigned int) person.m_health.getHealthStatus();
262  person_data[j].m_disease_counter = (unsigned int) person.m_health.getDiseaseCounter();
263  person_data[j].m_on_vacation = person.m_is_on_vacation;
264  person_index++;
265  }
266 
267  // Select a hyperslab in the dataset.
268  DataSpace filespace = DataSpace(dataset.getSpace());
269  hsize_t offset[1] = {person_index - selected_dims[0]};
270  filespace.selectHyperslab(H5S_SELECT_SET, selected_dims, offset);
271 
272  // Define memory space.
273  DataSpace memspace = DataSpace(1, selected_dims, NULL);
274 
275  // Write data to the selected portion of the dataset.
276  dataset.write(person_data, type_person_TD, memspace, filespace);
277  memspace.close();
278  filespace.close();
279  }
280 
281  dataset.close();
282  dataspace.close();
283 }
284 
285 
286 void Hdf5Saver::saveTravellers(Group& group, const Simulator& sim) const {
287 
288  using PersonType = Simulator::PersonType;
291 
292  const Agenda& travellers = sim.m_planner.getAgenda();
293  hsize_t dims[1] {sim.m_planner.size()};
294  CompType type_traveller = TravellerDataType::getCompType();
295 
296  DataSpace dataspace = DataSpace(1, dims);
297  DataSet dataset = DataSet(group.createDataSet("travellers", type_traveller, dataspace));
298  auto traveller_data = make_unique<std::vector<TravellerDataType>>(dims[0]);
299 
300 
301  // Obtain the travellers in a single vector, more convenient for index calculations
302  vector<Simulator::PersonType*> travellers_seq;
303  for (auto&& day : sim.m_planner.getAgenda()) {
304  for (auto&& traveller : *(day)) {
305  travellers_seq.push_back(traveller->getNewPerson());
306  }
307  }
308 
309  unsigned int current_index = 0;
310  unsigned int list_index = 0;
311 
312  for (auto&& day : travellers) {
313  const Block& current_day = *(day);
314  for (auto&& person: current_day) {
315  TravellerDataType traveller;
316 
317  string home_sim_name = person->getHomeSimulatorId();
318  string dest_sim_name = person->getDestinationSimulatorId();
319 
320  traveller.m_days_left = list_index;
321  traveller.m_home_sim_name = home_sim_name.c_str();
322  traveller.m_dest_sim_name = dest_sim_name.c_str();
323  traveller.m_home_sim_index = person->getHomeSimulatorIndex();
324  traveller.m_dest_sim_index = sim.m_population->m_original.size() +
325  (std::find(travellers_seq.begin(), travellers_seq.end(),
326  person->getNewPerson()) - travellers_seq.begin());
327 
328  PersonType original_person = person->getHomePerson();
329  #define setAttributeTraveller(attr_lhs, attr_rhs) traveller.attr_lhs = original_person.attr_rhs
330  setAttributeTraveller(m_orig_id, m_id);
331  setAttributeTraveller(m_age, m_age);
332  setAttributeTraveller(m_gender, m_gender);
333  setAttributeTraveller(m_orig_household_id, m_household_id);
334  setAttributeTraveller(m_orig_school_id, m_school_id);
335  setAttributeTraveller(m_orig_work_id, m_work_id);
336  setAttributeTraveller(m_orig_prim_comm_id, m_primary_community_id);
337  setAttributeTraveller(m_orig_sec_comm_id, m_secondary_community_id);
338  setAttributeTraveller(m_start_infectiousness, m_health.getStartInfectiousness());
339  setAttributeTraveller(m_start_symptomatic, m_health.getStartSymptomatic());
340  traveller.m_time_infectiousness = original_person.m_health.getEndInfectiousness() -
341  original_person.m_health.getStartInfectiousness();
342  traveller.m_time_symptomatic = original_person.m_health.getEndSymptomatic() -
343  original_person.m_health.getStartSymptomatic();
344 
345  PersonType current_person = *person->getNewPerson();
346  traveller.m_participant = current_person.m_is_participant;
347  traveller.m_health_status = (unsigned int) current_person.m_health.getHealthStatus();;
348  traveller.m_disease_counter = (unsigned int) current_person.m_health.getDiseaseCounter();;
349  traveller.m_new_id = current_person.m_id;
350  traveller.m_new_household_id = current_person.m_household_id;
351  traveller.m_new_school_id = current_person.m_school_id;
352  traveller.m_new_work_id = current_person.m_work_id;
353  traveller.m_new_prim_comm_id = current_person.m_primary_community_id;
354  traveller.m_new_sec_comm_id = current_person.m_secondary_community_id;
355 
356  (*traveller_data)[current_index++] = traveller;
357  }
358  list_index++;
359  }
360  #undef setAttributeTraveller
361 
362  if (sim.m_planner.size() != 0)
363  dataset.write(traveller_data->data(), TravellerDataType::getCompType());
364  dataset.close();
365  dataspace.close();
366 }
367 
368 
369 void Hdf5Saver::saveTimestepMetadata(H5File& file, unsigned int total_amt, unsigned int current, bool create) const {
370  DataSet dataset_amt;
371  if (create == true) {
372  hsize_t dims[1] {1};
373  DataSpace dataspace = DataSpace(1, dims);
374  dataset_amt = file.createDataSet("amt_timesteps", PredType::NATIVE_UINT, dataspace);
375  } else {
376  dataset_amt = file.openDataSet("amt_timesteps");
377  }
378  unsigned int amt_timesteps[1] {total_amt};
379  dataset_amt.write(amt_timesteps, PredType::NATIVE_UINT);
380  dataset_amt.close();
381 
382 
383  DataSet dataset_last;
384  if (create == true) {
385  hsize_t dims[1] {1};
386  DataSpace dataspace = DataSpace(1, dims);
387  dataset_last = file.createDataSet("last_timestep", PredType::NATIVE_UINT, dataspace);
388  } else {
389  dataset_last = file.openDataSet("last_timestep");
390  }
391  unsigned int last_timestep[1] {current};
392  dataset_last.write(last_timestep, PredType::NATIVE_UINT);
393  dataset_last.close();
394 }
395 
396 
397 void Hdf5Saver::saveRngState(Group& group, const Simulator& sim) const {
398  hsize_t dims[1] {1};
399  DataSpace dataspace = DataSpace(1, dims);
400  DataSet dataset = DataSet(group.createDataSet("randomgen", StrType(0, H5T_VARIABLE), dataspace));
401 
402  stringstream ss;
403  ss << *sim.m_rng;
404  string cppString = ss.str();
405  const char* rng_state[1] {cppString.c_str()};
406 
407  dataset.write(rng_state, StrType(0, H5T_VARIABLE));
408  dataset.close();
409  dataspace.close();
410 }
411 
412 
413 void Hdf5Saver::saveCalendar(Group& group, const Simulator& sim) const {
414  hsize_t dims[1] {1};
415  CompType typeCalendar = CalendarDataType::getCompType();
416  DataSpace dataspace = DataSpace(1, dims);
417  DataSet dataset = DataSet(group.createDataSet("calendar", typeCalendar, dataspace));
418 
419  stringstream ss;
420  ss << sim.m_calendar->getYear() << "-" << sim.m_calendar->getMonth() << "-" << sim.m_calendar->getDay();
421  string save_date = ss.str();
422 
423  CalendarDataType calendar[1];
424  calendar[0].m_day = sim.m_calendar->getSimulationDay();
425  calendar[0].m_date = save_date.c_str();
426  dataset.write(calendar, typeCalendar);
427 
428  dataset.close();
429  dataspace.close();
430 }
431 
432 
433 void Hdf5Saver::saveConfigs(H5File& file, const ptree& pt_config) const {
434  hsize_t dims[1] {1};
435  Group group(file.createGroup("/Configuration"));
436  DataSpace dataspace = DataSpace(1, dims);
437 
438  CompType type_conf_data = ConfigDataType::getCompType();
439  DataSet dataset = DataSet(group.createDataSet(H5std_string("configuration"), type_conf_data, dataspace));
440  ConfigDataType configData[1];
441 
442  auto getStringXmlPtree = [](const ptree xml) {
443  ostringstream oss;
444  boost::property_tree::xml_parser::write_xml(oss, xml);
445  return oss.str();
446  };
447 
448  // Searches file in DataDir
449  auto getPtreeXmlFile = [](string filename) {
450  const auto filepath {InstallDirs::getDataDir() /= filename};
451  if (!is_regular_file(filepath))
452  throw std::runtime_error(string(__func__) + "> File " + filepath.string() + " not present/regular.");
453  ptree tree;
454  xml_parser::read_xml(filepath.string(), tree, boost::property_tree::xml_parser::trim_whitespace);
455  return tree;
456  };
457 
458  string content_config = getStringXmlPtree(pt_config);
459  configData[0].m_config_content = content_config.c_str();
460  string content_disease = getStringXmlPtree(getPtreeXmlFile(pt_config.get<string>("run.disease.config")));
461  configData[0].m_disease_content = content_disease.c_str();
462  string content_age = getStringXmlPtree(getPtreeXmlFile(pt_config.get<string>("run.age_contact_matrix_file")));
463  configData[0].m_age_contact_content = content_age.c_str();
464 
465 
466  ptree json_tree;
467  string filename = pt_config.get<string>("run.holidays");
468  const auto filepath {InstallDirs::getDataDir() /= filename};
469  if (!is_regular_file(filepath))
470  throw std::runtime_error(string(__func__) + "> File " + filepath.string() + " not present/regular.");
471  json_parser::read_json(filepath.string(), json_tree);
472 
473  ostringstream oss;
474  json_parser::write_json(oss, json_tree);
475  string content_holidays = oss.str();
476  configData[0].m_holidays_content = content_holidays.c_str();
477 
478 
479  dataset.write(configData, type_conf_data);
480  dataset.close();
481  dataspace.close();
482  group.close();
483 }
484 
485 
486 }
Person< BehaviourPolicy, BeliefPolicy > PersonType
Definition: Simulator.h:72
static CompType getCompType()
list< unique_ptr< Block >> Agenda
Definition: SimplePlanner.h:26
Interface for install directory queries.
Header file for the Calendar class.
Time Dependent Person DataType.
Definition: NoBehaviour.h:17
static boost::filesystem::path getDataDir()
Utility method: get path to the directory for data files.
Utilities for the project.
Definition: Infector.h:36
virtual void update(const Simulator &sim)
Update function which is called by the subject.
Definition: Hdf5Saver.h:87
Header file for the Person class.
static CompType getCompType()
vector< unique_ptr< T >> Block
Definition: SimplePlanner.h:25
Header file for the core Population class.
static CompType getCompType()
Definition: ConfigDataType.h:9
Header file for the Saver class for the checkpointing functionality.
#define setAttributePerson(attr_lhs, attr_rhs)
static CompType getCompType()
#define setAttributeTraveller(attr_lhs, attr_rhs)
Header for the core Cluster class.
static CompType getCompType()
void forceSave(const Simulator &sim, int timestep=-1)
Forces a save to the hdf5 file, with an optional timestep argument which specifies a new timestep sav...
Definition: Hdf5Saver.h:90