Stride Reference Manual  1.0
Hdf5Loader.cpp
Go to the documentation of this file.
1 
6 #include <boost/date_time/gregorian/greg_date.hpp>
7 #include <boost/property_tree/xml_parser.hpp>
8 #include <iomanip>
9 #include "Hdf5Loader.h"
10 #include "calendar/Calendar.h"
11 #include "util/InstallDirs.h"
12 #include "pop/Population.h"
18 #include "sim/SimulatorBuilder.h"
19 #include "pop/PopulationBuilder.h"
20 #include "core/Cluster.h"
21 #include "util/etc.h"
22 
23 #include <algorithm>
24 #include <vector>
25 #include <string>
26 
27 using namespace H5;
28 using namespace boost::property_tree;
29 using namespace std;
30 using namespace boost::filesystem;
31 using namespace stride::util;
32 
33 namespace stride {
34 
35 
36 Hdf5Loader::Hdf5Loader(const char* filename) :
37  m_filename(filename) {
38 
39  try {
40  this->loadConfigs();
41  } catch (FileIException error) {
42  error.printError();
43  }
44 }
45 
46 void Hdf5Loader::loadConfigs() {
47  H5File file(m_filename, H5F_ACC_RDONLY, H5P_DEFAULT, H5P_DEFAULT);
48 
49  DataSet dataset = DataSet(file.openDataSet("Configuration/configuration"));
50  ConfigDataType configData[1];
51  dataset.read(configData, ConfigDataType::getCompType());
52  dataset.close();
53  file.close();
54 
55  auto getPropTree = [](string xml_content, ptree& dest_pt) {
56  istringstream iss;
57  iss.str(xml_content);
58  xml_parser::read_xml(iss, dest_pt, boost::property_tree::xml_parser::trim_whitespace);
59  };
60 
61  getPropTree(configData[0].m_config_content, m_pt_config);
62  getPropTree(configData[0].m_disease_content, m_pt_disease);
63  getPropTree(configData[0].m_age_contact_content, m_pt_contact);
64 }
65 
66 
67 void Hdf5Loader::loadFromTimestep(unsigned int timestep, std::shared_ptr<Simulator> sim) const {
68  H5File file(m_filename, H5F_ACC_RDONLY, H5P_DEFAULT, H5P_DEFAULT);
69 
70  stringstream ss;
71  ss << "/Timestep_" << std::setfill('0') << std::setw(6) << timestep;
72  string dataset_name = ss.str();
73 
74 
75  this->loadCalendar(file, dataset_name, sim);
76  this->loadPersonTDData(file, dataset_name, sim);
77  this->loadTravellers(file, dataset_name, sim);
78 
79  // Sort the population by id first, in order to increase the speed of cluster reordening
80  auto sortByID = [](const Simulator::PersonType& lhs, const Simulator::PersonType& rhs) {
81  return lhs.getId() < rhs.getId();
82  };
83  std::sort(sim->m_population->m_original.begin(), sim->m_population->m_original.end(), sortByID);
84 
85  this->loadClusters(file, dataset_name + "/household_clusters", sim->m_households, sim);
86  this->loadClusters(file, dataset_name + "/school_clusters", sim->m_school_clusters, sim);
87  this->loadClusters(file, dataset_name + "/work_clusters", sim->m_work_clusters, sim);
88  this->loadClusters(file, dataset_name + "/primary_community_clusters", sim->m_primary_community, sim);
89  this->loadClusters(file, dataset_name + "/secondary_community_clusters", sim->m_secondary_community, sim);
90 
91  this->updateClusterImmuneIndices(sim);
92 
93  if (sim->m_rng != nullptr) {
94  this->loadRngState(file, dataset_name, sim);
95  }
96 
97  file.close();
98 }
99 
100 
101 unsigned int Hdf5Loader::getLastSavedTimestep() const {
102  H5File file(m_filename, H5F_ACC_RDONLY, H5P_DEFAULT, H5P_DEFAULT);
103  DataSet dataset = DataSet(file.openDataSet("last_timestep"));
104  unsigned int data[1];
105  dataset.read(data, PredType::NATIVE_UINT);
106  dataset.close();
107  file.close();
108  return data[0];
109 }
110 
111 
112 void Hdf5Loader::updateClusterImmuneIndices(std::shared_ptr<Simulator> sim) const {
113  for (auto cluster : sim->m_households) {
114  cluster.m_index_immune = cluster.m_members.size() - 1;
115  }
116  for (auto cluster : sim->m_school_clusters) {
117  cluster.m_index_immune = cluster.m_members.size() - 1;
118  }
119  for (auto cluster : sim->m_work_clusters) {
120  cluster.m_index_immune = cluster.m_members.size() - 1;
121  }
122  for (auto cluster : sim->m_primary_community) {
123  cluster.m_index_immune = cluster.m_members.size() - 1;
124  }
125  for (auto cluster : sim->m_secondary_community) {
126  cluster.m_index_immune = cluster.m_members.size() - 1;
127  }
128 }
129 
130 void Hdf5Loader::loadCalendar(H5File& file, string dataset_name, shared_ptr<Simulator> sim) const {
131  DataSet dataset = DataSet(file.openDataSet(dataset_name + "/calendar"));
132  CalendarDataType calendar[1];
133  dataset.read(calendar, CalendarDataType::getCompType());
134  dataset.close();
135 
136  sim->m_calendar->m_day = calendar[0].m_day;
137  sim->m_calendar->m_date = boost::gregorian::from_simple_string(calendar[0].m_date);
138 }
139 
140 
141 void Hdf5Loader::loadTravellers(H5File& file, string dataset_name, shared_ptr<Simulator> sim) const {
142  try {
143  DataSet dataset = DataSet(file.openDataSet(dataset_name + "/travellers"));
144  DataSpace dataspace = dataset.getSpace();
145  hsize_t dims_travellers[1];
146  dataspace.getSimpleExtentDims(dims_travellers, NULL);
147  const unsigned int amt_travellers = dims_travellers[0];
148  dataspace.close();
149 
150  if (amt_travellers == 0) {
151  dataset.close();
152  return;
153  }
154 
155  auto travellers = make_unique<std::vector<TravellerDataType>>(amt_travellers);
156 
157  dataset.read(travellers->data(), TravellerDataType::getCompType());
158 
159  // TODO if necessary, count the travellers for each day, reserve in the planner
160 
161  // The travellers are saved in order of their index in the new simulator
162  // Therefore, we can just iterate over them and add them without worrying about changing the original order
163  for (auto object : *travellers) {
164 
165  // First of all, add the person to the population (visitors)
167  object.m_new_id, object.m_age,
168  object.m_new_household_id, object.m_new_school_id, object.m_new_work_id,
169  object.m_new_prim_comm_id, object.m_new_sec_comm_id,
170  object.m_start_infectiousness, object.m_start_symptomatic,
171  object.m_time_infectiousness, object.m_time_symptomatic
172  );
173  person.m_health.m_status = HealthStatus(object.m_health_status);
174  person.m_health.m_disease_counter = object.m_disease_counter;
175  person.m_is_participant = object.m_participant;
176 
177  sim->m_population->m_visitors.add(object.m_days_left, person);
178 
179  // Secondly, add the traveller to the planner in the simulator
180 
181  // Construct the person from the original simulator (his health does not matter, since his new one is used and returned eventually)
183  object.m_orig_id, object.m_age,
184  object.m_orig_household_id, object.m_orig_school_id, object.m_orig_work_id,
185  object.m_orig_prim_comm_id, object.m_orig_sec_comm_id,
186  object.m_start_infectiousness, object.m_start_symptomatic,
187  object.m_time_infectiousness, object.m_time_symptomatic
188  );
189 
190  string home_sim_name = object.m_home_sim_name;
191  string dest_sim_name = object.m_dest_sim_name;
192 
194  original_person, sim->m_population->m_visitors.getModifiableDay(object.m_days_left)->back().get(),
195  home_sim_name, dest_sim_name, object.m_home_sim_index);
196 
197  traveller.getNewPerson()->setOnVacation(false);
198  sim->m_planner.add(object.m_days_left, traveller);
199 
200 
201  // Finally, add the traveller to clusters
202  Simulator::PersonType* added_person = sim->m_population->m_visitors.getModifiableDay(
203  object.m_days_left)->back().get();
204 
205  sim->m_work_clusters.at(object.m_new_work_id).addPerson(added_person);
206  sim->m_primary_community.at(object.m_new_prim_comm_id).addPerson(added_person);
207  sim->m_secondary_community.at(object.m_new_sec_comm_id).addPerson(added_person);
208 
209 
210  // Since the travellers are ordered, it is safe to set these values every iteration
211  sim->m_next_id = object.m_new_id + 1;
212  sim->m_next_hh_id = object.m_new_household_id + 1;
213 
214  }
215 
216  } catch (DataSetIException e) {
217  // The dataset does not exist, no traveller information was stored.
218  return;
219  }
220 }
221 
222 
223 void Hdf5Loader::loadPersonTDData(H5File& file, string dataset_name, shared_ptr<Simulator> sim) const {
224  DataSet dataset = DataSet(file.openDataSet(dataset_name + "/person_time_dependent"));
225  unsigned long dims[1] = {sim->m_population.get()->m_original.size()};
226  CompType type_person_TD = PersonTDDataType::getCompType();
227 
228  for (unsigned int i = 0; i < dims[0]; i++) {
229  PersonTDDataType person[1];
230  hsize_t dim_sub[1] = {1};
231 
232  DataSpace memspace(1, dim_sub, NULL);
233 
234  hsize_t offset[1] = {i};
235  hsize_t count[1] = {1};
236  hsize_t stride[1] = {1};
237  hsize_t block[1] = {1};
238 
239  DataSpace dataspace = dataset.getSpace();
240  dataspace.selectHyperslab(H5S_SELECT_SET, count, offset, stride, block);
241  dataset.read(person, type_person_TD, memspace, dataspace);
242  sim->m_population->m_original.at(i).m_is_participant = person[0].m_participant;
243  sim->m_population->m_original.at(i).m_health.m_status = HealthStatus(person[0].m_health_status);
244  sim->m_population->m_original.at(i).m_health.m_disease_counter = person[0].m_disease_counter;
245  sim->m_population->m_original.at(i).m_is_on_vacation = person[0].m_on_vacation;
246  memspace.close();
247  dataspace.close();
248  }
249  dataset.close();
250 }
251 
252 
253 void Hdf5Loader::loadRngState(H5File& file, string dataset_name, shared_ptr<Simulator> sim) const {
254  DataSet dataset = DataSet(file.openDataSet(dataset_name + "/randomgen"));
255  std::string rng_state;
256  dataset.read(rng_state, StrType(0, H5T_VARIABLE));
257  dataset.close();
258 
259  sim->m_rng->setState(rng_state);
260 }
261 
262 
263 void Hdf5Loader::loadClusters(H5File& file, std::string full_dataset_name, std::vector<Cluster>& cluster,
264  std::shared_ptr<Simulator> sim) const {
265  std::shared_ptr<Population> pop = sim->m_population;
266 
267  DataSet dataset = DataSet(file.openDataSet(full_dataset_name));
268  DataSpace dataspace = dataset.getSpace();
269  hsize_t dims_clusters[1];
270  dataspace.getSimpleExtentDims(dims_clusters, NULL);
271  const unsigned int amtIds = dims_clusters[0];
272  dataspace.close();
273 
274  unsigned int cluster_data[amtIds];
275  unsigned int index = 0;
276 
277  dataset.read(cluster_data, PredType::NATIVE_UINT);
278  dataset.close();
279 
280  // Collect all the travellers in a single vector for convenience
281  vector<Simulator::PersonType*> travellers;
282  for (auto&& day : sim->m_planner.getAgenda()) {
283  for (auto&& traveller : *(day)) {
284  travellers.push_back(traveller->getNewPerson());
285  }
286  }
287 
288  for (unsigned int i = 0; i < cluster.size(); i++) {
289  for (unsigned int j = 0; j < cluster.at(i).getSize(); j++) {
290  unsigned int id = cluster_data[index++];
291 
292  if (id < pop->m_original.size()) {
293  Simulator::PersonType* person = &pop->m_original.at(id);
294  cluster.at(i).m_members.at(j).first = person;
295  } else {
296  // Get the pointer to the person via the travel planner
297  auto traveller = std::find_if(
298  travellers.begin(),
299  travellers.end(),
300  [&id](Simulator::PersonType* traveller) -> bool {
301  return (traveller)->getId() == id;
302  });
303  cluster.at(i).m_members.at(j).first = (*traveller);
304  }
305 
306  }
307  }
308 }
309 
310 void Hdf5Loader::extractConfigs(string filename) {
311  // Check for the existence/validity of the hdf5 file.
312  bool hdf5_file_exists = exists(system_complete(filename));
313  if (!hdf5_file_exists) {
314  throw runtime_error(string(__func__) + "> Hdf5 file " +
315  system_complete(filename).string() + " does not exist.");
316  }
317  const auto file_path_hdf5 = canonical(system_complete(filename));
318  if (!is_regular_file(file_path_hdf5)) {
319  throw runtime_error(string(__func__) + "> Hdf5 file " +
320  system_complete(filename).string() + " is not a regular file.");
321  }
322 
323  H5File file(filename, H5F_ACC_RDONLY, H5P_DEFAULT, H5P_DEFAULT);
324 
325  DataSet dataset = DataSet(file.openDataSet("Configuration/configuration"));
326  ConfigDataType configData[1];
327  dataset.read(configData, ConfigDataType::getCompType());
328  dataset.close();
329  file.close();
330 
331  auto writeToFile = [](string filename, string content) {
332  std::ofstream file;
333  file.open(filename.c_str());
334  file << content;
335  file.close();
336  };
337 
338  writeToFile(filename + "_config.xml", configData[0].m_config_content);
339  writeToFile(filename + "_disease.xml", configData[0].m_disease_content);
340  writeToFile(filename + "_contact.xml", configData[0].m_age_contact_content);
341  writeToFile(filename + "_holidays.json", configData[0].m_holidays_content);
342 }
343 
344 
345 }
Person< BehaviourPolicy, BeliefPolicy > PersonType
Definition: Simulator.h:72
static CompType getCompType()
Interface for install directory queries.
Traveller< PersonType > TravellerType
Definition: Simulator.h:74
void loadFromTimestep(unsigned int timestep, shared_ptr< Simulator > sim) const
Load from timestep, if the specified timestep is present in the hdf5 file.
Definition: Hdf5Loader.h:93
Header file for the Calendar class.
Time Dependent Person DataType.
Definition: NoBehaviour.h:17
Utilities for the project.
Definition: Infector.h:36
static void extractConfigs(string filename)
Extract the configuration files saved in the hdf5 file.
Definition: Hdf5Loader.h:109
Header for the SimulatorBuilder class.
static CompType getCompType()
Header file for the core Population class.
Header file for the Loader class for the checkpointing functionality.
static CompType getCompType()
Definition: ConfigDataType.h:9
STL namespace.
HealthStatus
Definition: Health.h:19
Header for the core Cluster class.
Initialize populations.
unsigned int getLastSavedTimestep() const
Retrieves the last saved timestep index in the hdf5 file.
Definition: Hdf5Loader.h:105
static CompType getCompType()