sasze  0.0-0.dev0.dirty.el6
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Groups
sasze_ingest.c
Go to the documentation of this file.
1 /*******************************************************************************
2 *
3 * COPYRIGHT (C) 2012 Battelle Memorial Institute. All Rights Reserved.
4 *
5 ********************************************************************************
6 *
7 * Authors:
8 *
9 * name: Yan Shi
10 * phone: (509) 375-6858
11 * email: yan.shi@pnnl.gov
12 *
13 * name: Brian Ermold
14 * phone: (509) 375-2277
15 * email: brian.ermold@pnnl.gov
16 *
17 *******************************************************************************/
18 
19 /** @file sasze_ingest.c
20  * SASZE Ingest Main Source File.
21  */
22 
23 #include "dsproc3.h"
24 #include "sasze_ingest.h"
25 
26 #include "../config.h"
27 static const char *_Version = PACKAGE_NAME"-"PACKAGE_VERSION;
28 
29 /**
30  * Initialize the SASZE Ingest process.
31  *
32  * This function will:
33  * - create the SaszeData structure
34  * - get the output datastream IDs
35  * - add the raw data file patterns
36  *
37  * If an error occurs in this function it will be appended to the log and
38  * error mail messages, and the process status will be set appropriately.
39  *
40  * @retval data void pointer to the SaszeData structure
41  * @retval NULL if an error occurred
42  */
43 void *sasze_init_process(void)
44 {
45  SaszeData *data;
46  const char *conf_root;
47  char pattern[64];
48  int in_dsid;
49  char conf_file[PATH_MAX];
50 
51  const char *patterns[] = {
52  "HiSun_nir_1s",
53  "HiSun_vis_1s",
54  "LowSun_vis_1s",
55  "LowSun_nir_1s",
56  "HiSun_ nir_1s",
57  "HiSun_ vis_1s",
58  "LowSun_ vis_1s",
59  "LowSun_ nir_1s",
60  "day_vis_1s",
61  "day_nir_1s"
62  };
63 
64  int npatterns = sizeof(patterns)/sizeof(const char *);
65 
66  /************************************************************
67  * Create the SaszeData structure
68  *************************************************************/
69 
70  data = (SaszeData *)calloc(1, sizeof(SaszeData));
71  if (!data) {
72  DSPROC_ERROR( DSPROC_ENOMEM,
73  "Failed to allocate memory for SaszeData structure.\n");
74  return ((void *)NULL);
75  }
76 
77  data->proc_name = dsproc_get_name();
78  data->site = dsproc_get_site();
79  data->facility = dsproc_get_facility();
80 
81  /************************************************************
82  * Add input datastream file patterns
83  *************************************************************/
84 
85  in_dsid = dsproc_get_input_datastream_id("sasze", "00");
86  if (in_dsid < 0) {
87  free(data);
88  return((void *)NULL);
89  }
90 
91  if (!dsproc_add_datastream_file_patterns(in_dsid, npatterns, patterns, 0)) {
92  free(data);
93  return((void *)NULL);
94  }
95 
96  dsproc_set_file_name_compare_function(
97  in_dsid, sas_raw_file_name_time_compare);
98 
99  /************************************************************
100  * Get raw data output datastream ID
101  *************************************************************/
102 
103  data->raw_dsid = dsproc_get_output_datastream_id("sasze", "00");
104  if (data->raw_dsid < 0) {
105  free(data);
106  return((void *)NULL);
107  }
108 
109  dsproc_set_rename_preserve_dots(data->raw_dsid, 3);
110 
111  /************************************************************
112  * Initialize the raw data structures
113  *************************************************************/
114 
115  if (!(data->nir_raw = sas_init_raw_data(SASHENIR)) ||
116  !(data->vis_raw = sas_init_raw_data(SASHEVIS))) {
117 
118  free(data);
119  return ((void *)NULL);
120  }
121 
122  /************************************************************
123  * Get the output datastream IDs
124  *************************************************************/
125 
126  data->nir_a0_dsid = dsproc_get_output_datastream_id("saszenir", "a0");
127  data->nir_a1_dsid = dsproc_get_output_datastream_id("saszenir", "a1");
128  data->vis_a0_dsid = dsproc_get_output_datastream_id("saszevis", "a0");
129  data->vis_a1_dsid = dsproc_get_output_datastream_id("saszevis", "a1");
130  data->fb_dsid = dsproc_get_output_datastream_id("saszefilterbands", "a1");
131 
132  if (data->nir_a0_dsid < 0 ||
133  data->nir_a1_dsid < 0 ||
134  data->vis_a0_dsid < 0 ||
135  data->vis_a1_dsid < 0 ||
136  data->fb_dsid < 0) {
137 
138  free(data);
139  return ((void *)NULL);
140  }
141 
142  /************************************************************
143  * Exclude attributes from the DOD metadata checks.
144  *************************************************************/
145 
146  if (!sasze_exclude_from_dod_compare()) {
147  free(data);
148  return ((void *)NULL);
149  }
150 
151  /************************************************************
152  * Create the file list used to track the NIR files to
153  * process when a VIS file is found.
154  *************************************************************/
155 
156  data->nir_file_list = sas_create_file_list();
157  if (!data->nir_file_list) {
158  free(data);
159  return((void *)NULL);
160  }
161 
162  /************************************************************
163  * Get the full path to the conf file directory
164  *************************************************************/
165 
166  conf_root = getenv("INGEST_HOME");
167 
168  if (!conf_root) {
169  DSPROC_ERROR(
170  "Could Not Find INGEST_HOME Environment Variable",
171  "Could not determine path to configuration files."
172  " -> the INGEST_HOME environment variable was not found\n");
173  free(data);
174  return((void *)NULL);
175  }
176 
177  snprintf(data->conf_dir, PATH_MAX,
178  "%s/conf/ingest/sasze/%ssasze%s",
179  conf_root, data->site, data->facility);
180 
181  /************************************************************
182  * Get the list of NIR responsivity files
183  *************************************************************/
184 
185  sprintf(pattern, "^%ssaszenir%s\\.[0-9]{8}\\.",
186  data->site, data->facility);
187 
188  data->nir_resp_dirlist = sas_get_config_dirlist(
189  data->conf_dir, pattern);
190 
191  if (!data->nir_resp_dirlist) {
192  free(data);
193  return((void *)NULL);
194  }
195 
196  /************************************************************
197  * Get the list of VIS responsivity files
198  *************************************************************/
199 
200  sprintf(pattern, "^%ssaszevis%s\\.[0-9]{8}\\.",
201  data->site, data->facility);
202 
203  data->vis_resp_dirlist = sas_get_config_dirlist(
204  data->conf_dir, pattern);
205 
206  if (!data->vis_resp_dirlist) {
207  free(data);
208  return((void *)NULL);
209  }
210 
211  /************************************************************
212  * Load the Gueymard table of TOA Io values
213  *************************************************************/
214 
215  snprintf(conf_file, PATH_MAX, "%ssasze%s.gueymard2003",
216  data->site, data->facility);
217 
218  data->toaIo_table = sas_load_config_table(
219  data->conf_dir, conf_file, 2, data->toaIo_table);
220 
221  if (!data->toaIo_table) {
222  free(data);
223  return((void *)NULL);
224  }
225 
226  return ((void *)data);
227 }
228 
229 /**
230  * Finish the SASZE Ingest process.
231  *
232  * This function frees all memory used by the SaszeData structure.
233  *
234  * @param user_data void pointer to the SaszeData structure
235  */
236 void sasze_finish_process(void *user_data)
237 {
238  SaszeData *data = (SaszeData*)user_data;
239 
240  if (data) {
241 
242  if (data->nir_resp_dirlist) dirlist_free(data->nir_resp_dirlist);
243  if (data->vis_resp_dirlist) dirlist_free(data->vis_resp_dirlist);
244 
245  if (data->nir_raw) sas_free_raw_data(data->nir_raw);
246  if (data->vis_raw) sas_free_raw_data(data->vis_raw);
247 
248  if (data->nir_file_list) sas_free_file_list(data->nir_file_list);
249 
250  if (data->fb_times) free(data->fb_times);
251  if (data->fb_nir_records) free(data->fb_nir_records);
252  if (data->fb_vis_records) free(data->fb_vis_records);
253 
254  if (data->toaIo_table) sas_free_config_table(data->toaIo_table);
255 
256  free(data);
257  }
258 
259  return;
260 }
261 
262 /**
263  * Process a set of SASZE data files.
264  *
265  * If an error occurs in this function it will be appended to the log and
266  * error mail messages, and the process status will be set appropriately.
267  *
268  * @param data pointer to the SaszeData structure
269  * @param input_dir path to the file
270  * @param nir_file nir file name
271  * @param vis_file vis file name
272  *
273  * @retval 1 if successful
274  * @retval -1 if a fatal error occurred
275  */
276 int sasze_process_file_set(
277  SaszeData *data,
278  const char *input_dir,
279  const char *nir_file,
280  const char *vis_file)
281 {
282  SasRawData *nir_raw = (SasRawData *)NULL;
283  SasRawData *vis_raw = (SasRawData *)NULL;
284  int status;
285 
286  /* Read in the raw NIR data and store the a0 data. */
287 
288  if (nir_file) {
289 
290  dsproc_set_input_source(nir_file);
291 
292  nir_raw = data->nir_raw;
293  status = sas_read_data(input_dir, nir_file, nir_raw, data->raw_dsid);
294  if (status < 0) return(-1);
295 
296  if (status == 0) {
297  nir_file = (const char *)NULL;
298  nir_raw = (SasRawData *)NULL;
299  }
300  else {
301  status = sas_store_a0_data(nir_raw, data->nir_a0_dsid);
302  if (status < 0) return(-1);
303  }
304  }
305 
306  /* Read in the raw VIS data and store the a0 data. */
307 
308  if (vis_file) {
309 
310  dsproc_set_input_source(vis_file);
311 
312  vis_raw = data->vis_raw;
313  status = sas_read_data(input_dir, vis_file, vis_raw, data->raw_dsid);
314  if (status < 0) return(-1);
315 
316  if (status == 0) {
317  vis_file = (const char *)NULL;
318  vis_raw = (SasRawData *)NULL;
319  }
320  else {
321  status = sas_store_a0_data(vis_raw, data->vis_a0_dsid);
322  if (status < 0) return(-1);
323  }
324  }
325 
326  /* Create and store the calibrated and filterband datasets. */
327 
328  if (nir_raw || vis_raw) {
329  status = sasze_store_data(data, nir_raw, vis_raw);
330  if (status < 0) return(-1);
331  }
332 
333  /* Rename the raw files */
334 
335  if (nir_file) {
336 
337  if (!dsproc_rename(data->raw_dsid,
338  input_dir, nir_file, nir_raw->begin_time, nir_raw->end_time)) {
339 
340  return(-1);
341  }
342  }
343 
344  if (vis_file) {
345 
346  if (!dsproc_rename(data->raw_dsid,
347  input_dir, vis_file, vis_raw->begin_time, vis_raw->end_time)) {
348 
349  return(-1);
350  }
351  }
352 
353  return(1);
354 }
355 
356 /**
357  * Process a SASZE data file.
358  *
359  * If an error occurs in this function it will be appended to the log and
360  * error mail messages, and the process status will be set appropriately.
361  *
362  * @param user_data void pointer to the SaszeData structure
363  * @param input_dir path to the file
364  * @param file_name file name
365  *
366  * @retval 1 if successful
367  * @retval 0 if the current file should be skipped
368  * @retval -1 if a fatal error occurred
369  */
370 int sasze_process_file(
371  void *user_data,
372  const char *input_dir,
373  const char *file_name)
374 {
375  SaszeData *data = (SaszeData *)user_data;
376  SasFileList *nir_list = data->nir_file_list;
377  const char *nir_file;
378  time_t nir_time;
379  const char *vis_file;
380  time_t vis_time;
381  int status;
382  int fi;
383 
384  if (strstr(file_name, "SASZe") &&
385  strstr(file_name, "1s")) {
386 
387  if (strstr(file_name, "nir")) {
388 
389  /* Add the SASZE NIR file to the list */
390 
391  if (!sas_add_file_to_list(nir_list, file_name)) {
392  return(-1);
393  }
394 
395  return(0);
396  }
397  else if (strstr(file_name, "vis")) {
398 
399  /* Process the SASZE VIS and NIR files */
400 
401  vis_file = file_name;
402  vis_time = sas_raw_file_name_time(vis_file);
403 
404  for (fi = 0; fi < nir_list->nfiles; ++fi) {
405 
406  nir_file = nir_list->files[fi];
407  nir_time = sas_raw_file_name_time(nir_file);
408 
409  if (nir_time == vis_time) {
410 
411  status = sasze_process_file_set(
412  data, input_dir, nir_file, vis_file);
413 
414  vis_file = (const char *)NULL;
415  }
416  else {
417  status = sasze_process_file_set(
418  data, input_dir, nir_file, NULL);
419  }
420 
421  if (status < 0) {
422  return(-1);
423  }
424  }
425 
426  if (vis_file) {
427 
428  status = sasze_process_file_set(
429  data, input_dir, NULL, vis_file);
430 
431  if (status < 0) {
432  return(-1);
433  }
434  }
435 
436  sas_clear_file_list(nir_list);
437 
438  return(1);
439  }
440  }
441 
442  DSPROC_ERROR( NULL,
443  "Unknown file name format: %s\n", file_name);
444 
445  return(-1);
446 }
447 
448 /**
449  * SASZE Ingest main function.
450  *
451  * @param argc number of command line arguments
452  * @param argv command line arguments
453  *
454  * @retval 0 successful
455  * @retval 1 failure
456  */
457 int main(int argc, char *argv[])
458 {
459  const char *proc_names[] = {
460  "sasze"
461  };
462 
463  int nproc_names = sizeof(proc_names)/sizeof(const char *);
464  int exit_value;
465 
466  dsproc_set_init_process_hook(sasze_init_process);
467  dsproc_set_finish_process_hook(sasze_finish_process);
468  dsproc_set_process_file_hook(sasze_process_file);
469 
470  exit_value = dsproc_main(
471  argc, argv,
472  PM_INGEST,
473  _Version,
474  nproc_names,
475  proc_names);
476 
477  return(exit_value);
478 }