sasze  0.0-0.dev0.dirty.el6
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Groups
sasze_store_data.c
1 /*******************************************************************************
2 *
3 * COPYRIGHT (C) 2012 Battelle Memorial Institute. All Rights Reserved.
4 *
5 ********************************************************************************
6 *
7 * Authors:
8 *
9 * name: Brian Ermold
10 * phone: (509) 375-2277
11 * email: brian.ermold@pnnl.gov
12 *
13 *******************************************************************************/
14 
15 #include <math.h>
16 
17 #include "sasze_ingest.h"
18 
19 /** VIS filterband wavelengths. */
20 static int gVisWavelens[] = {
21  340, 355, 368, 387, 408, 415, 440, 500, 532,
22  615, 650, 673, 762, 870, 910, 935, 940, 975 };
23 
24 /** Number of VIS filterband wavelengths. */
25 static int gNumVisWavelens = sizeof(gVisWavelens)/sizeof(int);
26 
27 /** NIR filterband wavelengths. */
28 static int gNirWavelens[] = {
29  976, 1020, 1030, 1050, 1064, 1125, 1175,
30  1225, 1370, 1440, 1500, 1550, 1637 };
31 
32 /** Number of NIR filterband wavelengths. */
33 static int gNumNirWavelens = sizeof(gNirWavelens)/sizeof(int);
34 
35 /**
36  * Exclude attributes from the metadata checks.
37  *
38  * Some attributes can change if a NIR or VIS record is missing
39  * but we do not want the output file to split when this happens.
40  *
41  * @retval 1 if successful
42  * @retval 0 if an error occurred
43  */
44 int sasze_exclude_from_dod_compare(void)
45 {
46  char var_name[64];
47  double wavelength;
48  int gwi;
49  const char *att_name;
50  const char *att_names[] = {
51  "serial_number_spectrometer_nir",
52  "serial_number_spectrometer_vis"
53  };
54 
55  if (!dsproc_exclude_from_dod_compare(NULL, 0, 2, att_names)) {
56  return(0);
57  }
58 
59  /* Exclude the earth-sun distance variable */
60 
61  strcpy(var_name, "earth_sun_distance");
62 
63  if (!dsproc_exclude_from_dod_compare(var_name, 1, 0, NULL)) {
64  return(0);
65  }
66 
67  /* Exclude the VIS zenith_radiance_* spectrometer_pixel attributes */
68 
69  att_name = "spectrometer_pixel";
70 
71  for (gwi = 0; gwi < gNumVisWavelens; ++gwi) {
72 
73  wavelength = gVisWavelens[gwi];
74 
75  sprintf(var_name, "zenith_radiance_%dnm", (int)wavelength);
76  if (!dsproc_exclude_from_dod_compare(var_name, 0, 1, &att_name)) {
77  return(0);
78  }
79 
80  sprintf(var_name, "zenith_transmittance_%dnm", (int)wavelength);
81  if (!dsproc_exclude_from_dod_compare(var_name, 0, 1, &att_name)) {
82  return(0);
83  }
84  }
85 
86  /* Exclude the NIR zenith_radiance_* spectrometer_pixel attributes */
87 
88  for (gwi = 0; gwi < gNumNirWavelens; ++gwi) {
89 
90  wavelength = gNirWavelens[gwi];
91 
92  sprintf(var_name, "zenith_radiance_%dnm", (int)wavelength);
93  if (!dsproc_exclude_from_dod_compare(var_name, 0, 1, &att_name)) {
94  return(0);
95  }
96 
97  sprintf(var_name, "zenith_transmittance_%dnm", (int)wavelength);
98  if (!dsproc_exclude_from_dod_compare(var_name, 0, 1, &att_name)) {
99  return(0);
100  }
101  }
102 
103  return(1);
104 }
105 
106 /**
107  * Load the response functions for the specified dataset.
108  *
109  * @param data pointer to the SaszeData structure
110  * @param raw pointer to the SasRawData structure
111  *
112  * @retval 1 if successful
113  * @retval 0 if an error occurred
114  */
115 int sasze_load_resp_table(SaszeData *data, SasRawData *raw)
116 {
117  int debug = dsproc_get_debug_level();
118  time_t data_time = raw->records[0].time;
119 // float int_time = raw->records[0].t_int;
120 // char int_time_string[16];
121  int nfiles;
122  char **file_list;
123  time_t file_time;
124  char ts1[32];
125  int fi;
126 
127  /* Initialize variables */
128 
129  if (raw->file_type == SASHEVIS) {
130  nfiles = data->vis_resp_dirlist->nfiles;
131  file_list = data->vis_resp_dirlist->file_list;
132  }
133  else { /* SASHENIR */
134  nfiles = data->nir_resp_dirlist->nfiles;
135  file_list = data->nir_resp_dirlist->file_list;
136  }
137 
138  /* Files look like: pvcsaszenirM1.20120720.900ms.dat */
139 
140 // sprintf(int_time_string, ".%dms.dat", (int)int_time);
141 
142  for (fi = nfiles - 1; fi > -1; --fi) {
143 // if (strstr(file_list[fi], int_time_string)) {
144  file_time = sas_conf_file_name_time(file_list[fi]);
145  if (data_time >= file_time) {
146  break;
147  }
148 // }
149  }
150 
151  if (fi < 0) {
152 /*
153  DSPROC_ERROR( "Could Not Find Required Configuration File",
154  "Could not find required configuration file for:\n"
155  " -> data time: %s\n"
156  " -> integration time: %dms\n",
157  format_secs1970(data_time, ts1), (int)int_time);
158 */
159  DSPROC_ERROR( "Could Not Find Required Configuration File",
160  "Could not find required configuration file for: %s\n",
161  format_secs1970(data_time, ts1));
162 
163  return(0);
164  }
165 
166  /* Load the respose function config file */
167 
168  raw->resp_table = sas_load_config_table(
169  data->conf_dir, file_list[fi], 8, raw->resp_table);
170 
171  if (!raw->resp_table) {
172  return(0);
173  }
174 
175  /* Get response function values for each wavelength */
176 
177  sas_get_config_values_double(raw->resp_table,
178  1, (size_t)raw->nwavelens, raw->wavelens,
179  2, raw->resp_vals);
180 
181  if (debug) {
182  DSPROC_DEBUG_LV1(
183  "Using response function file: %s\n", file_list[fi]);
184  }
185 
186  return(1);
187 }
188 
189 /**
190  * Calibrate the raw data.
191  *
192  * @param data pointer to the SaszeData structure
193  * @param raw pointer to the SasRawData structure
194  *
195  * @retval 1 if successful
196  * @retval 0 if no dark counts were found
197  * @retval -1 if an error occurred
198  */
199 int sasze_calibrate_data(SaszeData *data, SasRawData *raw)
200 {
201  double *resp_vals = raw->resp_vals;
202  double *dark_avg = raw->dark_avg;
203  double *dark_sum2 = raw->dark_sum2;
204  double *dark_std = raw->dark_std;
205  int *ndarks = raw->ndarks;
206  float int_time;
207  double dark;
208  int max_ndarks;
209  float *spec;
210  int ci, ri, wi;
211 
212  /* Allocate memory for the cal_times array if necessary */
213 
214  if (raw->nrecs > raw->cal_max_ntimes) {
215 
216  raw->cal_times = realloc(raw->cal_times, raw->nrecs * sizeof(time_t));
217 
218  if (!raw->cal_times) {
219  DSPROC_ERROR( DSPROC_ENOMEM,
220  "Could not allocate memory for calibrated times array.\n");
221  return(-1);
222  }
223 
224  raw->cal_max_ntimes = raw->nrecs;
225  }
226 
227  /* Load the response functions */
228 
229  if (!sasze_load_resp_table(data, raw)) {
230  return(-1);
231  }
232 
233  /* Compute averages of dark counts */
234 
235  memset(dark_avg, 0, raw->nwavelens * sizeof(double));
236  memset(dark_sum2, 0, raw->nwavelens * sizeof(double));
237  memset(dark_std, 0, raw->nwavelens * sizeof(double));
238  memset(ndarks, 0, raw->nwavelens * sizeof(int));
239 
240  max_ndarks = 0;
241 
242  for (ri = 0; ri < raw->nrecs; ++ri) {
243 
244  if (raw->records[ri].shutter_open_tf == 0) {
245 
246  spec = raw->records[ri].spec;
247  int_time = raw->records[ri].t_int;
248 
249  for (wi = 0; wi < raw->nwavelens; ++wi) {
250 
251  if (spec[wi] > -9998.0) {
252  dark = (double)spec[wi] / (double)int_time;
253  dark_avg[wi] += dark;
254  dark_sum2[wi] += dark * dark;
255  ndarks[wi] += 1;
256  }
257  }
258 
259  max_ndarks += 1;
260  }
261  }
262 
263  if (!max_ndarks) {
264  return(0);
265  }
266 
267  for (wi = 0; wi < raw->nwavelens; ++wi) {
268 
269  if (ndarks[wi] == 0) {
270  dark_avg[wi] = -9999.0;
271  dark_std[wi] = -9999.0;
272  }
273  else {
274  dark_avg[wi] /= ndarks[wi];
275  dark_std[wi] = (dark_sum2[wi] / ndarks[wi])
276  - (dark_avg[wi] * dark_avg[wi]);
277  }
278  }
279 
280  /* Subtract darks and divide by the response function */
281 
282  for (ri = 0, ci = 0; ri < raw->nrecs; ++ri) {
283 
284  spec = raw->records[ri].spec;
285  int_time = raw->records[ri].t_int;
286 
287  if (raw->records[ri].shutter_open_tf) {
288 
289  raw->cal_times[ci++] = raw->records[ri].time;
290 
291  for (wi = 0; wi < raw->nwavelens; ++wi) {
292 
293  if ((resp_vals[wi] <= 0.0) ||
294  (dark_avg[wi] < -9998.0) ||
295  (spec[wi] < -9998.0)) {
296 
297  spec[wi] = -9999.0;
298  }
299  else {
300  spec[wi] /= int_time;
301  spec[wi] -= (float)dark_avg[wi];
302  spec[wi] /= (float)resp_vals[wi];
303  }
304  }
305  }
306  }
307 
308  raw->cal_ntimes = ci;
309 
310  return(1);
311 }
312 
313 /**
314  * Set the runtime attributes for a calibrated dataset.
315  *
316  * @param raw pointer to the SasRawData structure
317  * @param dataset output dataset
318  *
319  * @retval 1 if successful
320  * @retval 0 if an error occurred
321  */
322 int sasze_set_calibrated_metadata(
323  SasRawData *raw,
324  CDSGroup *dataset)
325 {
326  CDSVar *var;
327  const char *chrp;
328  char serial_number[64];
329 
330  /* Set the wavelength dimension length */
331 
332  if (!dsproc_set_dim_length(dataset, "wavelength", raw->nwavelens)) {
333  return(0);
334  }
335 
336  /* Set global attributes */
337 
338  SET_DS_ATT_IF_FOUND(dataset, 0,
339  "model_number", "%s", raw->model_number);
340 
341  SET_DS_ATT_IF_FOUND(dataset, 0,
342  "serial_number_spectrometer", "%s", raw->serial_number);
343 
344  SET_DS_ATT_IF_FOUND(dataset, 0,
345  "serial_number_collector", "%s", raw->collector_sn);
346 
347  SET_DS_ATT_IF_FOUND(dataset, 0,
348  "operation_mode", "%s", raw->op_mode);
349 
350  /* Set the default attribute values that have not already been
351  * defined in the dataset. */
352 
353  if (!sas_set_default_att_values(dataset)) {
354  return(0);
355  }
356 
357  /* Set the responsivity variable attributes */
358 
359  // % Spectrometer_SN: 0911146U1
360 
361  strcpy(serial_number, "Unknown");
362  chrp = strstr(raw->resp_table->header, "Spectrometer_SN");
363  if (chrp) {
364  chrp = strchr(chrp, ' ');
365  if (chrp) {
366  sscanf(chrp, " %s", serial_number);
367  }
368  }
369 
370  GET_REQUIRED_DS_VAR(dataset, "responsivity", var, 0);
371 
372  SET_DS_ATT_IF_FOUND(var, 0,
373  "spectrometer_serial_number", "%s", serial_number);
374 
375  SET_DS_ATT_IF_FOUND(var, 0,
376  "responsivity_file", "%s", raw->resp_table->conf_file);
377 
378  /* Set the static variable data */
379 
380  SET_DS_VAR_DATA(dataset, "wavelength",
381  CDS_DOUBLE, raw->nwavelens, raw->wavelens, var, 0);
382 
383  SET_DS_VAR_DATA(dataset, "responsivity",
384  CDS_DOUBLE, raw->nwavelens, raw->resp_vals, var, 0);
385 
386 
387  return(1);
388 }
389 
390 /**
391  * Store calibrated SASZE data.
392  *
393  * @param raw pointer to the SasRawData structure
394  * @param dsid output datastream ID
395  *
396  * @retval nstored number of samples stored
397  * @retval -1 if an error occurred
398  */
399 int sasze_store_calibrated_data(SasRawData *raw, int dsid)
400 {
401  SasRecord *record;
402  CDSGroup *dataset;
403  size_t nwavelens;
404  float *specra;
405  int nstored;
406  int vi, ri, si;
407 
408  int nvars;
409  CDSVar **vars;
410  const char *var_names[] = {
411  "mio_temperature_mems",
412  "mio_temperature_mems_fahr",
413  "mio_temperature_trh",
414  "mio_rh",
415  "collector_temperature",
416  "collector_rh",
417  "collector_dewpoint",
418  "chiller_temperature",
419  "chiller_rh",
420  "chiller_dewpoint",
421  "collector_x_tilt",
422  "collector_y_tilt",
423  "collector_x_tilt_std",
424  "collector_y_tilt_std",
425  "band_azimuth",
426  "solar_azimuth",
427  "solar_zenith",
428  "inner_band_angle",
429  "inner_band_scattering_angle",
430  "clock_ticks",
431  "bench_temperature",
432  "ad_temperature",
433  "integration_time",
434  "number_of_scans",
435  "zenith_radiance",
436  NULL
437  };
438 
439  if (!raw->cal_ntimes) return(0);
440 
441  /* Create the dataset */
442 
443  record = &(raw->records[0]);
444 
445  dataset = dsproc_create_output_dataset(dsid, record->time, 1);
446  if (!dataset) {
447  return(-1);
448  }
449 
450  /* Set the metadata in the output dataset */
451 
452  if (!sasze_set_calibrated_metadata(raw, dataset)) {
453  return(-1);
454  }
455 
456  /* Set sample times */
457 
458  if (!dsproc_set_sample_times(dataset, 0, raw->cal_ntimes, raw->cal_times)) {
459  return(-1);
460  }
461 
462  /* Get the list of variable pointers in the order we want them. */
463 
464  nvars = dsproc_get_dataset_vars(dataset, var_names, 1, &vars, NULL, NULL);
465  if (nvars < 0) {
466  return(-1);
467  }
468 
469  /* Allocate memory for all variable data */
470 
471  for (vi = 0; vi < nvars; vi++) {
472  if (vars[vi]) {
473  if (!dsproc_alloc_var_data(vars[vi], 0, raw->cal_ntimes)) {
474  return(-1);
475  }
476  }
477  }
478 
479  /* Set the time varying data in the dataset */
480 
481  /* Copy the data from the SasRawData records to the
482  * variables in the output dataset */
483 
484  nwavelens = raw->nwavelens;
485  specra = vars[24]->data.fp;
486 
487  for (ri = 0, si = 0; ri < raw->nrecs; ++ri) {
488 
489  record = &(raw->records[ri]);
490 
491  if (record->shutter_open_tf) {
492 
493  vars[0]->data.fp[si] = record->temp_mio_p;
494  vars[1]->data.fp[si] = record->temp_mio_p_f;
495  vars[2]->data.fp[si] = record->temp_mio_trh;
496  vars[3]->data.fp[si] = record->rh_mio;
497  vars[4]->data.fp[si] = record->temp_collector;
498  vars[5]->data.fp[si] = record->rh_collector;
499  vars[6]->data.fp[si] = record->t_dewpt_collector;
500  vars[7]->data.fp[si] = record->temp_chiller;
501  vars[8]->data.fp[si] = record->rh_chiller;
502  vars[9]->data.fp[si] = record->t_dewpt_chiller;
503  vars[10]->data.fp[si] = record->x_tilt;
504  vars[11]->data.fp[si] = record->y_tilt;
505  vars[12]->data.fp[si] = record->x_tilt_stddev;
506  vars[13]->data.fp[si] = record->y_tilt_stddev;
507  vars[14]->data.fp[si] = record->band_az_n;
508  vars[15]->data.fp[si] = record->solar_azimuth;
509  vars[16]->data.fp[si] = record->solar_zenith;
510  vars[17]->data.fp[si] = record->inner_band_angle;
511  vars[18]->data.fp[si] = record->inner_band_scat_ang;
512  vars[19]->data.fp[si] = record->spectrometer_clock_ticks;
513  vars[20]->data.fp[si] = record->t_avantes_bench;
514  vars[21]->data.fp[si] = record->t_avantes_ad;
515  vars[22]->data.fp[si] = record->t_int;
516  vars[23]->data.ip[si] = record->n_avg;
517 
518  memcpy(specra, record->spec, nwavelens * sizeof(float));
519  specra += nwavelens;
520 
521  ++si;
522  }
523  }
524 
525  nstored = dsproc_store_dataset(dsid, 0);
526  return(nstored);
527 }
528 
529 /**
530  * Set the runtime attributes for a filterband dataset.
531  *
532  * @param data pointer to the SaszeData structure
533  * @param nir_raw pointer to the NIR SasRawData structure
534  * @param vis_raw pointer to the VIS SasRawData structure
535  * @param dataset output dataset
536  *
537  * @retval 1 if successful
538  * @retval 0 if an error occurred
539  */
540 int sasze_set_filterband_metadata(
541  SaszeData *data,
542  SasRawData *nir_raw,
543  SasRawData *vis_raw,
544  CDSGroup *dataset)
545 {
546  int ntoa = data->toaIo_table->nrows;
547  double *toa_wavelens = data->toaIo_table->cols[0];
548  double *toa_Ios = data->toaIo_table->cols[1];
549  CDSVar *var;
550  float sun_distance;
551  char *value;
552 
553  /* Set the Gueymard TOA values */
554 
555  if (!dsproc_set_dim_length(dataset, "wavelength", ntoa)) {
556  return(0);
557  }
558 
559  SET_DS_VAR_DATA(dataset, "wavelength",
560  CDS_DOUBLE, ntoa, toa_wavelens, var, 0);
561 
562  SET_DS_VAR_DATA(dataset, "solar_spectrum",
563  CDS_DOUBLE, ntoa, toa_Ios, var, 0);
564 
565  /* Set the Earth-Sun distance. These do not vary enough over the course
566  * of a day so we only need to report the first one. */
567 
568  if (vis_raw) {
569  sun_distance = vis_raw->records[0].earth_sun_dist;
570  }
571  else {
572  sun_distance = nir_raw->records[0].earth_sun_dist;
573  }
574 
575  SET_DS_VAR_DATA(dataset, "earth_sun_distance",
576  CDS_FLOAT, 1, &sun_distance, var, 0);
577 
578  /* Set global attributes */
579 
580  value = (vis_raw) ? vis_raw->model_number : nir_raw->model_number;
581  SET_DS_ATT_IF_FOUND(dataset, 0,
582  "model_number", "%s", value);
583 
584  value = (vis_raw) ? vis_raw->collector_sn : nir_raw->collector_sn;
585  SET_DS_ATT_IF_FOUND(dataset, 0,
586  "serial_number_collector", "%s", value);
587 
588  value = (nir_raw) ? nir_raw->serial_number : "-9999";
589  SET_DS_ATT_IF_FOUND(dataset, 0,
590  "serial_number_spectrometer_nir", "%s", value);
591 
592  value = (vis_raw) ? vis_raw->serial_number : "-9999";
593  SET_DS_ATT_IF_FOUND(dataset, 0,
594  "serial_number_spectrometer_vis", "%s", value);
595 
596  /* Set the default attribute values that have not already been
597  * defined in the dataset. */
598 
599  if (!sas_set_default_att_values(dataset)) {
600  return(0);
601  }
602 
603  return(1);
604 }
605 
606 /**
607  * Store data for a SASZE filterband datastream.
608  *
609  * @param data pointer to the SaszeData structure
610  * @param nir_raw pointer to the SasRawData structure
611  * @param vis_raw pointer to the SasRawData structure
612  * @param dsid output datastream ID
613  *
614  * @retval nstored number of samples stored
615  * @retval -1 if an error occurred
616  */
617 int sasze_store_filterband_data(
618  SaszeData *data,
619  SasRawData *nir_raw,
620  SasRawData *vis_raw,
621  int dsid)
622 {
623  int max_ntimes;
624  int ntimes;
625  time_t *times;
626  SasRecord **nir_records;
627  SasRecord **vis_records;
628  SasRecord *record;
629  int nir_done, vis_done;
630  time_t nir_time, vis_time;
631  int nri, vri, ti, vi;
632 
633  SasRawData *raw;
634  CDSVar *var;
635  int nwavelens;
636  int *wavelens;
637  int wavelen;
638  int vis_rad_wis[gNumVisWavelens];
639  int nir_rad_wis[gNumNirWavelens];
640  int *rad_wis;
641  float *vis_rad_datap[gNumVisWavelens];
642  float *nir_rad_datap[gNumNirWavelens];
643  float **rad_datap;
644  float *vis_trans_datap[gNumVisWavelens];
645  float *nir_trans_datap[gNumNirWavelens];
646  float **trans_datap;
647  float rad;
648  float sdist2;
649  float cossza;
650  float toaIo;
651  int i, gwi, wi;
652 
653  CDSGroup *dataset;
654  char var_name[64];
655  int nstored;
656  int nvars;
657  CDSVar **vars;
658  const char *var_names[] = {
659  "mio_temperature_mems",
660  "mio_temperature_mems_fahr",
661  "mio_temperature_trh",
662  "mio_rh",
663  "collector_temperature",
664  "collector_rh",
665  "collector_dewpoint",
666  "chiller_temperature",
667  "chiller_rh",
668  "chiller_dewpoint",
669  "collector_x_tilt",
670  "collector_y_tilt",
671  "collector_x_tilt_std",
672  "collector_y_tilt_std",
673  "solar_azimuth",
674  "solar_zenith",
675 
676  "bench_temperature_nir",
677  "ad_temperature_nir",
678  "clock_ticks_nir",
679  "integration_time_nir",
680  "number_of_scans_nir",
681 
682  "bench_temperature_vis",
683  "ad_temperature_vis",
684  "clock_ticks_vis",
685  "integration_time_vis",
686  "number_of_scans_vis",
687 
688  NULL
689  };
690 
691  /* Allocate memory for the filterband time and record arrays */
692 
693  max_ntimes = (nir_raw) ? nir_raw->nrecs : 0;
694  max_ntimes += (vis_raw) ? vis_raw->nrecs : 0;
695  if (max_ntimes == 0) return(0);
696 
697  if (max_ntimes > data->fb_max_ntimes) {
698 
699  data->fb_times = realloc(
700  data->fb_times, max_ntimes * sizeof(time_t));
701 
702  data->fb_nir_records = realloc(
703  data->fb_nir_records, max_ntimes * sizeof(SasRecord *));
704 
705  data->fb_vis_records = realloc(
706  data->fb_vis_records, max_ntimes * sizeof(SasRecord *));
707 
708  if (!data->fb_times ||
709  !data->fb_nir_records ||
710  !data->fb_nir_records) {
711 
712  DSPROC_ERROR( DSPROC_ENOMEM,
713  "Could not allocate memory for filterband times array.\n");
714 
715  return(-1);
716  }
717 
718  data->fb_max_ntimes = max_ntimes;
719  }
720 
721  times = data->fb_times;
722  nir_records = data->fb_nir_records;
723  vis_records = data->fb_vis_records;
724 
725  /* Create the arrays of merged sample times and records */
726 
727  nir_done = (nir_raw) ? 0 : 1;
728  vis_done = (vis_raw) ? 0 : 1;
729  nri = vri = ti = 0;
730 
731  while (!vis_done || !nir_done) {
732 
733  /* Skip past NIR dark count records */
734 
735  while (!nir_done) {
736  if (nir_raw->records[nri].shutter_open_tf) break;
737  if (++nri == nir_raw->nrecs) nir_done = 1;
738  }
739 
740  /* Skip past VIS dark count records */
741 
742  while (!vis_done) {
743  if (vis_raw->records[vri].shutter_open_tf) break;
744  if (++vri == vis_raw->nrecs) vis_done = 1;
745  }
746 
747  /* Set time and record pointers for this dataset record */
748 
749  nir_records[ti] = (SasRecord *)NULL;
750  vis_records[ti] = (SasRecord *)NULL;
751 
752  if (!nir_done && !vis_done) {
753 
754  nir_time = nir_raw->records[nri].time;
755  vis_time = vis_raw->records[vri].time;
756 
757  if (nir_time < vis_time) {
758  times[ti] = nir_time;
759  nir_records[ti] = &(nir_raw->records[nri++]);
760  }
761  else if (vis_time < nir_time) {
762  times[ti] = vis_time;
763  vis_records[ti] = &(vis_raw->records[vri++]);
764  }
765  else { /* vis_time == nir_time */
766  times[ti] = nir_time;
767  nir_records[ti] = &(nir_raw->records[nri++]);
768  vis_records[ti] = &(vis_raw->records[vri++]);
769  }
770  ++ti;
771  }
772  else if (!nir_done) {
773  times[ti] = nir_raw->records[nri].time;
774  nir_records[ti] = &(nir_raw->records[nri++]);
775  ++ti;
776  }
777  else if (!vis_done) {
778  times[ti] = vis_raw->records[vri].time;
779  vis_records[ti] = &(vis_raw->records[vri++]);
780  ++ti;
781  }
782 
783 /*
784 printf("%d ", (int)times[ti]);
785 if (vis_records[ti]) printf("X ");
786 else printf(" ");
787 if (nir_records[ti]) printf("X");
788 else printf(" ");
789 printf("\n");
790 */
791 
792  /* Check if we are at the end of the nir and/or vis records */
793 
794  if (!nir_done && nri == nir_raw->nrecs) nir_done = 1;
795  if (!vis_done && vri == vis_raw->nrecs) vis_done = 1;
796  }
797 
798  ntimes = ti;
799 
800  if (ntimes == 0) return(0);
801 
802  /* Create the dataset */
803 
804  dataset = dsproc_create_output_dataset(dsid, times[0], 1);
805  if (!dataset) return(-1);
806 
807  /* Set the metadata in the output dataset */
808 
809  if (!sasze_set_filterband_metadata(data, nir_raw, vis_raw, dataset)) {
810  return(-1);
811  }
812 
813  /* Set sample times */
814 
815  if (!dsproc_set_sample_times(dataset, 0, ntimes, times)) {
816  return(-1);
817  }
818 
819  /* Get the list of variable pointers in the order we want them. */
820 
821  nvars = dsproc_get_dataset_vars(dataset, var_names, 1, &vars, NULL, NULL);
822  if (nvars < 0) return(-1);
823 
824  /* Allocate memory for all variable data */
825 
826  for (vi = 0; vi < nvars; vi++) {
827  if (vars[vi]) {
828  if (!dsproc_alloc_var_data(vars[vi], 0, ntimes)) {
829  return(-1);
830  }
831  }
832  }
833 
834  /* Initialize memory and variables for the
835  * radiance and transmittance variables */
836 
837  for (i = 0; i < 2; ++i) {
838 
839  if (i == 0) {
840  /* Initialize VIS radiance variables */
841  raw = vis_raw;
842  nwavelens = gNumVisWavelens;
843  wavelens = gVisWavelens;
844  rad_wis = vis_rad_wis;
845  rad_datap = vis_rad_datap;
846  trans_datap = vis_trans_datap;
847  }
848  else {
849  /* Initialize NIR radiance variables */
850  raw = nir_raw;
851  nwavelens = gNumNirWavelens;
852  wavelens = gNirWavelens;
853  rad_datap = nir_rad_datap;
854  rad_wis = nir_rad_wis;
855  trans_datap = nir_trans_datap;
856  }
857 
858  /* Get Gueymard TOA Io values for each wavelength */
859 
860  if (raw) {
861  sas_get_config_values_double(data->toaIo_table,
862  0, (size_t)raw->nwavelens, raw->wavelens,
863  1, raw->toaIos);
864  }
865 
866  /* Loop over the desired radiance wavelengths */
867 
868  for (gwi = 0; gwi < nwavelens; ++gwi) {
869 
870  /* Get the wavelength index */
871 
872  wavelen = wavelens[gwi];
873 
874  if (raw) {
875  wi = sas_find_nearest_double(
876  raw->nwavelens, raw->wavelens, (double)wavelen);
877  }
878  else {
879  wi = -9999;
880  }
881 
882  rad_wis[gwi] = wi;
883 
884  /* Allocate memory for the radiance variable
885  * and get the pointer to its data array. */
886 
887  sprintf(var_name, "zenith_radiance_%dnm", wavelen);
888 
889  ALLOC_DS_VAR(dataset, var_name, rad_datap[gwi], ntimes, var, -1);
890 
891  SET_DS_ATT_IF_FOUND(var, -1, "spectrometer_pixel", "%d", wi);
892 
893  /* Allocate memory for the transmittance variable
894  * and get the pointer to its data array. */
895 
896  sprintf(var_name, "zenith_transmittance_%dnm", wavelen);
897 
898  ALLOC_DS_VAR(dataset, var_name, trans_datap[gwi], ntimes, var, -1);
899 
900  SET_DS_ATT_IF_FOUND(var, -1, "spectrometer_pixel", "%d", wi);
901  }
902  }
903 
904  /* Store the data in the dataset */
905 
906  for (ti = 0; ti < ntimes; ++ti) {
907 
908  record = (vis_records[ti]) ? vis_records[ti] : nir_records[ti];
909 
910  vars[0]->data.fp[ti] = record->temp_mio_p;
911  vars[1]->data.fp[ti] = record->temp_mio_p_f;
912  vars[2]->data.fp[ti] = record->temp_mio_trh;
913  vars[3]->data.fp[ti] = record->rh_mio;
914  vars[4]->data.fp[ti] = record->temp_collector;
915  vars[5]->data.fp[ti] = record->rh_collector;
916  vars[6]->data.fp[ti] = record->t_dewpt_collector;
917  vars[7]->data.fp[ti] = record->temp_chiller;
918  vars[8]->data.fp[ti] = record->rh_chiller;
919  vars[9]->data.fp[ti] = record->t_dewpt_chiller;
920  vars[10]->data.fp[ti] = record->x_tilt;
921  vars[11]->data.fp[ti] = record->y_tilt;
922  vars[12]->data.fp[ti] = record->x_tilt_stddev;
923  vars[13]->data.fp[ti] = record->y_tilt_stddev;
924  vars[14]->data.fp[ti] = record->solar_azimuth;
925  vars[15]->data.fp[ti] = record->solar_zenith;
926 
927  /* Set NIR variable data */
928 
929  if (nir_records[ti]) {
930  record = nir_records[ti];
931  vars[16]->data.fp[ti] = record->t_avantes_bench;
932  vars[17]->data.fp[ti] = record->t_avantes_ad;
933  vars[18]->data.fp[ti] = record->spectrometer_clock_ticks;
934  vars[19]->data.fp[ti] = record->t_int;
935  vars[20]->data.ip[ti] = record->n_avg;
936 
937  /* Set the NIR radiance and transmittance variable data */
938 
939  sdist2 = record->earth_sun_dist * record->earth_sun_dist;
940  cossza = cosf(DEG_TO_RAD(record->solar_zenith));
941 
942  for (gwi = 0; gwi < gNumNirWavelens; ++gwi) {
943  wi = nir_rad_wis[gwi];
944  if (wi == -9999 ||
945  record->spec[wi] == -9999.0 ||
946  record->solar_zenith > 89.0) {
947 
948  nir_rad_datap[gwi][ti] = MISSING;
949  nir_trans_datap[gwi][ti] = MISSING;
950  }
951  else {
952  rad = record->spec[wi];
953  toaIo = nir_raw->toaIos[wi];
954 
955  nir_rad_datap[gwi][ti] = rad;
956  nir_trans_datap[gwi][ti] = (PI * rad * sdist2)
957  / (1000 * cossza * toaIo);
958  }
959  }
960  }
961  else {
962  vars[16]->data.fp[ti] = MISSING;
963  vars[17]->data.fp[ti] = MISSING;
964  vars[18]->data.fp[ti] = MISSING;
965  vars[19]->data.fp[ti] = MISSING;
966  vars[20]->data.ip[ti] = MISSING;
967  for (gwi = 0; gwi < gNumNirWavelens; ++gwi) {
968  nir_rad_datap[gwi][ti] = MISSING;
969  nir_trans_datap[gwi][ti] = MISSING;
970  }
971  }
972 
973  /* Set VIS variable data */
974 
975  if (vis_records[ti]) {
976  record = vis_records[ti];
977  vars[21]->data.fp[ti] = record->t_avantes_bench;
978  vars[22]->data.fp[ti] = record->t_avantes_ad;
979  vars[23]->data.fp[ti] = record->spectrometer_clock_ticks;
980  vars[24]->data.fp[ti] = record->t_int;
981  vars[25]->data.ip[ti] = record->n_avg;
982 
983  /* Set the NIR radiance and transmittance variable data */
984 
985  sdist2 = record->earth_sun_dist * record->earth_sun_dist;
986  cossza = cosf(DEG_TO_RAD(record->solar_zenith));
987 
988  for (gwi = 0; gwi < gNumVisWavelens; ++gwi) {
989  wi = vis_rad_wis[gwi];
990 
991  if (wi == -9999 ||
992  record->spec[wi] == -9999.0 ||
993  record->solar_zenith > 89.0) {
994 
995  vis_rad_datap[gwi][ti] = MISSING;
996  vis_trans_datap[gwi][ti] = MISSING;
997  }
998  else {
999 
1000  rad = record->spec[wi];
1001  toaIo = vis_raw->toaIos[wi];
1002 
1003  vis_rad_datap[gwi][ti] = rad;
1004  vis_trans_datap[gwi][ti] = (PI * rad * sdist2)
1005  / (1000 * cossza * toaIo);
1006  }
1007  }
1008  }
1009  else {
1010  vars[21]->data.fp[ti] = MISSING;
1011  vars[22]->data.fp[ti] = MISSING;
1012  vars[23]->data.fp[ti] = MISSING;
1013  vars[24]->data.fp[ti] = MISSING;
1014  vars[25]->data.ip[ti] = MISSING;
1015  for (gwi = 0; gwi < gNumVisWavelens; ++gwi) {
1016  vis_rad_datap[gwi][ti] = MISSING;
1017  vis_trans_datap[gwi][ti] = MISSING;
1018  }
1019  }
1020  }
1021 
1022  nstored = dsproc_store_dataset(dsid, 0);
1023  return(nstored);
1024 }
1025 
1026 /**
1027  * Store data for the SASZE output datastreams.
1028  *
1029  * @param data pointer to the SaszeData structure
1030  * @param nir_raw pointer to the NIR SasRawData structure
1031  * @param vis_raw pointer to the VIS SasRawData structure
1032  *
1033  * @retval 1 if successful
1034  * @retval -1 if an error occurred
1035  */
1036 int sasze_store_data(SaszeData *data, SasRawData *nir_raw, SasRawData *vis_raw)
1037 {
1038  int nir_dsid = data->nir_a1_dsid;
1039  int vis_dsid = data->vis_a1_dsid;
1040  int fb_dsid = data->fb_dsid;
1041  int status;
1042 
1043  if (nir_raw) {
1044 
1045  status = sasze_calibrate_data(data, nir_raw);
1046  if (status < 0) return(-1);
1047 
1048  if (status == 0) {
1049 
1050  DSPROC_WARNING(
1051  "No dark counts found in file: %s\n",
1052  nir_raw->file_name);
1053 
1054  nir_raw = (SasRawData *)NULL;
1055  }
1056  else if (nir_raw->cal_ntimes == 0) {
1057 
1058  DSPROC_WARNING(
1059  "No data found to calibrate in file: %s\n",
1060  nir_raw->file_name);
1061 
1062  nir_raw = (SasRawData *)NULL;
1063  }
1064  else {
1065  status = sasze_store_calibrated_data(nir_raw, nir_dsid);
1066  if (status < 0) return(-1);
1067  }
1068  }
1069 
1070  if (vis_raw) {
1071 
1072  status = sasze_calibrate_data(data, vis_raw);
1073  if (status < 0) return(-1);
1074 
1075  if (status == 0) {
1076 
1077  DSPROC_WARNING(
1078  "No dark counts found in file: %s\n",
1079  vis_raw->file_name);
1080 
1081  vis_raw = (SasRawData *)NULL;
1082  }
1083  else if (vis_raw->cal_ntimes == 0) {
1084 
1085  DSPROC_WARNING(
1086  "No data found to calibrate in file: %s\n",
1087  vis_raw->file_name);
1088 
1089  vis_raw = (SasRawData *)NULL;
1090  }
1091  else {
1092  status = sasze_store_calibrated_data(vis_raw, vis_dsid);
1093  if (status < 0) return(-1);
1094  }
1095  }
1096 
1097  if (nir_raw || vis_raw) {
1098  status = sasze_store_filterband_data(data, nir_raw, vis_raw, fb_dsid);
1099  if (status < 0) return(-1);
1100  }
1101 
1102  return(1);
1103 }