libdsproc3  2.0
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Groups
dsproc3_internal.h
Go to the documentation of this file.
1 /*******************************************************************************
2 *
3 * COPYRIGHT (C) 2010 Battelle Memorial Institute. All Rights Reserved.
4 *
5 ********************************************************************************
6 *
7 * Author:
8 * name: Brian Ermold
9 * phone: (509) 375-2277
10 * email: brian.ermold@pnl.gov
11 *
12 ********************************************************************************
13 *
14 * REPOSITORY INFORMATION:
15 * $Revision: 80311 $
16 * $Author: ermold $
17 * $Date: 2017-08-31 22:40:18 +0000 (Thu, 31 Aug 2017) $
18 *
19 ********************************************************************************
20 *
21 * NOTE: DOXYGEN is used to generate documentation for this file.
22 *
23 *******************************************************************************/
24 
25 /** @file dsproc3_internal.h
26  * Internal libdsproc3 functions and structures
27  */
28 
29 #ifndef _DSPROC3_INTERNAL_H
30 #define _DSPROC3_INTERNAL_H
31 
32 #include "dsdb3.h"
33 #include "ncds3.h"
34 
35 /** DSPROC library name. */
36 #define DSPROC_LIB_NAME "libdsproc3"
37 
38 /******************************************************************************/
39 /**
40  * @defgroup INTERNAL_DSPROC_MAIN Internal: Process Framework
41  */
42 /*@{*/
43 
44 void dsproc_initialize(
45  int argc,
46  char **argv,
47  ProcModel proc_model,
48  const char *proc_version,
49  int nproc_names,
50  const char **proc_names);
51 
52 int dsproc_start_processing_loop(
53  time_t *interval_begin,
54  time_t *interval_end);
55 
56 int dsproc_retrieve_data(
57  time_t begin_time,
58  time_t end_time,
59  CDSGroup **ret_data);
60 
61 int dsproc_merge_retrieved_data(void);
62 
63 int dsproc_transform_data(
64  CDSGroup **trans_data);
65 
66 int dsproc_create_output_datasets(void);
67 
68 int dsproc_store_output_datasets(void);
69 
70 int dsproc_finish(void);
71 
72 int dsproc_force_rename_bad(
73  const char *file_path,
74  const char *file_name);
75 
76 int dsproc_is_fatal(int last_errno);
77 
78 /*@}*/
79 
80 /******************************************************************************/
81 /**
82  * @defgroup INTERNAL_DSPROC_MESSAGES Internal: Process Messages
83  */
84 /*@{*/
85 
86 void dsproc_bad_file_warning(
87  const char *func,
88  const char *src_file,
89  int src_line,
90  const char *file_name,
91  const char *format, ...);
92 
93 void dsproc_bad_line_warning(
94  const char *func,
95  const char *src_file,
96  int src_line,
97  const char *file_name,
98  int line_num,
99  const char *format, ...);
100 
101 void dsproc_bad_record_warning(
102  const char *func,
103  const char *src_file,
104  int src_line,
105  const char *file_name,
106  int rec_num,
107  const char *format, ...);
108 
109 void dsproc_debug(
110  const char *func,
111  const char *file,
112  int line,
113  int level,
114  const char *format, ...);
115 
116 void dsproc_error(
117  const char *func,
118  const char *file,
119  int line,
120  const char *status,
121  const char *format, ...);
122 
123 void dsproc_log(
124  const char *func,
125  const char *file,
126  int line,
127  const char *format, ...);
128 
129 void dsproc_mentor_mail(
130  const char *func,
131  const char *file,
132  int line,
133  const char *format, ...);
134 
135 void dsproc_warning(
136  const char *func,
137  const char *file,
138  int line,
139  const char *format, ...);
140 
141 void dsproc_reset_warning_count(void);
142 void dsproc_set_max_warnings(int max_warnings);
143 
144 /*@}*/
145 
146 /******************************************************************************/
147 /**
148  * @defgroup INTERNAL_DSPROC_CONTROL Internal: Process Control
149  */
150 /*@{*/
151 
152 void dsproc_abort(
153  const char *func,
154  const char *file,
155  int line,
156  const char *status,
157  const char *format, ...);
158 
159 void dsproc_enable_asynchronous_mode(void);
160 
161 void dsproc_disable(const char *message);
162 void dsproc_disable_db_updates(void);
163 void dsproc_disable_dynamic_metric_vars(int flag);
164 void dsproc_disable_lock_file(void);
165 void dsproc_disable_mail_messages(void);
166 
167 int dsproc_get_asynchrounous_mode(void);
168 int dsproc_get_dynamic_dods_mode(void);
169 int dsproc_get_force_mode(void);
170 int dsproc_get_real_time_mode(void);
171 int dsproc_get_reprocessing_mode(void);
172 
173 void dsproc_set_dynamic_dods_mode(int mode);
174 void dsproc_set_force_mode(int mode);
175 int dsproc_set_log_dir(const char *log_dir);
176 int dsproc_set_log_file(const char *log_file);
177 int dsproc_set_log_id(const char *log_id);
178 void dsproc_set_max_runtime(int max_runtime);
179 void dsproc_set_processing_interval(time_t begin_time, time_t end_time);
180 void dsproc_set_real_time_mode(int mode, float max_wait);
181 void dsproc_set_reprocessing_mode(int mode);
182 void dsproc_set_retriever_time_offsets(
183  int ds_id,
184  time_t begin_offset,
185  time_t end_offset);
186 
187 /*@}*/
188 
189 /******************************************************************************/
190 /**
191  * @defgroup INTERNAL_DSPROC_INFO Internal: Process Information
192  */
193 /*@{*/
194 
195 const char *adi_version(void);
196 const char *dsproc_lib_version(void);
197 
198 const char *dsproc_get_input_dir(void);
199 const char *dsproc_get_input_file(void);
200 const char *dsproc_get_input_source(void);
201 
202 const char *dsproc_get_status(void);
203 const char *dsproc_get_type(void);
204 const char *dsproc_get_version(void);
205 
206 time_t dsproc_get_max_run_time(void);
207 time_t dsproc_get_start_time(void);
208 time_t dsproc_get_time_remaining(void);
209 time_t dsproc_get_min_valid_time(void);
210 time_t dsproc_get_data_interval(void);
211 time_t dsproc_get_processing_interval(time_t *begin, time_t *end);
212 
213 void dsproc_set_input_dir(const char *input_dir);
214 void dsproc_set_input_source(const char *input_file);
215 void dsproc_set_status(const char *status);
216 
217 /*@}*/
218 
219 /******************************************************************************/
220 /**
221  * @defgroup INTERNAL_DSPROC_DATASTREAM Internal: DataStreams
222  */
223 /*@{*/
224 
225 /** Check for overlap with previously processed data before storing new data. */
226 #define DS_OVERLAP_CHECK 0x001
227 
228 /** Run standard QC checks before storing new data. */
229 #define DS_STANDARD_QC 0x002
230 
231 /** Preserve distinct observations when retrieving and storing data. */
232 #define DS_PRESERVE_OBS 0x004
233 
234 /** Replace NaN and Inf values with missing values when data is stored. */
235 #define DS_FILTER_NANS 0x008
236 
237 /** Do not merge multiple observations in retrieved data. */
238 #define DS_DISABLE_MERGE 0x010
239 
240 /** Skip the transformation logic for all variables in this datastream. */
241 #define DS_SKIP_TRANSFORM 0x020
242 
243 /** Consolidate the transformation QC bits for all variables when mapped
244  * to the output dataset. */
245 #define DS_ROLLUP_TRANS_QC 0x040
246 
247 /** Enable scan mode for datastreams that are not expected to be continuous. */
248 #define DS_SCAN_MODE 0x080
249 
250 /** Loop over observations (or files) instead of time intervals. */
251 #define DS_OBS_LOOP 0x100
252 
253 /**
254  * DataStream File Formats.
255  */
256 typedef enum {
257 
258  DSF_NETCDF = 1, /**< netcdf data file format */
259  DSF_CSV = 2, /**< csv data file format */
260  DSF_RAW = 10, /**< generic raw data format */
261  DSF_PNG = 11, /**< png image format */
262  DSF_JPG = 12 /**< jpg image format */
263 
264 } DSFormat;
265 
266 int dsproc_init_datastream(
267  const char *site,
268  const char *facility,
269  const char *dsc_name,
270  const char *dsc_level,
271  DSRole role,
272  const char *path,
273  DSFormat format,
274  int flags);
275 
276 void dsproc_set_datastream_flags(int ds_id, int flags);
277 void dsproc_set_datastream_format(int ds_id, DSFormat format);
278 int dsproc_set_output_format(DSFormat format);
279 
280 void dsproc_unset_datastream_flags(int ds_id, int flags);
281 
282 void dsproc_update_datastream_data_stats(
283  int ds_id,
284  int num_records,
285  const timeval_t *begin_time,
286  const timeval_t *end_time);
287 
288 void dsproc_update_datastream_file_stats(
289  int ds_id,
290  double file_size,
291  const timeval_t *begin_time,
292  const timeval_t *end_time);
293 
294 int dsproc_validate_datastream_data_time(
295  int ds_id,
296  const timeval_t *data_time);
297 
298 /*@}*/
299 
300 /******************************************************************************/
301 /**
302  * @defgroup INTERNAL_DSPROC_DATASTREAM_FILES Internal: DataStream Files
303  */
304 /*@{*/
305 
306 void dsproc_close_untouched_files(void);
307 
308 void dsproc_set_max_open_files(int ds_id, int max_open);
309 
310 /*@}*/
311 
312 /******************************************************************************/
313 /**
314  * @defgroup INTERNAL_DSPROC_DATASET Internal: Datasets
315  */
316 /*@{*/
317 
318 /** Flag specifying that all bad and indeterminate bits in the input
319  * QC variable should be consolidated into single bad or indeterminate
320  * bits in the output QC variable.
321  */
322 #define MAP_ROLLUP_TRANS_QC 0x1
323 
324 CDSGroup *dsproc_create_output_dataset(
325  int ds_id,
326  time_t data_time,
327  int set_location);
328 
329 int dsproc_dataset_pass_through(
330  CDSGroup *in_dataset,
331  CDSGroup *out_dataset,
332  int flags);
333 
334 int dsproc_map_datasets(
335  CDSGroup *in_parent,
336  CDSGroup *out_dataset,
337  int flags);
338 
339 int dsproc_map_var(
340  CDSVar *in_var,
341  size_t in_sample_start,
342  size_t sample_count,
343  CDSVar *out_var,
344  size_t out_sample_start,
345  int flags);
346 
347 int dsproc_set_dataset_location(CDSGroup *dataset);
348 
349 void dsproc_set_map_time_range(
350  time_t begin_time,
351  time_t end_time);
352 
353 void dsproc_set_map_timeval_range(
354  timeval_t *begin_time,
355  timeval_t *end_time);
356 
357 int dsproc_store_dataset(
358  int ds_id,
359  int newfile);
360 
361 /*@}*/
362 
363 /******************************************************************************/
364 /**
365  * @defgroup INTERNAL_DATASET_COMPARE Internal: Dataset Compare
366  */
367 /*@{*/
368 
369 int dsproc_compare_dod_versions(
370  CDSGroup *prev_ds,
371  CDSGroup *curr_ds,
372  int warn);
373 
374 int dsproc_compare_dod_dims(
375  CDSGroup *prev_ds,
376  CDSGroup *curr_ds,
377  int warn);
378 
379 int dsproc_compare_dod_atts(
380  CDSGroup *prev_ds,
381  CDSGroup *curr_ds,
382  int warn);
383 
384 int dsproc_compare_dod_vars(
385  CDSGroup *prev_ds,
386  CDSGroup *curr_ds,
387  int warn);
388 
389 int dsproc_compare_dods(
390  CDSGroup *prev_ds,
391  CDSGroup *curr_ds,
392  int warn);
393 
394 int dsproc_exclude_from_dod_compare(
395  const char *var_name,
396  int exclude_data,
397  int natts,
398  const char **att_names);
399 
400 /*@}*/
401 
402 /******************************************************************************/
403 /**
404  * @defgroup INTERNAL_DATASET_FETCH Internal: Dataset Fetch
405  */
406 /*@{*/
407 
408 int dsproc_fetch_dataset(
409  int ds_id,
410  timeval_t *begin_timeval,
411  timeval_t *end_timeval,
412  size_t nvars,
413  const char **var_names,
414  int merge_obs,
415  CDSGroup **dataset);
416 
417 timeval_t *dsproc_fetch_timevals(
418  int ds_id,
419  timeval_t *begin_timeval,
420  timeval_t *end_timeval,
421  size_t *ntimevals,
422  timeval_t *timevals);
423 
424 /*@}*/
425 
426 /******************************************************************************/
427 /**
428  * @defgroup INTERNAL_DATASET_FILTERS Internal: Dataset Filters
429  */
430 /*@{*/
431 
432 void dsproc_disable_nan_filter_warnings(void);
433 
434 int dsproc_filter_var_nans(CDSVar *var);
435 int dsproc_filter_dataset_nans(CDSGroup *dataset, int warn);
436 
437 /** Flag to resest overlap filtering back to duplicate records only */
438 #define FILTER_DUPS_ONLY 0x00
439 /** Flag to filter records that are not in chronological order */
440 #define FILTER_TIME_SHIFTS 0x01
441 /** Flag to filter records that have the same times but different data values */
442 #define FILTER_OVERLAPS 0x02
443 /** Same as FILTER_TIME_SHIFTS | FILTER_OVERLAPS */
444 #define FILTER_ALL FILTER_TIME_SHIFTS | FILTER_OVERLAPS
445 
446 void dsproc_set_overlap_filtering_mode(int mode);
447 
448 /*@}*/
449 
450 /******************************************************************************/
451 /**
452  * @defgroup INTERNAL_DATASET_QC Internal: Dataset QC
453  */
454 /*@{*/
455 
456 int dsproc_exclude_from_standard_qc_checks(
457  const char *var_name);
458 
459 int dsproc_qc_delta_checks(
460  CDSVar *var,
461  CDSVar *qc_var,
462  CDSVar *prev_var,
463  CDSVar *prev_qc_var,
464  int delta_flag,
465  int bad_flags);
466 
467 int dsproc_qc_limit_checks(
468  CDSVar *var,
469  CDSVar *qc_var,
470  int missing_flag,
471  int min_flag,
472  int max_flag);
473 
474 int dsproc_qc_time_checks(
475  CDSVar *time_var,
476  CDSVar *qc_time_var,
477  timeval_t *prev_timeval,
478  int lteq_zero_flag,
479  int min_delta_flag,
480  int max_delta_flag);
481 
482 int dsproc_standard_qc_checks(
483  int ds_id,
484  CDSGroup *dataset);
485 
486 int dsproc_solar_position_qc_check(
487  CDSGroup *dataset,
488  int solar_flag);
489 
490 int dsproc_qc_solar_position(
491  CDSVar *qc_var,
492  int solar_flag);
493 
494 /*@}*/
495 
496 /******************************************************************************/
497 /**
498  * @defgroup INTERNAL_DSPROC_DATASTREAM_DOD Internal: DataStream DOD
499  */
500 /*@{*/
501 
502 int dsproc_check_for_dsdod_update(
503  int ds_id,
504  time_t data_time);
505 
506 CDSAtt *dsproc_get_dsdod_att(
507  int ds_id,
508  const char *var_name,
509  const char *att_name);
510 
511 CDSDim *dsproc_get_dsdod_dim(
512  int ds_id,
513  const char *dim_name);
514 
515 void *dsproc_get_dsdod_att_value(
516  int ds_id,
517  const char *var_name,
518  const char *att_name,
519  CDSDataType type,
520  size_t *length,
521  void *value);
522 
523 char *dsproc_get_dsdod_att_text(
524  int ds_id,
525  const char *var_name,
526  const char *att_name,
527  size_t *length,
528  char *value);
529 
530 size_t dsproc_get_dsdod_dim_length(
531  int ds_id,
532  const char *dim_name);
533 
534 const char *dsproc_get_dsdod_version(
535  int ds_id,
536  int *major,
537  int *minor,
538  int *micro);
539 
540 int dsproc_set_dsdod_att_value(
541  int ds_id,
542  const char *var_name,
543  const char *att_name,
544  CDSDataType type,
545  size_t length,
546  void *value);
547 
548 int dsproc_set_dsdod_att_text(
549  int ds_id,
550  const char *var_name,
551  const char *att_name,
552  const char *format, ...);
553 
554 int dsproc_set_dsdod_dim_length(
555  int ds_id,
556  const char *dim_name,
557  size_t dim_length);
558 
559 int dsproc_set_runtime_metadata(int ds_id, CDSGroup *cds);
560 
561 int dsproc_update_datastream_dsdod(int ds_id, time_t data_time);
562 int dsproc_update_datastream_dsdods(time_t data_time);
563 
564 /*@}*/
565 
566 /******************************************************************************/
567 /**
568  * @defgroup INTERNAL_DSPROC_DSDB Internal: DSDB
569  */
570 /*@{*/
571 
572 int dsproc_db_connect(void);
573 void dsproc_db_disconnect(void);
574 
575 int dsproc_get_datastream_properties(int ds_id, DSProp ***dsprops);
576 int dsproc_get_datastream_property(
577  int ds_id,
578  const char *var_name,
579  const char *prop_name,
580  time_t data_time,
581  const char **prop_value);
582 
583 int dsproc_get_input_ds_classes(DSClass ***ds_classes);
584 int dsproc_get_output_ds_classes(DSClass ***ds_classes);
585 
586 int dsproc_get_location(ProcLoc **proc_loc);
587 int dsproc_get_site_description(const char **site_desc);
588 
589 int dsproc_get_config_value(
590  const char *config_key,
591  char **config_value);
592 
593 /*@}*/
594 
595 /******************************************************************************/
596 /**
597  * @defgroup INTERNAL_DSPROC_DQRDB Internal: DQRDB
598  */
599 /*@{*/
600 
601 int dsproc_dqrdb_connect(void);
602 void dsproc_dqrdb_disconnect(void);
603 
604 void dsproc_free_dqrs(DQR **dqrs);
605 int dsproc_get_dqrs(
606  const char *site,
607  const char *facility,
608  const char *dsc_name,
609  const char *dsc_level,
610  const char *var_name,
611  time_t start_time,
612  time_t end_time,
613  DQR ***dqrs);
614 
615 /*@}*/
616 
617 /******************************************************************************/
618 /**
619  * @defgroup INTERNAL_DSPROC_QC_UTILS Internal: QC Utilities */
620 /*@{*/
621 
622 unsigned int dsproc_get_qc_assessment_mask(
623  CDSVar *qc_var,
624  const char *assessment,
625  int *nfound,
626  int *max_bit_num);
627 
628 /*@}*/
629 
630 /******************************************************************************/
631 /**
632  * @defgroup INTERNAL_DSPROC_TRANSFORMS Internal: Transformation
633  */
634 /*@{*/
635 
636 /**
637  * Transform .
638  */
639 typedef struct {
640 
641  CDSVar *ret_coord_var; /**< pointer to the retrieved variable that
642  will be used for the coordinate variable
643  in the transformed dataset */
644 
645  int ret_dsid; /**< datastream ID of the ret_coord_var. */
646 
647  const char *name; /**< coordinate dimension name. */
648  const char *data_type; /**< user defined data type. */
649  const char *units; /**< user defined units. */
650 
651  double start; /**< dimension start value */
652  double length; /**< dimension length */
653  double interval; /**< dimension interval */
654 
655  const char *trans_type; /**< transformation type */
656  double trans_range; /**< range value used by the transformation */
657  double trans_align; /**< bin alignment used by the transformation */
658 
659 } TransDimInfo;
660 
661 int dsproc_get_trans_dim_info(
662  CDSVar *ret_var,
663  int dim_index,
664  TransDimInfo *dim_info);
665 
666 int dsproc_get_trans_qc_rollup_bits(
667  CDSVar *qc_var,
668  unsigned int *bad_flag,
669  unsigned int *ind_flag);
670 
671 int dsproc_is_transform_qc_var(CDSVar *qc_var);
672 
673 int dsproc_load_transform_params_file(
674  CDSGroup *group,
675  const char *site,
676  const char *facility,
677  const char *name,
678  const char *level);
679 
680 int dsproc_set_coordsys_trans_param(
681  const char *coordsys_name,
682  const char *field_name,
683  const char *param_name,
684  CDSDataType type,
685  size_t length,
686  void *value);
687 
688 /*@}*/
689 
690 /******************************************************************************/
691 /**
692  * @defgroup DSPROC_DEPRECATED Deprecated
693  */
694 /*@{*/
695 
696 /*
697 void dsproc_ingest_main(
698  int argc,
699  char *argv[],
700  char *version_tag,
701  char *proc_name,
702  const char **(*valid_proc_names)(int *nproc_names),
703  void *(*init_process)(void),
704  void (*finish_process)(void *user_data),
705  int (*process_file)(void *user_data,
706  const char *input_dir, const char *file_name));
707 */
708 
709 void dsproc_vap_main(
710  int argc,
711  char *argv[],
712  char *version_tag,
713  char *proc_name,
714  const char **(*valid_proc_names)(int *nproc_names),
715  void *(*init_process)(void),
716  void (*finish_process)(void *user_data),
717  int (*process_data)(void *user_data,
718  time_t begin_date, time_t end_date,
719  CDSGroup *ret_data));
720 
721 void dsproc_transform_main(
722  int argc,
723  char *argv[],
724  char *version_tag,
725  char *proc_name,
726  const char **(*valid_proc_names)(int *nproc_names),
727  void *(*init_process)(void),
728  void (*finish_process)(void *user_data),
729  int (*process_data)(void *user_data,
730  time_t begin_date, time_t end_date,
731  CDSGroup *trans_data));
732 
733 CDSGroup *dsproc_create_dataset(
734  int ds_id,
735  time_t data_time,
736  int set_location);
737 
738 int dsproc_trans_dataset_pass_through(
739  CDSGroup *trans_cds,
740  CDSGroup *out_dataset,
741  int flags);
742 
743 CDSGroup *dsproc_get_trans_ds_by_group_name(
744  const char *coordsys_name,
745  const char *ret_group_name);
746 
747 /*@}*/
748 
749 #endif /* _DSPROC3_INTERNAL_H */