dsproc3_internal.h File Reference

Internal libdsproc3 functions and structures. More...

#include "dsdb3.h"
#include "ncds3.h"

Go to the source code of this file.

Data Structures
struct	TransDimInfo
	Transform . More...

Macros
#define	DSPROC_LIB_NAME   "libdsproc3"
	DSPROC library name.
#define	DS_OVERLAP_CHECK   0x001
	Check for overlap with previously processed data before storing new data.
#define	DS_STANDARD_QC   0x002
	Run standard QC checks before storing new data.
#define	DS_PRESERVE_OBS   0x004
	Preserve distinct observations when retrieving and storing data.
#define	DS_FILTER_NANS   0x008
	Replace NaN and Inf values with missing values when data is stored.
#define	DS_DISABLE_MERGE   0x010
	Do not merge multiple observations in retrieved data.
#define	DS_SKIP_TRANSFORM   0x020
	Skip the transformation logic for all variables in this datastream.
#define	DS_ROLLUP_TRANS_QC   0x040
	Consolidate the transformation QC bits for all variables when mapped to the output dataset.
#define	DS_SCAN_MODE   0x080
	Enable scan mode for datastreams that are not expected to be continuous.
#define	DS_OBS_LOOP   0x100
	Loop over observations (or files) instead of time intervals.
#define	MAP_ROLLUP_TRANS_QC   0x1
	Flag specifying that all bad and indeterminate bits in the input QC variable should be consolidated into single bad or indeterminate bits in the output QC variable.
#define	FILTER_DUPS_ONLY   0x00
	Flag to resest overlap filtering back to duplicate records only.
#define	FILTER_TIME_SHIFTS   0x01
	Flag to filter records that are not in chronological order.
#define	FILTER_OVERLAPS   0x02
	Flag to filter records that have the same times but different data values.
#define	FILTER_ALL   FILTER_TIME_SHIFTS | FILTER_OVERLAPS
	Same as FILTER_TIME_SHIFTS | FILTER_OVERLAPS.

Enumerations
enum	DSFormat {   DSF_NETCDF = 1,   DSF_CSV = 2,   DSF_RAW = 10,   DSF_PNG = 11,   DSF_JPG = 12 }
	DataStream File Formats. More...

Functions
void	dsproc_initialize (int argc, char **argv, ProcModel proc_model, const char *proc_version, int nproc_names, const char **proc_names)
	Initialize a data system process.
int	dsproc_start_processing_loop (time_t *interval_begin, time_t *interval_end)
	Start a processing interval loop.
int	dsproc_retrieve_data (time_t begin_time, time_t end_time, CDSGroup **ret_data)
	Get input data using retriever information.
int	dsproc_merge_retrieved_data (void)
	Merge observations in the _DSProc->ret_data group.
int	dsproc_transform_data (CDSGroup **trans_data)
	Run the data transformation logic.
int	dsproc_create_output_datasets (void)
	Create all output datasets.
int	dsproc_store_output_datasets (void)
	Store all output datasets.
int	dsproc_finish (void)
	Finish a data system process.
int	dsproc_force_rename_bad (const char *file_path, const char *file_name)
	Force a file to be renamed as bad using 19700101 as the timestamp.
int	dsproc_is_fatal (int last_errno)
	Check if the last status was a fatal error.
void	dsproc_bad_file_warning (const char *func, const char *src_file, int src_line, const char *file_name, const char *format,...)
	Append a 'Bad File' message to the log file and warning mail message.
void	dsproc_bad_line_warning (const char *func, const char *src_file, int src_line, const char *file_name, int line_num, const char *format,...)
	Append a 'Bad Line' message to the log file and warning mail message.
void	dsproc_bad_record_warning (const char *func, const char *src_file, int src_line, const char *file_name, int rec_num, const char *format,...)
	Append a 'Bad Record' message to the log file and warning mail message.
void	dsproc_debug (const char *func, const char *file, int line, int level, const char *format,...)
	Generate a debug message.
void	dsproc_error (const char *func, const char *file, int line, const char *status, const char *format,...)
	Generate an error message.
void	dsproc_log (const char *func, const char *file, int line, const char *format,...)
	Append a message to the log file.
void	dsproc_mentor_mail (const char *func, const char *file, int line, const char *format,...)
	Send a mail message to the mentor of the software or data.
void	dsproc_warning (const char *func, const char *file, int line, const char *format,...)
	Generate a warning message.
void	dsproc_reset_warning_count (void)
	Reset the warning message counter.
void	dsproc_set_max_warnings (int max_warnings)
	Set the maximum number of warning messages to allow between dataset stores.
void	dsproc_abort (const char *func, const char *file, int line, const char *status, const char *format,...)
	Abort the process and exit cleanly.
void	dsproc_enable_asynchronous_mode (void)
	Enable asynchronous processing mode.
void	dsproc_disable (const char *message)
	Disable the datasystem process.
void	dsproc_disable_db_updates (void)
	Disable the database updates.
void	dsproc_disable_dynamic_metric_vars (int flag)
	Set flag used to disable dynamic metric vars.
void	dsproc_disable_lock_file (void)
	Disable the creation of the process lock file.
void	dsproc_disable_mail_messages (void)
	Disable the mail messages.
int	dsproc_get_asynchrounous_mode (void)
	Get the asynchronous processing mode.
int	dsproc_get_dynamic_dods_mode (void)
	Get the dynamic dods mode.
int	dsproc_get_force_mode (void)
	Get the force mode.
int	dsproc_get_real_time_mode (void)
	Get the real time mode.
int	dsproc_get_reprocessing_mode (void)
	Get the reprocessing mode.
void	dsproc_set_dynamic_dods_mode (int mode)
	Set Dynamic DODs mode.
void	dsproc_set_force_mode (int mode)
	Set the force mode.
int	dsproc_set_log_dir (const char *log_dir)
	Set Log file directory.
int	dsproc_set_log_file (const char *log_file)
	Set the name of the log file to use.
int	dsproc_set_log_id (const char *log_id)
	Replace timestamp in log file name with Log ID.
void	dsproc_set_max_runtime (int max_runtime)
	Set the maximum runtime allowed for the process.
void	dsproc_set_processing_interval (time_t begin_time, time_t end_time)
	Set the begin and end times for the current processing interval.
void	dsproc_set_real_time_mode (int mode, float max_wait)
	Set the real time mode.
void	dsproc_set_reprocessing_mode (int mode)
	Set the reprocessing mode.
void	dsproc_set_retriever_time_offsets (int ds_id, time_t begin_offset, time_t end_offset)
	Set the time offsets to use when retrieving data.
const char *	adi_version (void)
	ADI version.
const char *	dsproc_lib_version (void)
	libdsproc3 library version.
const char *	dsproc_get_input_dir (void)
	Get the input directory being used by an Ingest process.
const char *	dsproc_get_input_file (void)
	Get the current input file being processed by an Ingest.
const char *	dsproc_get_input_source (void)
	Get the full path to the input file being processed by an Ingest.
const char *	dsproc_get_status (void)
	Get the process status.
const char *	dsproc_get_type (void)
	Get the process type.
const char *	dsproc_get_version (void)
	Get the process version.
time_t	dsproc_get_max_run_time (void)
	Get the process max run time.
time_t	dsproc_get_start_time (void)
	Get the process start time.
time_t	dsproc_get_time_remaining (void)
	Get the time remaining until the max run time is reached.
time_t	dsproc_get_min_valid_time (void)
	Get the minimum valid data time for the process.
time_t	dsproc_get_data_interval (void)
	Get the expected data interval.
time_t	dsproc_get_processing_interval (time_t *begin, time_t *end)
	Get the begin and end times of the current processing interval.
void	dsproc_set_input_dir (const char *input_dir)
	Set the input directory used to create the input_source attribute.
void	dsproc_set_input_source (const char *input_file)
	Set the input file used to create the input_source attribute.
void	dsproc_set_status (const char *status)
	Set the process status.
int	dsproc_init_datastream (const char *site, const char *facility, const char *dsc_name, const char *dsc_level, DSRole role, const char *path, DSFormat format, int flags)
	Initialize a new datastream.
void	dsproc_set_datastream_flags (int ds_id, int flags)
	Set the control flags for a datastream.
void	dsproc_set_datastream_format (int ds_id, DSFormat format)
	Set the data format of a datastream.
int	dsproc_set_output_format (DSFormat format)
	Set the data format for all output datastreams.
void	dsproc_unset_datastream_flags (int ds_id, int flags)
	Unset the control flags for a datastream.
void	dsproc_update_datastream_data_stats (int ds_id, int num_records, const timeval_t *begin_time, const timeval_t *end_time)
	Update datastream data stats.
void	dsproc_update_datastream_file_stats (int ds_id, double file_size, const timeval_t *begin_time, const timeval_t *end_time)
	Update datastream file stats.
int	dsproc_validate_datastream_data_time (int ds_id, const timeval_t *data_time)
	Validate datastream data time.
void	dsproc_close_untouched_files (void)
	Close all open datastream files that haven't been touched since the last time this function was called.
void	dsproc_set_max_open_files (int ds_id, int max_open)
	Set the maximum number of files that can be held open.
CDSGroup *	dsproc_create_output_dataset (int ds_id, time_t data_time, int set_location)
	Create an output dataset.
int	dsproc_dataset_pass_through (CDSGroup *in_dataset, CDSGroup *out_dataset, int flags)
	Pass data from one dataset to another.
int	dsproc_map_datasets (CDSGroup *in_parent, CDSGroup *out_dataset, int flags)
	Map data from input datasets to output datasets.
int	dsproc_map_var (CDSVar *in_var, size_t in_sample_start, size_t sample_count, CDSVar *out_var, size_t out_sample_start, int flags)
	Map an input variable to an output variable.
int	dsproc_set_dataset_location (CDSGroup *dataset)
	Set the location variables for a dataset.
void	dsproc_set_map_time_range (time_t begin_time, time_t end_time)
	Set the time range to use in subsequent calls to dsproc_map_datasets().
void	dsproc_set_map_timeval_range (timeval_t *begin_time, timeval_t *end_time)
	Set the time range to use in subsequent calls to dsproc_map_datasets().
int	dsproc_store_dataset (int ds_id, int newfile)
	Store an output dataset.
int	dsproc_compare_dod_versions (CDSGroup *prev_ds, CDSGroup *curr_ds, int warn)
	Compare the DOD versions of two datasets.
int	dsproc_compare_dod_dims (CDSGroup *prev_ds, CDSGroup *curr_ds, int warn)
	Compare the DOD dimensions of two datasets.
int	dsproc_compare_dod_atts (CDSGroup *prev_ds, CDSGroup *curr_ds, int warn)
	Compare the DOD attributes of two datasets.
int	dsproc_compare_dod_vars (CDSGroup *prev_ds, CDSGroup *curr_ds, int warn)
	Compare the DOD variables of two datasets.
int	dsproc_compare_dods (CDSGroup *prev_ds, CDSGroup *curr_ds, int warn)
	Compare the DODs of two datasets.
int	dsproc_exclude_from_dod_compare (const char *var_name, int exclude_data, int natts, const char **att_names)
	Exclude attributes and/or static data from dod compare.
int	dsproc_fetch_dataset (int ds_id, timeval_t *begin_timeval, timeval_t *end_timeval, size_t nvars, const char **var_names, int merge_obs, CDSGroup **dataset)
	Fetch a dataset from previously stored data.
timeval_t *	dsproc_fetch_timevals (int ds_id, timeval_t *begin_timeval, timeval_t *end_timeval, size_t *ntimevals, timeval_t *timevals)
	Fetch the times of previously stored data.
void	dsproc_disable_nan_filter_warnings (void)
	Disable the warning messages from the NaN/Inf Filter.
int	dsproc_filter_var_nans (CDSVar *var)
	Replace NaN and Inf values in a variable with missing values.
int	dsproc_filter_dataset_nans (CDSGroup *dataset, int warn)
	Replace NaN and Inf values in a dataset with missing values.
void	dsproc_set_overlap_filtering_mode (int mode)
	Filter overlapping data records.
int	dsproc_exclude_from_standard_qc_checks (const char *var_name)
	Exclude a variable from the standard QC checks.
int	dsproc_qc_delta_checks (CDSVar *var, CDSVar *qc_var, CDSVar *prev_var, CDSVar *prev_qc_var, int delta_flag, int bad_flags)
	Perform QC delta checks.
int	dsproc_qc_limit_checks (CDSVar *var, CDSVar *qc_var, int missing_flag, int min_flag, int max_flag)
	Perform QC limit checks.
int	dsproc_qc_time_checks (CDSVar *time_var, CDSVar *qc_time_var, timeval_t *prev_timeval, int lteq_zero_flag, int min_delta_flag, int max_delta_flag)
	Perform QC time checks.
int	dsproc_standard_qc_checks (int ds_id, CDSGroup *dataset)
	Perform all standard QC checks.
int	dsproc_solar_position_qc_check (CDSGroup *dataset, int solar_flag)
	Performs the solar position qc checks for appropriate variables.
int	dsproc_qc_solar_position (CDSVar *qc_var, int solar_flag)
	Perform QC solar position check on the wc variable.
int	dsproc_check_for_dsdod_update (int ds_id, time_t data_time)
	Check if a datastream DOD needs to be updated.
CDSAtt *	dsproc_get_dsdod_att (int ds_id, const char *var_name, const char *att_name)
	Get a datastream DOD attribute.
CDSDim *	dsproc_get_dsdod_dim (int ds_id, const char *dim_name)
	Get a datastream DOD dimension.
void *	dsproc_get_dsdod_att_value (int ds_id, const char *var_name, const char *att_name, CDSDataType type, size_t *length, void *value)
	Get a copy of a datastream DOD attribute value.
char *	dsproc_get_dsdod_att_text (int ds_id, const char *var_name, const char *att_name, size_t *length, char *value)
	Get a copy of a datastream DOD attribute value.
size_t	dsproc_get_dsdod_dim_length (int ds_id, const char *dim_name)
	Get the length of a datastream DOD dimension.
const char *	dsproc_get_dsdod_version (int ds_id, int *major, int *minor, int *micro)
	Get the version of the datastream DOD currently loaded.
int	dsproc_set_dsdod_att_value (int ds_id, const char *var_name, const char *att_name, CDSDataType type, size_t length, void *value)
	Set a datastream DOD attribute value.
int	dsproc_set_dsdod_att_text (int ds_id, const char *var_name, const char *att_name, const char *format,...)
	Set a datastream DOD attribute value.
int	dsproc_set_dsdod_dim_length (int ds_id, const char *dim_name, size_t dim_length)
	Set a datastream DOD dimension length.
int	dsproc_set_runtime_metadata (int ds_id, CDSGroup *cds)
	Set the runtime metadata for a datastream in the specified dataset.
int	dsproc_update_datastream_dsdod (int ds_id, time_t data_time)
	Update the DSDOD for a datastream.
int	dsproc_update_datastream_dsdods (time_t data_time)
	Update the DSDODs for all datastreams.
int	dsproc_db_connect (void)
	Connect to the database.
void	dsproc_db_disconnect (void)
	Disconnect from the database.
int	dsproc_get_datastream_properties (int ds_id, DSProp ***dsprops)
	Get datastream properties.
int	dsproc_get_datastream_property (int ds_id, const char *var_name, const char *prop_name, time_t data_time, const char **prop_value)
	Get a datastream property for a specified time.
int	dsproc_get_input_ds_classes (DSClass ***ds_classes)
	Get input datastream classes.
int	dsproc_get_output_ds_classes (DSClass ***ds_classes)
	Get output datastream classes.
int	dsproc_get_location (ProcLoc **proc_loc)
	Get process location.
int	dsproc_get_site_description (const char **site_desc)
	Get site description.
int	dsproc_get_config_value (const char *config_key, char **config_value)
	Get process config value.
int	dsproc_dqrdb_connect (void)
	Connect to the DQR database.
void	dsproc_dqrdb_disconnect (void)
	Disconnect from the DQR database.
void	dsproc_free_dqrs (DQR **dqrs)
	Free all memory used by an array of pointers to DQR structures.
int	dsproc_get_dqrs (const char *site, const char *facility, const char *dsc_name, const char *dsc_level, const char *var_name, time_t start_time, time_t end_time, DQR ***dqrs)
	Get the DQRs for a datastream variable.
unsigned int	dsproc_get_qc_assessment_mask (CDSVar *qc_var, const char *assessment, int *nfound, int *max_bit_num)
	Get the QC mask for a specific QC assessment.
int	dsproc_get_trans_dim_info (CDSVar *ret_var, int dim_index, TransDimInfo *dim_info)
	Internal: Get information about a dimension in the target coordinate system.
int	dsproc_get_trans_qc_rollup_bits (CDSVar *qc_var, unsigned int *bad_flag, unsigned int *ind_flag)
	Internal: Get the output bits to use when consolidating transformation QC bits.
int	dsproc_is_transform_qc_var (CDSVar *qc_var)
	Internal: Check if a QC variable is from a transformation process.
int	dsproc_load_transform_params_file (CDSGroup *group, const char *site, const char *facility, const char *name, const char *level)
	Internal: Load a transformation parameters file.
int	dsproc_set_coordsys_trans_param (const char *coordsys_name, const char *field_name, const char *param_name, CDSDataType type, size_t length, void *value)
	Set the value of a coordinate system transformation parameter.
void	dsproc_vap_main (int argc, char *argv[], char *version_tag, char *proc_name, const char **(*valid_proc_names)(int *nproc_names), void *(*init_process)(void), void(*finish_process)(void *user_data), int(*process_data)(void *user_data, time_t begin_date, time_t end_date, CDSGroup *ret_data))
	Deprecated: Run retriever VAP without transformation logic.
void	dsproc_transform_main (int argc, char *argv[], char *version_tag, char *proc_name, const char **(*valid_proc_names)(int *nproc_names), void *(*init_process)(void), void(*finish_process)(void *user_data), int(*process_data)(void *user_data, time_t begin_date, time_t end_date, CDSGroup *trans_data))
	Deprecated: Run VAP using transformation logic.
CDSGroup *	dsproc_create_dataset (int ds_id, time_t data_time, int set_location)
	Deprecated: use dsproc_create_output_dataset() instead.
int	dsproc_trans_dataset_pass_through (CDSGroup *trans_cds, CDSGroup *out_dataset, int flags)
	Deprecated: use dsproc_map_datasets() instead.
CDSGroup *	dsproc_get_trans_ds_by_group_name (const char *coordsys_name, const char *ret_group_name)
	Deprecated: Find a transformation datastream group.

---

Detailed Description

Internal libdsproc3 functions and structures.

Definition in file dsproc3_internal.h.

---

Macro Definition Documentation

#define DSPROC_LIB_NAME   "libdsproc3"

DSPROC library name.

Definition at line 36 of file dsproc3_internal.h.