dsproc3.h armutils.h cds3.h dsproc3_internal.h dsproc.c dsproc_csv2cds.c dsproc_csv_ingest_config.c dsproc_csv_parser.c dsproc_dataset_atts.c dsproc_dataset_compare.c dsproc_dataset_dims.c dsproc_dataset_fetch.c dsproc_dataset_filters.c dsproc_dataset_store.c dsproc_dataset_times.c dsproc_dataset_vars.c dsproc_datasets.c dsproc_datastream_dod.c dsproc_datastream_files.c dsproc_datastreams.c dsproc_deprecated.c dsproc_dqrdb.c dsproc_dsdb.c dsproc_exec.c dsproc_file_utils.c dsproc_hooks.c dsproc_main.c dsproc_map_data.c dsproc_merge_obs.c dsproc_messages.c dsproc_outlier_filters.c dsproc_parse_args.c dsproc_print.c dsproc_qc_utils.c dsproc_rename.c dsproc_retriever.c dsproc_solar_position.c dsproc_standard_qc.c dsproc_time_utils.c dsproc_trans_params.c dsproc_transform.c dsproc_update_stored_metadata.c dsproc_var_tag.c dsproc_version.c VarTarget VarDQR CSVParser CSVStrMap CSV2CDSMap CSVTimeCol CSVFieldMap CSVConf DSP_RETRIEVER 0x001 Flag specifying that the data retrieval process should be run. DSP_RETRIEVER_REQUIRED 0x002 Flag specifying if a retriever definition is required. DSP_TRANSFORM 0x004 Flag specifying that the data transformation process should be run. DSP_INGEST 0x100 Flag specifying that this is an ingest process. DSPROC_ERROR status ... dsproc_error(__func__, __FILE__, __LINE__, status, __VA_ARGS__) Convenience macro for the dsproc_error() function. Usage: DSPROC_ERROR(const char *status, const char *format, ...)See printf for a complete description of the format string. DSPROC_WARNING ... dsproc_warning(__func__, __FILE__, __LINE__, __VA_ARGS__) Convenience macro for the dsproc_warning() function. Usage: DSPROC_WARNING(const char *format, ...)See printf for a complete description of the format string. DSPROC_LOG ... dsproc_log(__func__, __FILE__, __LINE__, __VA_ARGS__) Convenience macro for the dsproc_log() function. Usage: DSPROC_LOG(const char *format, ...)See printf for a complete description of the format string. DSPROC_BAD_FILE_WARNING file_name ... dsproc_bad_file_warning(__func__, __FILE__, __LINE__, file_name, __VA_ARGS__) Convenience macro for the dsproc_bad_file_warning() function. Usage: DSPROC_BAD_FILE_WARNING(const char *file_name, const char *format, ...)See printf for a complete description of the format string. DSPROC_BAD_LINE_WARNING file_name line_num ... dsproc_bad_line_warning(__func__, __FILE__, __LINE__, file_name, line_num, __VA_ARGS__) Convenience macro for the dsproc_bad_line_warning() function. Usage: DSPROC_BAD_LINE_WARNING(const char *file_name, int line_num, const char *format, ...)See printf for a complete description of the format string. DSPROC_BAD_RECORD_WARNING file_name rec_num ... dsproc_bad_record_warning(__func__, __FILE__, __LINE__, file_name, rec_num, __VA_ARGS__) Convenience macro for the dsproc_bad_record_warning() function. Usage: DSPROC_BAD_RECORD_WARNING(const char *file_name, int rec_num, const char *format, ...)See printf for a complete description of the format string. DSPROC_MENTOR_MAIL ... dsproc_mentor_mail(__func__, __FILE__, __LINE__, __VA_ARGS__) Convenience macro for the dsproc_mentor_mail() function. Usage: DSPROC_MENTOR_MAIL(const char *format, ...)See printf for a complete description of the format string. DSPROC_DEBUG_LV1 ... if (msngr_debug_level || msngr_provenance_level) \ dsproc_debug(__func__, __FILE__, __LINE__, 1, __VA_ARGS__) Convenience macro for the dsproc_debug() function: level == 1. Usage: DSPROC_DEBUG_LV1(const char *format, ...)See printf for a complete description of the format string. DSPROC_DEBUG_LV2 ... if (msngr_debug_level || msngr_provenance_level) \ dsproc_debug(__func__, __FILE__, __LINE__, 2, __VA_ARGS__) Convenience macro for the dsproc_debug() function: level == 2. Usage: DSPROC_DEBUG_LV2(const char *format, ...)See printf for a complete description of the format string. DSPROC_DEBUG_LV3 ... if (msngr_debug_level || msngr_provenance_level) \ dsproc_debug(__func__, __FILE__, __LINE__, 3, __VA_ARGS__) Convenience macro for the dsproc_debug() function: level == 3. Usage: DSPROC_DEBUG_LV3(const char *format, ...)See printf for a complete description of the format string. DSPROC_DEBUG_LV4 ... if (msngr_debug_level || msngr_provenance_level) \ dsproc_debug(__func__, __FILE__, __LINE__, 4, __VA_ARGS__) Convenience macro for the dsproc_debug() function: level == 4. Usage: DSPROC_DEBUG_LV4(const char *format, ...)See printf for a complete description of the format string. DSPROC_DEBUG_LV5 ... if (msngr_debug_level || msngr_provenance_level) \ dsproc_debug(__func__, __FILE__, __LINE__, 5, __VA_ARGS__) Convenience macro for the dsproc_debug() function: level == 5. Usage: DSPROC_DEBUG_LV5(const char *format, ...)See printf for a complete description of the format string. DSPROC_ABORT status ... dsproc_abort(__func__, __FILE__, __LINE__, status, __VA_ARGS__) Convenience macro for the dsproc_abort() function. Usage: DSPROC_ABORT(const char *status, const char *format, ...)See printf for a complete description of the format string. VAR_SKIP_TRANSFORM 0x1 Flag instructing the transform logic to ignore this variable. VAR_ROLLUP_TRANS_QC 0x2 Consolidate the transformation QC bits when mapped to the output dataset. DSPROC_SUCCESS "Successful" Successful. DSPROC_ENOMEM "Memory Allocation Error" Memory Allocation Error. DSPROC_EFORK "Could Not Create Fork For New Process" Could Not Create Fork For New Process. DSPROC_ENODATA "No Input Data Found" No Input Data Found. DSPROC_ENOOUTDATA "No Output Data Created" No Output Data Found. DSPROC_EINITSIGS "Could Not Initialize Signal Handlers" Could Not Initialize Signal Handlers. DSPROC_ERUNTIME "Maximum Run Time Limit Exceeded" Maximum Run Time Limit Exceeded. DSPROC_EFORCE "Could Not Force Process To Continue" Could Not Force Process To Continue. DSPROC_EDSPATH "Could Not Determine Path To Datastream" Could Not Determine Path To Datastream. DSPROC_ELOGSPATH "Could Not Determine Path To Logs Directory" Could Not Determine Path To Logs Directory. DSPROC_EACCESS "Could Not Access File or Directory" Could Not Open Log File. DSPROC_ELOGOPEN "Could Not Open Log File" Could Not Open Log File. DSPROC_EPROVOPEN "Could Not Open Provenance Log" Could Not Open Provenance Log. DSPROC_EMAILINIT "Could Not Initialize Mail" Could Not Initialize Mail. DSPROC_EDBERROR "Database Error (see log file)" Database Error (see log file) DSPROC_EDBCONNECT "Database Connection Error" Database Connection Error. DSPROC_EDQRDBERROR "DQR Database Error (see log file)" DQR Database Error (see log file) DSPROC_EDQRDBCONNECT "DQR Database Connection Error" DQR Database Connection Error. DSPROC_EDIRLIST "Could Not Get Directory Listing" Directory List Error. DSPROC_EREGEX "Regular Expression Error" Directory List Error. DSPROC_EBADINDSC "Invalid Input Datastream Class" Invalid Input Datastream Class. DSPROC_ENOINDSC "Could Not Find Input Datastream Class In Database" Could Not Find Input Datastream Class In Database. DSPROC_ETOOMANYINDSC "Too Many Input Datastreams Defined In Database" Too Many Input Datastream Classes In Database. DSPROC_EBADOUTDSC "Invalid Output Datastream Class" Invalid Output Datastream Class. DSPROC_EBADDSID "Invalid Datastream ID" Invalid Datastream ID. DSPROC_EBADOUTFORMAT "Invalid Output Datastream Format" Invalid Output Datastream Format. DSPROC_EMINTIME "Found Data Time Before Minimum Valid Time" Found Data Time Before Minimum Valid Time. DSPROC_EFUTURETIME "Found Data Time In The Future" Found Data Time In The Future. DSPROC_ETIMEORDER "Invalid Time Order" Invalid Time Order. DSPROC_ETIMEOVERLAP "Found Overlapping Data Times" Found Overlapping Data Times. DSPROC_EBASETIME "Could Not Get Base Time For Time Variable" Invalid Base Time. DSPROC_EGLOBALATT "Invalid Global Attribute Value" Invalid Global Attribute Value. DSPROC_ETIMEVARTYPE "Invalid Data Type For Time Variable" Invalid Data Type For Time Variable. DSPROC_EQCVARTYPE "Invalid Data Type For QC Variable" Invalid Data Type For QC Variable. DSPROC_EQCSAMPLESIZE "Invalid QC Variable Sample Size" Invalid QC Variable Sample Size. DSPROC_EQCVARDIMS "Invalid QC Variable Dimensions" Invalid QC Variable Dimensions. DSPROC_EVARTYPE "Invalid Data Type For Variable" Invalid Data Type For Variable. DSPROC_ESAMPLESIZE "Invalid Variable Sample Size" Invalid Variable Sample Size. DSPROC_EDATAATTTYPE "Data Attribute Has Invalid Data Type" Data Attribute Has Invalid Data Type. DSPROC_EFILECOPY "Could Not Copy File" Could Not Copy File. DSPROC_EFILEMOVE "Could Not Move File" Could Not Move File. DSPROC_EFILEOPEN "Could Not Open File" Could Not Open File. DSPROC_EFILEREAD "Could Not Read From File" Could Not Read From File. DSPROC_EFILEWRITE "Could Not Write To File" Could Not Write To File. DSPROC_EFILESTATS "Could Not Get File Stats" Could Not Get File Stats. DSPROC_EUNLINK "Could Not Delete File" Could Not Delete File. DSPROC_ENOSRCFILE "Source File Does Not Exist" Source File Does Not Exist. DSPROC_ENOFILETIME "Could Not Determine File Time" Could Not Determine File Time. DSPROC_EDESTDIRMAKE "Could Not Create Destination Directory" Could Not Create Destination Directory. DSPROC_ETIMECALC "Time Calculation Error" Time Calculation Error. DSPROC_EFILEMD5 "Could Not Get File MD5" Could Not Get File MD5. DSPROC_EMD5CHECK "Source And Destination File MD5s Do Not Match" Source And Destination File MD5s Do Not Match. DSPROC_ECDSALLOCVAR "Could Not Allocate Memory For Dataset Variable" Could Not Allocate Memory For Dataset Variable. DSPROC_ECDSCOPYVAR "Could Not Copy Dataset Variable" Could Not Copy Dataset Variable. DSPROC_ECLONEVAR "Could Not Clone Dataset Variable" Could Not Copy Dataset Variable. DSPROC_ECDSDEFVAR "Could Not Define Dataset Variable" Could Not Define Dataset Variable. DSPROC_ECDSDELVAR "Could Not Delete Dataset Variable" Could Not Delete Dataset Variable. DSPROC_ECDSCOPY "Could Not Copy Dataset Metadata" Could Not Copy Dataset Metadata. DSPROC_ECDSCHANGEATT "Could Not Change Attribute Value In Dataset" Could Not Change Attribute Value In Dataset. DSPROC_ECDSSETATT "Could Not Set Attribute Value In Dataset" Could Not Set Attribute Value In Dataset. DSPROC_ECDSSETDIM "Could Not Set Dimension Length In Dataset" Could Not Set Dimension Length In Dataset. DSPROC_ECDSSETDATA "Could Not Set Variable Data In Dataset" Could Not Set Variable Data In Dataset. DSPROC_ECDSSETTIME "Could Not Set Time Values In Dataset" Could Not Set Time Values In Dataset. DSPROC_ECDSGETTIME "Could Not Get Time Values From Dataset" Could Not Get Time Values From Dataset. DSPROC_EMERGE "Could Not Merge Datasets" Could Not Merge Datasets. DSPROC_EBOUNDSVAR "Invalid Cell Boundary Variable or Definition" Invalid Cell Boundary Variable or Definition. DSPROC_ENODOD "DOD Not Defined In Database" DOD Not Defined In Database. DSPROC_ENORETRIEVER "Could Not Find Retriever Definition In Database" Could Not Find Data Retriever Information. DSPROC_EBADRETRIEVER "Invalid Retriever Definition" Invalid Retriever Definition. DSPROC_EREQVAR "Required Variable Missing From Dataset" Required Variable Missing From Dataset. DSPROC_EREQATT "Required Attribute Variable Missing From Variable or Dataset" Required Attribute Variable Missing From Variable or Dataset. DSPROC_ERETRIEVER "Could Not Retrieve Input Data" Could Not Retrieve Input Data. DSPROC_ENCCREATE "Could Not Create NetCDF File" Could Not Create NetCDF File. DSPROC_ENCOPEN "Could Not Open NetCDF File" Could Not Open NetCDF File. DSPROC_ENCCLOSE "Could Not Close NetCDF File" Could Not Close NetCDF File. DSPROC_ENCSYNC "Could Not Sync NetCDF File" Could Not Close NetCDF File. DSPROC_ENCREAD "Could Not Read From NetCDF File" NetCDF File Read Error. DSPROC_ENCWRITE "Could Not Write To NetCDF File" NetCDF File Write Error. DSPROC_ENOTRANSFORM "Could Not Find Data Transform Information" Could Not Find Data Transform Information. DSPROC_ETRANSFORM "Could Not Transform Input Data" Could Not Find Data Transform Information. DSPROC_ETRANSQCVAR "Could Not Create Consolidated Transformation QC Variable" Could Not Create Consolidated Transformation QC Variable. DSPROC_ETRANSPARAMLOAD "Could Not Load Transform Parameters File" Could Not Load Transform Parameters File. DSPROC_EVARMAP "Could Not Map Input Variable To Output Variable" Could Not Map Input Variable To Output Variable. DSPROC_ECSVPARSER "Could Not Parse CSV File" Could Not Parse Input CSV File. DSPROC_ECSVCONF "Could Not Read CSV Ingest Configuration File" Could Not Read CSV Ingest Configuration File. DSPROC_ECSV2CDS "Could Not Map Input CSV Data To Output Dataset" Could Not Map Input CSV Data To Output Dataset. CSV_OVERWRITE 0x1 Mapping flag to specify that existing data should be overwritten. CSV_CHECK_DATA_CONF 0x01 Flag used by csv_read_conf_file() to check for config files under the root directory defined by the CONF_DATA environment variable. ProcModel PM_GENERIC DSP_RETRIEVER | DSP_TRANSFORM Generic VAP process. The retriever definition will be used if it exists in the database but it is not required for the process to run. This will also run the transform logic for any variables that are found in the retrieved data that have been tagged with a coordinate system name. PM_RETRIEVER_VAP DSP_RETRIEVER | DSP_RETRIEVER_REQUIRED Retriever only VAP. This VAP requires a retriever definition to be specified in the database, but will bypass the transformation logic. PM_TRANSFORM_VAP DSP_RETRIEVER | DSP_RETRIEVER_REQUIRED | DSP_TRANSFORM Transformation VAP. This VAP requires a retriever definition to be specified in the database, and will run the transformation logic. PM_INGEST DSP_INGEST Ingest Process. This is an Ingest process that loops over all raw files in the input datastream directory. PM_RETRIEVER_INGEST DSP_INGEST | DSP_RETRIEVER | DSP_RETRIEVER_REQUIRED Ingest/VAP Hybrid Process that bypasses the transform logic. This is an Ingest process that uses the PM_RETRIEVER_VAP processing model, but is designed to run in real-time like an ingest without the need for the '-b begin_time' command line argument. The standard VAP -b/-e command line options can still be used for reprocessing. PM_TRANSFORM_INGEST DSP_INGEST | DSP_RETRIEVER | DSP_RETRIEVER_REQUIRED | DSP_TRANSFORM Ingest/VAP Hybrid Process that uses the transform logic. This is an Ingest process that uses the PM_TRANSFORM_VAP processing model, but is designed to run in real-time like an ingest without the need for the '-b begin_time' command line argument. The standard VAP -b/-e command line options can still be used for reprocessing. Process Models. LogInterval LOG_MONTHLY 0 create monthly log files LOG_DAILY 1 create daily log files LOG_RUN 2 create one log file per run Log File Intervals. SplitMode SPLIT_ON_STORE 0 Always create a new file when data is stored. SPLIT_ON_HOURS 1 Split start is the hour of the day for the first split [0-23], and split interval is in hours. SPLIT_ON_DAYS 2 Split start is the day of the month for the first split [1-31], and split interval is in days. SPLIT_ON_MONTHS 3 Split start is the month of the year for the first split [1-12], and split interval is in months. SPLIT_NONE 4 Always append output to the previous file unless otherwise specified in the call to dsproc_store_dataset. Output File Splitting Mode. DSRole DSR_INPUT 1 input datastream DSR_OUTPUT 2 output datastream DataStream Roles. void void dsproc_freeopts (void) dsproc_freeopts void Free memory used by user defined command line options. This function can be called after dsproc_main() returns to free all internal memory used by the user defined command line options.dsproc_setopt(), dsproc_getopt() int int dsproc_getopt (const char *option, const char **value) dsproc_getopt const char * option const char ** value Get user defined command line option. Check if a user defined option was set on the command line and get the value if applicable.Example (see dsproc_setopt() for example that sets these options): constchar*strval; intfound; found=dsproc_getopt("o",&strval); if(found){ printf("Option'-o'or'--my-option'wassetbyuser:'%s'\n", strval); } found=dsproc_getopt("p",&strval); if(found){ printf("Option'-p'wassetbyuser.\n"); } found=dsproc_getopt("long-only",&strval); if(found){ printf("Option'--long-only'wassetbyuser.\n"); } option The short or long option. value output: pointer to the value. 1 if the option was specified on the command line 0 if the option was not specified on the command line dsproc_setopt(), dsproc_freeopts() int int dsproc_setopt (const char short_opt, const char *long_opt, const char *arg_name, const char *opt_desc) dsproc_setopt const char short_opt const char * long_opt const char * arg_name const char * opt_desc Set user defined command line option. This function must be called before calling dsproc_main.Example (see dsproc_getopt() for example that gets these options): if(!dsproc_setopt('o',"my-option","value", "Userdefinedoptionwithbothshortandlongoption\n" "switchesthatalsotakesauserdefinedargument.")){ exit(1); } if(!dsproc_setopt('p',NULL,NULL, "Userdefinedoptionwithonlyashortoptionswitch.\n")){ exit(1); } if(!dsproc_setopt('\0',"--long-only",NULL, "Userdefinedoptionwithonlyalongoptionswitch.")){ exit(1); } short_opt - Short options are single letters and are prefixed by a single dash on the command line. Multiple short options can be grouped behind a single dash. Specify NULL for this argument if a short option should not be used. long_opt - Long options are prefixed by two consecutive dashes on the command line. Specify NULL for this argument if a long option should not be used. arg_name - A single word description of the option argument to be used in the help message, or NULL if this option does not take an argument on the command line. opt_desc - A brief description of the option to be used in the help message. 1 if successful 0 if the option has already been used, or a memory allocation error occurred dsproc_getopt(), dsproc_freeopts() _DSProcOpts void void dsproc_use_nc_extension (void) dsproc_use_nc_extension void Set the default NetCDF file extension to 'nc' for output files. The NetCDF file extension used by ARM has historically been "cdf". This function can be used to change it to the new prefered extension of "nc", and must be called before calling dsproc_main(). int int dsproc_main (int argc, char **argv, ProcModel proc_model, const char *proc_version, int nproc_names, const char **proc_names) dsproc_main int argc char ** argv ProcModel proc_model const char * proc_version int nproc_names const char ** proc_names Datasystem Process Main Function. argc - command line argument count argv - command line argument vector proc_model - processing model to use proc_version - process version nproc_names - number of valid process names proc_names - list of valid process names suggested program exit value (0 = successful, 1 = failure) dsproc_initialize dsproc_finish dsproc_db_disconnect PM_INGEST void void dsproc_set_init_process_hook (void *(*_init_process_hook)(void)) dsproc_set_init_process_hook void *(*)(void) _init_process_hook init_process_hook VAP/Ingest: Set hook function to call when a process is first initialized. This function must be called from the main function before dsproc_main() is called.The specified init_process_hook() function will be called once just before the main data processing loop begins and before the initial database connection is closed.The init_process_hook() function does not take any arguments, but it must return: void pointer to a user defined data structure or value that will be passed in as user_data to all other hook functions.(void *)1 if no user data is returned.NULL if a fatal error occurred and the process should exit. init_process_hook - function to call when the process is initialized void void dsproc_set_finish_process_hook (void(*finish_process_hook)(void *user_data)) dsproc_set_finish_process_hook void(*)(void *user_data) finish_process_hook VAP/Ingest: Set hook function to call before a process finishes. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified finish_process_hook() function will be called once just after the main data processing loop finishes. This function should be used to clean up any temporary files used, and to free any memory used by the structure returned by the init_process_hook() function.The finish_process_hook function must take the following argument: void *user_data: value returned by the init_process_hook() function finish_process_hook - function to call before the process finishes void void dsproc_set_process_data_hook (int(*process_data)(void *user_data, time_tbegin_date, time_tend_date, CDSGroup *input_data)) dsproc_set_process_data_hook int(*)(void *user_data, time_tbegin_date, time_tend_date, CDSGroup *input_data) process_data process_data_hook VAP: Set the main data processing function. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified process_data_hook function will be called once per processing interval just after the output datasets are created, but before they are stored to disk.The process_data_hook function must take the following arguments: void *user_data: value returned by the init_process_hook() functiontime_t begin_date: the begin time of the current processing intervaltime_t end_date: the end time of the current processing intervalCDSGroup *input_data: pointer to the parent CDSGroup containing the input data. And must return: 1 if processing should continue normally0 if processing should skip the current processing interval and continue on to the next one.-1 if a fatal error occurred and the process should exit. process_data_hook - the main data processing function void void dsproc_set_pre_retrieval_hook (int(*pre_retrieval_hook)(void *user_data, time_tbegin_date, time_tend_date)) dsproc_set_pre_retrieval_hook int(*)(void *user_data, time_tbegin_date, time_tend_date) pre_retrieval_hook VAP: Set hook function to call before data is retrieved. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified pre_retrieval_hook function will be called once per processing interval just prior to data retrieval, and must take the following arguments:The pre_retrieval_hook function must take the following arguments: void *user_data: value returned by the init_process_hook() functiontime_t begin_date: the begin time of the current processing intervaltime_t end_date: the end time of the current processing interval And must return: 1 if processing should continue normally0 if processing should skip the current processing interval and continue on to the next one.-1 if a fatal error occurred and the process should exit. pre_retrieval_hook - function to call before data is retrieved void void dsproc_set_post_retrieval_hook (int(*post_retrieval_hook)(void *user_data, time_tbegin_date, time_tend_date, CDSGroup *ret_data)) dsproc_set_post_retrieval_hook int(*)(void *user_data, time_tbegin_date, time_tend_date, CDSGroup *ret_data) post_retrieval_hook VAP: Set hook function to call after data is retrieved. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified post_retrieval_hook function will be called once per processing interval just after data retrieval, but before the retrieved observations are merged and QC is applied.The post_retrieval_hook function must take the following arguments: void *user_data: value returned by the init_process_hook() functiontime_t begin_date: the begin time of the current processing intervaltime_t end_date: the end time of the current processing intervalCDSGroup *ret_data: pointer to the parent CDSGroup containing the retrieved data And must return: 1 if processing should continue normally0 if processing should skip the current processing interval and continue on to the next one.-1 if a fatal error occurred and the process should exit. post_retrieval_hook - function to call after data is retrieved void void dsproc_set_pre_transform_hook (int(*pre_transform_hook)(void *user_data, time_tbegin_date, time_tend_date, CDSGroup *ret_data)) dsproc_set_pre_transform_hook int(*)(void *user_data, time_tbegin_date, time_tend_date, CDSGroup *ret_data) pre_transform_hook VAP: Set hook function to call before the data is transformed. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified pre_transform_hook function will be called once per processing interval just prior to data transformation, and after the retrieved observations are merged and QC is applied.The pre_transform_hook function must take the following arguments: void *user_data: value returned by the init_process_hook() functiontime_t begin_date: the begin time of the current processing intervaltime_t end_date: the end time of the current processing intervalCDSGroup *ret_data: pointer to the parent CDSGroup containing the retrieved data And must return: 1 if processing should continue normally0 if processing should skip the current processing interval and continue on to the next one.-1 if a fatal error occurred and the process should exit. pre_transform_hook - function to call before the data is transformed void void dsproc_set_post_transform_hook (int(*post_transform_hook)(void *user_data, time_tbegin_date, time_tend_date, CDSGroup *trans_data)) dsproc_set_post_transform_hook int(*)(void *user_data, time_tbegin_date, time_tend_date, CDSGroup *trans_data) post_transform_hook VAP: Set hook function to call after the data is transformed. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified post_transform_hook function will be called once per processing interval just after data transformation, but before the output datasets are created.The post_transform_hook function must take the following arguments: void *user_data: value returned by the init_process_hook() functiontime_t begin_date: the begin time of the current processing intervaltime_t end_date: the end time of the current processing intervalCDSGroup *trans_data: pointer to the parent CDSGroup containing the transformed data And must return: 1 if processing should continue normally0 if processing should skip the current processing interval and continue on to the next one.-1 if a fatal error occurred and the process should exit. post_transform_hook - function to call after the data is transformed void void dsproc_set_quicklook_hook (int(*quicklook_hook)(void *user_data, time_tbegin_date, time_tend_date)) dsproc_set_quicklook_hook int(*)(void *user_data, time_tbegin_date, time_tend_date) quicklook_hook VAP or Ingest: Set hook function to call after all data is stored. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified quicklook_hook function will be called once per processing interval just after all data is stored.The quicklook_hook function must take the following arguments: void *user_data: value returned by the init_process_hook() functiontime_t begin_date: the begin time of the current processing intervaltime_t end_date: the end time of the current processing interval And must return: 1 if processing should continue normally0 if processing should skip the current processing interval and continue on to the next one.-1 if a fatal error occurred and the process should exit. quicklook_hook - function to call after all data is stored void void dsproc_set_process_file_hook (int(*process_file_hook)(void *user_data, const char *input_dir, const char *file_name)) dsproc_set_process_file_hook int(*)(void *user_data, const char *input_dir, const char *file_name) process_file_hook Ingest: Set the main file processing function. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified process_file_hook function will be called once for every file found in the input directory, and it must take the following arguments: void *user_data: value returned by the init_process_hook() functionconst char *input_dir: full path to the input directoryconst char *file_name: name of the file to process And must return: 1 if processing should continue normally0 if processing should skip the current file and continue on to the next one.-1 if a fatal error occurred and the process should exit. process_file_hook - the main file processing function void void dsproc_set_custom_qc_hook (int(*custom_qc_hook)(void *user_data, intds_id, CDSGroup *dataset)) dsproc_set_custom_qc_hook int(*)(void *user_data, intds_id, CDSGroup *dataset) custom_qc_hook Ingest: Set the custom QC function. This function must be called from the main function before dsproc_main() is called, or from the init_process_hook() function.The specified custom_qc_hook function will be called just after the standard QC checks are applied when the data is stored, and it must take the following arguments: void *user_data: value returned by the init_process_hook() function int ds_id: datastream ID CDSGroup *dataset: pointer to the dataset And must return: 1 if processing should continue normally0 if processing should skip the current dataset and continue-1 if a fatal error occurred and the process should exit custom_qc_hook - function used to apply custom QC checks int int dsproc_station_view_hook (void *user_data, time_t begin_date, time_t end_date, CDSGroup *trans_data) dsproc_station_view_hook void * user_data time_t begin_date time_t end_date CDSGroup * trans_data int int dsproc_get_debug_level (void) dsproc_get_debug_level void Get the current debug level. msngr_debug_level const char * const char* dsproc_get_site (void) dsproc_get_site void Get the process site. site name _DSProc const char * const char* dsproc_get_facility (void) dsproc_get_facility void Get the process facility. facility _DSProc const char * const char* dsproc_get_name (void) dsproc_get_name void Get the process name. process name _DSProc void void dsproc_set_datastream_split_mode (int ds_id, SplitMode split_mode, double split_start, double split_interval) dsproc_set_datastream_split_mode int ds_id SplitMode split_mode double split_start double split_interval Set the file splitting mode for output files. Default for VAPs: always create a new file when data is stored split_mode = SPLIT_ON_STOREsplit_start = ignoredsplit_interval = ignored Default for Ingests: daily files that split at midnight split_mode = SPLIT_ON_HOURSsplit_start = 0split_interval = 24 ds_id - datastream ID split_mode - the file splitting mode (see SplitMode) split_start - the start of the split interval (see SplitMode) split_interval - the split interval (see SplitMode) _DSProc msngr_debug_level msngr_provenance_level SPLIT_ON_STORE DEBUG_LV1 DSPROC_LIB_NAME SPLIT_ON_HOURS SPLIT_ON_DAYS SPLIT_ON_MONTHS SPLIT_NONE void void dsproc_set_log_interval (LogInterval interval, int use_begin_time) dsproc_set_log_interval LogInterval interval int use_begin_time Set Log file interval. interval - log file interval: LOG_MONTHLY = create monthly log filesLOG_DAILY = create daily log filesLOG_RUN = create one log file per run use_begin_time - VAP Only: flag indicating if the begin time specified on the command line should be used for the log file time. void void dsproc_set_processing_interval_offset (time_t offset) dsproc_set_processing_interval_offset time_t offset Set the offset to apply to the processing interval. This function can be used to shift the processing interval and should be called from either the init-process or pre-retrieval hook function. offset - offset in seconds DEBUG_LV1 DSPROC_LIB_NAME _DSProc void void dsproc_set_trans_qc_rollup_flag (int flag) dsproc_set_trans_qc_rollup_flag int flag Set the global transformation QC rollup flag. This function should typically be called from the users init_process function, but must be called before the post-transform hook returns.Setting this flag to true specifies that all bad and indeterminate bits in transformation QC variables should be consolidated into a single bad or indeterminate bit when they are mapped to the output datasets. This bit consolidation will only be done if the input and output QC variables have the appropriate bit descriptions: The input transformation QC variables will be determined by checking the tag names in the bit description attributes. These must be in same order as the transformation would define them. The output QC variables must contain two bit descriptions for the bad and indeterminate bits to use, and these bit descriptions must begin with the following text: "Transformation could not finish""Transformation resulted in an indeterminate outcome" An alternative to calling this function is to set the "ROLLUP TRANS QC" flags for the output datastreams and/or retrieved variables. See dsproc_set_datastream_flags() and dsproc_set_var_flags(). These options should not typically be needed, however, because the internal mapping logic will determine when it is appropriate to do the bit consolidation. flag - transformation QC rollup flag (1 = TRUE, 0 = FALSE) DEBUG_LV1 DSPROC_LIB_NAME int int dsproc_get_datastream_id (const char *site, const char *facility, const char *dsc_name, const char *dsc_level, DSRole role) dsproc_get_datastream_id const char * site const char * facility const char * dsc_name const char * dsc_level DSRole role Get the ID of a datastream. site - site name, or NULL to find first match facility - facility name, or NULL to find first match dsc_name - datastream class name dsc_level - datastream class level role - specifies input or output datastream datastream ID-1 if the datastream has not beed defined _DSProc int int dsproc_get_input_datastream_id (const char *dsc_name, const char *dsc_level) dsproc_get_input_datastream_id const char * dsc_name const char * dsc_level Get the ID of an input datastream. This function will generate an error if the specified datastream class has not been defined in the database as an input for this process.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dsc_name - datastream class name dsc_level - datastream class level datastream ID-1 if an error occurs dsproc_get_datastream_id DSR_INPUT ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_EBADINDSC int int dsproc_get_input_datastream_ids (int **ids) dsproc_get_input_datastream_ids int ** ids Get the IDs of all input datastreams. This function will return an array of all input datastream ids. The memory used by the returned array is dynamically allocated and must be freed by the calling process.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ids - output: pointer to a dymanically allocated array of input datastream ids number of input datastreams-1 if a memory allocation error occurs _DSProc ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM DSR_INPUT int int dsproc_get_output_datastream_id (const char *dsc_name, const char *dsc_level) dsproc_get_output_datastream_id const char * dsc_name const char * dsc_level Get the ID of an output datastream. This function will generate an error if the specified datastream class has not been defined in the database as an output for this process.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dsc_name - datastream class name dsc_level - datastream class level datastream ID-1 if an error occurs dsproc_get_datastream_id DSR_OUTPUT ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_EBADOUTDSC int int dsproc_get_output_datastream_ids (int **ids) dsproc_get_output_datastream_ids int ** ids Get the IDs of all output datastreams. This function will return an array of all output datastream ids. The memory used by the returned array is dynamically allocated and must be freed by the calling process.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ids - output: pointer to a dymanically allocated array of output datastream ids number of output datastreams-1 if a memory allocation error occurs _DSProc ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM DSR_OUTPUT const char * const char* dsproc_datastream_name (int ds_id) dsproc_datastream_name int ds_id Returns the fully qualified datastream name. The returned datastream name belongs to the internal datastream structure and must not be freed or altered by the calling process. ds_id - datastream ID pointer to the datastream nameNULL if the datastream ID is not valid _DSProc const char * const char* dsproc_datastream_site (int ds_id) dsproc_datastream_site int ds_id Returns the datastream site. The returned site code belongs to the internal datastream structure and must not be freed or altered by the calling process. ds_id - datastream ID pointer to the datastream site codeNULL if the datastream ID is not valid _DSProc const char * const char* dsproc_datastream_facility (int ds_id) dsproc_datastream_facility int ds_id Returns the datastream facility. The returned facility code belongs to the internal datastream structure and must not be freed or altered by the calling process. ds_id - datastream ID pointer to the datastream facility codeNULL if the datastream ID is not valid _DSProc const char * const char* dsproc_datastream_class_name (int ds_id) dsproc_datastream_class_name int ds_id Returns the datastream class name. The returned datastream class name belongs to the internal datastream structure and must not be freed or altered by the calling process. ds_id - datastream ID pointer to the datastream class nameNULL if the datastream ID is not valid _DSProc const char * const char* dsproc_datastream_class_level (int ds_id) dsproc_datastream_class_level int ds_id Returns the datastream class level. The returned datastream class level belongs to the internal datastream structure and must not be freed or altered by the calling process. ds_id - datastream ID pointer to the datastream class levelNULL if the datastream ID is not valid _DSProc const char * const char* dsproc_datastream_path (int ds_id) dsproc_datastream_path int ds_id Returns the path to the datastream directory. The returned path belongs to the internal datastream structure and must not be freed or altered by the calling process. ds_id - datastream ID pointer to the path to the datastream directoryNULL if the datastream path has not been set _DSProc int int dsproc_add_datastream_file_patterns (int ds_id, int npatterns, const char **patterns, int ignore_case) dsproc_add_datastream_file_patterns int ds_id int npatterns const char ** patterns int ignore_case Add datastream file patterns. This function adds file patterns to look for when creating the list of files in the datastream directory. By default all files in the directory will be listed.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ds_id - datastream ID npatterns - number of file patterns patterns - list of extended regex file patterns (man regcomp) ignore_case - ingnore case in file patterns 1 if successful0 if an error occurred _DSProc msngr_debug_level msngr_provenance_level DEBUG_LV1 DSPROC_LIB_NAME int int dsproc_find_datastream_files (int ds_id, time_t begin_time, time_t end_time, char ***file_list) dsproc_find_datastream_files int ds_id time_t begin_time time_t end_time char *** file_list Find all files in a datastream directory for a specified time range. This function will return a list of all files in a datastream directory containing data for the specified time range. This search will include the begin_time but exclude the end_time. That is, it will find files that include data greter than or equal to the begin time, and less than the end time.If the begin_time is not specified, the file containing data for the time just prior to the end_time will be returned.If the end_time is not specified, the file containing data for the time just after the begin_time will be returned.The memory used by the returned file list is dynamically allocated and and must be freed using the dsproc_free_file_list() function.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ds_id - datastream ID begin_time - beginning of the time range to search end_time - end of the time range to search file_list - output: pointer to the NULL terminated file list nfiles the number of files found -1 if an error occurred _DSProc msngr_debug_level msngr_provenance_level DEBUG_LV1 DSPROC_LIB_NAME format_secs1970 ERROR dsproc_set_status DSPROC_EBADDSID DSPROC_EDSPATH DSPROC_ENOFILETIME DSPROC_ENOMEM void void dsproc_free_file_list (char **file_list) dsproc_free_file_list char ** file_list Free a null terminated list of file names. file_list - null terminated list of file names int int dsproc_get_datastream_files (int ds_id, char ***file_list) dsproc_get_datastream_files int ds_id char *** file_list Get the list of files in a datastream directory. By default the returned list will be sorted using the qsort_numeric_strcmp() function for input datastreams with a datalevel starting with a 0. The qsort_strcmp() function will be used for all other datastreams. A different file name compare function can be set using dsproc_set_file_name_compare_function().The memory used by the returned file list is dynamically allocated and and must be freed using the dsproc_free_file_list() function.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ds_id - datastream ID file_list - output: pointer to the NULL terminated list of file names number of files-1 if an error occurred _DSProc DEBUG_LV1 DSPROC_LIB_NAME ERROR dsproc_set_status DSPROC_ENOMEM DSPROC_EDSPATH int int dsproc_set_datastream_path (int ds_id, const char *path) dsproc_set_datastream_path int ds_id const char * path Set the path to the datastream directory. Default datastream path set if path == NULL: dsenv_get_collection_dir() for level 0 input datastreamsdsenv_get_input_datastream_dir() for all other input datastreamsdsenv_get_output_datastream_dir() for output datastreams If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ds_id - datastream ID path - path to the datastream directory, or NULL to set the default datastream path 1 if successful0 if an error occurs _DSProc DSR_INPUT dsenv_get_collection_dir ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_EDSPATH dsenv_get_input_datastream_dir dsenv_get_output_datastream_dir DSPROC_ENOMEM msngr_debug_level msngr_provenance_level DEBUG_LV1 dsproc_set_file_name_compare_function qsort_numeric_strcmp qsort_strcmp dsproc_set_file_name_time_function void void dsproc_set_file_name_compare_function (int ds_id, int(*function)(const void *, const void *)) dsproc_set_file_name_compare_function int ds_id int(*)(const void *, const void *) function Set the file name compare function used to sort the file list. ds_id - datastream ID function - the file name compare function used to sort the file list _DSProc void void dsproc_set_file_name_time_function (int ds_id, time_t(*function)(const char *)) dsproc_set_file_name_time_function int ds_id time_t(*)(const char *) function Set the function used to parse the time from a file name. ds_id - datastream ID function - the function used to parse the time from a file name _DSProc void void dsproc_set_datastream_file_extension (int ds_id, const char *extension) dsproc_set_datastream_file_extension int ds_id const char * extension Set the datastream file extension. ds_id - datastream ID extension - file extension _DSProc DEBUG_LV1 DSPROC_LIB_NAME const char * const char* dsproc_dataset_name (CDSGroup *dataset) dsproc_dataset_name CDSGroup * dataset Returns the dataset name. The returned name belongs to the dataset structure and must not be freed or altered by the calling process. dataset pointer to the dataset name pointer to the dataset name NULL if the specified dataset is NULL CDSGroup::name const char * const char* dsproc_get_dataset_version (CDSGroup *dataset, int *major, int *minor, int *micro) dsproc_get_dataset_version CDSGroup * dataset int * major int * minor int * micro Get the DOD version of a dataset. The memory used by the return string belongs to the global dod_version attribute in the dataset and must not be altered or freed. dataset pointer to the dataset major output: major version number minor output: minor version number micro output: reserved for micro version number (this not currently implemented but I can see a need for it coming soon...) DOD version of the dataset (as a character string)NULL if the dod_version attribute was not found cds_get_att CDSAtt::type CDS_CHAR parse_version_string CDSAtt::value CDSData::cp CDSGroup * CDSGroup* dsproc_get_output_dataset (int ds_id, int obs_index) dsproc_get_output_dataset int ds_id int obs_index Get an output dataset. This function will return a pointer to the output dataset for the specifed datastream and observation. The obs_index should always be zero unless observation based processing is being used. This is because all input observations should have been merged into a single observation in the output datasets. ds_id output datastream ID obs_index the index of the obervation to get the dataset for dataset pointer to the output dataset NULL if it does not exist Example: Get the dataset for an output datastream constchar*dsc_name="example"; constchar*dsc_level="c1"; intds_id; CDSGroup*dataset; ds_id=dsproc_get_output_datastream_id(dsc_name,dsc_level); dataset=dsproc_get_output_dataset(ds_id,0); _DSProc DSR_OUTPUT CDSGroup * CDSGroup* dsproc_get_retrieved_dataset (int ds_id, int obs_index) dsproc_get_retrieved_dataset int ds_id int obs_index Get a retrieved dataset. This function will return a pointer to the retrieved dataset for the specifed datastream and observation.The obs_index is used to specify which observation to get the dataset for. This value will typically be zero unless this function is called from a post_retrieval_hook() function, or the process is using observation based processing. In either of these cases the retrieved data will contain one "observation" for every file the data was read from on disk.It is also possible to have multiple observations in the retrieved data when a pre_transform_hook() is called if a dimensionality conflict prevented all observations from being merged. ds_id input datastream ID obs_index observation index (0 based indexing) dataset pointer to the retrieved dataset NULL if it does not exist Example: Loop over all retrieved datasets for an input datastream constchar*dsc_name="example"; constchar*dsc_level="b1"; intds_id; intobs_index; CDSGroup*dataset; ds_id=dsproc_get_input_datastream_id(dsc_name,dsc_level); for(obs_index=0;;obs_index++){ dataset=dsproc_get_retrieved_dataset(ds_id,obs_index); if(!dataset)break; } _DSProc DSR_INPUT CDSGroup::groups CDSGroup * CDSGroup* dsproc_get_transformed_dataset (const char *coordsys_name, int ds_id, int obs_index) dsproc_get_transformed_dataset const char * coordsys_name int ds_id int obs_index Get a transformed dataset. This function will return a pointer to the transformed dataset for the specifed coordinate system, datastream, and observation. The obs_index should always be zero unless observation based processing is being used. This is because all input observations should have been merged into a single observation in the transformed datasets. coordsys_name the name of the coordinate system as specified in the retriever definition or NULL if a coordinate system name was not specified. ds_id input datastream ID obs_index the index of the obervation to get the dataset for pointer to the output datasetNULL if it does not exist _DSProc DSR_INPUT cds_get_group CDSGroup::name CDSDim * CDSDim* dsproc_get_dim (CDSGroup *dataset, const char *name) dsproc_get_dim CDSGroup * dataset const char * name Get a dimension from a dataset. dataset - pointer to the dataset name - name of the dimension pointer to the dimensionNULL if the dimension does not exist cds_get_dim size_t size_t dsproc_get_dim_length (CDSGroup *dataset, const char *name) dsproc_get_dim_length CDSGroup * dataset const char * name Get the length of a dimension in a dataset. dataset - pointer to the dataset name - name of the dimension dimension length0 if the dimension does not exist or has 0 length cds_get_dim CDSDim::length int int dsproc_set_dim_length (CDSGroup *dataset, const char *name, size_t length) dsproc_set_dim_length CDSGroup * dataset const char * name size_t length Set the length of a dimension in a dataset. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset - pointer to the dataset name - name of the dimension length - new length of the dimension 1 if successful0 if: the dimension does not existthe dimension definition is lockeddata has already been added to a variable using this dimension cds_get_dim CDSDim::def_lock ERROR DSPROC_LIB_NAME cds_get_object_path cds_change_dim_length dsproc_set_status DSPROC_ECDSSETDIM int int dsproc_change_att (void *parent, int overwrite, const char *name, CDSDataType type, size_t length, void *value) dsproc_change_att void * parent int overwrite const char * name CDSDataType type size_t length void * value Change an attribute for a dataset or variable. This function will define the specified attribute if it does not exist. If the attribute does exist and the overwrite flag is set, the data type and value will be changed.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. parent - pointer to the parent CDSGroup or CDSVar overwrite - overwrite flag (1 = TRUE, 0 = FALSE) name - attribute name type - attribute data type length - attribute length value - pointer to the attribute value 1 if successful0 if: the parent object is not a group or variablethe parent group or variable definition is lockedthe attribute definition is lockeda memory allocation error occurred cds_change_att dsproc_set_status DSPROC_ECDSCHANGEATT CDSAtt * CDSAtt* dsproc_get_att (void *parent, const char *name) dsproc_get_att void * parent const char * name Get an attribute from a dataset or variable. parent - pointer to the parent CDSGroup or CDSVar name - name of the attribute pointer to the attributeNULL if the attribute does not exist cds_get_att char * char* dsproc_get_att_text (void *parent, const char *name, size_t *length, char *value) dsproc_get_att_text void * parent const char * name size_t * length char * value Get a copy of an attribute value from a dataset or variable. This function will get a copy of an attribute value converted to a text string. If the data type of the attribute is not CDS_CHAR the cds_array_to_string() function is used to create the output string.Memory will be allocated for the returned string if the output string is NULL. In this case the calling process is responsible for freeing the allocated memory.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. parent - pointer to the parent CDSGroup or CDSVar name - name of the attribute length - pointer to the length of the output string input: length of the output stringignored if the output string is NULL output: number of characters written to the output string0 if the attribute value has zero length(size_t)-1 if a memory allocation error occurs value - pointer to the output string or NULL to dynamically allocate the memory needed. pointer to the output stringNULL if: the attribute does not exist or has zero length (length = 0)a memory allocation error occurs (length == (size_t)-1) cds_get_att cds_get_att_text dsproc_set_status DSPROC_ENOMEM void * void* dsproc_get_att_value (void *parent, const char *name, CDSDataType type, size_t *length, void *value) dsproc_get_att_value void * parent const char * name CDSDataType type size_t * length void * value Get a copy of an attribute value from a dataset or variable. This function will get a copy of an attribute value casted into the specified data type. The functions cds_string_to_array() and cds_array_to_string() are used to convert between text (CDS_CHAR) and numeric data types.Memory will be allocated for the returned array if the output array is NULL. In this case the calling process is responsible for freeing the allocated memory.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. parent - pointer to the parent CDSGroup or CDSVar name - name of the attribute type - data type of the output array length - pointer to the length of the output array input: length of the output arrayignored if the output array is NULL output: number of values written to the output array0 if the attribute value has zero length(size_t)-1 if a memory allocation error occurs value - pointer to the output array or NULL to dynamically allocate the memory needed. pointer to the output arrayNULL if: the attribute value has zero length (length == 0)a memory allocation error occurs (length == (size_t)-1) cds_get_att cds_get_att_value dsproc_set_status DSPROC_ENOMEM int int dsproc_set_att (void *parent, int overwrite, const char *name, CDSDataType type, size_t length, void *value) dsproc_set_att void * parent int overwrite const char * name CDSDataType type size_t length void * value Set the value of an attribute in a dataset or variable. This function will define the specified attribute if it does not exist. If the attribute does exist and the overwrite flag is set, the value will be set by casting the specified value into the data type of the attribute. The functions cds_string_to_array() and cds_array_to_string() are used to convert between text (CDS_CHAR) and numeric data types.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. parent - pointer to the parent CDSGroup or CDSVar overwrite - overwrite flag (1 = TRUE, 0 = FALSE) name - attribute name type - attribute data type length - attribute length value - pointer to the attribute value 1 if successful0 if: the parent object is not a group or variablethe parent group or variable definition is lockedthe attribute definition is lockeda memory allocation error occurred cds_set_att dsproc_set_status DSPROC_ECDSSETATT int int dsproc_set_att_text (void *parent, const char *name, const char *format,...) dsproc_set_att_text void * parent const char * name const char * format ... Set the value of an attribute in a dataset or variable. The cds_string_to_array() function will be used to set the attribute value if the data type of the attribute is not CDS_CHAR.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. parent - pointer to the parent CDSGroup or CDSVar name - name of the attribute format - format string (see printf) ... - arguments for the format string 1 if successful0 if: the attribute does not existthe attribute definition is lockeda memory allocation error occurred msngr_format_va_list ERROR DSPROC_LIB_NAME cds_get_object_path dsproc_set_status DSPROC_ENOMEM dsproc_set_att_value CDS_CHAR int int dsproc_set_att_value (void *parent, const char *name, CDSDataType type, size_t length, void *value) dsproc_set_att_value void * parent const char * name CDSDataType type size_t length void * value Set the value of an attribute in a dataset or variable. This function will set the value of an attribute by casting the specified value into the data type of the attribute. The functions cds_string_to_array() and cds_array_to_string() are used to convert between text (CDS_CHAR) and numeric data types.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. parent - pointer to the parent CDSGroup or CDSVar name - name of the attribute type - data type of the specified value length - length of the specified value value - pointer to the attribute value 1 if successful0 if: the attribute does not existthe attribute definition is lockeda memory allocation error occurred cds_get_att CDSAtt::def_lock ERROR DSPROC_LIB_NAME cds_get_object_path dsproc_set_status DSPROC_ECDSSETATT cds_set_att_value DSPROC_ENOMEM int int dsproc_set_att_value_if_null (void *parent, const char *name, CDSDataType type, size_t length, void *value) dsproc_set_att_value_if_null void * parent const char * name CDSDataType type size_t length void * value Set the value of an attribute if the current value is NULL. This function will check if the value for the specified attribute is NULL.If the attribute does not exist or the value is not NULL, nothing will be done and the function will return successfully.If the value is NULL, it will be set by casting the specified value into the data type of the attribute. The functions cds_string_to_array() and cds_array_to_string() are used to convert between text (CDS_CHAR) and numeric data types.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. parent - pointer to the parent CDSGroup or CDSVar name - name of the attribute type - data type of the specified value length - length of the specified value value - pointer to the attribute value 1 if successful0 if a memory allocation error occurred cds_get_att CDSAtt::length CDSAtt::def_lock cds_set_att_value dsproc_set_status DSPROC_ENOMEM int int dsproc_set_att_text_if_null (void *parent, const char *name, const char *format,...) dsproc_set_att_text_if_null void * parent const char * name const char * format ... Set the value of an attribute if the current value is NULL. This function will check if the value for the specified attribute is NULL.If the attribute does not exist or the value is not NULL, nothing will be done and the function will return successfully.If the value is NULL, it will be set to the specified value. The cds_string_to_array() function will be used to set the attribute value if the data type of the attribute is not CDS_CHAR.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. parent - pointer to the parent CDSGroup or CDSVar name - name of the attribute format - format string (see printf) ... - arguments for the format string 1 if successful0 if a memory allocation error occurred cds_get_att CDSAtt::length CDSAtt::def_lock msngr_format_va_list ERROR DSPROC_LIB_NAME cds_get_object_path dsproc_set_status DSPROC_ENOMEM cds_set_att_value CDS_CHAR int int dsproc_set_var_chunksizes (CDSVar *var, int *time_chunksize) dsproc_set_var_chunksizes CDSVar * var int * time_chunksize Set the _ChunkSizes attribute value for a variable. This function will use the lengths the variable's dimensions to set the value for the _ChunkSizes attribute if it has not already been defined.The time_chunksize argument can be used to specify the size that should be used for the time dimension. If the specified value is 0, the chunk size for the time dimension will be computed using the maximum number of samples per hour as determined from the parent dataset.If the size of the uncompressed chunk exceeds the maximum allowed size, the chunk size for the first dimension will be cut in half until the size of the chunk is within limits. This will continue on to the secondary dimesnions if necessary. By default the maximum allowed size of an uncompressed chunk is 4 MiB, this value can be changed using the cds_set_max_chunksize() function.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the CDSVar time_chunksize - input/output: chunk size to use for the time dimension. This will be computed and returned if the value is 0. 1 if successful0 if the variable does not have a _ChunkSizes attribute, the _ChunkSizes attribute value has already been set, or no time values were found in the parent dataset.-1 if an error occurred cds_get_att CDSAtt::length CDSAtt::type CDS_INT ERROR DSPROC_LIB_NAME cds_get_object_path cds_data_type_name CDSVar::ndims cds_data_type_size CDSVar::type CDSVar::dims CDSDim::name CDSVar::parent CDSDim::length DEBUG_LV1 CDSVar::name cds_array_to_string cds_set_att_value dsproc_set_status DSPROC_ECDSSETATT void void dsproc_set_max_chunksize (size_t max_chunksize) dsproc_set_max_chunksize size_t max_chunksize Set the maximum size of a chunk to use when setting _ChunkSizes. max_chunksize - maximum size of an uncompressed chunk in bytes int int dsproc_set_chunksizes (CDSGroup *dataset, int time_chunksize) dsproc_set_chunksizes CDSGroup * dataset int time_chunksize Set the values of all _ChunkSizes attributes that have not been defined. This function will call dsproc_set_var_chunksizes() for every variable in a dataset that has one or more dimensions.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset - pointer to the dataset time_chunksize - chunk size to use for the time dimension, or 0 to compute it. 1 if successful0 if an error occurred CDSGroup::nvars CDSGroup::vars CDSVar::ndims dsproc_set_var_chunksizes CDSVar * CDSVar* dsproc_clone_var (CDSVar *src_var, CDSGroup *dataset, const char *var_name, CDSDataType data_type, const char **dim_names, int copy_data) dsproc_clone_var CDSVar * src_var CDSGroup * dataset const char * var_name CDSDataType data_type const char ** dim_names int copy_data Create a clone of an existing variable. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. src_var - pointer to the source variable to clone dataset - dataset to create the new variable in, or NULL to create the variable in the same dataset the source variable belongs to. var_name - name to use for the new variable, or NULL to use the source variable name. data_type - data type to use for the new variable, or CDS_NAT to use the same data type as the source variable. dim_names - pointer to the list of corresponding dimension names in the dataset the new variable will be created in, or NULL if the dimension names are the same. copy_data - flag indicating if the data should be copied, (1 == TRUE, 0 == FALSE) pointer to the new variableNULL if: the variable already exists in the dataseta memory allocation error occurred CDSVar::parent CDSVar::name CDSVar::type CDSVar::ndims CDSVar::dims CDSDim::name cds_get_var ERROR DSPROC_LIB_NAME cds_get_object_path dsproc_set_status DSPROC_ECLONEVAR cds_get_dim cds_define_dim CDSDim::length CDSDim::is_unlimited cds_define_var DSPROC_ENOMEM cds_copy_var CDSVar::sample_count CDS_SKIP_DATA CDSVar * CDSVar* dsproc_define_var (CDSGroup *dataset, const char *name, CDSDataType type, int ndims, const char **dim_names, const char *long_name, const char *standard_name, const char *units, void *valid_min, void *valid_max, void *missing_value, void *fill_value) dsproc_define_var CDSGroup * dataset const char * name CDSDataType type int ndims const char ** dim_names const char * long_name const char * standard_name const char * units void * valid_min void * valid_max void * missing_value void * fill_value Define a new variable in an existing dataset. This function will define a new variable with all standard attributes. Any of the attribute values can be NULL to indicate that the attribute should not be created.Description of Attributes:long_name: This is a one line description of the variable and should be suitable to use as a plot title for the variable.standard_name: This is defined in the CF Convention and describes the physical quantities being represented by the variable. Please refer to the "CF Standard Names" section of the CF Convention for the table of standard names.units: This is the units string to use for the variable and must be recognized by the UDUNITS-2 libary.valid_min: The smallest value that should be considered to be a valid data value. The specified value must be the same data type as the variable.valid_max: The largest value that should be considered to be a valid data value. The specified value must be the same data type as the variable.missing_value: This comes from an older NetCDF convention and has been used by ARM for almost 2 decades. The specified value must be the same data type as the variable._FillValue: Most newer conventions specify the use of _FillValue over missing_value. The value of this attribute is also recognized by the NetCDF library and will be used to initialize the data values on disk when the variable is created. Tools like ncdump will also display fill values as _ so they can be easily identified in a text dump. The libdsproc3 library allows you to use both missing_value and _FillValue and they do not need to be the same. The specified value must be the same data type as the variable.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset - pointer to the dataset name - name of the variable type - data type of the variable ndims - number of variable dimensions dim_names - array of pointers to the dimension names long_name - string to use for the long_name attribute standard_name - string to use for the standard_name attribute units - string to use for the units attribute valid_min - void pointer to the valid_min valid_max - void pointer to the valid_max missing_value - void pointer to the missing_value fill_value - void pointer to the _FillValue pointer to the new variableNULL if an error occurred cds_define_var dsproc_set_status DSPROC_ECDSDEFVAR cds_define_att_text cds_define_att ERROR DSPROC_LIB_NAME cds_get_object_path cds_delete_var DSPROC_ENOMEM int int dsproc_delete_var (CDSVar *var) dsproc_delete_var CDSVar * var Delete a variable from a dataset. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable 1 if the variable was deleted (or the input var was NULL)0 if: the variable is lockedthe group is locked cds_delete_var dsproc_set_status DSPROC_ECDSDELVAR CDSVar * CDSVar* dsproc_get_bounds_var (CDSVar *coord_var) dsproc_get_bounds_var CDSVar * coord_var Get the boundary variable for a coordinate variable. coord_var - pointer to the coordinate variable pointer to the boundary variableNULL if not found cds_get_bounds_var CDSVar * CDSVar* dsproc_get_coord_var (CDSVar *var, int dim_index) dsproc_get_coord_var CDSVar * var int dim_index Get the coordinate variable for a variable's dimension. var - pointer to the variable dim_index - index of the dimension pointer to the coordinate variableNULL if not found cds_get_coord_var int int dsproc_get_dataset_vars (CDSGroup *dataset, const char **var_names, int required, CDSVar ***vars, CDSVar ***qc_vars, CDSVar ***aqc_vars) dsproc_get_dataset_vars CDSGroup * dataset const char ** var_names int required CDSVar *** vars CDSVar *** qc_vars CDSVar *** aqc_vars Get variables and companion QC variables from a dataset. If nvars is 0 or var_names is NULL, the output vars array will contain the pointers to the variables that are not companion QC variables. In this case the variables in the vars array will be in the same order they appear in the dataset. The following time and location variables will be excluded from this array: base_timetime_offsettimetime_boundslatlonalt If nvars and var_names are specified, the output vars array will contain an entry for every variable in the var_names list, and will be in the specified order. Variables that are not found in the dataset will have a NULL value if the required flag is set to 0. If the required flag is set to 1 and a variable does not exist, an error will be generated.If the qc_vars argument is not NULL it will contain the pointers to the companion qc_ variables. Likewise, if the aqc_vars argument is not NULL it will contain the pointers to the companion aqc_ variables. If a companion QC variable does not exist for a variable, the corresponding entry in the QC array will be NULL.The memory used by the returned arrays belongs to a 'dsproc_user_...' CDSVarGroup defined in the dataset and must NOT be freed by the calling process.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset - pointer to the dataset var_names - NULL terminated list of variable names required - specifies if all variables in the names list are required vars - output: pointer to the array of variables qc_vars - output: pointer to the array of companion qc_ variables aqc_vars - output: pointer to the array of companion aqc_ variables length of the output arrays-1 if an error occurred cds_get_vargroup cds_define_vargroup dsproc_set_status DSPROC_ENOMEM CDSGroup::nvars ERROR DSPROC_LIB_NAME cds_get_object_path cds_delete_vargroup CDSGroup::vars CDSVar::name cds_get_var cds_get_vararray CDSVarArray::vars cds_define_vararray DSPROC_EREQVAR cds_add_vararray_vars msngr_debug_level cds_print_vargroup CDS_SKIP_VARS CDSVar * CDSVar* dsproc_get_metric_var (CDSVar *var, const char *metric) dsproc_get_metric_var CDSVar * var const char * metric Get a companion metric variable for a variable. Known metrics at the time of this writing (so there may be others): "frac": the fraction of available input values used"std": the standard deviation of the calculated value var - pointer to the variable metric - name of the metric pointer to the metric variableNULL if not found CDSVar::name cds_get_var CDSVar::parent CDSVar * CDSVar* dsproc_get_output_var (int ds_id, const char *var_name, int obs_index) dsproc_get_output_var int ds_id const char * var_name int obs_index Get a variable from an output dataset. The obs_index should always be zero unless observation based processing is being used. This is because all input observations should have been merged into a single observation in the output datasets. ds_id - output datastream ID var_name - variable name obs_index - the index of the obervation to get the dataset for pointer to the output variableNULL if it does not exist dsproc_get_output_dataset cds_get_var CDSVar * CDSVar* dsproc_get_qc_var (CDSVar *var) dsproc_get_qc_var CDSVar * var Get the companion QC variable for a variable. var - pointer to the variable pointer to the QC variableNULL if not found CDSVar::name cds_get_var CDSVar::parent CDSVar * CDSVar* dsproc_get_retrieved_var (const char *var_name, int obs_index) dsproc_get_retrieved_var const char * var_name int obs_index Get a primary variable from the retrieved data. This function will find a variable in the retrieved data that was explicitly requested by the user in the retriever definition.The obs_index is used to specify which observation to pull the variable from. This value will typically be zero unless this function is called from a post_retrieval_hook() function, or the process is using observation based processing. In either of these cases the retrieved data will contain one "observation" for every file the data was read from on disk.It is also possible to have multiple observations in the retrieved data when a pre_transform_hook() is called if a dimensionality conflict prevented all observations from being merged. var_name - variable name obs_index - the index of the obervation to get the variable from pointer to the retrieved variableNULL if not found _DSProc CDSGroup::ngroups CDSGroup::groups cds_get_var cds_get_user_data CDSVar * CDSVar* dsproc_get_transformed_var (const char *var_name, int obs_index) dsproc_get_transformed_var const char * var_name int obs_index Get a primary variable from the transformed data. This function will find a variable in the transformed data that was explicitly requested by the user in the retriever definition.The obs_index is used to specify which observation to pull the variable from. This value will typically be zero unless the process is using observation based processing. If this is the case the transformed data will contain one "observation" for every file the data was read from on disk. var_name - variable name obs_index - the index of the obervation to get the variable from pointer to the transformed variableNULL if not found _DSProc CDSGroup::ngroups CDSGroup::groups cds_get_var cds_get_user_data CDSVar * CDSVar* dsproc_get_trans_coordsys_var (const char *coordsys_name, const char *var_name, int obs_index) dsproc_get_trans_coordsys_var const char * coordsys_name const char * var_name int obs_index Get a variable from a transformation coordinate system. Unlike the dsproc_get_transformed_var() function, this function will find any variable in the specified transformation coordinate system.The obs_index is used to specify which observation to pull the variable from. This value will typically be zero unless the process is using observation based processing. If this is the case the transformed data will contain one "observation" for every file the data was read from on disk. coordsys_name - coordinate system name var_name - variable name obs_index - the index of the obervation to get the variable from pointer to the transformed variableNULL if not found _DSProc CDSGroup::ngroups CDSGroup::groups CDSGroup::name cds_get_var CDSVar * CDSVar* dsproc_get_var (CDSGroup *dataset, const char *name) dsproc_get_var CDSGroup * dataset const char * name Get a variable from a dataset. dataset - pointer to the dataset name - name of the variable pointer to the variableNULL if the variable does not exist cds_get_var const char * const char* dsproc_var_name (CDSVar *var) dsproc_var_name CDSVar * var Returns the variable name. The returned name belongs to the variable structure and must not be freed or altered by the calling process. var - pointer to the variable pointer to the variable nameNULL if the specified variable is NULL CDSVar::name size_t size_t dsproc_var_sample_count (CDSVar *var) dsproc_var_sample_count CDSVar * var Returns the number of samples in a variable's data array. The sample_count for a variable is the number of samples stored along the variable's first dimension. var - pointer to the variable number of samples in the variable's data array CDSVar::sample_count size_t size_t dsproc_var_sample_size (CDSVar *var) dsproc_var_sample_size CDSVar * var Returns the sample size of a variable. Variables with less than 2 dimensions will always have a sample_size of 1. The sample_size for variables with 2 or more dimensions is the product of all the dimension lengths starting with the 2nd dimension. var - pointer to the variable sample size of the variable cds_var_sample_size void * void* dsproc_alloc_var_data (CDSVar *var, size_t sample_start, size_t sample_count) dsproc_alloc_var_data CDSVar * var size_t sample_start size_t sample_count Allocate memory for a variable's data array. This function will allocate memory as necessary to ensure that the variable's data array is large enough to store another sample_count samples starting from sample_start.The data type of the returned array will be the same as the variable’s data type. It is the responsibility of the calling process to cast the returned array into the proper data type. If the calling process does not know the data type of the variable, it can store the data in an array of a known type and then use the dsproc_set_var_data() function to cast this data into the variables data array. In this case the memory does not need to be preallocated and this function is not needed.The data array returned by this function belongs to the variable and will be freed when the variable is destroyed. The calling process must not attempt to free this memory.For multi-dimensional variables the data array is stored linearly in memory with the last dimension varying the fastest. See the dsproc_alloc_var_data_index() and/or dsproc_get_var_data_index() functions to get a "data index" into this array.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable sample_start - start sample (0 based indexing) sample_count - number of new samples pointer to the specifed start sample in the variable's data arrayNULL if: one of the variable's static dimensions has 0 lengththe variable has no dimensions, and sample_start is not equal to 0 or sample_count is not equal to 1.the first variable dimension is not unlimited, and sample_start + sample_count would exceed the dimension length.a memory allocation error occurred cds_alloc_var_data dsproc_set_status DSPROC_ECDSALLOCVAR void * void* dsproc_alloc_var_data_index (CDSVar *var, size_t sample_start, size_t sample_count) dsproc_alloc_var_data_index CDSVar * var size_t sample_start size_t sample_count Allocate memory for a variable's data array. This function is the same as dsproc_alloc_var_data() except that it returns a data index starting at the specified start sample (see dsproc_get_var_data_index()). For variables that have less than two dimensions this function is identical to dsproc_alloc_var_data(). It is up to the calling process to cast the return value of this function into the proper data type.The data index returned by this function belongs to the variable and will be freed when the variable is destroyed. The calling process must not attempt to free this memory.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable sample_start - start sample (0 based indexing) sample_count - number of new samples the data index into the variable's data array starting at the specified start sampleNULL if: one of the variable's static dimensions has 0 lengththe variable has no dimensions, and sample_start is not equal to 0 or sample_count is not equal to 1.the first variable dimension is not unlimited, and sample_start + sample_count would exceed the dimension length.a memory allocation error occurred cds_alloc_var_data_index dsproc_set_status DSPROC_ECDSALLOCVAR void * void* dsproc_get_var_data_index (CDSVar *var) dsproc_get_var_data_index CDSVar * var Get a data index for a multi-dimensional variable. This function will return a data index that can be used to access the data in a variable using the traditional x[i][j][k] syntax. It is up to the calling process to cast the return value of this function into the proper data type.Note: If the variable has less than 2 dimensions, the pointer to the variable’s data array will be returned.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable the data index into the variable’s data arrayNULL if: the pointer to the variable was NULLno data has been stored in the variable (var->sample_count == 0)a memory allocation error occurs CDSVar::sample_count cds_create_var_data_index dsproc_set_status DSPROC_ENOMEM void * void* dsproc_get_var_data (CDSVar *var, CDSDataType type, size_t sample_start, size_t *sample_count, void *missing_value, void *data) dsproc_get_var_data CDSVar * var CDSDataType type size_t sample_start size_t * sample_count void * missing_value void * data Get a copy of the data from a dataset variable. This function will get the data from a variable casted into the specified data type. All missing values used in the data will be converted to a single missing value appropriate for the requested data type. The missing value used will be the first value returned by cds_get_var_missing_values() if that value is within the range of the requested data type, otherwise, the default fill value for the requested data type will be used.Memory will be allocated for the returned data array if the output array is NULL. In this case the calling process is responsible for freeing the allocated memory. If an output data array is specified it must be large enough to hold (sample_count * cds_var_sample_size(var)) elements.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable type - data type of the output missing_value and data array sample_start - start sample (0 based indexing) sample_count - pointer to the sample_count input: length of the output arrayignored if the output array is NULL output: number of samples returned0 if no data for sample_start(size_t)-1 if a memory allocation error occurs missing_value - output: missing value data - pointer to the output data array or NULL to dynamically allocate the memory needed. pointer to the output data arrayNULL if: the pointer to the variable was NULLthe variable has no data for sample_start (sample_count == 0)a memory allocation error occurs (sample_count == (size_t)-1) cds_get_var_data dsproc_set_status DSPROC_ENOMEM int int dsproc_get_var_missing_values (CDSVar *var, void **values) dsproc_get_var_missing_values CDSVar * var void ** values Get the missing values for a CDS Variable. This function will return an array containing all values specified by the missing_value and _FillValue attributes (in that order), and will be the same data type as the variable. If the _FillValue attribute does not exist but a default fill value has been defined, it will be used instead.The memory used by the output array of missing values is dynamically allocated. It is the responsibility of the calling process to free this memory when it is no longer needed.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable values - output: pointer to the array of missing values, the data type of this array will be the same as the variable number of missing values0 if there are no missing or fill values-1 if a memory allocation error occurs cds_get_var_missing_values dsproc_set_status DSPROC_ENOMEM void * void* dsproc_init_var_data (CDSVar *var, size_t sample_start, size_t sample_count, int use_missing) dsproc_init_var_data CDSVar * var size_t sample_start size_t sample_count int use_missing Initialize the data values for a dataset variable. This function will make sure enough memory is allocated for the specified samples and initializing the data values to either the variable's missing value (use_missing == 1), or 0 (use_missing == 0).The search order for missing values is: missing_value attribute_FillValue attributevariable's default missing value If the variable does not have any missing or fill values defined the default fill value for the variable's data type will be used and the default fill value for the variable will be set.If the specified start sample is greater than the variable's current sample count, the hole between the two will be filled with the first missing or fill value defined for the variable.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable sample_start - start sample of the data to initialize (0 based indexing) sample_count - number of samples to initialize use_missing - flag indicating if the variables missing value should be used (1 == TRUE, 0 == fill with zeros) pointer to the specifed start sample in the variable's data arrayNULL if: the specified sample count is zeroone of the variable's static dimensions has 0 lengththe variable has no dimensions, and sample_start is not equal to 0 or sample_count is not equal to 1.the first variable dimension is not unlimited, and sample_start + sample_count would exceed the dimension length.a memory allocation error occurred cds_init_var_data dsproc_set_status DSPROC_ECDSALLOCVAR void * void* dsproc_init_var_data_index (CDSVar *var, size_t sample_start, size_t sample_count, int use_missing) dsproc_init_var_data_index CDSVar * var size_t sample_start size_t sample_count int use_missing Initialize the data values for a dataset variable. This function behaves the same as dsproc_init_var_data() except that it returns a data index starting at the specified start sample, see dsproc_get_var_data_index() for details. For variables that have less than two dimensions this function is identical to dsproc_init_var_data().The data index returned by this function belongs to the CDS variable and will be freed when the variable is destroyed. The calling process must not attempt to free this memory.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable sample_start - start sample of the data to initialize (0 based indexing) sample_count - number of samples to initialize use_missing - flag indicating if the variables missing value should be used (1 == TRUE, 0 == fill with zeros) the data index into the variables data array starting at the specified start sample.NULL if: the specified sample count is zeroone of the variable's static dimensions has 0 lengththe variable has no dimensions, and sample_start is not equal to 0 or sample_count is not equal to 1.the first variable dimension is not unlimited, and sample_start + sample_count would exceed the dimension length.a memory allocation error occurred cds_init_var_data_index dsproc_set_status DSPROC_ECDSALLOCVAR int int dsproc_set_bounds_data (CDSGroup *dataset, size_t sample_start, size_t sample_count) dsproc_set_bounds_data CDSGroup * dataset size_t sample_start size_t sample_count Set cell boundary data for all coordinate variables in a dataset. This function will call dsproc_set_bounds_var_data() for all variables in the specified dataset that have a bounds attribute defined.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset - pointer to the CDS variable sample_start - start sample along the unlimited dimension (0 based indexing) sample_count - number of samples along the unlimited dimension (0 for all available samples) 1 if successful0 if an error occurred. CDSGroup::nvars CDSGroup::vars cds_get_att CDSVar::ndims CDSVar::sample_count CDSVar::dims CDSDim::is_unlimited dsproc_set_bounds_var_data int int dsproc_set_bounds_var_data (CDSVar *coord_var, size_t sample_start, size_t sample_count) dsproc_set_bounds_var_data CDSVar * coord_var size_t sample_start size_t sample_count Set cell boundary data values for a dataset variable. This function can only be used to create the cell boundary data values for regular grids and requires: The variable has a bounds attribute defined that specifies the name of the boundary variable. The boundary variable has a bound_offsets attribute defined that specifies the cell boundary offsets from the data values. The number of boundary variable dimensions is one greater than than the number of variable dimensions. The boundary variable and bound_offsets attribute have the same data type as the variable. The bound_offsets attribute has the same length as the last dimension of the boundary variable. coord_var - pointer to the coordinate variable sample_start - start sample (0 based indexing) sample_count - number of samples (0 for all available samples) 1 if successful0 if the bounds and/or bound_offsets attributes are not defined, or the variable does not have any data for the specified sample_start.-1 if an error occurred. cds_set_bounds_var_data dsproc_set_status DSPROC_EBOUNDSVAR void * void* dsproc_set_var_data (CDSVar *var, CDSDataType type, size_t sample_start, size_t sample_count, void *missing_value, void *data) dsproc_set_var_data CDSVar * var CDSDataType type size_t sample_start size_t sample_count void * missing_value void * data Set the data values for a dataset variable. This function will set the data values of a variable by casting the values in the input data array into the data type of the variable. All missing values in the input data array will be converted to the first missing value used by the variable as returned by cds_get_var_missing_values(). If the variable does not have a missing_value or _FillValue attribute defined, the default fill value for the variable's data type will be used.For multi-dimensional variables, the specified data array must be stored linearly in memory with the last dimension varying the fastest.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable type - data type of the input missing_value and data array sample_start - start sample of the new data (0 based indexing) sample_count - number of new samples missing_value - pointer to the missing value used in the data array, or NULL if the data does not contain any missing values data - pointer to the input data array pointer to the specifed start sample in the variable's data arrayNULL if: one of the variable's static dimensions has 0 lengththe variable has no dimensions, and sample_start is not equal to 0 or sample_count is not equal to 1.the first variable dimension is not unlimited, and sample_start + sample_count would exceed the dimension length.a memory allocation error occurred cds_set_var_data dsproc_set_status DSPROC_ECDSSETDATA time_t time_t dsproc_get_base_time (void *cds_object) dsproc_get_base_time void * cds_object Get the base time of a dataset or time variable. This function will convert the units attribute of a time variable to seconds since 1970. If the input object is a CDSGroup, the specified dataset and then its parent datasets will be searched until a "time" or "time_offset" variable is found. cds_object - pointer to a CDSGroup or CDSVar base time in seconds since 1970 UTC0 if not found cds_get_base_time size_t size_t dsproc_get_time_range (void *cds_object, timeval_t *start_time, timeval_t *end_time) dsproc_get_time_range void * cds_object timeval_t * start_time timeval_t * end_time Get the time range of a dataset or time variable. This function will get the start and end times of a time variable. If the input object is a CDSGroup, the specified dataset and then its parent datasets will be searched until a "time" or "time_offset" variable is found. cds_object - pointer to a CDSGroup or CDSVar start_time - pointer to the timeval_t structure to store the start time in. end_time - pointer to the timeval_t structure to store the end time in. number of time values0 if no time values were found cds_get_time_range CDSVar * CDSVar* dsproc_get_time_var (void *cds_object) dsproc_get_time_var void * cds_object Get the time variable used by a dataset. This function will find the first dataset that contains either the "time" or "time_offset" variable and return a pointer to that variable. cds_object - pointer to a CDSGroup or CDSVar pointer to the time variableNULL if not found cds_find_time_var time_t * time_t* dsproc_get_sample_times (void *cds_object, size_t sample_start, size_t *sample_count, time_t *sample_times) dsproc_get_sample_times void * cds_object size_t sample_start size_t * sample_count time_t * sample_times Get the sample times for a dataset or time variable. This function will convert the data values of a time variable to seconds since 1970. If the input object is a CDSGroup, the specified dataset and then its parent datasets will be searched until a "time" or "time_offset" variable is found.Memory will be allocated for the returned array of sample times if the output array is NULL. In this case the calling process is responsible for freeing the allocated memory.Note: If the sample times can have fractional seconds the dsproc_get_sample_timevals() function should be used instead.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. cds_object - pointer to a CDSGroup or CDSVar sample_start - start sample (0 based indexing) sample_count - pointer to the sample_count input: length of the output arrayignored if the output array is NULL output: number of sample times returned0 if no data was found for sample_start(size_t)-1 if a memory allocation error occurs sample_times - pointer to the output array or NULL to dynamically allocate the memory needed. pointer to the array of sample times in seconds since 1970NULL if: there is no data for sample_start (sample_count == 0)an error occurred (sample_count == (size_t)-1) cds_get_sample_times dsproc_set_status DSPROC_ECDSGETTIME timeval_t * timeval_t* dsproc_get_sample_timevals (void *cds_object, size_t sample_start, size_t *sample_count, timeval_t *sample_times) dsproc_get_sample_timevals void * cds_object size_t sample_start size_t * sample_count timeval_t * sample_times Get the sample times for a dataset or time variable. This function will convert the data values of a time variable to an array of timeval_t structures. If the input object is a CDSGroup, the specified dataset and then its parent datasets will be searched until a "time" or "time_offset" variable is found.Memory will be allocated for the returned array of sample times if the output array is NULL. In this case the calling process is responsible for freeing the allocated memory.Note: Consider using the cds_get_sample_times() function if the sample times can not have fractional seconds.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. cds_object - pointer to a CDSGroup or CDSVar sample_start - start sample (0 based indexing) sample_count - pointer to the sample_count input: length of the output arrayignored if the output array is NULL output: number of sample times returned0 if no data was found for sample_start(size_t)-1 if a memory allocation error occurs sample_times - pointer to the output array or NULL to dynamically allocate the memory needed. pointer to the array of sample times in seconds since 1970NULL if: the variable has no data for sample_start (sample_count == 0)an error occurred (sample_count == (size_t)-1) cds_get_sample_timevals dsproc_set_status DSPROC_ECDSGETTIME int int dsproc_set_base_time (void *cds_object, const char *long_name, time_t base_time) dsproc_set_base_time void * cds_object const char * long_name time_t base_time Set the base time of a dataset or time variable. This function will set the base time for a time variable and adjust all attributes and data values as necessary. If the input object is one of the standard time variables ("time", "time_offset", or "base_time"), all standard time variables that exist in its parent dataset will also be updated. If the input object is a CDSGroup, the specified dataset and then its parent datasets will be searched until a "time" or "time_offset" variable is found. All standard time variables that exist in this dataset will then be updated.For the base_time variable the data value will be set and the "string" attribute will be set to the string representation of the base_time value. The "long_name" and "units" attributes will also be set to "Base time in Epoch" and "seconds since 1970-1-1 0:00:00 0:00" respectively.For the time_offset variable the "units" attribute will set to the string representation of the base_time value, and the "long_name" attribute will be set to "Time offset from base_time".For all other time variables the "units" attribute will be set to the string representation of the base_time value, and the "long_name" attribute will be set to the specified value. If a long_name attribute is not specified, the string "Time offset from midnight" will be used for base times of midnight, and "Sample times" will be used for all other base times.Any existing data in a time variable will also be adjusted for the new base_time value.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. cds_object - pointer to a CDSGroup or CDSVar long_name - string to use for the long_name attribute, or NULL to use the default value base_time - base time in seconds since 1970 1 if successful0 if an error occurred cds_set_base_time dsproc_set_status DSPROC_ECDSSETTIME int int dsproc_set_sample_times (void *cds_object, size_t sample_start, size_t sample_count, time_t *sample_times) dsproc_set_sample_times void * cds_object size_t sample_start size_t sample_count time_t * sample_times Set the sample times for a dataset or time variable. This function will set the data values for a time variable by subtracting the base time (as defined by the units attribute) and converting the remainder to the data type of the variable. If the input object is one of the standard time variables ("time", "time_offset", or "base_time"), all standard time variables that exist in its parent dataset will also be updated. If the input object is a CDSGroup, the specified dataset and then its parent datasets will be searched until a "time" or "time_offset" variable is found. All standard time variables that exist in this dataset will then be updated.If the specified sample_start value is 0 and a base time value has not already been set, the base time will be set using the time of midnight just prior to the first sample time.The data type of the time variable(s) must be CDS_SHORT, CDS_INT, CDS_FLOAT or CDS_DOUBLE. However, CDS_DOUBLE is usually recommended.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. cds_object - pointer to a CDSGroup or CDSVar sample_start - start sample (0 based indexing) sample_count - number of samples in the times array sample_times - pointer to the array of sample times in seconds since 1970 UTC. 1 if successful0 if an error occurred cds_set_sample_times dsproc_set_status DSPROC_ECDSSETTIME int int dsproc_set_sample_timevals (void *cds_object, size_t sample_start, size_t sample_count, timeval_t *sample_times) dsproc_set_sample_timevals void * cds_object size_t sample_start size_t sample_count timeval_t * sample_times Set the sample times for a dataset or time variable. This function will set the data values for a time variable by subtracting the base time (as defined by the units attribute) and converting the remainder to the data type of the variable.If the input object is one of the standard time variables: timetime_offsetbase_time All standard time variables that exist in its parent dataset will also be updated.If the input object is a CDSGroup, the specified dataset and then its parent datasets will be searched until a "time" or "time_offset" variable is found. All standard time variables that exist in this dataset will then be updated.If the specified sample_start value is 0 and a base time value has not already been set, the base time will be set using the time of midnight just prior to the first sample time.The data type of the time variable(s) must be either CDS_FLOAT or or CDS_DOUBLE. However, CDS_DOUBLE is usually recommended.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. cds_object - pointer to a CDSGroup or CDSVar sample_start - start sample (0 based indexing) sample_count - number of samples in the times array sample_times - pointer to the array of sample times in seconds since 1970 UTC. 1 if successful0 if an error occurred cds_set_sample_timevals dsproc_set_status DSPROC_ECDSSETTIME int int dsproc_consolidate_var_qc (CDSVar *in_qc_var, unsigned int bad_mask, CDSVar *out_qc_var, unsigned int bad_flag, unsigned int ind_flag, int reset) dsproc_consolidate_var_qc CDSVar * in_qc_var unsigned int bad_mask CDSVar * out_qc_var unsigned int bad_flag unsigned int ind_flag int reset Consolidate all QC bits for a variable into bad or indeterminate. This function will consolidate all the bad and indeterminate QC bits in the input QC variable into a single bad or indeterminate bit in the output QC variable. By default (reset == 0) the bit values of the output variable will be updated without affecting any of the bits that may have already been set. The output QC values can be reset to zero before setting the bad or indeterminate bit by specifying 1 for the reset parameter.The bad_mask can be determined using the bit assessment attributes by passing in 0 for it's value (see dsproc_get_bad_qc_mask()).The bad and indeteminate flag values are not the bit numbers. They are the actual numeric values of the bits. For example: bit_1 = pow(2,0) = (1 << 0) = 0x01 = 1bit_2 = pow(2,1) = (1 << 1) = 0x02 = 2bit_3 = pow(2,2) = (1 << 2) = 0x04 = 4bit_4 = pow(2,3) = (1 << 3) = 0x08 = 8bit_5 = pow(2,4) = (1 << 4) = 0x10 = 16bit_6 = pow(2,5) = (1 << 5) = 0x20 = 32bit_7 = pow(2,6) = (1 << 6) = 0x40 = 64bit_8 = pow(2,7) = (1 << 7) = 0x80 = 128 If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. in_qc_var - pointer to the input QC variable bad_mask - mask used to determine bad QC values in the input QC, or 0 to indicate that the bit assessment attributes should be used to determine it. out_qc_var - pointer to the output QC variable bad_flag - QC flag to use for bad values ind_flag - QC flag to use for indeterminate values reset - specifies if the output QC values should be reset to zero (0 = false, 1 = true) 1 if successful 0 if a fatal error occurred dsproc_get_bad_qc_mask CDSVar::type CDS_INT ERROR DSPROC_LIB_NAME cds_get_object_path dsproc_set_status DSPROC_EQCVARTYPE dsproc_var_sample_size CDSVar::sample_count DSPROC_EQCVARDIMS dsproc_init_var_data CDSVar::data CDSData::ip unsigned int unsigned int dsproc_get_bad_qc_mask (CDSVar *qc_var) dsproc_get_bad_qc_mask CDSVar * qc_var Get the QC mask used to determine bad QC values. This function will use the bit assessment attributes to create a mask with all bits set for bad assessment values. It will first check for field level bit assessment attributes, and then for the global attributes if they are not found. qc_var pointer to the QC variable qc_mask the QC mask with all bad bits set. dsproc_get_qc_assessment_mask int int dsproc_create_timestamp (time_t secs1970, char *timestamp) dsproc_create_timestamp time_t secs1970 char * timestamp Convert seconds since 1970 to a timestamp. This function will create a timestamp of the form: 'YYYYMMDD.hhmmss' The timestamp argument must be large enough to hold at least 16 characters (15 plus the null terminator).If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. secs1970 - seconds since 1970 timestamp - output: timestamp string in the form YYYYMMDD.hhmmss 1 if successful0 if an error occurred ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ETIMECALC int int dsproc_execvp (const char *file, char *const argv[], int flags) dsproc_execvp const char * file char *const argv [] int flags Wrapper to the execvp function. This function is a wrapper to the execvp function that redirects stdout and stderr to the process log file, and checks exit value of the spawned process for errors. If an error occurs it will be appended to the log and error mail messages, and the process status will be set appropriately.From the exec man page:The execvp() function provides an array of pointers to null-terminated strings that represent the argument list available to the new program. The first argument, by convention, should point to the filename associated with the file being executed. The array of pointers must be terminated by a NULL pointer.The execvp() function duplicates the actions of the shell in searching for an executable file if the specified filename does not contain a slash (/) character. The file is sought in the colon-separated list of directory pathnames specified in the PATH environment variable. If this variable isn't defined, the path list defaults to the current directory followed by the list of directories returned by confstr(_CS_PATH). (This confstr(3) call typically returns the value "/bin:/usr/bin".)The execvp() function takes the environment for the new process image from the external variable environ in the calling process. file - The name or path to the file that is to be executed. argv - A NULL terminated argument list. By convention the first argument should point to the name of the file being executed. flags - reserved for control flags exit_value exit value of the process, this is typically 0 for success and non-zero for errors. -1 if the process could not be executed msngr_get_log_file LogFile::fp ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM LOG DSPROC_EFORK DEBUG_LV1 int int dsproc_run_dq_inspector (int dsid, time_t begin_time, time_t end_time, const char *args[], int flags) dsproc_run_dq_inspector int dsid time_t begin_time time_t end_time const char * args [] int flags Run DQ Inspector. This function will run dq_inspector for the specified datastream and time range. The following arguments will be automatically added to the dq_inspector command line so they do not need to be specified by the calling process in the args[] array: -P -r read_path -d datastream -s start_date -e end_date If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately.A warning message will also be generated if dq_inpsector returns a non-zero exit value;Example quicklook_hook function: intadi_test3_quicklook_hook( void*user_data, time_tbegin_time, time_tend_time) { constchar*dsc_name="aditest3met"; constchar*dsc_level="c1"; intdsid; constchar*args[32]; intexit_value; intai; //prevent"unusedargument"compilerwarning user_data=user_data; dsid=dsproc_get_output_datastream_id(dsc_name,dsc_level); if(dsid<0)return(-1); //onlycreateplotforrhvariable ai=0; args[ai++]="-v"; args[ai++]="rh"; args[ai++]=NULL; exit_value=dsproc_run_dq_inspector( dsid,begin_time,end_time,args,0); if(exit_value<0)return(-1); if(exit_value>0)return(0); return(1); } dsid - Id of the output datastream. begin_time - Begin time of plot interval end_time - End time of plot interval args - Command line argument list to send to dq_instector. This list must be NULL terminated. flags - Reserved for control flags. Pass in 0 for this argument to maintain backward compatibility with future updates. exit_value dq_inspector exit value (0 == success). -1 if the process could not be executed dsproc_datastream_name dsproc_create_timestamp dsproc_execvp WARNING DSPROC_LIB_NAME int int dsproc_solar_position (time_t secs1970, double latitude, double longitude, double *ap_ra, double *ap_dec, double *altitude, double *refraction, double *azimuth, double *distance) dsproc_solar_position time_t secs1970 double latitude double longitude double * ap_ra double * ap_dec double * altitude double * refraction double * azimuth double * distance Calculate solar position. Wrapper function for solarposition, allowing UTC time as input.Author: Nels Larson, PNNLThis function employs the low precision formulas for the Sun's coordinates given in the "Astronomical Almanac" of 1990 to compute the Sun's apparent right ascension, apparent declination, altitude, atmospheric refraction correction applicable to the altitude, azimuth, and distance from Earth. The "Astronomical Almanac" (A. A.) states a precision of 0.01 degree for the apparent coordinates between the years 1950 and 2050, and an accuracy of 0.1 arc minute for refraction at altitudes of at least 15 degrees.The following assumptions and simplifications are made: refraction is calculated for standard atmosphere pressure and temperature at sea level.diurnal parallax is ignored, resulting in 0 to 9 arc seconds error in apparent position.diurnal aberration is also ignored, resulting in 0 to 0.02 second error in right ascension and 0 to 0.3 arc second error in declination.geodetic site coordinates are used, without correction for polar motion (maximum amplitude of 0.3 arc second) and local gravity anomalies.local mean sidereal time is substituted for local apparent sidereal time in computing the local hour angle of the Sun, resulting in an error of about 0 to 1 second of time as determined explicitly by the equation of the equinoxes. Right ascension is measured in hours from 0 to 24, and declination in degrees from 90 to -90.Altitude is measured from 0 degrees at the horizon to 90 at the zenith or -90 at the nadir.Azimuth is measured from 0 to 360 degrees starting at north and increasing toward the east at 90.The refraction correction should be added to the altitude if Earth's atmosphere is to be accounted for.Solar distance from Earth is in astronomical units, 1 a.u. representing the mean value.Revision by: Tim Shippert, PNNLAdded a smoother to refraction, so it varies linearly from it's value at alt == -1 to zero at alt == -2. This eliminates the discontinuity at alt == -1, and might even be kinda scientifically justified. secs1970 - Seconds since 1970 UTC latitude - Observation site geographic latitude. [degrees.fraction, North positive] longitude - Observation site geographic longitude. [degrees.fraction, East positive] ap_ra - output: Apparent solar right ascension. Argument can be NULL if the output value is not needed. [hours: 0.0 <= *ap_ra < 24.0] ap_dec - output: Apparent solar declination. Argument can be NULL if the output value is not needed. [degrees: -90.0 <= *ap_dec <= 90.0] altitude - output: Solar altitude, uncorrected for refraction. Argument can be NULL if the output value is not needed. [degrees: -90.0 <= *altitude <= 90.0] refraction - output: Refraction correction for solar altitude. Add this to altitude to compensate for refraction. Argument can be NULL if the output value is not needed. [degrees: 0.0 <= *refraction] azimuth - output: Solar azimuth. Argument can be NULL if the output value is not needed. [degrees: 0.0 <= *azimuth < 360.0, East is 90.0] distance - output: Distance of Sun from Earth (heliocentric-geocentric). Argument can be NULL if the output value is not needed. [astronomical units: 1 a.u. is mean distance] 1 if successful0 if an input parameter is out of bounds int int dsproc_flag_outliers_iqd (CDSGroup *dataset, const char *var_name, double window_width, int min_npoints, unsigned int skipped_flag, double bad_threshold, unsigned int bad_flag, double ind_threshold, unsigned int ind_flag, unsigned int analyze) dsproc_flag_outliers_iqd CDSGroup * dataset const char * var_name double window_width int min_npoints unsigned int skipped_flag double bad_threshold unsigned int bad_flag double ind_threshold unsigned int ind_flag unsigned int analyze Flag outliers using the Median and Interquartile Deviation Method (IQD). For this outlier detection method, the median of the residuals is calculated, along with the 25th percentile and the 75th percentile. The difference between the 25th and 75th percentile is the interquartile deviation (IQD). Then, the difference is calculated between each value and the residual median. If the value is a certain number of IQD away from the median of the residuals, that value is classified as an outlier. A typical default value for the threshold is 3.The 'analyze' option can be used to analyze the outlier detection results during development and help determine the best window width and threshold to use. Remember to set this value to 0 before creating a production release. Available options are: 1 = prints ranges of interquartile deviations from the median, and the number of points that fall within each range. 2 = for each point prints time offset, data value, window median, deviation from the median, interquartile deviation, and number of interquartile deviations the value is from the median. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset pointer to the dataset var_name name of the variable window_width width of window centered on data point (in seconds) min_npoints minimum number of values within window required to perform the test (default is 2). skipped_flag QC flag value to use for values that do not have enough points in window to perform the test. bad_threshold IQD factor used to flag outliers as bad bad_flag QC flag value to use for bad outliers ind_threshold IQD factor used to flag outliers as indeterminate ind_flag QC flag value to use for indeterminate outliers analyze Print statistics that may be helpfull during development (see above). 1 successful 0 a fatal error occurred DEBUG_LV1 DSPROC_LIB_NAME CDSGroup::name ERROR dsproc_set_status DSPROC_ENOMEM int int dsproc_flag_outliers_mean_dev (CDSGroup *dataset, const char *var_name, double window_width, int min_npoints, unsigned int skipped_flag, double bad_threshold, unsigned int bad_flag, double ind_threshold, unsigned int ind_flag, unsigned int analyze) dsproc_flag_outliers_mean_dev CDSGroup * dataset const char * var_name double window_width int min_npoints unsigned int skipped_flag double bad_threshold unsigned int bad_flag double ind_threshold unsigned int ind_flag unsigned int analyze Flag outliers using the Absolute Deviation from the Mean Method. For this outlier detection method values are flagged based on their absolute deviation from the mean of the surrounding values.The 'analyze' option can be used to analyze the outlier detection results during development and help determine the best window width and threshold to use. Remember to set this value to 0 before creating a production release. Available options are: 1 = prints ranges of absolute deviations from the mean, and the number of points that fall within each range. 2 = for each point prints time offset, data value, window mean, and absolute deviation from the mean. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset pointer to the dataset var_name name of the variable window_width width of window centered on data point (in seconds) min_npoints minimum number of values within window required to perform the test (default is 2). skipped_flag QC flag value to use for values that do not have enough points in window to perform the test. bad_threshold delta from mean used to flag outliers as bad bad_flag QC flag value to use for bad outliers ind_threshold delta from mean used to flag outliers as indeterminate ind_flag QC flag value to use for indeterminate outliers analyze Print statistics that may be helpfull during development (see above). 1 successful 0 a fatal error occurred DEBUG_LV1 DSPROC_LIB_NAME CDSGroup::name ERROR dsproc_set_status DSPROC_ENOMEM int int dsproc_flag_outliers_mean_mad (CDSGroup *dataset, const char *var_name, double window_width, int min_npoints, unsigned int skipped_flag, double bad_threshold, unsigned int bad_flag, double ind_threshold, unsigned int ind_flag, unsigned int analyze) dsproc_flag_outliers_mean_mad CDSGroup * dataset const char * var_name double window_width int min_npoints unsigned int skipped_flag double bad_threshold unsigned int bad_flag double ind_threshold unsigned int ind_flag unsigned int analyze Flag outliers using the Mean and Mean Absolute Deviation Method (MAD). For this outlier detection method the mean of the absolute deviations from the data's mean (MAD) is calculated. If a value is a certain number of MAD away from the mean of the residuals, that value is classified as an outlier. A typical default value for the threshold is 3.The MAD is more resilient to outliers in a data set than the standard deviation. In the standard deviation, the distances from the mean are squared, so large deviations are weighted more heavily, and thus outliers can heavily influence it. In the MAD, the deviations of a small number of outliers are irrelevant.The 'analyze' option can be used to analyze the outlier detection results during development and help determine the best window width and threshold to use. Remember to set this value to 0 before creating a production release. Available options are: 1 = prints ranges of mean absolute deviations from the mean, and the number of points that fall within each range. 2 = for each point prints time offset, data value, window mean, deviation from the mean, mean absolute deviation, and number of mean absolute deviations the value is from the mean. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset pointer to the dataset var_name name of the variable window_width width of window centered on data point (in seconds) min_npoints minimum number of values within window required to perform the test (default is 2). skipped_flag QC flag value to use for values that do not have enough points in window to perform the test. bad_threshold MAD factor used to flag outliers as bad bad_flag QC flag value to use for bad outliers ind_threshold MAD factor used to flag outliers as indeterminate ind_flag QC flag value to use for indeterminate outliers analyze Print statistics that may be helpfull during development (see above). 1 successful 0 a fatal error occurred DEBUG_LV1 DSPROC_LIB_NAME CDSGroup::name ERROR dsproc_set_status DSPROC_ENOMEM int int dsproc_flag_outliers_median_mad (CDSGroup *dataset, const char *var_name, double window_width, int min_npoints, unsigned int skipped_flag, double bad_threshold, unsigned int bad_flag, double ind_threshold, unsigned int ind_flag, unsigned int analyze) dsproc_flag_outliers_median_mad CDSGroup * dataset const char * var_name double window_width int min_npoints unsigned int skipped_flag double bad_threshold unsigned int bad_flag double ind_threshold unsigned int ind_flag unsigned int analyze Flag outliers using the Median and Median Absolute Deviation Method (MAD). For this outlier detection method the median of the absolute deviations from the data's median (MAD) is calculated. If a value is a certain number of MAD away from the median of the residuals, that value is classified as an outlier. A typical default value for the threshold is 3.The MAD is more resilient to outliers in a data set than the standard deviation. In the standard deviation, the distances from the mean are squared, so large deviations are weighted more heavily, and thus outliers can heavily influence it. In the MAD, the deviations of a small number of outliers are irrelevant.While this method is generally more effective than the mean and standard deviation method, it can be too aggressive in classifying values that are not really extremely different. Also, if more than 50% of the data points have the same value, MAD is computed to be 0, so any value different from the residual median is classified as an outlier.The 'analyze' option can be used to analyze the outlier detection results during development and help determine the best window width and threshold to use. Remember to set this value to 0 before creating a production release. Available options are: 1 = prints ranges of median absolute deviations from the median, and the number of points that fall within each range. 2 = for each point prints time offset, data value, window median, deviation from the median, median absolute deviation, and number of median absolute deviations the value is from the median. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset pointer to the dataset var_name name of the variable window_width width of window centered on data point (in seconds) min_npoints minimum number of values within window required to perform the test (default is 2). skipped_flag QC flag value to use for values that do not have enough points in window to perform the test. bad_threshold MAD factor used to flag outliers as bad bad_flag QC flag value to use for bad outliers ind_threshold MAD factor used to flag outliers as indeterminate ind_flag QC flag value to use for indeterminate outliers analyze Print statistics that may be helpfull during development (see above). 1 successful 0 a fatal error occurred DEBUG_LV1 DSPROC_LIB_NAME CDSGroup::name ERROR dsproc_set_status DSPROC_ENOMEM int int dsproc_flag_outliers_std (CDSGroup *dataset, const char *var_name, double window_width, int min_npoints, unsigned int skipped_flag, double bad_threshold, unsigned int bad_flag, double ind_threshold, unsigned int ind_flag, unsigned int analyze) dsproc_flag_outliers_std CDSGroup * dataset const char * var_name double window_width int min_npoints unsigned int skipped_flag double bad_threshold unsigned int bad_flag double ind_threshold unsigned int ind_flag unsigned int analyze Flag outliers using the Mean and Standard Deviation Method. For this outlier detection method, the mean and standard deviation of the residuals are calculated and compared. If a value is a certain number of standard deviations away from the mean, that data point is identified as an outlier. The specified number of standard deviations is called the threshold. A typical default value for the threshold is 3.This method can fail to detect outliers because the outliers increase the standard deviation. The more extreme the outlier, the more the standard deviation is affected.The 'analyze' option can be used to analyze the outlier detection results during development and help determine the best window width and threshold to use. Remember to set this value to 0 before creating a production release. Available options are: 1 = print ranges of standard deviations from the mean, and the number of points that fall within each range. 2 = for each point prints time offset, data value, window mean, deviation from the mean, standard deviation, and number of standard deviations the value is from the mean. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset pointer to the dataset var_name name of the variable window_width width of window centered on data point (in seconds) min_npoints minimum number of values within window required to perform the test (default is 2). skipped_flag QC flag value to use for values that do not have enough points in window to perform the test. bad_threshold STD factor used to flag outliers as bad bad_flag QC flag value to use for bad outliers ind_threshold STD factor used to flag outliers as indeterminate ind_flag QC flag value to use for indeterminate outliers analyze Print statistics that may be helpfull during development (see above). 1 successful 0 a fatal error occurred DEBUG_LV1 DSPROC_LIB_NAME CDSGroup::name ERROR dsproc_set_status DSPROC_ENOMEM int int dsproc_add_var_output_target (CDSVar *var, int ds_id, const char *var_name) dsproc_add_var_output_target CDSVar * var int ds_id const char * var_name Add an output target for a variable. This function will add an output target for the variable.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable ds_id - output datastream ID var_name - name of the variable in the output datastream 1 if successful0 if an error occurred _DSProc DSR_OUTPUT ERROR DSPROC_LIB_NAME cds_get_object_path dsproc_set_status DSPROC_EBADDSID msngr_debug_level msngr_provenance_level DEBUG_LV1 CDSVar::name DSPROC_ENOMEM int int dsproc_copy_var_tag (CDSVar *src_var, CDSVar *dest_var) dsproc_copy_var_tag CDSVar * src_var CDSVar * dest_var Copy a variable tag from one variable to another. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. src_var - pointer to the source variable dest_var - pointer to the destination variable 1 if successful0 if an error occurred cds_get_user_data STRDUP_FAILED cds_set_user_data ERROR DSPROC_LIB_NAME CDSVar::name dsproc_set_status DSPROC_ENOMEM void void dsproc_delete_var_tag (CDSVar *var) dsproc_delete_var_tag CDSVar * var Delete a variable tag. var - pointer to the variable DEBUG_LV1 DSPROC_LIB_NAME cds_get_object_path cds_delete_user_data const char * const char* dsproc_get_source_var_name (CDSVar *var) dsproc_get_source_var_name CDSVar * var Get the name of the source variable read in from the input file. The memory used by the returned variable name belongs to the internal variable tag and must not be freed or altered by the calling process. var - pointer to the variable pointer to the name of the source variableNULL if the variable was not explicitly requested by the user in the retriever definition cds_get_user_data const char * const char* dsproc_get_source_ds_name (CDSVar *var) dsproc_get_source_ds_name CDSVar * var Get the name of the input datastream the variable was retrieved from. The memory used by the returned datastream name belongs to the internal datastream structure and must not be freed or altered by the calling process. var - pointer to the variable pointer to the name of the input datastreamNULL if the variable was not explicitly requested by the user in the retriever definition cds_get_user_data int int dsproc_get_source_ds_id (CDSVar *var) dsproc_get_source_ds_id CDSVar * var Get the ID of the input datastream the variable was retrieved from. var - pointer to the variable input datastream ID-1 if the variable was not explicitly requested by the user in the retriever definition cds_get_user_data _DSProc const char * const char* dsproc_get_var_coordsys_name (CDSVar *var) dsproc_get_var_coordsys_name CDSVar * var Get the name of the transformation coordinate system. The memory used by the returned name belongs to the internal variable tag and must not be freed or altered by the calling process. var - pointer to the variable name of the coordinate systemNULL if a coordinate system was not specified for the variable cds_get_user_data int int dsproc_get_var_output_targets (CDSVar *var, VarTarget ***targets) dsproc_get_var_output_targets CDSVar * var VarTarget *** targets Get the output targets defined for the specified variable. The memory used by the output array belongs to the internal variable tag and must not be freed or altered by the calling process. var - pointer to the variable targets - output: pointer to the array of pointers to the VarTarget structures. number of variable targets0 if no variable targets have been defined cds_get_user_data int int dsproc_set_var_coordsys_name (CDSVar *var, const char *coordsys_name) dsproc_set_var_coordsys_name CDSVar * var const char * coordsys_name Set the coordinate system for a variable. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable coordsys_name - name of the coordinate system 1 if successful0 if an error occurred DEBUG_LV1 DSPROC_LIB_NAME cds_get_object_path ERROR CDSVar::name dsproc_set_status DSPROC_ENOMEM int int dsproc_set_var_flags (CDSVar *var, int flags) dsproc_set_var_flags CDSVar * var int flags Set the control flags for a variable. Control Flags: VAR_SKIP_TRANSFORM = Instruct the transform logic to ignore this variable. VAR_ROLLUP_TRANS_QC = Consolidate the transformation QC bits when they are mapped to the output dataset. var - pointer to the variable flags - flags to set 1 if successful0 if an error occurred msngr_debug_level msngr_provenance_level DEBUG_LV1 DSPROC_LIB_NAME cds_get_object_path VAR_SKIP_TRANSFORM VAR_ROLLUP_TRANS_QC int int dsproc_set_var_output_target (CDSVar *var, int ds_id, const char *var_name) dsproc_set_var_output_target CDSVar * var int ds_id const char * var_name Set the output target for a variable. This function will replace any previously specified output targets already set for the variable.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. var - pointer to the variable ds_id - output datastream ID var_name - name of the variable in the output datastream 1 if successful0 if an error occurred _DSProc DSR_OUTPUT ERROR DSPROC_LIB_NAME cds_get_object_path dsproc_set_status DSPROC_EBADDSID msngr_debug_level msngr_provenance_level DEBUG_LV1 CDSVar::name DSPROC_ENOMEM void void dsproc_unset_var_flags (CDSVar *var, int flags) dsproc_unset_var_flags CDSVar * var int flags Unset the control flags for a variable. See dsproc_set_var_flags() for flags and descriptions. var - pointer to the variable flags - flags to set msngr_debug_level msngr_provenance_level DEBUG_LV1 DSPROC_LIB_NAME cds_get_object_path VAR_SKIP_TRANSFORM cds_get_user_data int int dsproc_get_var_dqrs (CDSVar *var, VarDQR ***dqrs) dsproc_get_var_dqrs CDSVar * var VarDQR *** dqrs var_dqrs Get all available DQRs for the data stored in the specified variable. The memory used by the output array belongs to the internal variable tag and must not be freed by the calling process. var - pointer to the variable var_dqrs - output: pointer to the array of pointers to the VarDQR structures. number of DQRs found0 if no DQRs where found for the relevant time range, or the variable was not explicitely requested by the user in the retriever definition.-1 if an error occurred cds_get_user_data CDSVar::parent int int dsproc_dump_dataset (CDSGroup *dataset, const char *outdir, const char *prefix, time_t file_time, const char *suffix, int flags) dsproc_dump_dataset CDSGroup * dataset const char * outdir const char * prefix time_t file_time const char * suffix int flags Dump the contents of a dataset to a text file. This function will dump the contents of a dataset to a text file with the following name: prefix.YYYYMMDD.hhmmss.suffix If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. dataset - pointer to the dataset outdir - the output directory prefix - the prefix to use for the file name, or NULL to use the name of the dataset. file_time - the time to use to create the file timestamp, or 0 to use the first sample time in the dataset. suffix - the suffix portion of the file name flags - reserved for control flags 1 if successful0 if and error occurred make_path CDSGroup::name dsproc_get_time_range CDSGroup::ngroups CDSGroup::groups DEBUG_LV1 DSPROC_LIB_NAME cds_get_object_path ERROR dsproc_set_status DSPROC_EFILEOPEN cds_print int int dsproc_dump_output_datasets (const char *outdir, const char *suffix, int flags) dsproc_dump_output_datasets const char * outdir const char * suffix int flags Dump all output datasets to text files. This function will dump all output datasets to a text file with the following names: datastream.YYYYMMDD.hhmmss.suffix where: datastream = the output datastream nameYYYYMMDD = the date of the first sample in the datasethhmmss = the time of the first sample in the datasetsuffix = the user specified suffix in the function call If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. outdir - the output directory suffix - the suffix portion of the file name flags - reserved for control flags 1 if successful0 if and error occurred _DSProc DSR_OUTPUT dsproc_dump_dataset int int dsproc_dump_retrieved_datasets (const char *outdir, const char *suffix, int flags) dsproc_dump_retrieved_datasets const char * outdir const char * suffix int flags Dump all retrieved datasets to a text file. This function will dump all retrieved data to a text file with the following name: {site}{process}{facility}.YYYYMMDD.hhmmss.suffix where: site = the site name as specified on the command lineprocess = the name of the process as defined in the databasefacility = the facility name as specified on the command lineYYYYMMDD = the start date of the current processing intervalhhmmss = the start time of the current processing intervalsuffix = the user specified suffix in the function call If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. outdir - the output directory suffix - the suffix portion of the file name flags - reserved for control flags 1 if successful0 if and error occurred _DSProc dsproc_dump_dataset int int dsproc_dump_transformed_datasets (const char *outdir, const char *suffix, int flags) dsproc_dump_transformed_datasets const char * outdir const char * suffix int flags Dump all transformed datasets to a text file. This function will dump all transformed data to a text file with the following name: {site}{process}{facility}.YYYYMMDD.hhmmss.suffix where: site = the site name as specified on the command lineprocess = the name of the process as defined in the databasefacility = the facility name as specified on the command lineYYYYMMDD = the start date of the current processing intervalhhmmss = the start time of the current processing intervalsuffix = the user specified suffix in the function call If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. outdir - the output directory suffix - the suffix portion of the file name flags - reserved for control flags 1 if successful0 if and error occurred _DSProc dsproc_dump_dataset int int dsproc_copy_file (const char *src_file, const char *dest_file) dsproc_copy_file const char * src_file const char * dest_file Copy a file. This function will copy a file and use MD5 checking to validate the file copy was successful. It will also add a "copying file" message to the log file.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. src_file - full path to the source file dest_file - full path to the destination file 1 if successfull0 if an error occurred LOG DSPROC_LIB_NAME file_copy FC_CHECK_MD5 dsproc_set_status DSPROC_EFILECOPY int int dsproc_move_file (const char *src_file, const char *dest_file) dsproc_move_file const char * src_file const char * dest_file Move a file. This function will first attempt to rename the file. If the rename fails because the file is being moved across file systems, it will be copied to the destination file and the source file deleted. MD5 checking will be used if the file needs to be copied across file systems.This function will also add a "moving file" message to the log file.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. src_file - full path to the source file dest_file - full path to the destination file 1 if the file was moved successfully0 if an error occurred LOG DSPROC_LIB_NAME file_move FC_CHECK_MD5 dsproc_set_status DSPROC_EFILEMOVE FILE * FILE* dsproc_open_file (const char *file) dsproc_open_file const char * file Open a file for reading. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. file - full path to the file pointer to the open fileNULL if an error occurred DEBUG_LV1 DSPROC_LIB_NAME ERROR dsproc_set_status DSPROC_EFILEOPEN int int dsproc_rename (int ds_id, const char *file_path, const char *file_name, time_t begin_time, time_t end_time) dsproc_rename int ds_id const char * file_path const char * file_name time_t begin_time time_t end_time Rename a data file. This function will rename a raw data file into the datastream directory using the datastream name and begin_time to give it a fully qualified ARM name.The begin_time will be validated using: dsproc_validate_datastream_data_time() If the end_time is specified, this function will verify that it is greater than the begin_time and is not in the future. If only one record was found in the raw file, the end_time argument must be set to NULL.If the output file exists and has the same MD5 as the input file, the input file will be removed and a warning message will be generated.If the output file exists and has a different MD5 than the input file, the rename will fail.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ds_id - datastream ID file_path - path to the directory the file is in file_name - name of the file to rename begin_time - time of the first record in the file end_time - time of the last record in the file 1 if successful0 if an error occurred int int dsproc_rename_tv (int ds_id, const char *file_path, const char *file_name, const timeval_t *begin_time, const timeval_t *end_time) dsproc_rename_tv int ds_id const char * file_path const char * file_name const timeval_t * begin_time const timeval_t * end_time Rename a data file. This function is the same as dsproc_rename() except that it takes timevals for the begin and end times.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ds_id - datastream ID file_path - path to the directory the file is in file_name - name of the file to rename begin_time - time of the first record in the file end_time - time of the last record in the file 1 if successful0 if an error occurred int int dsproc_rename_bad (int ds_id, const char *file_path, const char *file_name, time_t file_time) dsproc_rename_bad int ds_id const char * file_path const char * file_name time_t file_time Rename a bad data file. This function works the same as dsproc_rename() except that the extension "bad" will be used in the output file name.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ds_id - datastream ID file_path - path to the directory the file is in file_name - name of the file to rename file_time - the time used to rename the file 1 if successful0 if an error occurred int int dsproc_set_preserve_dots_from_name (int ds_id, const char *file_name) dsproc_set_preserve_dots_from_name int ds_id const char * file_name Determine the portion of the original file name to preserve. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. ds_id datastream ID file_name name of the raw data file 1 if successful 0 if a pattern matching error occurred _DSProc re_compile DSPROC_ERROR re_execute re_free DEBUG_LV1 DSPROC_LIB_NAME void void dsproc_set_rename_preserve_dots (int ds_id, int preserve_dots) dsproc_set_rename_preserve_dots int ds_id int preserve_dots Set the portion of file names to preserve when they are renamed. This is the number of dots from the end of the file name and specifies the portion of the original file name to preserve when it is renamed.Default value set if preserve_dots < 0: 2 for level '0' datastreams0 for all other datastreams ds_id - output datastream ID preserve_dots - portion of original file name to preserve _DSProc DEBUG_LV1 DSPROC_LIB_NAME int int dsproc_count_csv_delims (const char *strp, char delim) dsproc_count_csv_delims const char * strp char delim Count the number of delimiters in a string. Delimiters inside single or double quoted strings will not be matched. strp pointer to the delimited string delim delimiter character nfound number of delimiters found. dsproc_skip_csv_whitespace dsproc_find_csv_delim char * char* dsproc_find_csv_delim (const char *strp, char delim) dsproc_find_csv_delim const char * strp char delim Find the next delimiter in a string. Delimiters inside single or double quoted strings will not be matched. strp pointer to the string delim delimiter to search for strp pointer to the next delimiter in the string NULL if not found char * char* dsproc_skip_csv_whitespace (const char *strp, char delim) dsproc_skip_csv_whitespace const char * strp char delim Skip white-space characters that do not match the delimiter. If a delimiter is specified, it will not be treated as a white-space character unless it is a normal space ' '. strp pointer to the first character in the string delim delimiter character strp pointer to the next non white-space character int int dsproc_split_csv_string (char *strp, char delim, int length, char **list) dsproc_split_csv_string char * strp char delim int length char ** list Split a delimited string into list of strings. The pointers in the output list point to the substrings in the input string and have been null terminated. All leading and trailing spaces, and surrounding quotes have also been removed from the substrings.The pointers in the output list will only be valid as long as the memory used by the input string is not altered or freed. strp pointer to the delimited string delim delimiter character length length of the output list list output: pointers to the substrings in the input string. nfound number of substrings returned in the output list, or the number that would have beed returned if the output list had been big enough. dsproc_skip_csv_whitespace dsproc_find_csv_delim void void dsproc_free_csv_parser (CSVParser *csv) dsproc_free_csv_parser CSVParser * csv Free all memory used by a CSVParser structure. csv pointer to the CSVParser structure CSVParser::file_name CSVParser::file_path CSVParser::file_data CSVParser::lines CSVParser::headers CSVParser::free_header CSVParser::nfields_alloced CSVParser::header_data CSVParser::rec_buff CSVParser::values CSVParser::ft_patterns retime_list_free CSVParser::ft_result CSVParser::tc_names CSVParser::ntc CSVParser::tc_patterns CSVParser::tc_index CSVParser::tvs char ** char** dsproc_get_csv_column_headers (CSVParser *csv, int *nfields) dsproc_get_csv_column_headers CSVParser * csv int * nfields Get the array of string pointers to the column headers in a CSV file. The memory used by the returned array of strings belongs to the CSVParser structure and must not be freed by the calling process. csv pointer to the CSVParser structure nfields output: number of header fields. headers array of string pointers to the column headers NULL if the column headers have not been parsed CSVParser::nfields CSVParser::headers char ** char** dsproc_get_csv_field_strvals (CSVParser *csv, const char *name) dsproc_get_csv_field_strvals CSVParser * csv const char * name Get the array of string pointers for a field in a CSV file. The memory used by the returned array of strings belongs to the CSVParser structure and must not be freed by the calling process. csv pointer to the CSVParser structure name column name as specified in the header. vals array of string pointers to the column values NULL if the column name was not found CSVParser::nfields CSVParser::headers CSVParser::values timeval_t * timeval_t* dsproc_get_csv_timevals (CSVParser *csv, int *nrecs) dsproc_get_csv_timevals CSVParser * csv int * nrecs Get the array of record times after parsing a CSV file. The memory used by the returned array of timevals belongs to the CSVParser structure and must not be freed by the calling process. csv pointer to the CSVParser structure nrecs output: the number of records found in the file times array of timevals in seconds since 1970 NULL if no records have been parsed from the file CSVParser::nrecs CSVParser::tvs time_t time_t dsproc_get_csv_file_name_time (CSVParser *csv, const char *name, RETimeRes *result) dsproc_get_csv_file_name_time CSVParser * csv const char * name RETimeRes * result Get the time from a CSV file name. This function must be called after the CSVParser structure has been initialized using the dsproc_init_csv_parser() function and the file name time string patterns have been set using dsproc_set_csv_file_time_patterns().If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSVParser structure name CSV file name result if not NULL the complete result from the pattern match will be stored in this RETimeRes structure. time time in seconds since 1970 -1 if an error occurred CSVParser::ft_patterns ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ECSVPARSER retime_list_execute RETimeList::retimes RETimeList::npatterns RETime::tspattern retime_get_secs1970 char * char* dsproc_get_next_csv_line (CSVParser *csv) dsproc_get_next_csv_line CSVParser * csv Get the next line from the file loaded into the CSVParser structure. The memory used by the returned line belongs to the CSVParser structure and must not be freed by the calling process. csv pointer to the CSVParser structure line the next line in the file loaded into memory NULL if the end of file was reached CSVParser::linenum CSVParser::nlines CSVParser::linep CSVParser::lines CSVParser * CSVParser* dsproc_init_csv_parser (CSVParser *csv) dsproc_init_csv_parser CSVParser * csv Initialize a CSVParser structure. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSV Data structure to initialize, or NULL to create a new CSVParser structure. csv pointer to the CSVParser structure NULL if an error occurred CSVParser::file_path CSVParser::file_name CSVParser::nlines CSVParser::linep CSVParser::linenum CSVParser::nfields CSVParser::nrecs CSVParser::tc_index CSVParser::tro_offset ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM CSVParser::delim CSVParser::nlines_guess CSVParser::nfields_guess int int dsproc_load_csv_file (CSVParser *csv, const char *path, const char *name) dsproc_load_csv_file CSVParser * csv const char * path const char * name Load a CSV data file into a CSVParser structure. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSVParser structure created by dsproc_init_csv_parser() path path to the location of the file name the name of the file nlines number of lines read from the file -1 if an error occurred CSVParser::nlines dsproc_init_csv_parser CSVParser::nlines_alloced CSVParser::nlines_guess CSVParser::lines ERROR DSPROC_LIB_NAME CSVParser::file_name dsproc_set_status DSPROC_ENOMEM CSVParser::file_path CSVParser::ft_result CSVParser::file_stats DSPROC_EFILESTATS CSVParser::nbytes_alloced CSVParser::file_data DSPROC_EFILEOPEN DSPROC_EFILEREAD dsproc_find_csv_delim int int dsproc_parse_csv_header (CSVParser *csv, const char *linep) dsproc_parse_csv_header CSVParser * csv const char * linep Parse a header line. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSVParser structure linep pointer to the line to parse, or NULL to parse the current line. nfields number of header fields -1 if a memory allocation error occurs CSVParser::delim CSVParser::linenum CSVParser::lines dsproc_count_csv_delims CSVParser::header_data ERROR DSPROC_LIB_NAME CSVParser::file_name dsproc_set_status DSPROC_ENOMEM CSVParser::nlines CSVParser::headers CSVParser::free_header CSVParser::nfields_alloced dsproc_split_csv_string DSPROC_ECSVPARSER CSVParser::nfields int int dsproc_parse_csv_record (CSVParser *csv, char *linep, int flags) dsproc_parse_csv_record CSVParser * csv char * linep int flags Parse a record line. If the number of values found on the current line does not match the number of header fields, a warning message will be generated.The records values in the internal CSVParser structure point to the substrings in the input string and have been null terminated. All leading and trailing spaces, and surrounding quotes have also been removed from the substrings.The input string must not be altered or freed after calling this function until the record values are no longer needed by the calling process. csv pointer to the CSVParser structure linep pointer to the line to parse, or NULL to parse from the beginning of the current line. flags reserved for control flags, set to 0 to maintain backward compatibility with future updates. 1 if successful 0 if the record time has an invalid format, or the number of values found on the line does not match the number of header fields -1 if an error occurred CSVParser::delim CSVParser::linenum CSVParser::lines CSVParser::nrecs CSVParser::nrecs_alloced CSVParser::nfields ERROR DSPROC_LIB_NAME CSVParser::file_name dsproc_set_status DSPROC_ENOMEM dsproc_split_csv_string CSVParser::rec_buff DSPROC_BAD_RECORD_WARNING CSVParser::values CSVParser::ntc int int dsproc_print_csv (FILE *fp, CSVParser *csv) dsproc_print_csv FILE * fp CSVParser * csv Print CSV header and records. If the number of values found on the current line does not match the number of header fields, a warning message will be generated. fp pointer to the output stream csv pointer to the CSVParser structure 1 if successful 0 if no data was found in the CSV table -1 if a write error occurred dsproc_print_csv_header dsproc_print_csv_record int int dsproc_print_csv_header (FILE *fp, CSVParser *csv) dsproc_print_csv_header FILE * fp CSVParser * csv Print CSV header data. If the number of values found on the current line does not match the number of header fields, a warning message will be generated. fp pointer to the output stream csv pointer to the CSVParser structure 1 if successful 0 if no header was found in the CSV table -1 if a write error occurred CSVParser::delim CSVParser::nfields CSVParser::headers ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_EFILEWRITE int int dsproc_print_csv_record (FILE *fp, CSVParser *csv) dsproc_print_csv_record FILE * fp CSVParser * csv Print CSV record data. If the number of values found on the current line does not match the number of header fields, a warning message will be generated. fp pointer to the output stream csv pointer to the CSVParser structure 1 if successful 0 if no data was found in the CSV table -1 if a write error occurred CSVParser::delim CSVParser::nfields CSVParser::nrecs CSVParser::values ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_EFILEWRITE int int dsproc_set_csv_column_name (CSVParser *csv, int index, const char *name) dsproc_set_csv_column_name CSVParser * csv int index const char * name Set or change a column name in the header. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSVParser structure index column index name column name 1 if successful 0 if a memory allocation error occurs CSVParser::nfields_guess CSVParser::nlines ERROR DSPROC_LIB_NAME CSVParser::file_name dsproc_set_status DSPROC_ENOMEM CSVParser::headers CSVParser::free_header CSVParser::nfields void void dsproc_set_csv_delimiter (CSVParser *csv, char delim) dsproc_set_csv_delimiter CSVParser * csv char delim Set the column delimiter. csv pointer to the CSVParser structure delim column delimiter CSVParser::delim int int dsproc_set_csv_base_time (CSVParser *csv, time_t base_time) dsproc_set_csv_base_time CSVParser * csv time_t base_time Set the base time to use for record times. This option is used when the records times are relative to a base time. csv pointer to the CSVParser structure base_time base time in seconds since 1970 1 if successful 0 if an error occurred CSVParser::base_tm ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ECSVPARSER int int dsproc_set_csv_file_time_patterns (CSVParser *csv, int npatterns, const char **patterns) dsproc_set_csv_file_time_patterns CSVParser * csv int npatterns const char ** patterns Specify the pattern to use to parse the date/time from the file name. The time string patterns can containing a mixture of regex and time format codes similar to the strptime function. The time format codes recognized by this function begin with a % and are followed by one of the following characters: 'C' century number (year/100) as a 2-digit integer'd' day number in the month (1-31).'e' day number in the month (1-31).'h' hour * 100 + minute (0-2359)'H' hour (0-23)'j' day number in the year (1-366).'m' month number (1-12)'M' minute (0-59)'n' arbitrary whitespace'o' time offset in seconds's' seconds since Epoch, 1970-01-01 00:00:00 +0000 (UTC)'S' second (0-60; 60 may occur for leap seconds)'t' arbitrary whitespace'y' year within century (0-99)'Y' year with century as a 4-digit integer'' a literal "%" character An optional 0 character can be used between the % and format code to specify that the number must be zero padded. For example, '%0d' specifies that the day range is 01 to 31.See the regex(7) man page for the descriptions of the regex patterns.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSVParser structure npatterns number of possible patterns to check patterns list of possible time string patterns 1 if successful 0 if an error occurred CSVParser::ft_patterns retime_list_free retime_list_compile ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ECSVPARSER void void dsproc_set_csv_time_offset (CSVParser *csv, time_t time_offset) dsproc_set_csv_time_offset CSVParser * csv time_t time_offset Set the time offset to apply to record times. csv pointer to the CSVParser structure time_offset base time in seconds since 1970 CSVParser::time_offset int int dsproc_set_csv_time_patterns (CSVParser *csv, const char *name, int npatterns, const char **patterns) dsproc_set_csv_time_patterns CSVParser * csv const char * name int npatterns const char ** patterns Specify the pattern to use to parse a date/time column. The time string patterns can containing a mixture of regex and time format codes similar to the strptime function. The time format codes recognized by this function begin with a % and are followed by one of the following characters: 'C' century number (year/100) as a 2-digit integer'd' day number in the month (1-31).'e' day number in the month (1-31).'h' hour * 100 + minute (0-2359)'H' hour (0-23)'j' day number in the year (1-366).'m' month number (1-12)'M' minute (0-59)'n' arbitrary whitespace's' seconds since Epoch, 1970-01-01 00:00:00 +0000 (UTC)'S' second (0-60; 60 may occur for leap seconds)'t' arbitrary whitespace'y' year within century (0-99)'Y' year with century as a 4-digit integer'' a literal "%" character An optional 0 character can be used between the % and format code to specify that the number must be zero padded. For example, '%0d' specifies that the day range is 01 to 31.See the regex(7) man page for the descriptions of the regex patterns.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSVParser structure name name of the time column npatterns number of possible patterns to check patterns list of possible time string patterns 1 if successful 0 if an error occurred CSVParser::ntc CSVParser::tc_index CSVParser::tc_names CSVParser::tc_patterns retime_list_free retime_list_compile ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM int int dsproc_map_csv_to_cds (CSVParser *csv, int csv_start, int csv_count, CSV2CDSMap *map, CDSGroup *cds, int cds_start, int flags) dsproc_map_csv_to_cds CSVParser * csv int csv_start int csv_count CSV2CDSMap * map CDSGroup * cds int cds_start int flags Map CSVParser data to variables in a CDSGroup. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSVParser structure csv_start index of the start record in the CSVParser structure csv_count number of records to map (0 for all) map pointer to the CSV2CDS Map structure cds pointer to the CDSGroup structure cds_start index of the start record in the CDSGroup flags control flags 1 if successful 0 if an error occurred CSVParser::nrecs ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM dsproc_map_csv_to_cds_by_index int int dsproc_map_csv_to_cds_by_index (CSVParser *csv, int *csv_indexes, int csv_count, CSV2CDSMap *map, CDSGroup *cds, int cds_start, int flags) dsproc_map_csv_to_cds_by_index CSVParser * csv int * csv_indexes int csv_count CSV2CDSMap * map CDSGroup * cds int cds_start int flags Map CSVParser data to variables in a CDSGroup using CSV record indexes. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. csv pointer to the CSVParser structure csv_indexes indexes of the CSV records csv_count number of indexes map pointer to the CSV2CDS Map structure cds pointer to the CDSGroup structure cds_start index of the start record in the CDSGroup flags control flags 1 if successful 0 if an error occurred dsproc_get_dynamic_dods_mode DEBUG_LV1 DSPROC_LIB_NAME CSVParser::file_name CDSGroup::name CSV2CDSMap::csv_name CSV2CDSMap::cds_name CSV2CDSMap::csv_units CSV2CDSMap::csv_missings CSV2CDSMap::str_map CSV2CDSMap::str_to_dbl CSV2CDSMap::set_data dsproc_get_csv_field_strvals ERROR dsproc_set_status DSPROC_ECSV2CDS cds_get_var cds_get_dim cds_define_dim cds_get_object_path cds_define_var CDS_DOUBLE CDSDim::name CDS_FLOAT cds_define_att_text cds_define_att CDSVar::sample_count CSV_OVERWRITE DEBUG_LV2 cds_get_var_units cds_get_unit_converter CDSData::vp cds_get_var_missing_values CDSVar::name DSPROC_ENOMEM cds_get_default_fill_value CDSVar::type cds_alloc_var_data cds_var_sample_size cds_data_type_size CDS_CHAR CDSData::cp CDS_BYTE CSV_MAP_TO_CDS CDSData::bp CDS_SHORT CDSData::sp CDS_INT CDSData::ip CDSData::fp CDSData::dp cds_convert_units int int dsproc_add_csv_str_to_dbl_function (CSV2CDSMap *map, const char *csv_name, double(*str_to_dbl)(const char *strval, int *status)) dsproc_add_csv_str_to_dbl_function CSV2CDSMap * map const char * csv_name double(*)(const char *strval, int *status) str_to_dbl Add a string to double conversion function to a CSV2CDS Mapping structure. The str_to_double function must return a double and set the status value to non-zero if successful or 0 for an invalid input string. map pointer to the CSV2CDS Map structure csv_name name of the column in the csv file str_to_dbl function used to convert csv string to a double. 1 if the specified csv_name was found 0 if the specified csv_name was not found CSV2CDSMap::csv_name CSV2CDSMap::str_to_dbl int int dsproc_add_csv_field_map (CSVConf *conf, const char *out_name, const char *col_name, int nargs, const char **args) dsproc_add_csv_field_map CSVConf * conf const char * out_name const char * col_name int nargs const char ** args Add an entry to the field map. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. conf pointer to CSVConf structure to populate out_name name of the output variable, or NULL to use the column name col_name name of the CSV column nargs length of args list args list of additional arguments in the following order (specify NULL or empty string for no value): unitscomma separated list of missing value strings 1 if successful 0 if an error occurred CSVConf::field_nmaps CSVConf::field_maps CSVFieldMap::out_name CSVFieldMap::col_name CSVFieldMap::units CSVFieldMap::missings CSVFieldMap::missbuf dsproc_count_csv_delims CSVFieldMap::nmissings dsproc_split_csv_string ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM int int dsproc_add_csv_file_name_patterns (CSVConf *conf, int npatterns, const char **patterns) dsproc_add_csv_file_name_patterns CSVConf * conf int npatterns const char ** patterns Add file name patterns to a CSVConf structure. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. conf pointer to CSVConf structure to populate npatterns number of patterns patterns list of extended regex patterns, see man regex(7) 1 if successful 0 if an error occurred CSVConf::fn_npatterns CSVConf::fn_patterns ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM int int dsproc_add_csv_file_time_patterns (CSVConf *conf, int npatterns, const char **patterns) dsproc_add_csv_file_time_patterns CSVConf * conf int npatterns const char ** patterns Add file time patterns to a CSVConf structure. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. conf pointer to CSVConf structure to populate npatterns number of patterns patterns list of extended regex patterns, see man regex(7) 1 if successful 0 if an error occurred CSVConf::ft_npatterns CSVConf::ft_patterns ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM int int dsproc_add_csv_time_column_patterns (CSVConf *conf, const char *name, int npatterns, const char **patterns) dsproc_add_csv_time_column_patterns CSVConf * conf const char * name int npatterns const char ** patterns Add time column patterns to a CSVConf structure. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. conf pointer to CSVConf structure to populate name name of the time column npatterns number of patterns patterns list of time patterns 1 if successful 0 if an error occurred CSVConf::time_ncols CSVConf::time_cols CSVTimeCol::name CSVTimeCol::npatterns CSVTimeCol::patterns ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM int int dsproc_append_csv_header_line (CSVConf *conf, const char *string) dsproc_append_csv_header_line CSVConf * conf const char * string Append a string to the end of the header line. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. conf pointer to CSVConf structure to populate string string to append to the header line 1 if successful 0 if an error occurred CSVConf::header_line ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM void void dsproc_clear_csv_field_maps (CSVConf *conf) dsproc_clear_csv_field_maps CSVConf * conf Clear the time column patterns in a CSVConf structure. conf pointer to CSVConf structure to populate CSVConf::field_maps CSVConf::field_nmaps CSVFieldMap::out_name CSVFieldMap::col_name CSVFieldMap::units CSVFieldMap::missings CSVFieldMap::missbuf void void dsproc_clear_csv_file_name_patterns (CSVConf *conf) dsproc_clear_csv_file_name_patterns CSVConf * conf Clear the file name patterns in a CSVConf structure. conf pointer to CSVConf structure to populate CSVConf::fn_patterns CSVConf::fn_npatterns void void dsproc_clear_csv_file_time_patterns (CSVConf *conf) dsproc_clear_csv_file_time_patterns CSVConf * conf Clear the file name patterns in a CSVConf structure. conf pointer to CSVConf structure to populate CSVConf::ft_patterns CSVConf::ft_npatterns void void dsproc_clear_csv_time_column_patterns (CSVConf *conf) dsproc_clear_csv_time_column_patterns CSVConf * conf Clear the time column patterns in a CSVConf structure. conf pointer to CSVConf structure to populate CSVConf::time_cols CSVConf::time_ncols CSVTimeCol::name CSVTimeCol::patterns CSVTimeCol::npatterns int int dsproc_configure_csv_parser (CSVConf *conf, CSVParser *csv) dsproc_configure_csv_parser CSVConf * conf CSVParser * csv Free memory used by a CSVConf structure. conf pointer to CSVConf structure csv pointer to CSVParser 1 if successful 0 if an error occured CSVConf::delim dsproc_set_csv_delimiter CSVConf::ft_npatterns dsproc_set_csv_file_time_patterns CSVConf::ft_patterns CSVConf::time_ncols CSVConf::time_cols dsproc_set_csv_time_patterns CSVTimeCol::name CSVTimeCol::npatterns CSVTimeCol::patterns CSV2CDSMap * CSV2CDSMap* dsproc_create_csv_to_cds_map (CSVConf *conf, CSVParser *csv, CDSGroup *cds, int flags) dsproc_create_csv_to_cds_map CSVConf * conf CSVParser * csv CDSGroup * cds int flags Create a CSV2CDS Map. The memory used by the returned CSV2CDS Map is dynamically allocated and must be freed using the dsproc_free_csv_to_cds_map().If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. conf pointer to the CSVConf structure csv pointer to the CSVParser structure cds pointer to the CDSGroup structure flags reserved for control flags map pointer to the CSV2CDS Map structure NULL if an error occurred CSVParser::nfields dsproc_get_dataset_vars CSVParser::ntc CSVParser::tc_names CSVParser::headers CSVConf::field_maps CSVConf::field_nmaps CSV2CDSMap::csv_name CSVFieldMap::col_name CSVFieldMap::out_name CSV2CDSMap::cds_name CSVFieldMap::units CSV2CDSMap::csv_units CSVFieldMap::nmissings CSVFieldMap::missings CSV2CDSMap::csv_missings dsproc_free_csv_to_cds_map ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM void void dsproc_free_csv_conf (CSVConf *conf) dsproc_free_csv_conf CSVConf * conf Free memory used by a CSVConf structure. conf pointer to CSVConf structure CSVConf::proc CSVConf::site CSVConf::fac CSVConf::name CSVConf::level CSVConf::file_name CSVConf::file_path CSVConf::search_paths CSVConf::search_npaths CSVConf::dirlist dirlist_free dsproc_clear_csv_file_name_patterns dsproc_clear_csv_file_time_patterns CSVConf::header_line CSVConf::header_tag dsproc_clear_csv_time_column_patterns dsproc_clear_csv_field_maps CSVConf::split_interval void void dsproc_free_csv_to_cds_map (CSV2CDSMap *map) dsproc_free_csv_to_cds_map CSV2CDSMap * map Free the memory used by a CSV2CDS Map. map pointer to the CSV2CDS Map structure CSV2CDSMap::csv_name CSV2CDSMap::csv_missings CSVConf * CSVConf* dsproc_init_csv_conf (const char *name, const char *level) dsproc_init_csv_conf const char * name const char * level Initialize a new CSVConf structure. The memory used by the returned structure is dynamically allocated and must be freed using dsproc_free_csv_conf().If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. name the base name of the conf file level the data level of the conf file, or NULL conf pointer to the CSVConf structure NULL if an error occurred dsproc_get_name dsproc_get_site dsproc_get_facility CSVConf::proc CSVConf::site CSVConf::fac CSVConf::name CSVConf::level dsproc_free_csv_conf ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM int int dsproc_load_csv_conf (CSVConf *conf, time_t data_time, int flags) dsproc_load_csv_conf CSVConf * conf time_t data_time int flags Load the CSV Configuration file into a CVSConf structure. The frist time this function is called the data_time argument must be set to 0. This will load the main conf file containing the the file name patterns and all default configuration settings. It will also set the path to look for time varying conf files in subsequent calls to this function.If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. conf Pointer to the CSVConf structure. data_time The start time of the data being processed, this should be determined from the file name. Specify 0 to find the main conf file containing the the file name patterns and all default configuration settings. flags Control Flags: - CSV_CHECK_DATA_CONF check for config files under the root directory defined by the CONF_DATA environment variable. 1 if successful 0 if a file was not found, or it has already been loaded -1 if error occurred CSVConf::file_path CSVConf::file_name msngr_debug_level dsproc_print_csv_conf ERROR DSPROC_LIB_NAME dsproc_set_status DSPROC_ENOMEM int int dsproc_print_csv_conf (FILE *fp, CSVConf *conf) dsproc_print_csv_conf FILE * fp CSVConf * conf Print the contents of a CSVConf structure. If an error occurs in this function it will be appended to the log and error mail messages, and the process status will be set appropriately. fp pointer to the output stream conf pointer to the CSVConf structure 1 if successful 0 if successful CSVConf::fn_patterns CSVConf::fn_npatterns CSVConf::ft_patterns CSVConf::ft_npatterns CSVConf::delim CSVConf::header_line CSVConf::header_tag CSVConf::header_linenum CSVConf::header_nlines CSVConf::exp_ncols CSVConf::time_cols CSVConf::time_ncols CSVTimeCol::name CSVTimeCol::npatterns CSVTimeCol::patterns CSVConf::split_interval CSVConf::field_maps CSVConf::field_nmaps CSVFieldMap::out_name CSVFieldMap::col_name CSVFieldMap::units CSVFieldMap::nmissings CSVFieldMap::missings Header file for libdsproc3. /******************************************************************************* * *COPYRIGHT(C)2010BattelleMemorialInstitute.AllRightsReserved. * ******************************************************************************** * *Author: *name:BrianErmold *phone:(509)375-2277 *email:brian.ermold@pnl.gov * ******************************************************************************** * *REPOSITORYINFORMATION: *$Revision:81662$ *$Author:ermold$ *$Date:2017-10-2716:09:46+0000(Fri,27Oct2017)$ * ******************************************************************************** * *NOTE:DOXYGENisusedtogeneratedocumentationforthisfile. * *******************************************************************************/ /**@filedsproc3.h *Headerfileforlibdsproc3 */ #ifndef_DSPROC3_H #define_DSPROC3_H #include"armutils.h" #include"cds3.h" /******************************************************************************/ /** *@defgroupDSPROC_MAINProcessMain */ /*@{*/ /**Flagspecifyingthatthedataretrievalprocessshouldberun.*/ #defineDSP_RETRIEVER0x001 /**Flagspecifyingifaretrieverdefinitionisrequired.*/ #defineDSP_RETRIEVER_REQUIRED0x002 /**Flagspecifyingthatthedatatransformationprocessshouldberun.*/ #defineDSP_TRANSFORM0x004 /**Flagspecifyingthatthisisaningestprocess.*/ #defineDSP_INGEST0x100 /** *ProcessModels */ typedefenum{ /**GenericVAPprocess. * *Theretrieverdefinitionwillbeusedifitexistsinthedatabase *butitisnotrequiredfortheprocesstorun.Thiswillalsorunthe *transformlogicforanyvariablesthatarefoundintheretrieved *datathathavebeentaggedwithacoordinatesystemname. */ PM_GENERIC=DSP_RETRIEVER|DSP_TRANSFORM, /**RetrieveronlyVAP. * *ThisVAPrequiresaretrieverdefinitiontobespecifiedinthe *database,butwillbypassthetransformationlogic. */ PM_RETRIEVER_VAP=DSP_RETRIEVER|DSP_RETRIEVER_REQUIRED, /**TransformationVAP. * *ThisVAPrequiresaretrieverdefinitiontobespecifiedinthe *database,andwillrunthetransformationlogic. */ PM_TRANSFORM_VAP=DSP_RETRIEVER|DSP_RETRIEVER_REQUIRED|DSP_TRANSFORM, /**IngestProcess. * *ThisisanIngestprocessthatloopsoverallrawfilesinthe *inputdatastreamdirectory. */ PM_INGEST=DSP_INGEST, /**Ingest/VAPHybridProcessthatbypassesthetransformlogic. * *ThisisanIngestprocessthatusesthePM_RETRIEVER_VAPprocessing *model,butisdesignedtoruninreal-timelikeaningestwithoutthe *needforthe'-bbegin_time'commandlineargument.Thestandard *VAP-b/-ecommandlineoptionscanstillbeusedforreprocessing. */ PM_RETRIEVER_INGEST=DSP_INGEST|DSP_RETRIEVER|DSP_RETRIEVER_REQUIRED, /**Ingest/VAPHybridProcessthatusesthetransformlogic. * *ThisisanIngestprocessthatusesthePM_TRANSFORM_VAPprocessing *model,butisdesignedtoruninreal-timelikeaningestwithoutthe *needforthe'-bbegin_time'commandlineargument.Thestandard *VAP-b/-ecommandlineoptionscanstillbeusedforreprocessing. */ PM_TRANSFORM_INGEST=DSP_INGEST|DSP_RETRIEVER|DSP_RETRIEVER_REQUIRED|DSP_TRANSFORM }ProcModel; voiddsproc_freeopts(void); intdsproc_getopt( constchar*option, constchar**value); intdsproc_setopt( constcharshort_opt, constchar*long_opt, constchar*arg_name, constchar*opt_desc); voiddsproc_use_nc_extension(void); intdsproc_main( intargc, char**argv, ProcModelproc_model, constchar*proc_version, intnproc_names, constchar**proc_names); voiddsproc_set_init_process_hook( void*(*_init_process_hook)(void)); voiddsproc_set_finish_process_hook( void(*finish_process_hook)(void*user_data)); voiddsproc_set_process_data_hook( int(*process_data)( void*user_data, time_tbegin_date, time_tend_date, CDSGroup*input_data)); voiddsproc_set_pre_retrieval_hook( int(*pre_retrieval_hook)( void*user_data, time_tbegin_date, time_tend_date)); voiddsproc_set_post_retrieval_hook( int(*post_retrieval_hook)( void*user_data, time_tbegin_date, time_tend_date, CDSGroup*ret_data)); voiddsproc_set_pre_transform_hook( int(*pre_transform_hook)( void*user_data, time_tbegin_date, time_tend_date, CDSGroup*ret_data)); voiddsproc_set_post_transform_hook( int(*post_transform_hook)( void*user_data, time_tbegin_date, time_tend_date, CDSGroup*trans_data)); voiddsproc_set_quicklook_hook( int(*quicklook_hook)( void*user_data, time_tbegin_date, time_tend_date)); voiddsproc_set_process_file_hook( int(*process_file_hook)( void*user_data, constchar*input_dir, constchar*file_name)); voiddsproc_set_custom_qc_hook( int(*custom_qc_hook)( void*user_data, intds_id, CDSGroup*dataset)); //Stationview intdsproc_station_view_hook( void*user_data, time_tbegin_date, time_tend_date, CDSGroup*trans_data); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_MESSAGESProcessMessages */ /*@{*/ /** *Conveniencemacroforthedsproc_error()function. * *Usage:DSPROC_ERROR(constchar*status,constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_ERROR(status,...)\ dsproc_error(__func__,__FILE__,__LINE__,status,__VA_ARGS__) /** *Conveniencemacroforthedsproc_warning()function. * *Usage:DSPROC_WARNING(constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_WARNING(...)\ dsproc_warning(__func__,__FILE__,__LINE__,__VA_ARGS__) /** *Conveniencemacroforthedsproc_log()function. * *Usage:DSPROC_LOG(constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_LOG(...)\ dsproc_log(__func__,__FILE__,__LINE__,__VA_ARGS__) /** *Conveniencemacroforthedsproc_bad_file_warning()function. * *Usage:DSPROC_BAD_FILE_WARNING(constchar*file_name,constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_BAD_FILE_WARNING(file_name,...)\ dsproc_bad_file_warning(__func__,__FILE__,__LINE__,file_name,__VA_ARGS__) /** *Conveniencemacroforthedsproc_bad_line_warning()function. * *Usage:DSPROC_BAD_LINE_WARNING(constchar*file_name,intline_num,constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_BAD_LINE_WARNING(file_name,line_num,...)\ dsproc_bad_line_warning(__func__,__FILE__,__LINE__,file_name,line_num,__VA_ARGS__) /** *Conveniencemacroforthedsproc_bad_record_warning()function. * *Usage:DSPROC_BAD_RECORD_WARNING(constchar*file_name,intrec_num,constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_BAD_RECORD_WARNING(file_name,rec_num,...)\ dsproc_bad_record_warning(__func__,__FILE__,__LINE__,file_name,rec_num,__VA_ARGS__) /** *Conveniencemacroforthedsproc_mentor_mail()function. * *Usage:DSPROC_MENTOR_MAIL(constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_MENTOR_MAIL(...)\ dsproc_mentor_mail(__func__,__FILE__,__LINE__,__VA_ARGS__) /** *Conveniencemacroforthedsproc_debug()function:level==1. * *Usage:DSPROC_DEBUG_LV1(constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_DEBUG_LV1(...)\ if(msngr_debug_level||msngr_provenance_level)\ dsproc_debug(__func__,__FILE__,__LINE__,1,__VA_ARGS__) /** *Conveniencemacroforthedsproc_debug()function:level==2. * *Usage:DSPROC_DEBUG_LV2(constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_DEBUG_LV2(...)\ if(msngr_debug_level||msngr_provenance_level)\ dsproc_debug(__func__,__FILE__,__LINE__,2,__VA_ARGS__) /** *Conveniencemacroforthedsproc_debug()function:level==3. * *Usage:DSPROC_DEBUG_LV3(constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_DEBUG_LV3(...)\ if(msngr_debug_level||msngr_provenance_level)\ dsproc_debug(__func__,__FILE__,__LINE__,3,__VA_ARGS__) /** *Conveniencemacroforthedsproc_debug()function:level==4. * *Usage:DSPROC_DEBUG_LV4(constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_DEBUG_LV4(...)\ if(msngr_debug_level||msngr_provenance_level)\ dsproc_debug(__func__,__FILE__,__LINE__,4,__VA_ARGS__) /** *Conveniencemacroforthedsproc_debug()function:level==5. * *Usage:DSPROC_DEBUG_LV5(constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_DEBUG_LV5(...)\ if(msngr_debug_level||msngr_provenance_level)\ dsproc_debug(__func__,__FILE__,__LINE__,5,__VA_ARGS__) intdsproc_get_debug_level(void); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_INFOProcessInformation */ /*@{*/ constchar*dsproc_get_site(void); constchar*dsproc_get_facility(void); constchar*dsproc_get_name(void); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_CONTROLProcessControl */ /*@{*/ /** *Conveniencemacroforthedsproc_abort()function. * *Usage:DSPROC_ABORT(constchar*status,constchar*format,...) * *Seeprintfforacompletedescriptionoftheformatstring. */ #defineDSPROC_ABORT(status,...)\ dsproc_abort(__func__,__FILE__,__LINE__,status,__VA_ARGS__) /** *LogFileIntervals. */ typedefenum{ LOG_MONTHLY=0,/**<createmonthlylogfiles*/ LOG_DAILY=1,/**<createdailylogfiles*/ LOG_RUN=2,/**<createonelogfileperrun*/ }LogInterval; /** *OutputFileSplittingMode. */ typedefenum{ SPLIT_ON_STORE=0,/**<Alwayscreateanewfilewhendataisstored.*/ SPLIT_ON_HOURS=1,/**<Splitstartisthehourofthedayforthefirst split[0-23],andsplitintervalisinhours.*/ SPLIT_ON_DAYS=2,/**<Splitstartisthedayofthemonthforthefirst split[1-31],andsplitintervalisindays.*/ SPLIT_ON_MONTHS=3,/**<Splitstartisthemonthoftheyearforthefirst split[1-12],andsplitintervalisinmonths.*/ SPLIT_NONE=4/**<Alwaysappendoutputtothepreviousfile unlessotherwisespecifiedinthecallto dsproc_store_dataset.*/ }SplitMode; voiddsproc_set_datastream_split_mode( intds_id, SplitModesplit_mode, doublesplit_start, doublesplit_interval); voiddsproc_set_log_interval(LogIntervalinterval,intuse_begin_time); voiddsproc_set_processing_interval_offset(time_toffset); voiddsproc_set_trans_qc_rollup_flag(intflag); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_DATASTREAMDataStreams */ /*@{*/ /** *DataStreamRoles */ typedefenum{ DSR_INPUT=1,/**<inputdatastream*/ DSR_OUTPUT=2/**<outputdatastream*/ }DSRole; intdsproc_get_datastream_id( constchar*site, constchar*facility, constchar*dsc_name, constchar*dsc_level, DSRolerole); intdsproc_get_input_datastream_id( constchar*dsc_name, constchar*dsc_level); intdsproc_get_input_datastream_ids(int**ids); intdsproc_get_output_datastream_id( constchar*dsc_name, constchar*dsc_level); intdsproc_get_output_datastream_ids(int**ids); constchar*dsproc_datastream_name(intds_id); constchar*dsproc_datastream_site(intds_id); constchar*dsproc_datastream_facility(intds_id); constchar*dsproc_datastream_class_name(intds_id); constchar*dsproc_datastream_class_level(intds_id); constchar*dsproc_datastream_path(intds_id); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_DATASTREAM_FILESDataStreamFiles */ /*@{*/ intdsproc_add_datastream_file_patterns( intds_id, intnpatterns, constchar**patterns, intignore_case); intdsproc_find_datastream_files( intds_id, time_tbegin_time, time_tend_time, char***file_list); voiddsproc_free_file_list(char**file_list); intdsproc_get_datastream_files(intds_id,char***file_list); intdsproc_set_datastream_path(intds_id,constchar*path); voiddsproc_set_file_name_compare_function( intds_id, int(*function)(constvoid*,constvoid*)); voiddsproc_set_file_name_time_function( intds_id, time_t(*function)(constchar*)); voiddsproc_set_datastream_file_extension( intds_id, constchar*extension); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_DATASETDatasets */ /*@{*/ constchar*dsproc_dataset_name(CDSGroup*dataset); constchar*dsproc_get_dataset_version( CDSGroup*dataset, int*major, int*minor, int*micro); CDSGroup*dsproc_get_output_dataset( intds_id, intobs_index); CDSGroup*dsproc_get_retrieved_dataset( intds_id, intobs_index); CDSGroup*dsproc_get_transformed_dataset( constchar*coordsys_name, intds_id, intobs_index); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_DATASET_DIMSDatasetDimensions */ /*@{*/ CDSDim*dsproc_get_dim( CDSGroup*dataset, constchar*name); size_tdsproc_get_dim_length( CDSGroup*dataset, constchar*name); intdsproc_set_dim_length( CDSGroup*dataset, constchar*name, size_tlength); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_DATASET_ATTSDatasetAttributes */ /*@{*/ intdsproc_change_att( void*parent, intoverwrite, constchar*name, CDSDataTypetype, size_tlength, void*value); CDSAtt*dsproc_get_att( void*parent, constchar*name); char*dsproc_get_att_text( void*parent, constchar*name, size_t*length, char*value); void*dsproc_get_att_value( void*parent, constchar*name, CDSDataTypetype, size_t*length, void*value); intdsproc_set_att( void*parent, intoverwrite, constchar*name, CDSDataTypetype, size_tlength, void*value); intdsproc_set_att_text( void*parent, constchar*name, constchar*format,...); intdsproc_set_att_value( void*parent, constchar*name, CDSDataTypetype, size_tlength, void*value); intdsproc_set_att_value_if_null( void*parent, constchar*name, CDSDataTypetype, size_tlength, void*value); intdsproc_set_att_text_if_null( void*parent, constchar*name, constchar*format,...); intdsproc_set_var_chunksizes( CDSVar*var, int*time_chunksize); voiddsproc_set_max_chunksize( size_tmax_chunksize); intdsproc_set_chunksizes( CDSGroup*dataset, inttime_chunksize); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_DATASET_VARSDatasetVariables */ /*@{*/ CDSVar*dsproc_clone_var( CDSVar*src_var, CDSGroup*dataset, constchar*var_name, CDSDataTypedata_type, constchar**dim_names, intcopy_data); CDSVar*dsproc_define_var( CDSGroup*dataset, constchar*name, CDSDataTypetype, intndims, constchar**dim_names, constchar*long_name, constchar*standard_name, constchar*units, void*valid_min, void*valid_max, void*missing_value, void*fill_value); intdsproc_delete_var( CDSVar*var); CDSVar*dsproc_get_bounds_var(CDSVar*coord_var); CDSVar*dsproc_get_coord_var( CDSVar*var, intdim_index); intdsproc_get_dataset_vars( CDSGroup*dataset, constchar**var_names, intrequired, CDSVar***vars, CDSVar***qc_vars, CDSVar***aqc_vars); CDSVar*dsproc_get_metric_var( CDSVar*var, constchar*metric); CDSVar*dsproc_get_output_var( intds_id, constchar*var_name, intobs_index); CDSVar*dsproc_get_qc_var( CDSVar*var); CDSVar*dsproc_get_retrieved_var( constchar*var_name, intobs_index); CDSVar*dsproc_get_transformed_var( constchar*var_name, intobs_index); CDSVar*dsproc_get_trans_coordsys_var( constchar*coordsys_name, constchar*var_name, intobs_index); CDSVar*dsproc_get_var( CDSGroup*dataset, constchar*name); constchar*dsproc_var_name(CDSVar*var); size_tdsproc_var_sample_count(CDSVar*var); size_tdsproc_var_sample_size(CDSVar*var); /* Needtoadd: intdsproc_var_is_unlimited(CDSVar*var); CDSGroup*dsproc_var_parent(CDSVar*var); intdsproc_var_shape(CDSVar*var,size_t**lengths); */ /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_VAR_DATADatasetVariableData */ /*@{*/ void*dsproc_alloc_var_data( CDSVar*var, size_tsample_start, size_tsample_count); void*dsproc_alloc_var_data_index( CDSVar*var, size_tsample_start, size_tsample_count); void*dsproc_get_var_data_index( CDSVar*var); void*dsproc_get_var_data( CDSVar*var, CDSDataTypetype, size_tsample_start, size_t*sample_count, void*missing_value, void*data); intdsproc_get_var_missing_values( CDSVar*var, void**values); void*dsproc_init_var_data( CDSVar*var, size_tsample_start, size_tsample_count, intuse_missing); void*dsproc_init_var_data_index( CDSVar*var, size_tsample_start, size_tsample_count, intuse_missing); intdsproc_set_bounds_data( CDSGroup*dataset, size_tsample_start, size_tsample_count); intdsproc_set_bounds_var_data( CDSVar*coord_var, size_tsample_start, size_tsample_count); void*dsproc_set_var_data( CDSVar*var, CDSDataTypetype, size_tsample_start, size_tsample_count, void*missing_value, void*data); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_TIME_DATADatasetTimeData */ /*@{*/ time_tdsproc_get_base_time(void*cds_object); size_tdsproc_get_time_range( void*cds_object, timeval_t*start_time, timeval_t*end_time); CDSVar*dsproc_get_time_var(void*cds_object); time_t*dsproc_get_sample_times( void*cds_object, size_tsample_start, size_t*sample_count, time_t*sample_times); timeval_t*dsproc_get_sample_timevals( void*cds_object, size_tsample_start, size_t*sample_count, timeval_t*sample_times); intdsproc_set_base_time( void*cds_object, constchar*long_name, time_tbase_time); intdsproc_set_sample_times( void*cds_object, size_tsample_start, size_tsample_count, time_t*sample_times); intdsproc_set_sample_timevals( void*cds_object, size_tsample_start, size_tsample_count, timeval_t*sample_times); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_QC_UTILSQCUtilities */ /*@{*/ intdsproc_consolidate_var_qc( CDSVar*in_qc_var, unsignedintbad_mask, CDSVar*out_qc_var, unsignedintbad_flag, unsignedintind_flag, intreset); unsignedintdsproc_get_bad_qc_mask(CDSVar*qc_var); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_MISC_UTILITIESMiscellaneousUtilities */ /*@{*/ intdsproc_create_timestamp( time_tsecs1970, char*timestamp); intdsproc_execvp( constchar*file, char*constargv[], intflags); intdsproc_run_dq_inspector( intdsid, time_tbegin_time, time_tend_time, constchar*args[], intflags); intdsproc_solar_position( time_tsecs1970, doublelatitude, doublelongitude, double*ap_ra, double*ap_dec, double*altitude, double*refraction, double*azimuth, double*distance); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_OUTLIER_FILTERSOutlierFilters */ /*@{*/ intdsproc_flag_outliers_iqd( CDSGroup*dataset, constchar*var_name, doublewindow_width, intmin_npoints, unsignedintskipped_flag, doublebad_threshold, unsignedintbad_flag, doubleind_threshold, unsignedintind_flag, unsignedintanalyze); intdsproc_flag_outliers_mean_dev( CDSGroup*dataset, constchar*var_name, doublewindow_width, intmin_npoints, unsignedintskipped_flag, doublebad_threshold, unsignedintbad_flag, doubleind_threshold, unsignedintind_flag, unsignedintanalyze); intdsproc_flag_outliers_mean_mad( CDSGroup*dataset, constchar*var_name, doublewindow_width, intmin_npoints, unsignedintskipped_flag, doublebad_threshold, unsignedintbad_flag, doubleind_threshold, unsignedintind_flag, unsignedintanalyze); intdsproc_flag_outliers_median_mad( CDSGroup*dataset, constchar*var_name, doublewindow_width, intmin_npoints, unsignedintskipped_flag, doublebad_threshold, unsignedintbad_flag, doubleind_threshold, unsignedintind_flag, unsignedintanalyze); intdsproc_flag_outliers_std( CDSGroup*dataset, constchar*var_name, doublewindow_width, intmin_npoints, unsignedintskipped_flag, doublebad_threshold, unsignedintbad_flag, doubleind_threshold, unsignedintind_flag, unsignedintanalyze); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_VARTAGSVariableTags */ /*@{*/ /**Flaginstructingthetransformlogictoignorethisvariable.*/ #defineVAR_SKIP_TRANSFORM0x1 /**ConsolidatethetransformationQCbitswhenmappedtotheoutputdataset.*/ #defineVAR_ROLLUP_TRANS_QC0x2 /** *OutputVariableTarget. */ typedefstruct{ intds_id;/**<outputdatastreamID*/ constchar*var_name;/**<outputvariablename*/ }VarTarget; intdsproc_add_var_output_target( CDSVar*var, intds_id, constchar*var_name); intdsproc_copy_var_tag( CDSVar*src_var, CDSVar*dest_var); voiddsproc_delete_var_tag( CDSVar*var); constchar*dsproc_get_source_var_name(CDSVar*var); constchar*dsproc_get_source_ds_name(CDSVar*var); intdsproc_get_source_ds_id(CDSVar*var); constchar*dsproc_get_var_coordsys_name(CDSVar*var); intdsproc_get_var_output_targets( CDSVar*var, VarTarget***targets); intdsproc_set_var_coordsys_name( CDSVar*var, constchar*coordsys_name); intdsproc_set_var_flags(CDSVar*var,intflags); intdsproc_set_var_output_target( CDSVar*var, intds_id, constchar*var_name); voiddsproc_unset_var_flags(CDSVar*var,intflags); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_DQRSVariableDQRs */ /*@{*/ /** *VariableDQR. * *Atthetimeofthiswritingthe'code=>color=>code_desc'valueswere: * *--1=>None=>Presumednottoexist *-0=>Black=>Missing *-1=>White=>Notinspected *-2=>Green=>Good *-3=>Yellow=>Suspect *-4=>Red=>Incorrect *-5=>Transparent=>Doesnotaffectquality */ typedefstruct{ constchar*id;/**<DQRID*/ constchar*desc;/**<description*/ constchar*ds_name;/**<datastreamname*/ constchar*var_name;/**<variablename*/ intcode;/**<codenumber*/ constchar*color;/**<codecolor*/ constchar*code_desc;/**<codedescription*/ time_tstart_time;/**<starttimeinsecondssince1970*/ time_tend_time;/**<endtimeinsecondssince1970*/ size_tstart_index;/**<starttimeindexindataset*/ size_tend_index;/**<endtimeindexindataset*/ }VarDQR; intdsproc_get_var_dqrs(CDSVar*var,VarDQR***dqrs); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_PrintPrintFunctions */ /*@{*/ intdsproc_dump_dataset( CDSGroup*dataset, constchar*outdir, constchar*prefix, time_tfile_time, constchar*suffix, intflags); intdsproc_dump_output_datasets( constchar*outdir, constchar*suffix, intflags); intdsproc_dump_retrieved_datasets( constchar*outdir, constchar*suffix, intflags); intdsproc_dump_transformed_datasets( constchar*outdir, constchar*suffix, intflags); /* Needfunctions: voiddsproc_print_dataset_object( constchar*file_name, void*object); */ /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_STATUSProcessStatusDefinitions */ /*@{*/ /**Successful*/ #defineDSPROC_SUCCESS"Successful" /**MemoryAllocationError*/ #defineDSPROC_ENOMEM"MemoryAllocationError" /**CouldNotCreateForkForNewProcess*/ #defineDSPROC_EFORK"CouldNotCreateForkForNewProcess" /**NoInputDataFound*/ #defineDSPROC_ENODATA"NoInputDataFound" /**NoOutputDataFound*/ #defineDSPROC_ENOOUTDATA"NoOutputDataCreated" /**CouldNotInitializeSignalHandlers*/ #defineDSPROC_EINITSIGS"CouldNotInitializeSignalHandlers" /**MaximumRunTimeLimitExceeded*/ #defineDSPROC_ERUNTIME"MaximumRunTimeLimitExceeded" /**CouldNotForceProcessToContinue*/ #defineDSPROC_EFORCE"CouldNotForceProcessToContinue" /**CouldNotDeterminePathToDatastream*/ #defineDSPROC_EDSPATH"CouldNotDeterminePathToDatastream" /**CouldNotDeterminePathToLogsDirectory*/ #defineDSPROC_ELOGSPATH"CouldNotDeterminePathToLogsDirectory" /**CouldNotOpenLogFile*/ #defineDSPROC_EACCESS"CouldNotAccessFileorDirectory" /**CouldNotOpenLogFile*/ #defineDSPROC_ELOGOPEN"CouldNotOpenLogFile" /**CouldNotOpenProvenanceLog*/ #defineDSPROC_EPROVOPEN"CouldNotOpenProvenanceLog" /**CouldNotInitializeMail*/ #defineDSPROC_EMAILINIT"CouldNotInitializeMail" /**DatabaseError(seelogfile)*/ #defineDSPROC_EDBERROR"DatabaseError(seelogfile)" /**DatabaseConnectionError*/ #defineDSPROC_EDBCONNECT"DatabaseConnectionError" /**DQRDatabaseError(seelogfile)*/ #defineDSPROC_EDQRDBERROR"DQRDatabaseError(seelogfile)" /**DQRDatabaseConnectionError*/ #defineDSPROC_EDQRDBCONNECT"DQRDatabaseConnectionError" /**DirectoryListError*/ #defineDSPROC_EDIRLIST"CouldNotGetDirectoryListing" /**DirectoryListError*/ #defineDSPROC_EREGEX"RegularExpressionError" /**InvalidInputDatastreamClass*/ #defineDSPROC_EBADINDSC"InvalidInputDatastreamClass" /**CouldNotFindInputDatastreamClassInDatabase*/ #defineDSPROC_ENOINDSC"CouldNotFindInputDatastreamClassInDatabase" /**TooManyInputDatastreamClassesInDatabase*/ #defineDSPROC_ETOOMANYINDSC"TooManyInputDatastreamsDefinedInDatabase" /**InvalidOutputDatastreamClass*/ #defineDSPROC_EBADOUTDSC"InvalidOutputDatastreamClass" /**InvalidDatastreamID*/ #defineDSPROC_EBADDSID"InvalidDatastreamID" /**InvalidOutputDatastreamFormat*/ #defineDSPROC_EBADOUTFORMAT"InvalidOutputDatastreamFormat" /**FoundDataTimeBeforeMinimumValidTime*/ #defineDSPROC_EMINTIME"FoundDataTimeBeforeMinimumValidTime" /**FoundDataTimeInTheFuture*/ #defineDSPROC_EFUTURETIME"FoundDataTimeInTheFuture" /**InvalidTimeOrder*/ #defineDSPROC_ETIMEORDER"InvalidTimeOrder" /**FoundOverlappingDataTimes*/ #defineDSPROC_ETIMEOVERLAP"FoundOverlappingDataTimes" /**InvalidBaseTime*/ #defineDSPROC_EBASETIME"CouldNotGetBaseTimeForTimeVariable" /**InvalidGlobalAttributeValue*/ #defineDSPROC_EGLOBALATT"InvalidGlobalAttributeValue" /**InvalidDataTypeForTimeVariable*/ #defineDSPROC_ETIMEVARTYPE"InvalidDataTypeForTimeVariable" /**InvalidDataTypeForQCVariable*/ #defineDSPROC_EQCVARTYPE"InvalidDataTypeForQCVariable" /**InvalidQCVariableSampleSize*/ #defineDSPROC_EQCSAMPLESIZE"InvalidQCVariableSampleSize" /**InvalidQCVariableDimensions*/ #defineDSPROC_EQCVARDIMS"InvalidQCVariableDimensions" /**InvalidDataTypeForVariable*/ #defineDSPROC_EVARTYPE"InvalidDataTypeForVariable" /**InvalidVariableSampleSize*/ #defineDSPROC_ESAMPLESIZE"InvalidVariableSampleSize" /**DataAttributeHasInvalidDataType*/ #defineDSPROC_EDATAATTTYPE"DataAttributeHasInvalidDataType" /**CouldNotCopyFile*/ #defineDSPROC_EFILECOPY"CouldNotCopyFile" /**CouldNotMoveFile*/ #defineDSPROC_EFILEMOVE"CouldNotMoveFile" /**CouldNotOpenFile*/ #defineDSPROC_EFILEOPEN"CouldNotOpenFile" /**CouldNotReadFromFile*/ #defineDSPROC_EFILEREAD"CouldNotReadFromFile" /**CouldNotWriteToFile*/ #defineDSPROC_EFILEWRITE"CouldNotWriteToFile" /**CouldNotGetFileStats*/ #defineDSPROC_EFILESTATS"CouldNotGetFileStats" /**CouldNotDeleteFile*/ #defineDSPROC_EUNLINK"CouldNotDeleteFile" /**SourceFileDoesNotExist*/ #defineDSPROC_ENOSRCFILE"SourceFileDoesNotExist" /**CouldNotDetermineFileTime*/ #defineDSPROC_ENOFILETIME"CouldNotDetermineFileTime" /**CouldNotCreateDestinationDirectory*/ #defineDSPROC_EDESTDIRMAKE"CouldNotCreateDestinationDirectory" /**TimeCalculationError*/ #defineDSPROC_ETIMECALC"TimeCalculationError" /**CouldNotGetFileMD5*/ #defineDSPROC_EFILEMD5"CouldNotGetFileMD5" /**SourceAndDestinationFileMD5sDoNotMatch*/ #defineDSPROC_EMD5CHECK"SourceAndDestinationFileMD5sDoNotMatch" /**CouldNotAllocateMemoryForDatasetVariable*/ #defineDSPROC_ECDSALLOCVAR"CouldNotAllocateMemoryForDatasetVariable" /**CouldNotCopyDatasetVariable*/ #defineDSPROC_ECDSCOPYVAR"CouldNotCopyDatasetVariable" /**CouldNotCopyDatasetVariable*/ #defineDSPROC_ECLONEVAR"CouldNotCloneDatasetVariable" /**CouldNotDefineDatasetVariable*/ #defineDSPROC_ECDSDEFVAR"CouldNotDefineDatasetVariable" /**CouldNotDeleteDatasetVariable*/ #defineDSPROC_ECDSDELVAR"CouldNotDeleteDatasetVariable" /**CouldNotCopyDatasetMetadata*/ #defineDSPROC_ECDSCOPY"CouldNotCopyDatasetMetadata" /**CouldNotChangeAttributeValueInDataset*/ #defineDSPROC_ECDSCHANGEATT"CouldNotChangeAttributeValueInDataset" /**CouldNotSetAttributeValueInDataset*/ #defineDSPROC_ECDSSETATT"CouldNotSetAttributeValueInDataset" /**CouldNotSetDimensionLengthInDataset*/ #defineDSPROC_ECDSSETDIM"CouldNotSetDimensionLengthInDataset" /**CouldNotSetVariableDataInDataset*/ #defineDSPROC_ECDSSETDATA"CouldNotSetVariableDataInDataset" /**CouldNotSetTimeValuesInDataset*/ #defineDSPROC_ECDSSETTIME"CouldNotSetTimeValuesInDataset" /**CouldNotGetTimeValuesFromDataset*/ #defineDSPROC_ECDSGETTIME"CouldNotGetTimeValuesFromDataset" /**CouldNotMergeDatasets*/ #defineDSPROC_EMERGE"CouldNotMergeDatasets" /**InvalidCellBoundaryVariableorDefinition*/ #defineDSPROC_EBOUNDSVAR"InvalidCellBoundaryVariableorDefinition" /**DODNotDefinedInDatabase*/ #defineDSPROC_ENODOD"DODNotDefinedInDatabase" /**CouldNotFindDataRetrieverInformation*/ #defineDSPROC_ENORETRIEVER"CouldNotFindRetrieverDefinitionInDatabase" /**InvalidRetrieverDefinition*/ #defineDSPROC_EBADRETRIEVER"InvalidRetrieverDefinition" /**RequiredVariableMissingFromDataset*/ #defineDSPROC_EREQVAR"RequiredVariableMissingFromDataset" /**RequiredAttributeVariableMissingFromVariableorDataset*/ #defineDSPROC_EREQATT"RequiredAttributeVariableMissingFromVariableorDataset" /**CouldNotRetrieveInputData*/ #defineDSPROC_ERETRIEVER"CouldNotRetrieveInputData" /**CouldNotCreateNetCDFFile*/ #defineDSPROC_ENCCREATE"CouldNotCreateNetCDFFile" /**CouldNotOpenNetCDFFile*/ #defineDSPROC_ENCOPEN"CouldNotOpenNetCDFFile" /**CouldNotCloseNetCDFFile*/ #defineDSPROC_ENCCLOSE"CouldNotCloseNetCDFFile" /**CouldNotCloseNetCDFFile*/ #defineDSPROC_ENCSYNC"CouldNotSyncNetCDFFile" /**NetCDFFileReadError*/ #defineDSPROC_ENCREAD"CouldNotReadFromNetCDFFile" /**NetCDFFileWriteError*/ #defineDSPROC_ENCWRITE"CouldNotWriteToNetCDFFile" /**CouldNotFindDataTransformInformation*/ #defineDSPROC_ENOTRANSFORM"CouldNotFindDataTransformInformation" /**CouldNotFindDataTransformInformation*/ #defineDSPROC_ETRANSFORM"CouldNotTransformInputData" /**CouldNotCreateConsolidatedTransformationQCVariable*/ #defineDSPROC_ETRANSQCVAR"CouldNotCreateConsolidatedTransformationQCVariable" /**CouldNotLoadTransformParametersFile*/ #defineDSPROC_ETRANSPARAMLOAD"CouldNotLoadTransformParametersFile" /**CouldNotMapInputVariableToOutputVariable*/ #defineDSPROC_EVARMAP"CouldNotMapInputVariableToOutputVariable" /**CouldNotParseInputCSVFile*/ #defineDSPROC_ECSVPARSER"CouldNotParseCSVFile" /**CouldNotReadCSVIngestConfigurationFile*/ #defineDSPROC_ECSVCONF"CouldNotReadCSVIngestConfigurationFile" /**CouldNotMapInputCSVDataToOutputDataset*/ #defineDSPROC_ECSV2CDS"CouldNotMapInputCSVDataToOutputDataset" /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_FILE_UTILSFileUtilities */ /*@{*/ intdsproc_copy_file(constchar*src_file,constchar*dest_file); intdsproc_move_file(constchar*src_file,constchar*dest_file); FILE*dsproc_open_file(constchar*file); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_INGEST_RENAME_RAWIngest:RenameRaw */ /*@{*/ intdsproc_rename( intds_id, constchar*file_path, constchar*file_name, time_tbegin_time, time_tend_time); intdsproc_rename_tv( intds_id, constchar*file_path, constchar*file_name, consttimeval_t*begin_time, consttimeval_t*end_time); intdsproc_rename_bad( intds_id, constchar*file_path, constchar*file_name, time_tfile_time); intdsproc_set_preserve_dots_from_name(intds_id,constchar*file_name); voiddsproc_set_rename_preserve_dots(intds_id,intpreserve_dots); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_CSV_UTILITIESIngest:CSVUtilityFunctions */ /*@{*/ intdsproc_count_csv_delims(constchar*strp,chardelim); char*dsproc_find_csv_delim(constchar*strp,chardelim); char*dsproc_skip_csv_whitespace(constchar*strp,chardelim); intdsproc_split_csv_string(char*strp,chardelim,intlength,char**list); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_CSV_FILE_PARSINGIngest:CSVParser */ /*@{*/ /** *CSVParsingStructure. */ typedefstruct { char*file_path;/**<pathtothedirectorythefileisin*/ char*file_name;/**<nameofthefile*/ structstatfile_stats;/**<filestats*/ char*file_data;/**<inmemorycopyoftheparseddatafile*/ intnlines;/**<numberoflinesinthefile*/ char**lines;/**<arrayoflinepointers*/ intlinenum;/**<currentlinenumber*/ char*linep;/**<pointertothecurrentlineinmemory*/ char**headers;/**<pointerstotheheaderfields*/ char***values;/**<pointerstothefieldvalues*/ intnfields;/**<numberoffieldsperrecord*/ intnrecs;/**<numberofrecords*/ char*header_data;/**<parsedcopyofheaderline*/ int*free_header;/**<onlyusedwhenaddingheadersmanually*/ char**rec_buff;/**<bufferusedtoparserecordlines*/ intnbytes_alloced;/**<allocatedlengthofthefile_dataarray*/ intnlines_alloced;/**<allocatedlengthofthelinesarray*/ intnfields_alloced;/**<numberoffieldsallocated*/ intnrecs_alloced;/**<numberofrecordsallocated*/ chardelim;/**<CSVcolumndelimiter*/ intnlines_guess;/**<estimatednumberoflinesinafile*/ intnfields_guess;/**<onlyusedwhenaddingheadersmanually*/ RETimeList*ft_patterns;/**<compiledlistoffiletimepatterns*/ RETimeRes*ft_result;/**<filetimeusedinternally*/ intntc;/**<numberoftimecolumns*/ char**tc_names;/**<listoftimecolumnnames*/ RETimeList**tc_patterns;/**<compiledlistoftimestringpatterns*/ int*tc_index;/**<indexesoftimecolumns*/ timeval_t*tvs;/**<arrayofrecordtimes*/ time_ttime_offset;/**<offsettoapplytorecordtimes*/ structtmbase_tm;/**<basetimetouseforrecordtimes*/ inttro_threshold;/**<thresholdusedtodetecttimerollovers*/ time_ttro_offset;/**<offsetusedtotracktimerollovers*/ }CSVParser; voiddsproc_free_csv_parser(CSVParser*csv); char**dsproc_get_csv_column_headers(CSVParser*csv,int*nfields); char**dsproc_get_csv_field_strvals(CSVParser*csv,constchar*name); timeval_t*dsproc_get_csv_timevals(CSVParser*csv,int*nrecs); time_tdsproc_get_csv_file_name_time( CSVParser*csv, constchar*name, RETimeRes*result); char*dsproc_get_next_csv_line(CSVParser*csv); CSVParser*dsproc_init_csv_parser(CSVParser*csv); intdsproc_load_csv_file(CSVParser*csv,constchar*path,constchar*name); intdsproc_parse_csv_header(CSVParser*csv,constchar*linep); intdsproc_parse_csv_record(CSVParser*csv,char*linep,intflags); intdsproc_print_csv(FILE*fp,CSVParser*csv); intdsproc_print_csv_header(FILE*fp,CSVParser*csv); intdsproc_print_csv_record(FILE*fp,CSVParser*csv); intdsproc_set_csv_column_name( CSVParser*csv, intindex, constchar*name); voiddsproc_set_csv_delimiter( CSVParser*csv, chardelim); intdsproc_set_csv_base_time( CSVParser*csv, time_tbase_time); intdsproc_set_csv_file_time_patterns( CSVParser*csv, intnpatterns, constchar**patterns); voiddsproc_set_csv_time_offset( CSVParser*csv, time_ttime_offset); intdsproc_set_csv_time_patterns( CSVParser*csv, constchar*name, intnpatterns, constchar**patterns); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_CSV2CDSIngest:CSVtoCDSMappingFunctions */ /*@{*/ /**Mappingflagtospecifythatexistingdatashouldbeoverwritten*/ #defineCSV_OVERWRITE0x1 /** *Structureusedtomapastringtoadatavalue. */ typedefstruct{ constchar*strval;/**<stringvalueinCSVfile*/ doubledblval;/**<valuetouseinoutputdataset*/ }CSVStrMap; /** *StructureusedtomapCSVParserdatatoaCDSGroup. */ typedefstruct{ constchar*csv_name;/**<columnnameintheCSVfile*/ constchar*csv_units;/**<unitsstring*/ constchar**csv_missings;/**<listifmissingvaluesinCSVdata*/ constchar*cds_name;/**<variablenameintheCDSstructure*/ /**listofstringtodoublemappingstructures*/ CSVStrMap*str_map; /**functionusedtotranslateastringtoadouble*/ double(*str_to_dbl)(constchar*strval,int*status); /**advancedfunctionformappingCSVdatatoCDSvariabledata*/ int(*set_data)( constchar*csv_strval, constchar**csv_missings, CDSVar*cds_var, size_tcds_sample_size, CDSDatacds_missing, CDSDatacds_datap); }CSV2CDSMap; intdsproc_map_csv_to_cds( CSVParser*csv, intcsv_start, intcsv_count, CSV2CDSMap*map, CDSGroup*cds, intcds_start, intflags); intdsproc_map_csv_to_cds_by_index( CSVParser*csv, int*csv_indexes, intcsv_count, CSV2CDSMap*map, CDSGroup*cds, intcds_start, intflags); intdsproc_add_csv_str_to_dbl_function( CSV2CDSMap*map, constchar*csv_name, double(*str_to_dbl)(constchar*strval,int*status)); /*@}*/ /******************************************************************************/ /** *@defgroupDSPROC_CSV_INGEST_CONFIGIngest:CSVIngestConfig */ /*@{*/ /**Flagusedbycsv_read_conf_file()tocheckforconfigfilesunder *therootdirectorydefinedbytheCONF_DATAenvironmentvariable.*/ #defineCSV_CHECK_DATA_CONF0x01 /** *CSVTimeColumnNamesandPatterns. */ typedefstruct { constchar*name;/**<nameofthedate/timecolumn*/ intnpatterns;/**<numberofpossibletimestringpatterns*/ constchar**patterns;/**<listofpossiletimestringpatterns*/ }CSVTimeCol; /** *CSVFieldMapStructure. */ typedefstruct { constchar*out_name;/**<nameofthevariableintheoutputdataset*/ constchar*col_name;/**<nameofthecolumnintheinputCSVfile*/ constchar*units;/**<unitsusedintheCSVfile*/ intnmissings;/**<numberofmissingvaluesusedintheCSVfile*/ constchar**missings;/**<listofmissingvaluesusedintheCSVfile*/ char*missbuf;/**<bufferusedtoparsethestringofmissingvalues*/ }CSVFieldMap; /** *CSVConfigurationStructure. */ typedefstruct { /*Setbycsv_init_conffunction*/ constchar*proc;/**<theprocessname*/ constchar*site;/**<thesitename*/ constchar*fac;/**<thefacilityname*/ constchar*name;/**<theconffilebasename*/ constchar*level;/**<theconffilelevel*/ /*Setbycsv_read_conf_file*/ constchar*file_path;/**<pathtotheconfigurationfile*/ constchar*file_name;/**<nameoftheconfigurationfile*/ /*Usedtofindconfigurationfiles*/ intsearch_npaths;/**<numberofconffilesearchpaths*/ constchar**search_paths;/**<listofconffilesearchpaths*/ DirList*dirlist;/**<listoftimevaryingconffiles*/ /*Readfromconffile*/ intfn_npatterns;/**<numberofcsvfilenamepatterns*/ constchar**fn_patterns;/**<listofcsvfilenamepatterns*/ intft_npatterns;/**<numberofcsvfiletimepatterns*/ constchar**ft_patterns;/**<listofcsvfiletimepatterns*/ chardelim;/**<columndelimiter*/ constchar*header_line;/**<stringcontainingtheheaderline*/ constchar*header_tag;/**<stringidentifierfortheheaderline*/ intheader_linenum;/**<linenumberofthefirstheaderline*/ intheader_nlines;/**<numberofheaderlines*/ intexp_ncols;/**<expectednumberofcolumns*/ inttime_ncols;/**<numberoftimecolumns*/ CSVTimeCol*time_cols;/**<listoftimecolumns*/ intfield_nmaps;/**<numberofentriesinthefieldmap*/ CSVFieldMap*field_maps;/**<fieldmap*/ constchar*split_interval;/**<splitintervalforoutputfiles*/ }CSVConf; intdsproc_add_csv_field_map( CSVConf*conf, constchar*out_name, constchar*col_name, intnargs, constchar**args); intdsproc_add_csv_file_name_patterns( CSVConf*conf, intnpatterns, constchar**patterns); intdsproc_add_csv_file_time_patterns( CSVConf*conf, intnpatterns, constchar**patterns); intdsproc_add_csv_time_column_patterns( CSVConf*conf, constchar*name, intnpatterns, constchar**patterns); intdsproc_append_csv_header_line( CSVConf*conf, constchar*string); voiddsproc_clear_csv_field_maps(CSVConf*conf); voiddsproc_clear_csv_file_name_patterns(CSVConf*conf); voiddsproc_clear_csv_file_time_patterns(CSVConf*conf); voiddsproc_clear_csv_time_column_patterns(CSVConf*conf); intdsproc_configure_csv_parser(CSVConf*conf,CSVParser*csv); CSV2CDSMap*dsproc_create_csv_to_cds_map( CSVConf*conf, CSVParser*csv, CDSGroup*cds, intflags); voiddsproc_free_csv_conf(CSVConf*conf); voiddsproc_free_csv_to_cds_map( CSV2CDSMap*map); CSVConf*dsproc_init_csv_conf( constchar*name, constchar*level); intdsproc_load_csv_conf( CSVConf*conf, time_tdata_time, intflags); intdsproc_print_csv_conf( FILE*fp,CSVConf*conf); /*@}*/ #include"dsproc3_internal.h" #endif/*_DSPROC3_H*/