sas_utils.h

Go to the documentation of this file.

/*******************************************************************************
*
*  COPYRIGHT (C) 2012 Battelle Memorial Institute.  All Rights Reserved.
*
********************************************************************************
*
*  Authors:
*
*     name:  Brian D. Ermold
*     phone: (509) 375-2277
*     email: brian.ermold@pnnl.gov
*
*******************************************************************************/

/** @file sas_utils.h
 *  SAS Ingest Utility Functions Header.
 */

#ifndef _SAS_UTILS
#define _SAS_UTILS

#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <string.h>
#include <time.h>

#include "dsproc3.h"

/**
 *  @defgroup SAS_UTILS SAS Utilities
 */
/*@{*/

/** The constant PI */
#define PI        3.1415926

/** Missing value used in data values */
#define MISSING  -9999

/** Maximum number of wavelengths */
#define SAS_WL_MAX    2048

/** Maximum length of a line in an input file */
#define SAS_LINE_MAX  32768

/** Maximum number of data values per line */
#define SAS_VALS_MAX  4096

#ifndef DEG_TO_RAD
/** Convert degrees to radians */
#define DEG_TO_RAD(a)  (a * 0.017453292519943)
#endif

#ifndef RAD_TO_DEG
/** Convert radians to degrees */
#define RAD_TO_DEG(a)  (a * 57.29577951308232)
#endif

#ifndef INTERPOLATE
/** Macro to interpolate between two values */
#define INTERPOLATE(x,x0,y0,x1,y1)  (y0 + (x - x0) * ((y1 - y0)/(x1 - x0)))
#endif

/**
 *  Convenience macro for initializing a variable in an output
 *  dataset and getting the index into its data array.
 */
#define GET_REQUIRED_DS_VAR(dataset,varname,var,errval) \
if (!(var = dsproc_get_var(dataset, varname))) { \
    DSPROC_ERROR( \
        "Required Variable Not Found In Dataset", \
        "Variable '%s' not found in dataset: %s\n", \
        varname, dataset->name); \
    return(errval); \
}

/**
 *  Convenience macro for initializing a variable in an output
 *  dataset and getting the index into its data array.
 */
#define ALLOC_DS_VAR(dataset,varname,datap,length,var,errval) \
GET_REQUIRED_DS_VAR(dataset,varname,var,errval); \
if (!(datap = dsproc_alloc_var_data_index(var, 0, length))) { \
    return(errval); \
}

/**
 *  Convenience macro for setting variable data in an output dataset.
 */
#define SET_DS_VAR_DATA(dataset,varname,type,length,cvar,var,errval) \
GET_REQUIRED_DS_VAR(dataset,varname,var,errval); \
if (!dsproc_set_var_data(var,type, 0, length, NULL, cvar)) { \
    return(errval); \
}

/**
 *  Convenience macro for setting the value of an attribute
 *  if it exists in the specified CDSGroup or CDSVar.
 */
#define SET_DS_ATT_IF_FOUND(parent,errval,name,format,...) \
if (dsproc_get_att(parent, name)) { \
    if (!dsproc_set_att_text(parent, name, format, __VA_ARGS__)) { \
        return(errval); \
    } \
}

/**
 *  SAS File List.
 */
typedef struct {

    int    nfiles;   /**< number of files in the list       */
    char **files;    /**< list of file names                */
    int    nalloced; /**< allocated length of the file list */

} SasFileList;

/**
 *  SAS File Types.
 */
typedef enum
{
    SASHEVIS, /**< SASHE VIS Record */
    SASHENIR, /**< SASHE NIR Record */
    SASZEVIS, /**< SASZE VIS Record */
    SASZENIR  /**< SASZE NIR Record */

} SasFileType;

/**
 *  SAS Configuration Table.
 */
typedef struct
{
    char     conf_dir[PATH_MAX];  /**< full path to the conf files directory */
    char     conf_file[PATH_MAX]; /**< name of the conf file                 */

    char     header[2048]; /**< table file header */

    size_t   nrows;        /**< number of rows    */
    size_t   ncols;        /**< number of columns */
    double **cols;         /**< column vectors    */

    double  *buffer;       /**< buffer used to parse the lines in the file */
    size_t   rows_alloced; /**< number of rows allocated                   */
    size_t   cols_alloced; /**< number of columns allocated                */

} SasConfigTable;

/**
 *  SAS Data Record.
 */
typedef struct {
    time_t time;
    int    tag;
    float  earth_sun_dist;
    float  airmass;
    float  pressure_pa;
    float  temp_mio_p;
    float  temp_mio_p_f;
    float  temp_mio_trh;
    float  rh_mio;
    float  temp_collector;
    float  rh_collector;
    float  t_dewpt_collector;
    float  temp_chiller;
    float  rh_chiller;
    float  t_dewpt_chiller;
    float  x_tilt;
    float  y_tilt;
    float  x_tilt_stddev;
    float  y_tilt_stddev;
    float  band_az_raw;
    float  band_az_n;
    float  solar_azimuth;
    float  solar_zenith;
    float  inner_band_angle_raw;
    float  inner_band_angle;
    float  inner_band_scat_ang;
    float  outer_band_angle_raw;
    float  outer_band_angle;
    float  outer_band_scat_ang;
    short  shutter_open_tf;
    float  spectrometer_clock_ticks;
    float  t_avantes_bench;
    float  t_avantes_ad;
    float  t_int;
    int    n_avg;
    float *spec;
    int    nalloced;
} SasRecord;

/**
 *  SAS Raw Data Structure.
 */
typedef struct {

    SasFileType  file_type;            /**< SAS file type                    */
    const char  *file_name;            /**< name of the file being processed */
    char         full_path[PATH_MAX];  /**< full path to the raw file        */
    FILE        *fp;                   /**< current file pointer             */
    int          linenum;              /**< line number in the current file  */
    char         line[SAS_LINE_MAX];   /**< read line buffer                 */
    double       vals[SAS_VALS_MAX];   /**< parse line buffer                */

    time_t       begin_time;           /**< time of the first record         */
    time_t       end_time;             /**< time of the last record          */

    /* Raw Data Header */

    char         serial_number[64];    /**< serial number of spectrometer    */
    char         model_number[64];     /**< model number of system           */
    char         collector_sn[64];     /**< serial number of collector       */
    int          pixel_range[2];       /**< the pixel range                  */
    char         op_mode[4096];        /**< operating mode description       */

    int          nwavelens;            /**< number of wavelengths            */
    double       wavelens[SAS_WL_MAX]; /**< array of wavelengths             */

    /* Raw Data Records */

    int          nrecs;                /**< number of records read in        */
    SasRecord   *records;              /**< array of raw data records        */
    int          nalloced;             /**< number of records allocated      */
    time_t      *times;                /**< array of all record times        */
    int         *tags;                 /**< array of all record tags         */

    /* Used for SASHE Data Processing */

    int          max_recsets;          /**< number of record sets allocated  */
    int          nrecsets;             /**< number of record sets found      */
    int         *recsets;              /**< start indexes of the record sets */
    time_t      *rs_times;             /**< record set times                 */
    double      *rs_sza;               /**< solar zenith angles              */
    double      *rs_cossza;            /**< cosines of solar zenith angles   */
    double      *rs_coscor;            /**< cosine corrections               */
    int          rs_drk;               /**< index offset of dark counts      */
    int          rs_tot;               /**< index offset of total irradiance */
    int          rs_sb1;               /**< index offset of side blocked 1   */
    int          rs_blk;               /**< index offset of sun blocked      */
    int          rs_sb2;               /**< index offset of side blocked 2   */

    SasConfigTable *Io_calib_table;       /**< Io calibration table          */
    double          Io_calib[SAS_WL_MAX]; /**< Io calibration values         */

    /* Used for SASZE Data Processing */

    SasConfigTable *resp_table;         /**< response function config table  */

    int          cal_max_ntimes;        /**< number of cal_times allocated   */
    int          cal_ntimes;            /**< number of cal_times             */
    time_t      *cal_times;             /**< array of shutter open times     */

    double       dark_avg[SAS_WL_MAX];  /**< averages of darks counts        */
    double       dark_sum2[SAS_WL_MAX]; /**< sums of darks counts squared    */
    double       dark_std[SAS_WL_MAX];  /**< standard deviations of darks    */
    double       resp_vals[SAS_WL_MAX]; /**< response function values        */
    int          ndarks[SAS_WL_MAX];    /**< number of darks in average      */
    double       toaIos[SAS_WL_MAX];    /**< interpolated Gueymard Io values */

} SasRawData;

int     sas_add_file_to_list(SasFileList *list, const char *file_name);
void    sas_clear_file_list(SasFileList *list);

SasFileList *sas_create_file_list(void);

time_t  sas_conf_file_name_time(const char *file_name);
int     sas_conf_file_name_time_compare(const void *str1, const void *str2);

int     sas_find_nearest_double(int n, double *x, double v);
int     sas_find_nearest_float(int n, float *x, float v);

void    sas_free_config_table(SasConfigTable *table);

void    sas_free_file_list(SasFileList *list);

void    sas_free_raw_data(SasRawData *raw);

DirList *sas_get_config_dirlist(
            const char *conf_dir,
            const char *pattern);

void    sas_get_config_values_double(
            SasConfigTable *table,
            int             x_col,
            size_t          x_nvals,
            double         *x_vals,
            int             y_col,
            double         *y_vals);

void    sas_get_config_values_float(
            SasConfigTable *table,
            int             x_col,
            size_t          x_nvals,
            float          *x_vals,
            int             y_col,
            float          *y_vals);

void    sas_get_config_values_interp_double(
            SasConfigTable *table,
            int             x_col,
            size_t          x_nvals,
            double         *x_vals,
            int             y_col,
            double         *y_vals);

void    sas_get_config_values_interp_float(
            SasConfigTable *table,
            int             x_col,
            size_t          x_nvals,
            float          *x_vals,
            int             y_col,
            float          *y_vals);

SasRawData *sas_init_raw_data(SasFileType type);

SasConfigTable *sas_load_config_table(
            const char     *conf_dir,
            const char     *conf_file,
            size_t          ncols,
            SasConfigTable *table);

int     sas_print_config_table(const char *file, SasConfigTable *table);

time_t  sas_raw_file_name_time(const char *file_name);
int     sas_raw_file_name_time_compare(const void *str1, const void *str2);

int     sas_set_default_att_values(CDSGroup *dataset);

int     sas_string_to_doubles(char *string, int buflen, double *buffer);

/*@}*/

/**
 *  @defgroup SAS_READ_DATA SAS Read Data
 */
/*@{*/

int     sas_read_line(SasRawData *raw, char **line);
int     sas_parse_header(SasRawData *raw);
int     sas_parse_record(SasRawData *raw);
int     sas_read_data(
            const char *input_dir,
            const char *file_name,
            SasRawData *raw,
            int         raw_dsid);

/*@}*/

/**
 *  @defgroup SAS_STORE_DATA SAS Store Data
 */
/*@{*/

int     sas_set_a0_metadata(SasRawData *raw, CDSGroup *dataset);
int     sas_store_a0_data(SasRawData *raw, int dsid);

/*@}*/

#endif /* _SAS_UTILS */