eris_ifu_detlin_static.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
#include "eris_ifu_detlin_static.h"
#include "eris_ifu_error.h"
#include "eris_ifu_utils.h"
#include "eris_ifu_debug.h"
#include "eris_ifu_dfs.h"
#include "eris_utils.h"
#include "eris_ifu_functions.h"
#include "eris_pfits.h"
#include "eris_utils.h"
 
#include <hdrl.h>
#include <string.h>
#define FILE_NAME_SZ 256
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
cpl_error_code eris_ifu_detlin_load_frames(const cpl_frameset *frameset,
        int exposureCorrectionMode,
        hdrl_imagelist **hdrl_imglist_on,
        hdrl_imagelist **hdrl_imglist_off,
        cpl_vector **vec_dit_on,
        cpl_vector **vec_dit_off)
{
    cpl_error_code      ret                 = CPL_ERROR_NONE;
    const cpl_frame     *fr                 = NULL;
    const char          *fn                 = NULL;
    cpl_size            nr_dit_on           = 0,
            nr_dit_off          = 0;
    double_t            dit                 = 0.;
    hdrl_image          *tmp_img            = NULL;
    cpl_propertylist    *pl                 = NULL;
 
    cpl_ensure_code(frameset,           CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(hdrl_imglist_on,    CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(hdrl_imglist_off,   CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(vec_dit_on,         CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(vec_dit_off,        CPL_ERROR_NULL_INPUT);
 
    TRY
    {
        *hdrl_imglist_on = hdrl_imagelist_new();
        *hdrl_imglist_off = hdrl_imagelist_new();
        *vec_dit_on = cpl_vector_new(cpl_frameset_get_size(frameset));
        *vec_dit_off = cpl_vector_new(cpl_frameset_get_size(frameset));
 
        /*
         * Loop through all LINEARITY_LAMP frames and save on- and off-frames
         * accordingly in their cpl_imagelists
         */
        fr = cpl_frameset_find_const(frameset, ERIS_IFU_RAW_LIN);
        while (fr != NULL) {
            fn = cpl_frame_get_filename(fr);
            eris_ifu_file_exists(fn);
            tmp_img = eris_ifu_load_exposure_file(fn,
                    exposureCorrectionMode, NULL);
 
            pl = cpl_propertylist_load(fn, 0);
            dit = eris_ifu_get_dit(pl);
            CHECK_ERROR_STATE();
 
            if (eris_ifu_frame_is_on(fr)  == 1) {
                hdrl_imagelist_set(*hdrl_imglist_on, tmp_img, nr_dit_on);
                cpl_vector_set(*vec_dit_on, nr_dit_on++, dit);
            } else {
                hdrl_imagelist_set(*hdrl_imglist_off, tmp_img, nr_dit_off);
                cpl_vector_set(*vec_dit_off, nr_dit_off++, dit);
            }
 
            /* next frame */
            fr = cpl_frameset_find_const(frameset, NULL);
            eris_ifu_free_propertylist(&pl);
        }
        cpl_vector_set_size(*vec_dit_on, nr_dit_on);
        cpl_vector_set_size(*vec_dit_off, nr_dit_off);
 
        /*
         * check if there are twice as dits of on- than and dits of off-frames
         */
        if (nr_dit_on != 2*nr_dit_off) {
            cpl_msg_warning(cpl_func, "Found %lld on-frames and %lld off-frames, "
                    "but twice as many on-frmaes than off-frames are expected",
                    nr_dit_on, nr_dit_off);
        }
    }
    CATCH
    {
        CATCH_MSGS();
        ret = cpl_error_get_code();
        eris_ifu_free_hdrl_imagelist(hdrl_imglist_on);
        eris_ifu_free_hdrl_imagelist(hdrl_imglist_off);
        eris_ifu_free_vector(vec_dit_on);
        eris_ifu_free_vector(vec_dit_off);
    }
 
    eris_ifu_free_propertylist(&pl);
    eris_check_error_code("eris_ifu_detlin_load_frames");
    return ret;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static cpl_error_code
eris_detlin_qc_lin_log(hdrl_imagelist* fit_coef, cpl_image* chi2,
        cpl_image* dof, cpl_mask* mask, cpl_propertylist* linc_qclist)
{
 
    cpl_mask* bpm;
    cpl_size planes = hdrl_imagelist_get_size(fit_coef);
    cpl_image* image = NULL;
    cpl_image* error = NULL;
    char* name_o1;
    char* name_o2;
 
    for(cpl_size deg = 0; deg < planes; deg++)
    {
 
        image = hdrl_image_get_image(hdrl_imagelist_get(fit_coef, deg));
        error = hdrl_image_get_error(hdrl_imagelist_get(fit_coef, deg));
        cpl_image_power(error, 2.);
        cpl_image_multiply(error, chi2);
        cpl_image_divide(error, dof);
        cpl_image_power(error, 0.5);
 
        bpm = cpl_image_set_bpm(image, mask);
        const double coeff = cpl_image_get_median(image);
        cpl_image_set_bpm(image, bpm);
 
        name_o1 = cpl_sprintf("ESO QC LIN COEF%d", (int)deg);
 
        cpl_propertylist_append_double(linc_qclist, name_o1, coeff);
        cpl_propertylist_set_comment(linc_qclist, name_o1, ERIS_QC_LIN_COEF_C);
 
        cpl_free(name_o1);
        name_o1= NULL;
 
        name_o2 = cpl_sprintf("ESO QC LIN COEF%d ERR", (int)deg);
 
        bpm = cpl_image_set_bpm(error, mask);
        const double coeff_err = cpl_image_get_median(error);
        cpl_image_set_bpm(error, bpm);
 
        cpl_propertylist_append_double(linc_qclist, name_o2, coeff_err);
        cpl_propertylist_set_comment(linc_qclist, name_o2, ERIS_QC_LIN_COEF_ERR_C);
 
        cpl_free(name_o2);
        name_o2= NULL;
 
    }
    eris_check_error_code("eris_detlin_qc_lin_log");
    return cpl_error_get_code();
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
cpl_image* eris_ifu_detlin_compute_linearity(
        const cpl_parameterlist *parlist,
        const hdrl_imagelist    *imglist,
        const cpl_vector        *vec_dit,
        cpl_propertylist* qclog)
{
    hdrl_parameter  *hp         = NULL;
    cpl_image       *bpm_fit    = NULL;
 
    cpl_ensure(parlist, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(imglist, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(vec_dit, CPL_ERROR_NULL_INPUT, NULL);
 
    cpl_size        nr_img   = hdrl_imagelist_get_size(imglist),
            nr_vec  = cpl_vector_get_size(vec_dit);
    cpl_ensure(nr_img == nr_vec, CPL_ERROR_ILLEGAL_INPUT, NULL);
 
    TRY
    {
        // get bpm_fit-parameter
        hp = hdrl_bpm_fit_parameter_parse_parlist(parlist, REC_NAME_DETLIN);
 
        // calculate bpm_fit
        hdrl_bpm_fit_compute(hp, imglist, vec_dit, &bpm_fit);
 
        cpl_size sx = cpl_image_get_size_x(bpm_fit);
        cpl_size sy = cpl_image_get_size_y(bpm_fit);
        cpl_mask* mask = cpl_mask_new(sx, sy);
        int* pbpm = cpl_image_get_data(bpm_fit);
        cpl_binary* pmsk = cpl_mask_get_data(mask);
        for (cpl_size j = 0; j < sy; j++) {
            for (cpl_size i = 0; i < sx; i++) {
                if(pbpm[i+j*sx] != 0 ) {
                    pmsk[i+j*sx] = BAD_PIX;
                }
            }
        }
        /* flag pixels defined in bpm_fit into hdrl imagelist imglist */
        cpl_image* image;
        cpl_image* error;
        hdrl_image* hima;
 
        for(cpl_size i = 0; i < hdrl_imagelist_get_size(imglist); i++) {
            hima = hdrl_imagelist_get(imglist,i);
            image = hdrl_image_get_image(hima);
            cpl_image_reject_from_mask(image, mask);
            error = hdrl_image_get_error(hima);
            cpl_image_reject_from_mask(error, mask);
            //cpl_image_set_bpm(error, mask);
        }
 
        cpl_image * out_chi2 = NULL, * out_dof = NULL;
        hdrl_imagelist * out_coef = NULL;
        int order = hdrl_bpm_fit_parameter_get_degree(hp);
        hdrl_fit_polynomial_imagelist(imglist, vec_dit, order, &out_coef,
                &out_chi2, &out_dof);
 
        /* log fit coeff as QC params */
        eris_detlin_qc_lin_log(out_coef, out_chi2, out_dof, mask, qclog);
        hdrl_imagelist_delete(out_coef);
        cpl_image_delete(out_chi2);
        cpl_image_delete(out_dof);
        cpl_mask_delete(mask);
        eris_ifu_get_badpix_qc_from_ima(bpm_fit, qclog, "LINEARITY");
    }
    CATCH
    {
        CATCH_MSGS();
        eris_ifu_free_image(&bpm_fit);
    }
    eris_ifu_free_hdrl_parameter(&hp);
    eris_check_error_code("eris_ifu_detlin_compute_linearity");
    return bpm_fit;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
cpl_mask* eris_ifu_detlin_filter_mask(const cpl_mask *bpm,
        const cpl_parameterlist *parlist)
{
    cpl_mask            *filtered   = NULL;
    const cpl_parameter *p          = NULL;
    int                 pfx         = 0,
            pfy         = 0;
    const char          *pfm        = NULL;
    cpl_filter_mode     filter_mode = CPL_FILTER_CLOSING;
 
    cpl_ensure(bpm, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(parlist, CPL_ERROR_NULL_INPUT, NULL);
 
    TRY
    {
        p = cpl_parameterlist_find_const(parlist,
                REC_NAME_DETLIN".post-filter-x");
        pfx = cpl_parameter_get_int(p);
        CHECK_ERROR_STATE();
 
        p = cpl_parameterlist_find_const(parlist,
                REC_NAME_DETLIN".post-filter-y");
        pfy = cpl_parameter_get_int(p);
        CHECK_ERROR_STATE();
 
        p = cpl_parameterlist_find_const(parlist,
                REC_NAME_DETLIN".post-filter-mode");
        pfm = cpl_parameter_get_string(p);
 
        if (!strcmp(pfm, "closing")) {
            filter_mode = CPL_FILTER_CLOSING ;
        }
        else if (!strcmp(pfm, "dilation")) {
            filter_mode = CPL_FILTER_DILATION ;
        }
        else {
            BRK_WITH_ERROR_MSG(
                    CPL_ERROR_ILLEGAL_INPUT,
                    "Filter mode can only be \"closing\" or \"dilation\" (not %s)",
                    pfm);
        }
 
        /* Post Filtering */
        if ((pfx > 0) && (pfy > 0)) {
            BRK_IF_NULL(
                    filtered = hdrl_bpm_filter(bpm, pfx, pfy, filter_mode));
        }
    }
    CATCH
    {
        CATCH_MSGS();
        eris_ifu_free_mask(&filtered);
    }
    eris_check_error_code("eris_ifu_detlin_filter_mask");
    return filtered;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static int
eris_get_clean_mean_window(cpl_image* img,
        int llx, int lly, int urx, int ury,
        const int kappa, const int nclip,
        double* local_clean_mean,
        double* clean_stdev)
{
 
    double mean = 0;
    double stdev = 0;
 
    cpl_image * tmp = NULL;
    cpl_stats * stats = NULL;
    int i = 0;
 
    tmp = cpl_image_extract(img, llx, lly, urx, ury);
    cpl_image_accept_all(tmp);
    for(i = 0; i < nclip; i++) {
 
        cpl_stats_delete(stats);
        stats = cpl_stats_new_from_image(tmp, CPL_STATS_MEAN | CPL_STATS_STDEV);
        mean = cpl_stats_get_mean(stats);
        stdev = cpl_stats_get_stdev(stats);
 
        double threshold = kappa * stdev;
        double lo_cut = mean - threshold;
        double hi_cut = mean + threshold;
        cpl_image_accept_all(tmp);
        cpl_mask* mask = cpl_mask_threshold_image_create(tmp, lo_cut, hi_cut);
 
        cpl_mask_not(mask);
        cpl_image_reject_from_mask(tmp, mask);
        cpl_mask_delete(mask);
 
    }
    *local_clean_mean = mean;
    *clean_stdev = stdev;
    cpl_image_delete(tmp);
    cpl_stats_delete(stats);
    eris_check_error_code("eris_get_clean_mean_window");
    return 0;
 
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
cpl_table* eris_compute_gain(cpl_frameset* frameset)
{
 
    cpl_ensure(frameset,CPL_ERROR_NULL_INPUT, NULL);
 
    cpl_frame*    frm = NULL;
 
    cpl_image* img_on1 = NULL;
    cpl_image* img_on2 = NULL;
    cpl_image* img_on_dif = NULL;
    cpl_image* img_on_sub = NULL;
 
 
    cpl_image* img_of1 = NULL;
    cpl_image* img_of2 = NULL;
    cpl_image* img_of_dif = NULL;
    cpl_image* img_of_sub = NULL;
 
    cpl_table* res_tbl = NULL;
    cpl_vector* dit_on = NULL;
    cpl_vector* dit_of = NULL;
    cpl_vector* exptime_on = NULL;
    cpl_vector* exptime_of = NULL;
    cpl_propertylist* plist = NULL;
 
    int non = 0;
    int nof = 0;
    int nfr = 0;
    double avg_on1 = 0;
    double avg_on2 = 0;
    double avg_of1 = 0;
    double avg_of2 = 0;
    double std = 0;
 
    double sig_on_dif = 0;
    double sig_of_dif = 0;
    char* name = NULL;
    int i = 0;
    int m = 0;
 
    int llx = 270;
    int lly = 1000;
    int urx = 320;
    int ury = 1050;
    //int zone[4];
    double gain = 0;
    double dit_ref = 0;
    double dit_tmp = 0;
    double exptime_ref = 0;
    double exptime_tmp = 0;
    int kappa = 5;
    int nclip = 25;
    double centre = 0;
 
    cpl_size nff = cpl_frameset_get_size(frameset);
    cpl_frameset* son = cpl_frameset_new();
    cpl_frameset* sof = cpl_frameset_new();
    cpl_frame* frm_dup;
    /* separates on and off frames */
 
    for(i = 0; i < nff; i++) {
        frm = cpl_frameset_get_position(frameset, i);
        frm_dup = cpl_frame_duplicate(frm);
        if(eris_ifu_frame_is_on(frm)) {
            cpl_frameset_insert(son, frm_dup);
        } else {
            cpl_frameset_insert(sof, frm_dup);
        }
    }
 
    non = cpl_frameset_get_size(son);
    nof = cpl_frameset_get_size(sof);
    nfr = (non <= nof) ? non : nof;
 
    dit_on = cpl_vector_new(nfr);
    dit_of = cpl_vector_new(nfr);
    exptime_on = cpl_vector_new(nfr);
    exptime_of = cpl_vector_new(nfr);
 
    for(i = 0; i < nfr; i++) {
 
        frm = cpl_frameset_get_position(son, i);
        name = (char*) cpl_frame_get_filename(frm);
        plist = cpl_propertylist_load(name, 0);
        dit_ref = eris_pfits_get_dit(plist);
        exptime_ref = cpl_propertylist_get_double(plist, "EXPTIME");
        cpl_propertylist_delete(plist);
        cpl_vector_set(dit_on, i, dit_ref);
        cpl_vector_set(exptime_on, i, exptime_ref);
 
        frm = cpl_frameset_get_position(sof, i);
        name = (char*) cpl_frame_get_filename(frm);
        plist = cpl_propertylist_load(name, 0);
        dit_ref = eris_pfits_get_dit(plist);
        exptime_ref = cpl_propertylist_get_double(plist, "EXPTIME");
        cpl_propertylist_delete(plist);
        cpl_vector_set(dit_of, i, dit_ref);
        cpl_vector_set(exptime_of, i, exptime_ref);
 
    }
 
    res_tbl = cpl_table_new(nfr);
    cpl_table_new_column(res_tbl, "adu", CPL_TYPE_DOUBLE);
    cpl_table_new_column(res_tbl, "gain", CPL_TYPE_DOUBLE);
 
    for(i = 0; i < nfr; i++) {
        frm = cpl_frameset_get_position(son, i);
        name = (char*) cpl_frame_get_filename(frm);
        img_on1 = cpl_image_load(name, CPL_TYPE_DOUBLE, 0, 0);
 
        frm = cpl_frameset_get_position(sof, i);
        name = (char*) cpl_frame_get_filename(frm);
        img_of1 = cpl_image_load(name, CPL_TYPE_DOUBLE, 0, 0);
 
 
        dit_ref = cpl_vector_get(dit_on, i);
        exptime_ref = cpl_vector_get(exptime_on, i);
 
        for(m = 0; m < nfr; m++) {
            if(m != i) {
                frm = cpl_frameset_get_position(son, m);
                name = (char*) cpl_frame_get_filename(frm);
                dit_tmp = cpl_vector_get(dit_on, m);
                exptime_tmp = cpl_vector_get(exptime_on, m);
                if(dit_tmp == dit_ref && exptime_tmp == exptime_ref) {
                    /* eris_msg("m=%d i=%d\n", m, i); */
                    img_on2 = cpl_image_load(name, CPL_TYPE_DOUBLE, 0, 0);
                    frm = cpl_frameset_get_position(sof, m);
                    name = (char*) cpl_frame_get_filename(frm);
                    img_of2 = cpl_image_load(name, CPL_TYPE_DOUBLE, 0, 0);
 
                    img_on_dif = cpl_image_subtract_create(img_on1, img_on2);
                    img_of_dif = cpl_image_subtract_create(img_of1, img_of2);
 
                    img_on_sub = cpl_image_extract(img_on_dif, llx, lly, urx, ury);
                    img_of_sub = cpl_image_extract(img_of_dif, llx, lly, urx, ury);
 
                    eris_get_clean_mean_window(img_on1, llx, lly, urx, ury, kappa,
                            nclip, &avg_on1, &std);
                    eris_get_clean_mean_window(img_on2, llx, lly, urx, ury, kappa,
                            nclip, &avg_on2, &std);
                    eris_get_clean_mean_window(img_of1, llx, lly, urx, ury, kappa,
                            nclip, &avg_of1, &std);
                    eris_get_clean_mean_window(img_of2, llx, lly, urx, ury, kappa,
                            nclip, &avg_of2, &std);
                    /*
                 cpl_image_save(img_on_sub, "ima_on_sub.fits",
                                CPL_BPP_IEEE_FLOAT, NULL, CPL_IO_DEFAULT);
                 cpl_image_save(img_of_sub, "ima_of_sub.fits",
                                CPL_BPP_IEEE_FLOAT, NULL, CPL_IO_DEFAULT);
                     */
                     /*
               //worse accuracy
               eris_stat_rectangle(img_on_dif, kappa, nclip, &centre, &sig_on_dif);
               eris_stat_rectangle(img_of_dif, kappa, nclip, &centre, &sig_of_dif);
                      */
 
 
                    //better accuracy
                    eris_get_clean_mean_window(img_on_dif, llx, lly, urx, ury, kappa,
                            nclip, &centre, &sig_on_dif);
                    eris_get_clean_mean_window(img_of_dif, llx, lly, urx, ury, kappa,
                            nclip, &centre, &sig_of_dif);
                    cpl_image_delete(img_on2);
                    cpl_image_delete(img_of2);
                    cpl_image_delete(img_on_dif);
                    cpl_image_delete(img_of_dif);
                    cpl_image_delete(img_on_sub);
                    cpl_image_delete(img_of_sub);
 
                    gain = ( (avg_on1 + avg_on2) - (avg_of1 + avg_of2) ) /
                            ( (sig_on_dif * sig_on_dif) - (sig_of_dif * sig_of_dif) );
 
                    cpl_table_set_double(res_tbl, "gain", m, gain);
                    cpl_table_set_double(res_tbl, "adu", m,
                            ( (avg_on1 + avg_on2) / 2 - (avg_of1 + avg_of2) / 2) );
                    /* eris_msg("gain=%f ADU=%f\n",gain,
                  (avg_on1 + avg_on2) / 2 - (avg_of1 + avg_of2) / 2);
                  eris_msg("g=%f avg_on1=%f avg_on2=%f", gain, avg_on1, avg_on2);
                 eris_msg("avg_of1=%f avg_of2=%f sig_on_dif=%f sig_of_dif=%f",
                  avg_of1, avg_of2, sig_on_dif, sig_of_dif);
                     */
 
                }
            }
        }
        cpl_image_delete(img_on1);
        cpl_image_delete(img_of1);
    }
 
 
    /*
     eris_get_clean_mean_window(img_on_dif, llx, lly, urx, ury, kappa,
     nclip, &avg, &sig_on_dif);
     eris_get_clean_mean_window(img_of_dif, llx, lly, urx, ury, kappa,
     nclip, &avg, &sig_of_dif);
     */
 
    cpl_vector_delete(dit_on);
    cpl_vector_delete(dit_of);
    cpl_vector_delete(exptime_on);
    cpl_vector_delete(exptime_of);
    cpl_frameset_delete(son);
    cpl_frameset_delete(sof);
    eris_check_error_code("eris_compute_gain");
    return res_tbl;
 
}