eris_ifu_detlin.c

/* $Id: eris_ifu_detlin.c,v 0.1 2019-06-11 08:13:17 06:06:06 agudo Exp $
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
 
/*
 * $Author: agudo $
 * $Date: 2018-06-11 06:06:06  $
 * $Revision: 0.1 $
 * $Name: not supported by cvs2svn $
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>          /* allows the program compilation */
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 ----------------------------------------------------------------------------*/
#include <strings.h>
#include <string.h>
#include <stdio.h>
#include <math.h>
 
#include <cpl.h>       
#include <hdrl.h>
//#include <irplib_utils.h>
 
#include "eris_ifu_dfs.h"
#include "eris_ifu_error.h"
#include "eris_ifu_functions.h"
#include "eris_ifu_utils.h"
#include "eris_ifu_detlin_static.h"
 
/*-----------------------------------------------------------------------------
                            Static variables
 ----------------------------------------------------------------------------*/
static char eris_ifu_detlin_description[] = "\
This recipe computes detector non linearities and a bad pixel map.\n\
The input files are increasing intensity raw flats\n\
\n\
-----------------------------------------------------------------------------\n\
Input files:\n\
   DO CATG          Explanation                              Required #Frames\n\
   -------          -----------                              -------- -------\n\
   LINEARITY_LAMP   Lamp exposures                              Y       3-n  \n\
\n\
Output files:\n\
   DO CATG          Explanation\n\
   -------          -----------\n\
   BPM_DETLIN       Badpixel frame\n\
   BPM_DETLIN_FILT  Filtered badpixel frame\n\
-----------------------------------------------------------------------------\n\
\n\
Information on relevant parameters may be found with\n\
  esorex --params "REC_NAME_DETLIN"\n\
  esorex --help   "REC_NAME_DETLIN"\n";
 
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
cpl_recipe_define(eris_ifu_detlin, ERIS_BINARY_VERSION, "Alex Agudo Berbel",
                  PACKAGE_BUGREPORT, "2020",
                  "Detector's linearity & non linear bad pixels determination.",
                  eris_ifu_detlin_description);
 
/*-----------------------------------------------------------------------------
                                Functions code
 ----------------------------------------------------------------------------*/
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
static cpl_error_code eris_ifu_detlin_fill_parameterlist(cpl_parameterlist *pl)
{
    cpl_error_code      err         = CPL_ERROR_NONE;
    cpl_parameter       *p          = NULL;
    hdrl_parameter      *hp         = NULL;
    cpl_parameterlist   *pl_tmp     = NULL;
 
    cpl_ensure_code(pl, CPL_ERROR_NULL_INPUT);
 
    TRY
    {
        BRK_IF_ERROR(
            eris_ifu_add_std_params(pl, REC_NAME_DETLIN));
 
        BRK_IF_NULL(
            hp = hdrl_bpm_fit_parameter_create_rel_chi(2, 8.0, 8.0));
 
        BRK_IF_NULL(
            pl_tmp = hdrl_bpm_fit_parameter_create_parlist(REC_NAME_DETLIN,
                                                         "", hp));
        eris_ifu_free_hdrl_parameter(&hp);
        for (p = cpl_parameterlist_get_first(pl_tmp);
             p != NULL;
             p = cpl_parameterlist_get_next(pl_tmp))
        {
            BRK_IF_ERROR(
                cpl_parameterlist_append(pl, cpl_parameter_duplicate(p)));
        }
        eris_ifu_free_parameterlist(&pl_tmp);
 
        BRK_IF_NULL(
            p = cpl_parameter_new_value(REC_NAME_DETLIN".post-filter-x",
                                        CPL_TYPE_INT,
                                        "X Size of the post filtering kernel",
                                        REC_NAME_DETLIN,
                                        3));
        BRK_IF_ERROR(
            cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pfx"));
        BRK_IF_ERROR(
            cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV));
        BRK_IF_ERROR(
            cpl_parameterlist_append(pl, p));
 
        BRK_IF_NULL(
            p = cpl_parameter_new_value(REC_NAME_DETLIN".post-filter-y",
                                        CPL_TYPE_INT,
                                        "Y Size of the post filtering kernel",
                                        REC_NAME_DETLIN,
                                        3));
        BRK_IF_ERROR(
            cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pfy"));
        BRK_IF_ERROR(
            cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV));
        BRK_IF_ERROR(
            cpl_parameterlist_append(pl, p));
 
        BRK_IF_NULL(
            p = cpl_parameter_new_enum(REC_NAME_DETLIN".post-filter-mode",
                                       CPL_TYPE_STRING,
                                       "Post filtering mode",
                                       REC_NAME_DETLIN,
                                       "closing", 2,
                                       "closing", "dilation"));
        BRK_IF_ERROR(
            cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pfm"));
        BRK_IF_ERROR(
            cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV));
        BRK_IF_ERROR(
            cpl_parameterlist_append(pl, p));
    }
    CATCH
    {
        CATCH_MSGS();
        err = cpl_error_get_code();
    }
 
    return err;
}
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
static int
eris_ifu_detlin(cpl_frameset *frameset, const cpl_parameterlist *parlist)
{
    double              p                   = 0.;
    cpl_size            n                   = 0.;
    cpl_vector          *vec_dit_on         = NULL,
                        *vec_dit_off        = NULL;
    cpl_image           *bpm                = NULL,
                        *bpm_filtered       = NULL;
    cpl_mask            *bpm_mask           = NULL,
                        *bpm_mask_filtered  = NULL;
    cpl_propertylist    *qc_list            = NULL;
    hdrl_imagelist      *hdrl_imglist_on    = NULL,
                        *hdrl_imglist_off   = NULL;
    double              dit_on;
    struct stdParamStruct stdParams         = stdParamStructInit;
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(cpl_frameset_is_empty(frameset) == FALSE,
                    CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(cpl_frameset_count_tags(frameset, ERIS_IFU_RAW_LIN) > 2,
                    CPL_ERROR_ILLEGAL_INPUT);
 
    TRY
    {
        /* -----------------------------------------------------------------
         *   Setup
         * ----------------------------------------------------------------- */
//        /* --instrument */
//        BRK_IF_NULL(
//            instrument = cpl_parameter_get_string(
//                                cpl_parameterlist_find_const(parlist,
//                                "eris.eris_ifu_detlin.instrument")));
//        cpl_msg_info(cpl_func, "Processing %s frames", instrument);
 
        /* Identify the RAW and CALIB frames in the input frameset */
        eris_ifu_dfs_set_groups(frameset);
        eris_ifu_fetch_std_param(parlist, REC_NAME_DETLIN, &stdParams);
        qc_list = cpl_propertylist_new();
 
        char* pipe_id = cpl_sprintf("%s%s%s", PACKAGE, "/", PACKAGE_VERSION);
               cpl_propertylist *applist = cpl_propertylist_new();
               cpl_propertylist_update_string(applist, CPL_DFS_PRO_CATG,
                       ERIS_IFU_PRO_DETLIN_GAIN_INFO);
        cpl_table* gain_table = eris_compute_gain(frameset);
        cpl_dfs_save_propertylist(frameset, NULL, parlist, frameset, NULL,
                REC_NAME_DETLIN, applist, NULL,
           pipe_id, ERIS_IFU_PRO_DETLIN_GAIN_INFO_FN);
        cpl_table_save(gain_table, applist, applist,
                ERIS_IFU_PRO_DETLIN_GAIN_INFO_FN, CPL_IO_EXTEND);
        cpl_table_delete(gain_table);
        cpl_propertylist_delete(applist);
        cpl_free(pipe_id);
// Search for static bad pixels in stacks of flatfield frames
// with in/decreasing intensities. For each pixel position a curve is plotted
// of the pixel intensity in each plane against the clean mean of the whole
// plane.
// A polynomial is fit and the found coefficients are stored sequentially in
// a data cube. Then the deviation of the linear coefficients from the clean
// mean is computed for each pixel and the pixel are declared bad if the
// deviations exceed a threshold defined by a factor of the clean standard
// deviation.
// For the resting good pixels the non-linear coefficients are examined.
// If one coefficients deviate more than a user given threshold the pixel
// is also declared as bad. The data cubus with the fit results and
// a bad pixel mask image is stored
 
        cpl_msg_info(cpl_func, "Loading frames...");
        eris_ifu_detlin_load_frames(frameset, stdParams.rawImageCorrectionMask,
                &hdrl_imglist_on, &hdrl_imglist_off, &vec_dit_on, &vec_dit_off);
 
        /* save optional output */
        if ((stdParams.productDepth & 2) != 0) {
            cpl_msg_info(cpl_func, "Save additional products ...");
            eris_ifu_save_hdrl_imagelist_dbg(hdrl_imglist_on,
                    ERIS_IFU_PRO_DETLIN_DBG_CUBE_ON_FN, TRUE);
            eris_ifu_save_hdrl_imagelist_dbg(hdrl_imglist_off,
                    ERIS_IFU_PRO_DETLIN_DBG_CUBE_OFF_FN, TRUE);
        }
        /* subtract on- and off-frames */
        int onSize = (int) hdrl_imagelist_get_size(hdrl_imglist_on);
        int offSize = (int) hdrl_imagelist_get_size(hdrl_imglist_off);
        for (int ix=0; ix < onSize; ix++) {
            dit_on = cpl_vector_get(vec_dit_on, ix);
            int found = 0;
            for (int ox=0; ox < offSize; ox++) {
                if (dit_on == cpl_vector_get(vec_dit_off, ox)) {
                    found = 1;
                    hdrl_image_sub_image(hdrl_imagelist_get(hdrl_imglist_on, ix),
                        hdrl_imagelist_get_const(hdrl_imglist_off, ox));
                        break;
                }
            }
            if (found == 0) {
                cpl_msg_error(cpl_func, "No matching off-frame DIT found for"
                    "DIT %f", dit_on);
            }
        }
        eris_ifu_free_hdrl_imagelist(&hdrl_imglist_off);
        /* For some reason the bad pixel propagation generates a new bad pixel
         * that is not properly flagged in the hdrl image. Therefore adding this
         * lines synchronizes again the bpm of the data and error section. This
         * could happen if there are invalid pixel in the error section that
         * were not properly flagged at the beginning - like INFINITY */
 
        for (int ix=0; ix < onSize; ix++) {
            hdrl_image * himage =  hdrl_imagelist_get(hdrl_imglist_on, ix);
            cpl_image  * cimage_data  = hdrl_image_get_image(himage);
            cpl_image  * cimage_error = hdrl_image_get_error(himage);
 
            cpl_mask * dbpm = cpl_image_get_bpm(cimage_data);
            cpl_mask * ebpm = cpl_image_get_bpm(cimage_error);
            cpl_mask_or(dbpm, ebpm);
            cpl_mask_or(ebpm, dbpm);
        }
 
        /* save optional output */
        if ((stdParams.productDepth & 2) != 0) {
            cpl_msg_info(cpl_func, "Save additional products ...");
            eris_ifu_save_hdrl_imagelist_dbg(hdrl_imglist_on,
                    ERIS_IFU_PRO_DETLIN_DBG_CUBE_SUB_FN, TRUE);
        }
 
        cpl_msg_info(cpl_func, "Generating bad pixel map...");
        bpm = eris_ifu_detlin_compute_linearity(parlist, hdrl_imglist_on,
                                                vec_dit_on, qc_list);
 
        /* calc stats on bpm */
        bpm_mask = cpl_mask_threshold_image_create(bpm, 0, INT_MAX);
        n = cpl_mask_count(bpm_mask);
        p = (double)n /
                (double) (cpl_mask_get_size_x(bpm_mask) * cpl_mask_get_size_y(bpm_mask));
        CHECK_ERROR_STATE();
        cpl_msg_info(cpl_func, "   %lld bad pixels (%g%%)", n, p * 100.);
 
        cpl_msg_info(cpl_func, "Filtering bad pixel map...");
        bpm_mask_filtered = eris_ifu_detlin_filter_mask(bpm_mask, parlist);
 
        cpl_msg_info(cpl_func, "Saving bad pixel maps...");
        eris_ifu_save_image(frameset, qc_list, parlist, REC_NAME_DETLIN,
                            ERIS_IFU_PRO_DETLIN_BPM, ERIS_IFU_PRO_DETLIN_BPM_FN,
                            CPL_TYPE_USHORT, bpm);
        cpl_propertylist_delete(qc_list);
        qc_list = cpl_propertylist_new();
        bpm_filtered = cpl_image_new_from_mask(bpm_mask_filtered);
        eris_ifu_get_badpix_qc_from_ima(bpm_filtered, qc_list, "LINEARITY");
        eris_ifu_save_image(frameset, qc_list, parlist, REC_NAME_DETLIN,
                            ERIS_IFU_PRO_DETLIN_BPM_FILT,
                            ERIS_IFU_PRO_DETLIN_BPM_FILT_FN, CPL_TYPE_USHORT,
                            bpm_filtered);
    }
    CATCH
    {
        CATCH_MSGS();
    }
 
    eris_ifu_free_hdrl_imagelist(&hdrl_imglist_on);
    eris_ifu_free_vector(&vec_dit_on);
    eris_ifu_free_vector(&vec_dit_off);
    eris_ifu_free_image(&bpm);
    eris_ifu_free_mask(&bpm_mask);
    eris_ifu_free_image(&bpm_filtered);
    eris_ifu_free_mask(&bpm_mask_filtered);
    eris_ifu_free_propertylist(&qc_list);
    eris_ifu_free_std_param(&stdParams);
    if (cpl_memory_is_empty() == 0) {
        cpl_memory_dump();
    }
 
    return (int) cpl_error_get_code();
}