eris_nix_flat_twilight.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
/*
 * $Author$
 * $Date$
 * $Revision$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include <string.h>
 
#include "eris_utils.h"
#include "eris_nix_utils.h"
#include "eris_pfits.h"
#include "eris_dfs.h"
#include "eris_nix_dfs.h"
#include "eris_nix_master_bpm.h"
#include "eris_nix_master_dark.h"
#include "eris_nix_gain_linearity.h"
#include <hdrl.h>
 
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char eris_nix_flat_twilight_description[] =
"This recipe reduces a set of twilight frames to produce a \n"
ERIS_NIX_MASTER_FLAT_TWILIGHT_LOFREQ_PRO_CATG" result.\n"
"\n"
"Input files:\n"
"\n"
"  DO CATG                       Explanation             Req.  #Frames\n"
"  -------                       -----------             ---   -------\n"
"  "ERIS_NIX_RAW_FLAT_TWILIGHT_DO_CATG
               "                 frames with a            Y  >min-frames\n"
"                                range of twilight \n"
"                                images\n"
"  "ERIS_NIX_MASTER_DARK_IMG_PRO_CATG
             "                   a MASTER_DARK with       Y        1\n"
"                                matching detector \n"
"                                configuration\n"
"  "ERIS_NIX_MASTER_BPM_LAMP_PRO_CATG
                 "               a lamp MASTER_BPM        Y        1\n"
"  "ERIS_NIX_GAIN_PRO_CATG
           "                     DETMON gain information  N      0 or 1\n"
"                                e.g. in file \n"
"                                detmon_ir_lg_gain_table.fits\n"
"  "ERIS_NIX_COEFFS_CUBE_PRO_CATG
             "                   DETMON linearity curves  N      0 or 1\n"
"                                e.g. in file \n"
"                                detmon_ir_coeffs_cube.fits\n"
"  "ERIS_NIX_NL_BPM_PRO_CATG
           "                     DETMON non-linear bpm    N      0 or 1\n"
"                                e.g. in file \n"
"                                detmon_ir_lg_bpm.fits\n"
"\n"
"Output files:\n"
"\n"
"  DO CATG                       Explanation \n"
"  -------                       ----------- \n"
"  "ERIS_NIX_MASTER_FLAT_TWILIGHT_LOFREQ_PRO_CATG
                             "   The required LOFREQ twilight flatfield.\n"
"\n"
"  The output will be a FITS file named 'master_flat_twilight_lofreq.fits',\n"
"  with extensions:\n"
"  - DATA, with the flatfield data.\n"
"  - ERR, with the flatfield error plane.\n"
"  - DQ, with the flatfield data quality plane.\n"
"  - CONFIDENCE, with the flatfield confidence plane.\n"
"\n";
#define RECIPE_NAME "eris_nix_flat_twilight"
#define CONTEXT "eris."RECIPE_NAME
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
cpl_recipe_define(eris_nix_flat_twilight, ERIS_BINARY_VERSION,
                  "John Lightfoot",
                  PACKAGE_BUGREPORT, "2017",
                  "Calculate a MASTER_FLAT_TWILIGHT_LOFREQ",
                  eris_nix_flat_twilight_description);
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_flat_twilight_fill_parameterlist(
  cpl_parameterlist *self) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    cpl_parameterlist * bpm_parlist = NULL;
    cpl_parameterlist * collapse_parlist = NULL;
    hdrl_parameter    * filter_defaults = NULL;
    hdrl_parameter    * flat_defaults = NULL;
    cpl_parameterlist * flat_parlist = NULL;
    hdrl_parameter    * legendre_defaults = NULL;
    hdrl_parameter    * minmax_defaults = NULL;
    hdrl_parameter    * sigclip_defaults = NULL;
 
    /* generate the general parameter list for the flat-field calculation */
 
    flat_defaults = hdrl_flat_parameter_create(21, 21, HDRL_FLAT_FREQ_LOW);
    flat_parlist = hdrl_flat_parameter_create_parlist(CONTEXT, "flat",
                                                      flat_defaults);
 
    for (cpl_parameter * p = cpl_parameterlist_get_first(flat_parlist);
         p != NULL; p = cpl_parameterlist_get_next(flat_parlist)) {
 
        const char * pname = cpl_parameter_get_name(p);
        if (strstr(pname, "method")) {
            cpl_parameter_disable(p, CPL_PARAMETER_MODE_CLI);
        }
        cpl_parameter * duplicate = cpl_parameter_duplicate(p);
        cpl_parameterlist_append(self, duplicate);
    }
 
    cpl_parameter * cp = NULL;
    cp = cpl_parameter_new_value(CONTEXT".min_frames", CPL_TYPE_INT,
                                 "minimum acceptable number of twilight "
                                 "images", CONTEXT, 2);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "min-frames");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    /* coords of pixel whose value to be used for diagnostics during 
       reduction */
 
    cp = cpl_parameter_new_value(CONTEXT".x_probe", CPL_TYPE_INT,
                                 "x coord of diagnostic pixel",
                                 CONTEXT, -1);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "x-probe");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    cp = cpl_parameter_new_value(CONTEXT".y_probe", CPL_TYPE_INT,
                                 "y coord of diagnostic pixel",
                                 CONTEXT, -1);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "y-probe");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    cp = cpl_parameter_new_value(CONTEXT".threshold", CPL_TYPE_DOUBLE,
            "positive saturation level (for QC)",
            CONTEXT, 60000.);
       cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "threshold");
       cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
       cpl_parameterlist_append(self, cp);
 
       cp = cpl_parameter_new_value(CONTEXT".saturation_neg", CPL_TYPE_DOUBLE,
            "negative saturation level (for QC)", CONTEXT, -4.5e7);
       cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "saturation_neg");
       cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
       cpl_parameterlist_append(self, cp);
 
    /* cleanup */
 
    cpl_parameterlist_delete(bpm_parlist);
    cpl_parameterlist_delete(collapse_parlist);
    hdrl_parameter_delete(filter_defaults);
    hdrl_parameter_delete(flat_defaults);
    cpl_parameterlist_delete(flat_parlist);
    hdrl_parameter_delete(legendre_defaults);
    hdrl_parameter_delete(minmax_defaults);
    hdrl_parameter_delete(sigclip_defaults);
 
    return 0;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static hdrl_imagelist * eris_nix_reduce_twilight(
                                   located_imagelist * twilight_limlist,
                                   const master_dark * mdark,
                                   const gain_linearity * gain_lin,
                                   const master_bpm * master_bpm_lamp,
                                   const cpl_size x_probe,
                                   const cpl_size y_probe) {
 
    cpl_ensure(twilight_limlist, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(mdark, CPL_ERROR_NULL_INPUT, NULL);
 
    cpl_imagelist * flat_fit = NULL;
    cpl_image     * flat_fit_error = NULL;
    cpl_imagelist * twilight_imagelist = NULL;
    cpl_vector    * twilight_medians = NULL;
 
    hdrl_imagelist * result = hdrl_imagelist_new();
 
/*
    const int debug = x_probe > 0 && 
                      x_probe <= hdrl_image_get_size_x(
                      twilight_limlist->limages[0]->himage) &&
                      y_probe > 0 &&
                      y_probe <= hdrl_image_get_size_y(
                      twilight_limlist->limages[0]->himage);
*/
 
    twilight_imagelist = cpl_imagelist_new();
    twilight_medians = cpl_vector_new(twilight_limlist->size);
 
    for (cpl_size i = 0; i < twilight_limlist->size; i++) {
  
        /* do basic calibration: linearize, calculate error plane */
 
        int flag_mask = 0;
        flag_mask = ~flag_mask;
        enu_basic_calibrate(twilight_limlist->limages[i],
                            CPL_TRUE,
                            NULL,
                            mdark,
                            gain_lin,
                            NULL,
                            NULL,
                            master_bpm_lamp,
                            flag_mask, 
                            "noop",
                            0.0,
                            x_probe,
                            y_probe);
        enu_check_error_code("error performing basic calibration of twilight "
                             "frames");
 
        /* build the cpl_imagelist that will be use for fitting pixel response */
 
        cpl_imagelist_set(twilight_imagelist, hdrl_image_get_image(
                          twilight_limlist->limages[i]->himage), i);
        cpl_vector_set(twilight_medians, i, 
                       cpl_image_get_median(hdrl_image_get_image(
                       twilight_limlist->limages[i]->himage)));
    }
    flat_fit = cpl_fit_imagelist_polynomial(twilight_medians,
                                            twilight_imagelist,
                                            0, 1, CPL_FALSE, 
                                            CPL_TYPE_DOUBLE, 
                                            flat_fit_error);
    enu_check_error_code("error fitting twilight frames");
    hdrl_imagelist_set(result, hdrl_image_create(
                       cpl_imagelist_get(flat_fit, 1), NULL), 0); 
 
    /* release image from the cpl_imagelist so that they don't get
       deleted twice later */
 
    while (cpl_imagelist_get_size(twilight_imagelist) > 0) {
        cpl_imagelist_unset(twilight_imagelist, 0);
    } 
 
 cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        hdrl_imagelist_delete(result);
        result = NULL;
    }
    cpl_imagelist_delete(flat_fit);
    cpl_image_delete(flat_fit_error);
    cpl_imagelist_delete(twilight_imagelist);
    cpl_vector_delete(twilight_medians);
 
    return result;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int eris_nix_flat_twilight(cpl_frameset * frameset,
                                  const cpl_parameterlist * parlist) {
    cpl_mask             * bpm_mask = NULL;
    hdrl_parameter       * collapse_params = NULL;
    cpl_image            * confidence_lo = NULL;
    hdrl_parameter       * flat_params = NULL;
    cpl_propertylist     * flat_plist = NULL;
    const gain_linearity * gain_lin = NULL;
    master_bpm           * master_bpm_lamp = NULL;
    master_dark          * mdark = NULL;
    hdrl_image           * master_flat_lofreq = NULL;     
    cpl_mask             * old_mask = NULL;
    const cpl_parameter  * param = NULL;
    located_imagelist    * twilight_limlist = NULL; 
    hdrl_imagelist       * twilight_reduced = NULL;
    cpl_frameset         * used_frameset = NULL;
 
    enu_check_error_code("%s():%d: An error is already set: %s",
                         cpl_func, __LINE__, cpl_error_get_where());
 
    /* check input parameters */
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
 
    /* set the msg verbosity level from environment variable CPL_MSG_LEVEL */
 
    cpl_msg_set_level_from_env();
    cpl_msg_severity severity = cpl_msg_get_level();
    cpl_msg_info(cpl_func, "level %d", (int) severity);
 
    /* check required input tags are present */
    const int ntags = 6;
    const char* required_tags[6] = {
            ERIS_NIX_RAW_FLAT_TWILIGHT_DO_CATG,
            ERIS_NIX_NL_BPM_PRO_CATG,
            ERIS_NIX_COEFFS_CUBE_PRO_CATG,
            ERIS_NIX_GAIN_PRO_CATG,
            ERIS_NIX_MASTER_BPM_LAMP_PRO_CATG,
            ERIS_NIX_MASTER_DARK_IMG_PRO_CATG
    };
 
    const int nopt_tags = 2;
    const char* optional_tags[2] = {
            ERIS_NIX_MASTER_FLAT_LAMP_HIFREQ_PRO_CATG,
            ERIS_NIX_MASTER_FLAT_LAMP_LOFREQ_PRO_CATG
    };
 
 
    cpl_ensure_code(CPL_ERROR_NONE ==
            eris_dfs_check_input_tags(frameset, required_tags, ntags, 1),
            CPL_ERROR_ILLEGAL_INPUT);
            eris_dfs_check_input_tags(frameset, optional_tags, nopt_tags, 0);
 
 
    /* retrieve input parameters for collapse, there is only one
       flat to collapse so this should not matter */
 
    collapse_params = hdrl_collapse_sigclip_parameter_create(3., 3., 5);
 
    /* retrieve input filter parameters for HI and LO flat-field */
 
    flat_params = hdrl_flat_parameter_parse_parlist(parlist,
                                                    CONTEXT".flat");
    cpl_size filter_size_x = hdrl_flat_parameter_get_filter_size_x(
                             flat_params);
    cpl_size filter_size_y = hdrl_flat_parameter_get_filter_size_y(
                             flat_params);
 
    /* retrieve minimum acceptable number of coadds */
 
    param = cpl_parameterlist_find_const(parlist, CONTEXT".min_frames");
    int min_frames = cpl_parameter_get_int(param);
 
    /* retrieve probe pixel coords */
 
    param = cpl_parameterlist_find_const(parlist, CONTEXT".x_probe");
    cpl_size x_probe = (cpl_size) cpl_parameter_get_int(param);
    param = cpl_parameterlist_find_const(parlist, CONTEXT".y_probe");
    cpl_size y_probe = (cpl_size) cpl_parameter_get_int(param);
    enu_check_error_code("Could not retrieve input parameters");
 
    /* identify the RAW and CALIB frames in the input frameset */
 
    eris_nix_dfs_set_groups(frameset);
    enu_check_error_code("Could not identify RAW and CALIB frames");    
 
    used_frameset = cpl_frameset_new();
 
    /* read the gain and linearity information, if available */
 
    int required = CPL_FALSE;
    gain_lin = engl_gain_linearity_load_from_frameset(frameset, 
               ERIS_NIX_GAIN_PRO_CATG, ERIS_NIX_COEFFS_CUBE_PRO_CATG,
               ERIS_NIX_NL_BPM_PRO_CATG, required, used_frameset);
    enu_check_error_code("failed to read gain/linearity information from SoF");
 
    /* read the mdark */
 
    mdark = en_master_dark_load_from_frameset(frameset,
                  ERIS_NIX_MASTER_DARK_IMG_PRO_CATG, used_frameset);
    enu_check_error_code("failed to read master dark from SoF");
 
    /* read the master_bpm */
 
    master_bpm_lamp = en_master_bpm_load_from_frameset(frameset,
                      ERIS_NIX_MASTER_BPM_LAMP_PRO_CATG, used_frameset,
                      CPL_TRUE);
    enu_check_error_code("failed to read master BPM from SoF");
 
    /* see what data the sof contains */
 
    twilight_limlist = enu_limlist_load_from_frameset(frameset,
                       ERIS_NIX_RAW_FLAT_TWILIGHT_DO_CATG, used_frameset);
    cpl_msg_info(cpl_func, "read %d "ERIS_NIX_RAW_FLAT_TWILIGHT_DO_CATG
                 " frames", (int) twilight_limlist->size);
    enu_check(twilight_limlist->size >= min_frames, CPL_ERROR_ILLEGAL_INPUT, 
              "%d or more "ERIS_NIX_RAW_FLAT_TWILIGHT_DO_CATG
              " frames required", min_frames);
 
    /* reduce raw data to get input to flatfield calculation */
 
    twilight_reduced = eris_nix_reduce_twilight(twilight_limlist,
                                                mdark,
                                                gain_lin,
                                                master_bpm_lamp,
                                                x_probe,
                                                y_probe);
    enu_check_error_code("failed basic reduction of twilight data");
 
    /* get the LO freq flatfield */
 
    master_flat_lofreq = enu_calc_flat(twilight_reduced,
                                       1,
                                       collapse_params,
                                       filter_size_x,
                                       filter_size_y,
                                       HDRL_FLAT_FREQ_LOW);
    enu_check_error_code("failed to calculate LOFREQ flat");
 
    /* create a LOFREQ confidence array, essentially the flat field with 
       bad pixels set to 0 and median pixel value normalised to 100 */
 
    confidence_lo = cpl_image_duplicate(hdrl_image_get_image_const(
                                        master_flat_lofreq));
    cpl_image_fill_rejected(confidence_lo, 0.0);
    cpl_image_accept_all(confidence_lo);
    enu_normalise_confidence(confidence_lo);
    enu_check_error_code("error computing LOFREQ confidence map");
    /* QC on FLAT_ON frames not using information from pixels of master_bpm */
 
    const char* saturation_limit = cpl_propertylist_get_string(gain_lin->plist,
            "ESO PRO REC1 PARAM25 VALUE");
    double saturation = atof(saturation_limit);
 
    cpl_propertylist* qclog = enu_raw_flats_qc(twilight_limlist,
            gain_lin->bpm, parlist, CONTEXT, saturation, CPL_FALSE, CPL_TRUE);
 
    /* now save the LOFREQ twilight flat */
   
    enu_flat_save(ERIS_NIX_MASTER_FLAT_TWILIGHT_LOFREQ_PRO_CATG,
                  master_flat_lofreq,
                  confidence_lo,
                  NULL,
                  frameset,
                  parlist,
                  "master_flat_twilight_lofreq.fits",
                  RECIPE_NAME, qclog);
    cpl_propertylist_delete(qclog);
    enu_check_error_code("Failed to save LOFREQ flat");
 
cleanup:
    cpl_mask_delete(bpm_mask);
    hdrl_parameter_delete(collapse_params);
    cpl_image_delete(confidence_lo);
    hdrl_parameter_delete(flat_params);
    cpl_propertylist_delete(flat_plist);
    engl_gain_linearity_delete((gain_linearity *) gain_lin);
    en_master_bpm_delete(master_bpm_lamp);
    en_master_dark_delete(mdark);
    hdrl_image_delete(master_flat_lofreq);
    cpl_mask_delete(old_mask);
    enu_located_imagelist_delete(twilight_limlist);
    hdrl_imagelist_delete(twilight_reduced);
    cpl_frameset_delete(used_frameset);
 
    return (int)cpl_error_get_code();
}