eris_nix_flat_lamp.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
/*
 * $Author$
 * $Date$
 * $Revision$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include <string.h>
#include <stdlib.h>
#include "eris_utils.h"
#include "eris_nix_utils.h"
#include "eris_pfits.h"
#include "eris_dfs.h"
#include "eris_nix_dfs.h"
#include "eris_nix_master_bpm.h"
#include "eris_nix_master_dark.h"
#include "eris_nix_gain_linearity.h"
#include <hdrl.h>
 
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char eris_nix_flat_lamp_description[] =
"This recipe reduces a set of calibration lamp frames to produce \n"
ERIS_NIX_MASTER_FLAT_LAMP_HIFREQ_PRO_CATG", "
ERIS_NIX_MASTER_FLAT_LAMP_LOFREQ_PRO_CATG" and\n"
ERIS_NIX_MASTER_BPM_LAMP_PRO_CATG" results.\n"
"\n"
"Input files:\n"
"  The recipe works either with a set of lamp-on/lamp-off pairs:\n"
"\n"
"  DO CATG                     Explanation             Req.  #Frames\n"
"  -------                     -----------             ---   -------\n"
"  "ERIS_NIX_RAW_FLAT_LAMP_ON_DO_CATG
              "                lamp-on frames          Y  >min-coadds\n"
"  "ERIS_NIX_RAW_FLAT_LAMP_OFF_DO_CATG
               "               lamp-off frames         Y  = #lamp-on\n"
"  "ERIS_NIX_MASTER_DARK_IMG_PRO_CATG
             "                 a MASTER_DARK with      Y        1\n"
"                              matching detector \n"
"                              configuration\n"
"  "ERIS_NIX_GAIN_PRO_CATG
           "                   DETMON gain             N      0 or 1\n"
"                              information e.g. in\n"
"                              file \n"
"                              detmon_ir_lg_gain_table.fits\n"
"  "ERIS_NIX_COEFFS_CUBE_PRO_CATG
             "                 DETMON linearity        N      0 or 1\n"
"                              curves e.g. in file \n"
"                              detmon_ir_coeffs_cube.fits\n"
"  "ERIS_NIX_NL_BPM_PRO_CATG
           "                   DETMON non-linear bpm   N      0 or 1\n"
"                              e.g. in file \n"
"                              detmon_ir_lg_bpm.fits\n"
"\n"
"  or with a series of lamp-on frames:\n"
"\n"
"  "ERIS_NIX_RAW_FLAT_LAMP_ON_DO_CATG
              "                lamp-on frames          Y  >min-coadds\n"
"  "ERIS_NIX_MASTER_DARK_IMG_PRO_CATG
             "                 a MASTER_DARK with      Y        1\n"
"                              matching detector \n"
"                              configuration\n"
"  "ERIS_NIX_GAIN_PRO_CATG
           "                   DETMON gain info        N      0 or 1\n"
"                              e.g. in file \n"
"                              detmon_ir_lg_gain_table.fits\n"
"  "ERIS_NIX_COEFFS_CUBE_PRO_CATG
             "                 DETMON linearity        N      0 or 1\n"
"                              curves e.g. in file \n"
"                              detmon_ir_coeffs_cube.fits\n"
"  "ERIS_NIX_NL_BPM_PRO_CATG
           "                   DETMON non-linear bpm   N      0 or 1\n"
"                              e.g. in file \n"
"                              detmon_ir_lg_bpm.fits\n"
"\n"
"Output files:\n"
"\n"
"  DO CATG                     Explanation \n"
"  -------                     ----------- \n"
"  "ERIS_NIX_MASTER_FLAT_LAMP_HIFREQ_PRO_CATG
                         "     The required HIFREQ flatfield.\n"
"  "ERIS_NIX_MASTER_FLAT_LAMP_LOFREQ_PRO_CATG
                         "     The required LOFREQ flatfield.\n"
"\n"
"  "ERIS_NIX_MASTER_BPM_LAMP_PRO_CATG
                 "             The required master BPM flatfield.\n"
"\n"
"  The HIFREQ output will be a FITS file named \n"
"  'master_flat_lamp_hifreq.fits', with extensions:\n"
"  - DATA, with the flatfield data.\n"
"  - ERR, with the flatfield error plane.\n"
"  - DQ, with the flatfield data quality plane.\n"
"  - COLD_BPM, with a bpm for the 'cold' pixels.\n"
"  - CONFIDENCE, with the flatfield confidence plane.\n"
"\n"
"  The LOFREQ output will be a FITS file named \n"
"  'master_flat_lamp_lofreq.fits', with extensions:\n"
"  - DATA, with the flatfield data.\n"
"  - ERR, with the flatfield error plane.\n"
"  - DQ, with the flatfield data quality plane.\n"
"  - CONFIDENCE, with the flatfield confidence plane.\n"
"\n"
"  The master BPM output will be a FITS file named \n"
"  'master_bpm_lamp.fits'. Common pixel values are:\n"
"  - 512, 'hot' pixel.\n"
"  - 1024, 'cold' pixel.\n"
"  - 32768, 'non-linear' pixel.\n"
"\n";
 
#define RECIPE_NAME "eris_nix_flat_lamp"
#define CONTEXT "eris."RECIPE_NAME
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
cpl_recipe_define(eris_nix_flat_lamp, ERIS_BINARY_VERSION, "John Lightfoot",
                  PACKAGE_BUGREPORT, "2017", "Calculate a "
                  ERIS_NIX_MASTER_FLAT_LAMP_HIFREQ_PRO_CATG", "
                  ERIS_NIX_MASTER_FLAT_LAMP_LOFREQ_PRO_CATG" and "
                  ERIS_NIX_MASTER_BPM_LAMP_PRO_CATG,
                  eris_nix_flat_lamp_description);
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_flat_lamp_fill_parameterlist(
  cpl_parameterlist *self) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    cpl_parameterlist * bpm_parlist = NULL;
    cpl_parameterlist * collapse_parlist = NULL;
    hdrl_parameter    * filter_defaults = NULL;
    hdrl_parameter    * flat_defaults = NULL;
    cpl_parameterlist * flat_parlist = NULL;
    hdrl_parameter    * legendre_defaults = NULL;
    hdrl_parameter    * minmax_defaults = NULL;
    hdrl_parameter    * sigclip_defaults = NULL;
 
    /* generate the general parameter list for the collapse */
    hdrl_parameter * mode_def =
        hdrl_collapse_mode_parameter_create(10., 1., 0., HDRL_MODE_MEDIAN, 0);
    sigclip_defaults = hdrl_collapse_sigclip_parameter_create(10.0, 10.0, 3);
    minmax_defaults = hdrl_collapse_minmax_parameter_create(50, 50);
    collapse_parlist = hdrl_collapse_parameter_create_parlist(CONTEXT,
                       "collapse", "MEDIAN", sigclip_defaults,
                    minmax_defaults,mode_def);
 
    /* add the subset of parameters to be used, i.e. leave out 'minmax' */
 
    for (const cpl_parameter * p = 
         cpl_parameterlist_get_first_const(collapse_parlist);
         p != NULL;
         p = cpl_parameterlist_get_next_const(collapse_parlist)) {
 
        const char * pname = cpl_parameter_get_name(p);
        if (strstr(pname, "minmax") == NULL) {
            cpl_parameter * duplicate = cpl_parameter_duplicate(p);
            cpl_parameterlist_append(self, duplicate);
        }
    }
 
    /* generate the general parameter list for the flat-field calculation */
 
    flat_defaults = hdrl_flat_parameter_create(21, 21, HDRL_FLAT_FREQ_HIGH);
    flat_parlist = hdrl_flat_parameter_create_parlist(CONTEXT, "flat",
                                                      flat_defaults);
 
    for (const cpl_parameter * p = 
         cpl_parameterlist_get_first(flat_parlist);
         p != NULL;
         p = cpl_parameterlist_get_next(flat_parlist)) {
        cpl_parameter * duplicate = cpl_parameter_duplicate(p);
        cpl_parameterlist_append(self, duplicate);
    }
 
    /* Get the general parameter list for the cold-pixel bpm */
 
    filter_defaults = hdrl_bpm_2d_parameter_create_filtersmooth(5.0, 20.0, 3,
                      CPL_FILTER_MEDIAN, CPL_BORDER_NOP, 21, 21);
    legendre_defaults = hdrl_bpm_2d_parameter_create_legendresmooth(4, 5, 6,
                        20, 21, 11, 12, 2, 10);
    bpm_parlist = hdrl_bpm_2d_parameter_create_parlist(CONTEXT, "coldpix",
                  "FILTER", filter_defaults, legendre_defaults);
 
    for (const cpl_parameter * p = 
         cpl_parameterlist_get_first_const(bpm_parlist);
         p != NULL;
         p = cpl_parameterlist_get_next_const(bpm_parlist)) {
        cpl_parameter * duplicate = cpl_parameter_duplicate(p);
        cpl_parameterlist_append(self, duplicate);
    }
 
    /* the minimum number of lamp_on - lamp_off images acceptable for the
       calculation of the flat */
 
    cpl_parameter * cp = NULL;
    cp = cpl_parameter_new_value(CONTEXT".min_coadds", CPL_TYPE_INT,
                                 "minimum acceptable number of (lamp_on - "
                                 "lamp_off) images", CONTEXT, 1);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "min-coadds");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    /* coords of pixel whose value to be used for diagnostics during 
       reduction */
 
    cp = cpl_parameter_new_value(CONTEXT".x_probe", CPL_TYPE_INT,
                                 "x coord of diagnostic pixel",
                                 CONTEXT, -1);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "x-probe");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    cp = cpl_parameter_new_value(CONTEXT".y_probe", CPL_TYPE_INT,
                                 "y coord of diagnostic pixel",
                                 CONTEXT, -1);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "y-probe");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    cp = cpl_parameter_new_value(CONTEXT".saturation_pos", CPL_TYPE_DOUBLE,
            "positive saturation level (for QC). If -1 uses the value "
            "of the parameter saturation_limit from the input GAIN_INFO table."
            "Else uses the value set by the user. Recommended values are:"
            "15000 for slow readout mode; 46500 for fast readout mode.",
            CONTEXT, -1.0);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "saturation_pos");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    cp = cpl_parameter_new_value(CONTEXT".saturation_neg", CPL_TYPE_DOUBLE,
            "negative saturation level (for QC)", CONTEXT, -4.5e7);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "saturation_neg");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
 
    /* cleanup */
 
    cpl_parameterlist_delete(bpm_parlist);
    cpl_parameterlist_delete(collapse_parlist);
    hdrl_parameter_delete(filter_defaults);
    hdrl_parameter_delete(flat_defaults);
    cpl_parameterlist_delete(flat_parlist);
    hdrl_parameter_delete(legendre_defaults);
    hdrl_parameter_delete(minmax_defaults);
    hdrl_parameter_delete(sigclip_defaults);
    hdrl_parameter_delete(mode_def);
    return 0;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static hdrl_imagelist * eris_nix_reduce_lamp_on(
                                       located_imagelist * lamp_on_limlist,
                                       const master_dark * master_drk,
                                       const gain_linearity * gain_lin,
                                       const cpl_size x_probe,
                                       const cpl_size y_probe) {
 
    cpl_ensure(lamp_on_limlist, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(master_drk, CPL_ERROR_NULL_INPUT, NULL);
 
    hdrl_imagelist * result = hdrl_imagelist_new();
 
    cpl_msg_info(cpl_func, "basic calibration of lamp-on data");
 
    for (int i = 0; i < lamp_on_limlist->size; i++) {
        enu_basic_calibrate(lamp_on_limlist->limages[i],
                            CPL_TRUE,
                            NULL,
                            master_drk,
                            gain_lin,
                            NULL,
                            NULL,
                            NULL,
                            0,
                            "noop",
                            0.0 , 
                            x_probe,
                            y_probe);
   
        enu_check_error_code("error performing basic calibration of lamp-on "
                             "frames");
 
        hdrl_imagelist_set(result,
                           hdrl_image_duplicate(lamp_on_limlist->
                           limages[i]->himage),
                           hdrl_imagelist_get_size(result));
    }
 
 cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        hdrl_imagelist_delete(result);
        result = NULL;
    }
    return result;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static hdrl_imagelist * eris_nix_reduce_lamp_on_off(
                                       located_imagelist * lamp_on_limlist,
                                       located_imagelist * lamp_off_limlist,
                                       const master_dark * master_drk,
                                       const gain_linearity * gain_lin,
                                       const cpl_size x_probe,
                                       const cpl_size y_probe) {
 
    cpl_ensure(lamp_on_limlist, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(lamp_off_limlist, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(master_drk, CPL_ERROR_NULL_INPUT, NULL);
 
    hdrl_imagelist * result = hdrl_imagelist_new();
 
    const int debug = x_probe > 0 && 
                      x_probe <= hdrl_image_get_size_x(
                      lamp_on_limlist->limages[0]->himage) &&
                      y_probe > 0 &&
                      y_probe <= hdrl_image_get_size_y(
                      lamp_on_limlist->limages[0]->himage);
 
    if (debug) {
        cpl_msg_info(cpl_func, "..     ON   (err) DQ      OFF  (err) DQ    ON-OFF (err) DQ");
    }
 
    enu_check(lamp_on_limlist->size == lamp_off_limlist->size, 
              CPL_ERROR_INCOMPATIBLE_INPUT,
              "different number of lamp-on and lamp-off frames");
 
    for (int i = 0; i < lamp_on_limlist->size; i++) {
  
        /* do basic calibration: linearize, calculate error plane */
 
        enu_basic_calibrate(lamp_on_limlist->limages[i], 
                            CPL_TRUE,
                            NULL,
                            master_drk,
                            gain_lin,
                            NULL,
                            NULL,
                            NULL,
                            0,
                            "noop",
                            0.0,
                            x_probe,
                            y_probe);
        enu_basic_calibrate(lamp_off_limlist->limages[i],
                            CPL_TRUE,
                            NULL,
                            master_drk,
                            gain_lin,
                            NULL,
                            NULL,
                            NULL,
                            0,
                            "noop",
                            0.0,
                            x_probe,
                            y_probe);
        enu_check_error_code("error performing basic calibration of lamp-on/off "
                             "frames");
 
        /* difference the on and off images and 'or' the bad pixel map with
           that of the master dark */
 
        hdrl_value hon = {0.0, 0.0};
        int ron = 0;
        hdrl_value hoff = {0.0, 0.0};
        int roff = 0;
        hdrl_value hres = {0.0, 0.0};
        int rres = 0;
        if (debug) {
            hon = hdrl_image_get_pixel(lamp_on_limlist->limages[i]->himage,
                                      x_probe, y_probe, &ron);
            hoff = hdrl_image_get_pixel(lamp_off_limlist->limages[i]->himage,
                                       x_probe, y_probe, &roff);
        }    
 
        hdrl_image_sub_image(lamp_on_limlist->limages[i]->himage,
                             lamp_off_limlist->limages[i]->himage);
        cpl_mask_or(hdrl_image_get_mask(lamp_on_limlist->limages[i]->
                    himage), master_drk->hot_bpm);
        hdrl_imagelist_set(result,
                           hdrl_image_duplicate(lamp_on_limlist->
                           limages[i]->himage),
                           hdrl_imagelist_get_size(result));
 
        if (debug) {
            hres = hdrl_image_get_pixel(lamp_on_limlist->limages[i]->
                                        himage, x_probe, y_probe, &rres);
            cpl_msg_info(cpl_func, ".. %8.2f(%5.2f) %d  %8.2f(%5.2f) %d  "
                         "%8.2f(%5.2f) %d",
                         hon.data, hon.error, ron,
                         hoff.data, hoff.error, roff, 
                         hres.data, hres.error, rres);
        }
    }
 
 cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        hdrl_imagelist_delete(result);
        result = NULL;
    }
    return result;
}
/*
static cpl_error_code
eris_lamp_flat_get_qc_sat(cpl_frameset* frameset, cpl_propertylist* qclog)
{
    cpl_frameset* raws = cpl_frameset_new();
 
    eris_dfs_extract_raw_frames(frameset, raws);
 
    cpl_frame* frm_mdark = cpl_frameset_find(frameset,
            ERIS_NIX_MASTER_DARK_IMG_PRO_CATG);
 
    double saturation_negative = - 4.5e7;
    double saturation = 15000;
    const char* frame_name;
    saturation=3000;
    cpl_size size = cpl_frameset_get_size(raws);
    for(cpl_size i = 0; i < size; i++) {
 
        cpl_frame* frame = cpl_frameset_get_position(raws, i);
 
        frame_name = cpl_frame_get_filename(frame);
        eris_get_sat_pix_qc_for_image(frame_name, frm_mdark, saturation,
                saturation_negative, i, qclog);
 
    }
 
    cpl_frameset_delete(raws);
 
    return cpl_error_get_code();
}
*/
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int eris_nix_flat_lamp(cpl_frameset * frameset,
                              const cpl_parameterlist * parlist) {
    cpl_mask             * bpm_mask = NULL;
    hdrl_parameter       * bpm_params = NULL;
    cpl_propertylist     * bpm_plist = NULL;
    cpl_mask             * cold_bpm = NULL;
    hdrl_parameter       * collapse_params = NULL;
    cpl_image            * confidence_hi = NULL;
    cpl_image            * confidence_lo = NULL;
    hdrl_parameter       * flat_params = NULL;
    cpl_propertylist     * flat_plist = NULL;
    const gain_linearity * gain_lin = NULL;
    hdrl_image           * illumination_flat = NULL;     
    located_imagelist    * lamp_off_limlist = NULL; 
    located_imagelist    * lamp_on_limlist = NULL; 
    hdrl_imagelist       * lamp_reduced_data = NULL;
    master_bpm           * master_bad_pix_map = NULL;
    master_dark          * master_drk = NULL;
    hdrl_image           * master_flat_hifreq = NULL;     
    hdrl_image           * master_flat_lofreq = NULL;     
    cpl_mask             * old_mask = NULL;
    const cpl_parameter  * param = NULL;
    cpl_mask             * un_illum = NULL;
    cpl_frameset         * used_frameset = NULL;
 
    enu_check_error_code("%s():%d: An error is already set: %s",
                         cpl_func, __LINE__, cpl_error_get_where());
 
    /* check input parameters */
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
 
    /* set the msg verbosity level from environment variable CPL_MSG_LEVEL */
 
    cpl_msg_set_level_from_env();
    cpl_msg_severity severity = cpl_msg_get_level();
    cpl_msg_info(cpl_func, "level %d", (int) severity);
 
    /* check required input tags are present */
    const int ntags = 5;
    const char* required_tags[5] = {ERIS_NIX_RAW_FLAT_LAMP_ON_DO_CATG,
            ERIS_NIX_NL_BPM_PRO_CATG,
            ERIS_NIX_COEFFS_CUBE_PRO_CATG,
            ERIS_NIX_GAIN_PRO_CATG,
            ERIS_NIX_MASTER_DARK_IMG_PRO_CATG
    };
    const int nopt_tags = 1;
       const char* optional_tags[1] = {ERIS_NIX_RAW_FLAT_LAMP_OFF_DO_CATG,
       };
 
    cpl_ensure_code(CPL_ERROR_NONE ==
            eris_dfs_check_input_tags(frameset, required_tags, ntags, 1),
            CPL_ERROR_ILLEGAL_INPUT);
 
    eris_dfs_check_input_tags(frameset, optional_tags, nopt_tags, 0);
 
 
 
    /* retrieve input parameters for collapse */
 
    collapse_params = hdrl_collapse_parameter_parse_parlist(parlist,
                      CONTEXT".collapse");
 
    /* retrieve input filter parameters for HI and LO flat-field */
 
    flat_params = hdrl_flat_parameter_parse_parlist(parlist,
                                                    CONTEXT".flat");
    cpl_size filter_size_x = hdrl_flat_parameter_get_filter_size_x(
                             flat_params);
    cpl_size filter_size_y = hdrl_flat_parameter_get_filter_size_y(
                             flat_params);
 
    /* retrieve input parameters for 'cold-pixel' bpm */
 
    bpm_params = hdrl_bpm_2d_parameter_parse_parlist(parlist,
                                                     CONTEXT".coldpix");
 
 
    /* retrieve minimum acceptable number of coadds */
 
    param = cpl_parameterlist_find_const(parlist, CONTEXT".min_coadds");
    int min_coadds = cpl_parameter_get_int(param);
 
    /* retrieve probe pixel coords */
 
    param = cpl_parameterlist_find_const(parlist, CONTEXT".x_probe");
    cpl_size x_probe = (cpl_size) cpl_parameter_get_int(param);
    param = cpl_parameterlist_find_const(parlist, CONTEXT".y_probe");
    cpl_size y_probe = (cpl_size) cpl_parameter_get_int(param);
 
    enu_check_error_code("Could not retrieve input parameters");
 
    /* identify the RAW and CALIB frames in the input frameset */
 
    eris_nix_dfs_set_groups(frameset);
    enu_check_error_code("Could not identify RAW and CALIB frames");    
 
    used_frameset = cpl_frameset_new();
 
    /* read the gain and linearity information, if available */
 
    int required = CPL_FALSE;
    gain_lin = engl_gain_linearity_load_from_frameset(frameset, 
               ERIS_NIX_GAIN_PRO_CATG, ERIS_NIX_COEFFS_CUBE_PRO_CATG,
               ERIS_NIX_NL_BPM_PRO_CATG, required, used_frameset);
    enu_check_error_code("failed to read gain/linearity information from SoF");
 
    /* read the master_dark */
 
    master_drk = en_master_dark_load_from_frameset(frameset,
                  ERIS_NIX_MASTER_DARK_IMG_PRO_CATG, used_frameset);
    enu_check_error_code("failed to read master dark from SoF");
 
    /* see what data the sof contains */
 
    lamp_on_limlist = enu_limlist_load_from_frameset(frameset,
                      ERIS_NIX_RAW_FLAT_LAMP_ON_DO_CATG, used_frameset);
    lamp_off_limlist = enu_limlist_load_from_frameset(frameset,
                      ERIS_NIX_RAW_FLAT_LAMP_OFF_DO_CATG, used_frameset);
 
    double saturation = 0;
 
 
    param = cpl_parameterlist_find_const(parlist, CONTEXT".saturation_pos");
 
    if(eris_param_has_changed(param)) {
        saturation = cpl_parameter_get_double(param);
    } else {
        const char* sat_limit = cpl_propertylist_get_string(gain_lin->plist,
                "ESO PRO REC1 PARAM25 VALUE");
        saturation = atof(sat_limit);
    }
    /* QC on FLAT_ON frames not using information from pixels of master_bad_pix_map */
    cpl_propertylist* qclog = enu_raw_flats_qc(lamp_on_limlist, gain_lin->bpm,
                                               parlist, CONTEXT, saturation,
                                               CPL_TRUE, CPL_FALSE);
 
    //eris_lamp_flat_get_qc_sat(frameset, qclog);
 
 
    /* reduce raw data to get input to flatfield calculation */
 
    if (lamp_on_limlist->size > 0 && lamp_off_limlist->size > 0) {
        lamp_reduced_data = eris_nix_reduce_lamp_on_off(lamp_on_limlist,
                                                        lamp_off_limlist,
                                                        master_drk,
                                                        gain_lin,
                                                        x_probe,
                                                        y_probe);
    } else if (lamp_on_limlist->size > 0 && lamp_off_limlist->size == 0) {
        lamp_reduced_data = eris_nix_reduce_lamp_on(lamp_on_limlist,
                                                    master_drk,
                                                    gain_lin,
                                                    x_probe,
                                                    y_probe);
    }
    enu_check_error_code("error doing basic calibration");
 
    /* get a basic flatfield for use in detecting an illumination mask */
 
    cpl_msg_info(cpl_func, "measuring illumination...");
    hdrl_imagelist * duplicate_list = hdrl_imagelist_duplicate(
                                      lamp_reduced_data);
    illumination_flat = enu_calc_flat(lamp_reduced_data,
                                      min_coadds,
                                      collapse_params,
                                      5, 5, HDRL_FLAT_FREQ_LOW);
    hdrl_imagelist_delete(duplicate_list);
 
    /* compute the 'bad-illumination' BPM and reject affected pixels in 
       the lamp data, otherwise will get nasty edge effects in the lofreq
       flat */
 
    un_illum = cpl_mask_threshold_image_create(hdrl_image_get_image(
                                               illumination_flat),
                                               -1000.0, 0.5);
    enu_check_error_code("error computing un-illuminated bpm");
 
    for (cpl_size i = 0; i < hdrl_imagelist_get_size(lamp_reduced_data); i++) {
        hdrl_image * temp = hdrl_imagelist_get(lamp_reduced_data, i);
        cpl_image * data = hdrl_image_get_image(temp);
        cpl_mask * data_mask = cpl_image_get_bpm(data); 
        cpl_binary * data_mask_data = cpl_mask_get_data(data_mask); 
        cpl_image * error = hdrl_image_get_error(temp);
        cpl_mask * error_mask = cpl_image_get_bpm(error);
        cpl_binary * error_mask_data = cpl_mask_get_data(error_mask);
 
        cpl_binary * illum_mask_data = cpl_mask_get_data(un_illum);
 
        cpl_size nx = cpl_image_get_size_x(data); 
        cpl_size ny = cpl_image_get_size_y(data);
 
        for (cpl_size p = 0; p < nx * ny; p++) {
            if (illum_mask_data[p] == CPL_BINARY_1) {
                data_mask_data[p] = CPL_BINARY_1;
                error_mask_data[p] = CPL_BINARY_1;
            }
        } 
    }
 
    /* get the HI freq flatfield - duplicate imagelist as it gets 
       overwritten */
 
    duplicate_list = hdrl_imagelist_duplicate(lamp_reduced_data);
    master_flat_hifreq = enu_calc_flat(duplicate_list,
                                       min_coadds,
                                       collapse_params,
                                       filter_size_x,
                                       filter_size_y,
                                       HDRL_FLAT_FREQ_HIGH);
    hdrl_imagelist_delete(duplicate_list);
 
    /* compute the 'cold-pixel' BPM, the output will not include
       pixels already set bad in the input image */
 
    cold_bpm = hdrl_bpm_2d_compute(master_flat_hifreq, bpm_params);
    enu_check_error_code("error computing cold-pixel bpm");
 
    /* construct the MASTER_BPM */
 
    master_bad_pix_map = en_master_bpm_create("eris_nix_flat_lamp", cold_bpm,
                                      BPM_DARK, NULL);
    en_master_bpm_set(master_bad_pix_map, master_drk->hot_bpm, BPM_HOT);
    en_master_bpm_set(master_bad_pix_map, un_illum, BPM_UNILLUMINATED);
    if (gain_lin) {
        en_master_bpm_set(master_bad_pix_map, gain_lin->bpm, BPM_NON_LINEAR);
    }
 
    /* create a bpm mask, find how many bad pixels overall */
    //bpm_plist = cpl_propertylist_new();
    //eris_nix_get_badpix_qc_from_ima(master_bad_pix_map, bpm_plist, "FLATLAMP");
 
    int flag_mask = 0;
    flag_mask = ~flag_mask;
    bpm_mask = en_master_bpm_get_mask(master_bad_pix_map, flag_mask);
    bpm_plist = cpl_propertylist_new();
    int nbad = cpl_mask_count(bpm_mask);
    cpl_propertylist_append_int(bpm_plist, "ESO QC NUMBER BAD PIXELS", nbad);
    cpl_size sx = cpl_mask_get_size_x(bpm_mask);
    cpl_size sy = cpl_mask_get_size_y(bpm_mask);
    double fraction = (double) nbad / (sx * sy);
    cpl_propertylist_append_double(bpm_plist, "ESO QC FRACTION BAD PIXELS",fraction);
    /* and save it */
 
    enu_dfs_save_bpm(ERIS_NIX_MASTER_BPM_LAMP_PRO_CATG,
                     frameset, parlist, used_frameset, master_bad_pix_map,
                     RECIPE_NAME, bpm_plist, PACKAGE "/" PACKAGE_VERSION,
                     "master_bpm_lamp.fits");
    enu_check_error_code("Failed to save MASTER_BPM");
 
 
 
    /* create a HIFREQ confidence array, essentially the flat field with 
       bad pixels set to 0 and median pixel value normalised to 100 */
 
    confidence_hi = cpl_image_duplicate(hdrl_image_get_image_const(
                                        master_flat_hifreq));
    cpl_image_fill_rejected(confidence_hi, 0.0);
    cpl_image_accept_all(confidence_hi);
    enu_normalise_confidence(confidence_hi);
    enu_check_error_code("error computing HIFREQ confidence map");
 
    /* now save the HIFREQ flat */
 
    enu_flat_save(ERIS_NIX_MASTER_FLAT_LAMP_HIFREQ_PRO_CATG,
                  master_flat_hifreq,
                  confidence_hi,
                  cold_bpm,
                  frameset,
                  parlist,
                  "master_flat_lamp_hifreq.fits",
                  RECIPE_NAME,
                  qclog);
    cpl_propertylist_delete(qclog);
    enu_check_error_code("Failed to save HIFREQ flat");
 
    /* similar for LOFREQ flat */
 
    /* except reject pixels flagged by the master bpm first */
 
    for (cpl_size i = 0; i < hdrl_imagelist_get_size(lamp_reduced_data); i++) {
        cpl_mask * current_mask = hdrl_image_get_mask(
                                  hdrl_imagelist_get(lamp_reduced_data, i));
        cpl_mask_or(current_mask, bpm_mask);
    }
 
    /* get the LO freq flatfield */
 
    duplicate_list = hdrl_imagelist_duplicate(lamp_reduced_data);
    master_flat_lofreq = enu_calc_flat(duplicate_list,
                                       min_coadds,
                                       collapse_params,
                                       filter_size_x,
                                       filter_size_y,
                                       HDRL_FLAT_FREQ_LOW);
    hdrl_imagelist_delete(duplicate_list);
 
    confidence_lo = cpl_image_duplicate(hdrl_image_get_image_const(
                                        master_flat_lofreq));
    cpl_image_fill_rejected(confidence_lo, 0.0);
    cpl_image_accept_all(confidence_lo);
    enu_normalise_confidence(confidence_lo);
    enu_check_error_code("error computing LOFREQ confidence map");
 
    enu_flat_save(ERIS_NIX_MASTER_FLAT_LAMP_LOFREQ_PRO_CATG,
                  master_flat_lofreq,
                  confidence_lo,
                  NULL,
                  frameset,
                  parlist,
                  "master_flat_lamp_lofreq.fits",
                  RECIPE_NAME,
                  NULL);
    enu_check_error_code("Failed to save LOFREQ flat");
 
cleanup:
    cpl_mask_delete(bpm_mask);
    cpl_propertylist_delete(bpm_plist);
    hdrl_parameter_delete(bpm_params);
    cpl_mask_delete(cold_bpm);
    hdrl_parameter_delete(collapse_params);
    cpl_image_delete(confidence_hi);
    cpl_image_delete(confidence_lo);
    hdrl_parameter_delete(flat_params);
    cpl_propertylist_delete(flat_plist);
    engl_gain_linearity_delete((gain_linearity *) gain_lin);
    hdrl_image_delete(illumination_flat);
    enu_located_imagelist_delete(lamp_off_limlist);
    enu_located_imagelist_delete(lamp_on_limlist);
    hdrl_imagelist_delete(lamp_reduced_data);
    en_master_bpm_delete(master_bad_pix_map);
    en_master_dark_delete(master_drk);
    hdrl_image_delete(master_flat_hifreq);
    hdrl_image_delete(master_flat_lofreq);
    cpl_mask_delete(old_mask);
    cpl_mask_delete(un_illum);
    cpl_frameset_delete(used_frameset);
 
    return (int)cpl_error_get_code();
}