eris_nix_flat_sky.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
/*
 * $Author$
 * $Date$
 * $Revision$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include <string.h>
 
#include "eris_utils.h"
#include "eris_nix_utils.h"
#include "eris_pfits.h"
#include "eris_dfs.h"
#include "eris_nix_dfs.h"
#include "eris_nix_master_bpm.h"
#include "eris_nix_master_dark.h"
#include "eris_nix_gain_linearity.h"
#include <hdrl.h>
 
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char eris_nix_flat_sky_description[] =
"This recipe reduces either:\n"
" a set of sky image frames taken at 2 different airmasses to produce\n"
" "ERIS_NIX_MASTER_FLAT_SKY_HIFREQ_PRO_CATG", "
ERIS_NIX_MASTER_FLAT_SKY_LOFREQ_PRO_CATG", and\n"
" "ERIS_NIX_MASTER_BPM_SKY_PRO_CATG" results,\n"
"or:\n"
" a set of sky LSS frames taken at 2 different airmasses to produce\n"
" "ERIS_NIX_MASTER_FLAT_LSS_HIFREQ_PRO_CATG" and "
ERIS_NIX_MASTER_BPM_LSS_PRO_CATG" results.\n"
"\n"
"Input files:\n"
"  The recipe works with a set of sky frames, equal numbers taken at \n"
"  two different airmasses:\n"
"\n"
"  DO CATG                    Explanation          Req.  #Frames\n"
"  -------                    -----------          ---   -------\n"
"  "ERIS_NIX_RAW_FLAT_SKY_DO_CATG
          "                   sky frames           Y  >2*min-coadds\n"
"        or\n"
"  "ERIS_NIX_RAW_FLAT_SKY_LSS_DO_CATG
              "               sky LSS frames       Y  >2*min-coadds\n"
"  "ERIS_NIX_MASTER_DARK_IMG_PRO_CATG
             "                a MASTER_DARK with   Y        1\n"
"                             matching detector \n"
"                             configuration\n"
"  "ERIS_NIX_GAIN_PRO_CATG
           "                  DETMON gain          N      0 or 1\n"
"                             information e.g. in\n"
"                             file \n"
"                             detmon_ir_lg_gain_table.fits\n"
"  "ERIS_NIX_COEFFS_CUBE_PRO_CATG
             "                DETMON linearity     N      0 or 1\n"
"                             curves e.g. in file \n"
"                             detmon_ir_coeffs_cube.fits\n"
"  "ERIS_NIX_NL_BPM_PRO_CATG
           "                  DETMON non-linear    N      0 or 1\n"
"                             bpm e.g. in file \n"
"                             detmon_ir_lg_bpm.fits\n"
"\n"
"Output files:\n"
"\n"
"  DO CATG                    Explanation \n"
"  -------                    ----------- \n"
"  "ERIS_NIX_MASTER_FLAT_SKY_HIFREQ_PRO_CATG
                        "     The required sky HIFREQ flatfield.\n"
"  "ERIS_NIX_MASTER_FLAT_SKY_LOFREQ_PRO_CATG
                        "     For imaging data only, the\n"
"                             required sky LOFREQ flatfield.\n"
"\n"
"  "ERIS_NIX_MASTER_BPM_SKY_PRO_CATG
                "             The required sky master BPM.\n"
"\n"
"  The HIFREQ output will be a FITS file named \n"
"  'master_flat_hifreq.fits', with extensions:\n"
"  - DATA, with the flatfield data.\n"
"  - ERR, with the flatfield error plane.\n"
"  - DQ, with the flatfield data quality plane.\n"
"  - COLD_BPM, with a bpm for the 'cold' pixels.\n"
"  - CONFIDENCE, with the flatfield confidence plane.\n"
"\n"
"  The LOFREQ output will be a FITS file named \n"
"  'master_flat_lofreq.fits', with extensions:\n"
"  - DATA, with the flatfield data.\n"
"  - ERR, with the flatfield error plane.\n"
"  - DQ, with the flatfield data quality plane.\n"
"  - CONFIDENCE, with the flatfield confidence plane.\n"
"\n"
"  The master BPM output will be a FITS file named \n"
"  'master_bpm.fits'. Common pixel values are:\n"
"  - 512, 'hot' pixel.\n"
"  - 1024, 'cold' pixel.\n"
"  - 32768, 'non-linear' pixel.\n"
"\n";
 
#define RECIPE_NAME "eris.eris_nix_flat_sky"
#define CONTEXT "eris."RECIPE_NAME
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
cpl_recipe_define(eris_nix_flat_sky, ERIS_BINARY_VERSION, "John Lightfoot",
                  PACKAGE_BUGREPORT, "2017",
                  "Calculate a MASTER_FLAT_SKY_HIFREQ/LOFREQ and "
                  "MASTER_SKY_BPM", eris_nix_flat_sky_description);
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_flat_sky_fill_parameterlist(
  cpl_parameterlist *self) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    cpl_parameterlist * bpm_parlist = NULL;
    cpl_parameterlist * collapse_parlist = NULL;
    hdrl_parameter    * filter_defaults = NULL;
    hdrl_parameter    * flat_defaults = NULL;
    cpl_parameterlist * flat_parlist = NULL;
    hdrl_parameter    * legendre_defaults = NULL;
    hdrl_parameter    * minmax_defaults = NULL;
    hdrl_parameter    * sigclip_defaults = NULL;
 
    /* generate the general parameter list for the collapse */
    hdrl_parameter * mode_def =
        hdrl_collapse_mode_parameter_create(10., 1., 0., HDRL_MODE_MEDIAN, 0);
    sigclip_defaults = hdrl_collapse_sigclip_parameter_create(10.0, 10.0, 3);
    minmax_defaults = hdrl_collapse_minmax_parameter_create(50, 50);
    collapse_parlist = hdrl_collapse_parameter_create_parlist(RECIPE_NAME,
                       "collapse", "MEDIAN", sigclip_defaults,
                                  minmax_defaults, mode_def);
 
    /* add the subset of parameters to be used, i.e. leave out 'minmax' */
 
    for (const cpl_parameter * p = 
         cpl_parameterlist_get_first_const(collapse_parlist);
         p != NULL;
         p = cpl_parameterlist_get_next_const(collapse_parlist)) {
 
        const char * pname = cpl_parameter_get_name(p);
        if (strstr(pname, "minmax") == NULL) {
            cpl_parameter * duplicate = cpl_parameter_duplicate(p);
            cpl_parameterlist_append(self, duplicate);
        }
    }
 
    /* generate the general parameter list for the flat-field calculation */
 
    flat_defaults = hdrl_flat_parameter_create(21, 21, HDRL_FLAT_FREQ_HIGH);
    flat_parlist = hdrl_flat_parameter_create_parlist(RECIPE_NAME, "flat",
                                                      flat_defaults);
 
    for (const cpl_parameter * p = 
         cpl_parameterlist_get_first(flat_parlist);
         p != NULL;
         p = cpl_parameterlist_get_next(flat_parlist)) {
        cpl_parameter * duplicate = cpl_parameter_duplicate(p);
        cpl_parameterlist_append(self, duplicate);
    }
 
    /* Get the general parameter list for the cold-pixel bpm */
 
    filter_defaults = hdrl_bpm_2d_parameter_create_filtersmooth(5.0, 20.0, 3,
                      CPL_FILTER_MEDIAN, CPL_BORDER_NOP, 21, 21);
    legendre_defaults = hdrl_bpm_2d_parameter_create_legendresmooth(4, 5, 6,
                        20, 21, 11, 12, 2, 10);
    bpm_parlist = hdrl_bpm_2d_parameter_create_parlist(RECIPE_NAME, "coldpix",
                  "FILTER", filter_defaults, legendre_defaults);
 
    for (const cpl_parameter * p = 
         cpl_parameterlist_get_first_const(bpm_parlist);
         p != NULL;
         p = cpl_parameterlist_get_next_const(bpm_parlist)) {
        cpl_parameter * duplicate = cpl_parameter_duplicate(p);
        cpl_parameterlist_append(self, duplicate);
    }
 
    /* the minimum number of lamp_on - lamp_off images acceptable for the
       calculation of the flat */
 
    cpl_parameter * cp = NULL;
    cp = cpl_parameter_new_value(RECIPE_NAME".min_coadds", CPL_TYPE_INT,
                                 "minimum acceptable number of (lamp_on - "
                                 "lamp_off) images", RECIPE_NAME, 1);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "min-coadds");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    /* coords of pixel whose value to be used for diagnostics during 
       reduction */
 
    cp = cpl_parameter_new_value(RECIPE_NAME".x_probe", CPL_TYPE_INT,
                                 "x coord of diagnostic pixel",
                                 RECIPE_NAME, -1);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "x-probe");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    cp = cpl_parameter_new_value(RECIPE_NAME".y_probe", CPL_TYPE_INT,
                                 "y coord of diagnostic pixel",
                                 RECIPE_NAME, -1);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "y-probe");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
 
 
    cp = cpl_parameter_new_value(CONTEXT".threshold", CPL_TYPE_DOUBLE,
            "positive saturation level (for QC)",
            CONTEXT, 60000.);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "threshold");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    cp = cpl_parameter_new_value(CONTEXT".saturation_neg", CPL_TYPE_DOUBLE,
            "negative saturation level (for QC)", CONTEXT, -4.5e7);
    cpl_parameter_set_alias(cp, CPL_PARAMETER_MODE_CLI, "saturation_neg");
    cpl_parameter_disable(cp, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, cp);
 
    /* cleanup */
 
    cpl_parameterlist_delete(bpm_parlist);
    cpl_parameterlist_delete(collapse_parlist);
    hdrl_parameter_delete(filter_defaults);
    hdrl_parameter_delete(flat_defaults);
    cpl_parameterlist_delete(flat_parlist);
    hdrl_parameter_delete(legendre_defaults);
    hdrl_parameter_delete(minmax_defaults);
    hdrl_parameter_delete(sigclip_defaults);
    hdrl_parameter_delete(mode_def);
    return 0;
}
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static hdrl_value hdrl_get_airmass_young(hdrl_value hvaCosZt)
{
    double aCosZt    = hvaCosZt.data,
           aCosZtErr = hvaCosZt.error;
 
    hdrl_value airmass = {(1.002432 * aCosZt * aCosZt + 0.148386 * aCosZt + 0.0096467)
                        / (aCosZt * aCosZt * aCosZt + 0.149864 * aCosZt * aCosZt + 0.0102963 * aCosZt
                           + 0.000303978),
                        aCosZtErr * fabs( ( (2. * 1.002432 * aCosZt + 0.148386) * (aCosZt * aCosZt * aCosZt + 0.149864 * aCosZt * aCosZt + 0.0102963 * aCosZt + 0.000303978)
                                        - (3. * aCosZt * aCosZt + 2. * 0.149864 * aCosZt + 0.0102963) * (1.002432 * aCosZt * aCosZt + 0.148386 * aCosZt + 0.0096467)
                                        ) / pow(aCosZt * aCosZt * aCosZt + 0.149864 * aCosZt * aCosZt + 0.0102963 * aCosZt + 0.000303978, 2) )};
 
    return airmass;
}
 
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_flat_sky_bin_by_airmass(
                                           cpl_frameset * frameset,
                                           const int min_coadds,
                                           cpl_frameset * used,
                                           located_imagelist ** low_z_limlist,
                                           located_imagelist ** high_z_limlist,
                                           int * lss) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(used, CPL_ERROR_NULL_INPUT);
 
    cpl_vector        * airmass = NULL;
    cpl_vector        * high_z_list = NULL;
    cpl_vector        * low_z_list = NULL;
    located_imagelist * sky_limlist = NULL;
    located_imagelist * sky_lss_limlist = NULL;
 
    /* read in the sky data */
 
    sky_limlist = enu_limlist_load_from_frameset(frameset,
                  ERIS_NIX_RAW_FLAT_SKY_DO_CATG, used);
    sky_lss_limlist = enu_limlist_load_from_frameset(frameset,
                      ERIS_NIX_RAW_FLAT_SKY_LSS_DO_CATG, used);
    enu_check_error_code("Failed to read sky data from frameset");
 
    cpl_msg_info(cpl_func, "%d SKY frames read",
                 (int) sky_limlist->size);
    cpl_msg_info(cpl_func, "%d SKY LSS frames read",
                 (int) sky_lss_limlist->size);
 
    /* is this imaging or LSS data */
 
    enu_check(!((sky_limlist->size > 0) && (sky_lss_limlist->size > 0)),
              CPL_ERROR_ILLEGAL_INPUT, "cannot combine imaging and LSS data");
    enu_check(!((sky_limlist->size == 0) && (sky_lss_limlist->size == 0)),
              CPL_ERROR_ILLEGAL_INPUT, "no sky flat data found");
 
    *lss = CPL_FALSE;
    if (sky_lss_limlist->size > 0) {
        *lss = CPL_TRUE;
        enu_located_imagelist_delete(sky_limlist);
        sky_limlist = sky_lss_limlist;
        sky_lss_limlist = NULL;
    }
 
    /* get the airmass of each frame, stored in different places for 
       NIX and NACO  */
 
    airmass = cpl_vector_new(sky_limlist->size);
    for (cpl_size i = 0; i < sky_limlist->size; i++) {
        double air =0;
        /*
        air = enu_get_airmass(sky_limlist->limages[i]->plist);
        // Alternative way to compute airmass using HDRL and FITS header info:
         * method 1:
        double dRA = cpl_propertylist_get_double(sky_limlist->limages[i]->plist, "RA");
        double dDEC = cpl_propertylist_get_double(sky_limlist->limages[i]->plist, "DEC");
        double dLST = cpl_propertylist_get_double(sky_limlist->limages[i]->plist, "LST");
        double exptime = cpl_propertylist_get_double(sky_limlist->limages[i]->plist, "EXPTIME");
        double geolat = cpl_propertylist_get_double(sky_limlist->limages[i]->plist, "ESO TEL GEOLAT");
        //168.839579      -29.50799       34614.068       -24.627
        //dRA=168.839579;
        //dDEC=-29.50799;
        //dLST=34614.068;
        //geolat=-24.627;
        //hdrl_airmass_approx type = HDRL_AIRMASS_APPROX_YOUNG;
        hdrl_airmass_approx type = HDRL_AIRMASS_APPROX_HARDIE;
        cpl_msg_info(cpl_func,"ra: %g dec: %g lst: %g exptime: %g geolat: %g",
                dRA, dDEC, dLST, exptime, geolat);
 
        double rel_err = 0.01;
        hdrl_value hva_ra = {dRA, fabs(dRA * rel_err)};
        hdrl_value hva_dec = {dDEC, fabs(dDEC * rel_err)};
        hdrl_value hva_lst = {dLST, fabs(dLST * rel_err)};
        hdrl_value hva_exptime = {exptime, fabs(exptime * rel_err)};
        hdrl_value hva_geolat = {geolat, fabs(geolat * rel_err)};
        hdrl_value hva_airmass = hdrl_utils_airmass(hva_ra, hva_dec, hva_lst,
                hva_exptime, hva_geolat, type);
        eris_print_rec_status(0);
        air = hva_airmass.data;
        END Method 1*/
        /* Method 2, using HDRL
        // Compute hour angle of the observation in degrees.
        hdrl_value HA = {hva_ra.data * 15. / 3600. - hva_ra.data,
                hva_lst.error * fabs(15. / 3600.) + hva_ra.error * fabs(-1.)};
 
        // Range adjustments.Angle between line of sight and meridian is needed.
        if (HA.data < -180.) HA.data += 360.;
        if (HA.data >  180.) HA.data -= 360.;
 
        // Convert angles from degrees to radians.
 
        hdrl_value delta     = {hva_dec.data       *      CPL_MATH_RAD_DEG,
                hva_dec.error      * fabs(CPL_MATH_RAD_DEG)};
 
        hdrl_value latitude  = {hva_geolat.data  *      CPL_MATH_RAD_DEG,
                hva_geolat.error * fabs(CPL_MATH_RAD_DEG)};
 
        hdrl_value hourangle = {HA.data         *      CPL_MATH_RAD_DEG,
                                HA.error        * fabs(CPL_MATH_RAD_DEG)};
 
        cpl_msg_warning(cpl_func,"delta: %g", delta.data);
        cpl_msg_warning(cpl_func,"latitude: %g", latitude.data);
        cpl_msg_warning(cpl_func,"hourangle: %g", hourangle.data);
 
        END Method 2*/
        /* Method 3:
         * Calculate airmass of the observation using the approximation given    *
         * by Young (1994). For non-zero exposure times these airmass values are *
         * averaged using the weights given by Stetson.                          */
 
 
        /*
        hdrl_value cosz = hdrl_get_zenith_distance(hourangle, delta, latitude);
        hdrl_value z    = {0., cosz.error};
        double zlimit   = 80.;
 
        cpl_msg_info(cpl_func,"cosz: %g",cosz.data);
        */
        /* TODO: TEMPORARY SOLUTION. As previous methods still have problems
         * in determining proper value of airmass and the the algorithm below
         * is only used to separate frames in groups of same airmass (equivalent
         * to groups of same tel.alt we use the next method 4, the simplest one
         * even if also this is not accurate.
         */
        double tel_alt = enu_get_tel_alt(sky_limlist->limages[i]->plist);
        double zenith_dist = 90.0 - tel_alt;
        hdrl_value hvaCosZt = {cos(zenith_dist), 0};
        hdrl_value hva_airm =  hdrl_get_airmass_young(hvaCosZt);
        //cpl_msg_info(cpl_func,"air1: %g",hva_airm.data);
        air = hva_airm.data;
        //cpl_msg_info(cpl_func,"airmass: %g",air);
 
 
        cpl_vector_set(airmass, i, air);
    }
 
    /* assign frames to high and low airmass bins */
 
    double max_z = cpl_vector_get_max(airmass);
    double min_z = cpl_vector_get_min(airmass);
    low_z_list = cpl_vector_new(sky_limlist->size);
    cpl_vector_fill(low_z_list, -1.0);
    cpl_size ilow = 0;  
    high_z_list = cpl_vector_new(sky_limlist->size);
    cpl_vector_fill(high_z_list, -1.0);
    cpl_size ihigh = 0;  
 
    for (cpl_size i = 0; i < cpl_vector_get_size(airmass); i++) {
        if (fabs(cpl_vector_get(airmass, i) - min_z) < 0.1) {
            cpl_vector_set(low_z_list, ilow, i);
            ilow++;
        } else if (fabs(cpl_vector_get(airmass, i) - max_z) < 0.1) {
            cpl_vector_set(high_z_list, ihigh, i);
            ihigh++;
        }
    }
 
    cpl_msg_info(cpl_func, "Found %d frames at high airmass", (int)ihigh);
    cpl_msg_info(cpl_func, "Found %d frames at low airmass", (int)ilow);
 
    /* check bins are filled as we expect, half of frames in each */
 
    enu_check(ilow >= min_coadds, CPL_ERROR_ILLEGAL_INPUT, 
              "SoF contains insufficient sky frames at lower airmass "
              "%d < %d", (int) ilow, min_coadds);
    enu_check(ilow == ihigh, CPL_ERROR_ILLEGAL_INPUT, 
              "SoF contains unequal numbers of high / low airmass frames "
              "%d != %d", (int) ihigh, (int) ilow);
 
    /* fill lists with the high and low z frames */
 
    *low_z_limlist = enu_located_imagelist_new(ihigh);
    for (cpl_size i = 0; i < ilow; i++) {
        enu_located_imagelist_insert(*low_z_limlist,
                                     enu_located_image_duplicate(
                                     sky_limlist->limages[(int)
                                     cpl_vector_get(low_z_list, i)]), i);
    }
 
    *high_z_limlist = enu_located_imagelist_new(ilow);
    for (cpl_size i = 0; i < ihigh; i++) {
        enu_located_imagelist_insert(*high_z_limlist,
                                     enu_located_image_duplicate(
                                     sky_limlist->limages[(int)
                                     cpl_vector_get(high_z_list, i)]), i);
    }
 
cleanup:
    cpl_vector_delete(airmass);
    cpl_vector_delete(high_z_list);
    cpl_vector_delete(low_z_list);
    enu_located_imagelist_delete(sky_limlist);
    enu_located_imagelist_delete(sky_lss_limlist);
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        enu_located_imagelist_delete(*high_z_limlist);
        *high_z_limlist = NULL;
        enu_located_imagelist_delete(*low_z_limlist);
        *low_z_limlist = NULL;
    }
 
    return cpl_error_get_code();
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static hdrl_imagelist * eris_nix_reduce_sky(located_imagelist * high_z_limlist,
                                            located_imagelist * low_z_limlist,
                                            const master_dark * mdark,
                                            const gain_linearity * gain_lin,
                                            const cpl_size x_probe,
                                            const cpl_size y_probe) {
 
 
    hdrl_imagelist * result = NULL;
    cpl_ensure(high_z_limlist, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(low_z_limlist, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(mdark, CPL_ERROR_NULL_INPUT, NULL);
 
    enu_check(high_z_limlist->size == low_z_limlist->size,
              CPL_ERROR_INCOMPATIBLE_INPUT,
              "different number of high z and low z frames");
 
    result = hdrl_imagelist_new();
 
/*
    const int debug = x_probe > 0 && 
                      x_probe <= hdrl_image_get_size_x(
                      high_z_limlist->limages[0]->himage) &&
                      y_probe > 0 &&
                      y_probe <= hdrl_image_get_size_y(
                      high_z_limlist->limages[0]->himage);
*/
  
 
 
    for (int i = 0; i < high_z_limlist->size; i++) {
 
        /* do basic calibration: linearize, calculate error plane */
 
        enu_basic_calibrate(high_z_limlist->limages[i],
                            CPL_TRUE,
                            NULL,
                            mdark,
                            gain_lin,
                            NULL,
                            NULL,
                            NULL,
                            0,
                            "noop",
                            0.0,
                            x_probe,
                            y_probe);
        enu_basic_calibrate(low_z_limlist->limages[i],
                            CPL_TRUE,
                            NULL,
                            mdark,
                            gain_lin,
                            NULL,
                            NULL,
                            NULL,
                            0,
                            "noop",
                            0.0,
                            x_probe,
                            y_probe);
        enu_check_error_code("error performing basic calibration of sky "
                             "frames");
 
        /* difference the high and low airmass images */
 
        hdrl_image_sub_image(high_z_limlist->limages[i]->himage,
                             low_z_limlist->limages[i]->himage);
        hdrl_imagelist_set(result,
                           hdrl_image_duplicate(
                           high_z_limlist->limages[i]->himage),
                           hdrl_imagelist_get_size(result));
    }
 
 cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
      if(result != NULL) {
    hdrl_imagelist_delete(result);
        result = NULL;
      }
    }
    return result;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int eris_nix_flat_sky(cpl_frameset * frameset,
                              const cpl_parameterlist * parlist) {
    cpl_mask             * bpm_mask = NULL;
    hdrl_parameter       * bpm_params = NULL;
    cpl_propertylist     * bpm_plist = NULL;
    cpl_mask             * cold_bpm = NULL;
    hdrl_parameter       * collapse_params = NULL;
    cpl_image            * confidence_hi = NULL;
    cpl_image            * confidence_lo = NULL;
    hdrl_parameter       * flat_params = NULL;
    cpl_propertylist     * flat_plist = NULL;
    const gain_linearity * gain_lin = NULL;
    located_imagelist    * high_z_limlist = NULL; 
    located_imagelist    * low_z_limlist = NULL; 
    master_bpm           * mbad_pix_map = NULL;
    master_dark          * mdark = NULL;
    hdrl_image           * master_flat_hifreq = NULL;     
    hdrl_image           * master_flat_lofreq = NULL;     
    const cpl_parameter  * param = NULL;
    hdrl_imagelist       * sky_reduced_data = NULL;
    cpl_frameset         * used = NULL;
 
    enu_check_error_code("%s():%d: An error is already set: %s",
                         cpl_func, __LINE__, cpl_error_get_where());
 
    /* check input parameters */
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
 
    /* set the msg verbosity level from environment variable CPL_MSG_LEVEL */
 
    cpl_msg_set_level_from_env();
    cpl_msg_severity severity = cpl_msg_get_level();
    cpl_msg_info(cpl_func, "level %d", (int) severity);
 
    /* check required input tags are present */
    const int ntags = 5;
    const char* required_tags[5] = {
            ERIS_NIX_RAW_FLAT_SKY_DO_CATG,
            ERIS_NIX_NL_BPM_PRO_CATG,
            ERIS_NIX_COEFFS_CUBE_PRO_CATG,
            ERIS_NIX_GAIN_PRO_CATG,
            ERIS_NIX_MASTER_DARK_IMG_PRO_CATG
    };
    cpl_ensure_code(CPL_ERROR_NONE ==
            eris_dfs_check_input_tags(frameset, required_tags, ntags, 1),
            CPL_ERROR_ILLEGAL_INPUT);
    /* retrieve input parameters for collapse */
 
    collapse_params = hdrl_collapse_parameter_parse_parlist(parlist,
                      RECIPE_NAME".collapse");
 
    /* retrieve input filter parameters for HI and LO flat-field */
 
    flat_params = hdrl_flat_parameter_parse_parlist(parlist,
                                                    RECIPE_NAME".flat");
    cpl_size filter_size_x = hdrl_flat_parameter_get_filter_size_x(
                             flat_params);
    cpl_size filter_size_y = hdrl_flat_parameter_get_filter_size_y(
                             flat_params);
 
    /* retrieve input parameters for 'cold-pixel' bpm */
 
    bpm_params = hdrl_bpm_2d_parameter_parse_parlist(parlist,
                                                     RECIPE_NAME".coldpix");
 
    /* retrieve minimum acceptable number of coadds */
 
    param = cpl_parameterlist_find_const(parlist, RECIPE_NAME".min_coadds");
    int min_coadds = cpl_parameter_get_int(param);
 
    /* retrieve probe pixel coords */
 
    param = cpl_parameterlist_find_const(parlist, RECIPE_NAME".x_probe");
    cpl_size x_probe = (cpl_size) cpl_parameter_get_int(param);
    param = cpl_parameterlist_find_const(parlist, RECIPE_NAME".y_probe");
    cpl_size y_probe = (cpl_size) cpl_parameter_get_int(param);
 
    enu_check_error_code("Could not retrieve input parameters");
 
    /* identify the RAW and CALIB frames in the input frameset */
 
    eris_nix_dfs_set_groups(frameset);
    enu_check_error_code("Could not identify RAW and CALIB frames");    
 
    used = cpl_frameset_new();
 
    /* read the gain and linearity information, if available */
 
    int required = CPL_FALSE;
    gain_lin = engl_gain_linearity_load_from_frameset(frameset, 
               ERIS_NIX_GAIN_PRO_CATG, ERIS_NIX_COEFFS_CUBE_PRO_CATG,
               ERIS_NIX_NL_BPM_PRO_CATG, required, used);
    enu_check_error_code("failed to read gain/linearity information from SoF");
 
    /* read the mdark */
 
    mdark = en_master_dark_load_from_frameset(frameset,
                  ERIS_NIX_MASTER_DARK_IMG_PRO_CATG, used);
    enu_check_error_code("failed to read master dark from SoF");
 
    /* read the SoF sky data into high and low airmass bins */
 
    int lss = CPL_FALSE;
    eris_nix_flat_sky_bin_by_airmass(frameset, 
                                     min_coadds,
                                     used,
                                     &low_z_limlist,
                                     &high_z_limlist,
                                     &lss);
 
    /* reduce raw data to get input to flatfield calculation */
 
    sky_reduced_data = eris_nix_reduce_sky(high_z_limlist,
                                           low_z_limlist,
                                           mdark,
                                           gain_lin,
                                           x_probe,
                                           y_probe);
 
    /* get the HI freq flatfield */
 
    for (cpl_size i = 0; i < hdrl_imagelist_get_size(sky_reduced_data); i++) {
        char * fname = cpl_sprintf("sky%d.fits", (int)i);
        cpl_mask_save(cpl_image_get_bpm(hdrl_image_get_image(hdrl_imagelist_get(sky_reduced_data, i))),
                       fname, NULL, CPL_IO_CREATE);
        cpl_free(fname);
    }
 
 
    master_flat_hifreq = enu_calc_flat(sky_reduced_data,
                                       min_coadds,
                                       collapse_params,
                                       filter_size_x,
                                       filter_size_y,
                                       HDRL_FLAT_FREQ_HIGH);
    enu_check_error_code("error computing hi-freq flatfield");
 
    /* compute the 'cold-pixel' BPM, the output will not include
       pixels already set bad in the input image */
 
    cpl_image_save(hdrl_image_get_image(master_flat_hifreq), "hifreq.fits",
                   CPL_TYPE_UNSPECIFIED, NULL, CPL_IO_CREATE);
    cpl_image_save(hdrl_image_get_error(master_flat_hifreq), "hifreq_error.fits",
                   CPL_TYPE_UNSPECIFIED, NULL, CPL_IO_CREATE);
 
    cold_bpm = hdrl_bpm_2d_compute(master_flat_hifreq, bpm_params);
    enu_check_error_code("error computing cold-pixel bpm");
 
    /* construct the MASTER_BPM */
 
    mbad_pix_map = en_master_bpm_create("eris_nix_flat_sky", cold_bpm,
                                      BPM_DARK, NULL);
    en_master_bpm_set(mbad_pix_map, mdark->hot_bpm, BPM_HOT);
    if (gain_lin) {
        en_master_bpm_set(mbad_pix_map, gain_lin->bpm, BPM_NON_LINEAR);
    }
 
    /* create a bpm mask, find how many bad pixels overall */
    //bpm_plist = cpl_propertylist_new();
    //eris_nix_get_badpix_qc_from_ima(mbad_pix_map, bpm_plist, "FLATSKY");
 
    int flag_mask = 0;
    flag_mask = ~flag_mask;
    bpm_mask = en_master_bpm_get_mask(mbad_pix_map, flag_mask);
    bpm_plist = cpl_propertylist_new();
    int nbad = cpl_mask_count(bpm_mask);
    cpl_propertylist_append_int(bpm_plist, "ESO QC NUMBER BAD PIXELS", nbad);
    cpl_size sx = cpl_mask_get_size_x(bpm_mask);
    cpl_size sy = cpl_mask_get_size_y(bpm_mask);
    double fraction = (double) nbad / (sx * sy);
    cpl_propertylist_append_double(bpm_plist, "ESO QC FRACTION BAD PIXELS",fraction);
    /* and save it */
 
    enu_dfs_save_bpm(ERIS_NIX_MASTER_BPM_SKY_PRO_CATG,
                     frameset, parlist, used, mbad_pix_map,
                     RECIPE_NAME, bpm_plist, PACKAGE "/" PACKAGE_VERSION,
                     "master_bpm.fits");
    enu_check_error_code("Failed to save MASTER_BPM");
 
    /* create a HIFREQ confidence array, essentially the flat field with 
       bad pixels set to 0 and median pixel value normalised to 100 */
 
    confidence_hi = cpl_image_duplicate(hdrl_image_get_image_const(
                                        master_flat_hifreq));
    cpl_image_fill_rejected(confidence_hi, 0.0);
    cpl_image_accept_all(confidence_hi);
    enu_normalise_confidence(confidence_hi);
    enu_check_error_code("error computing HIFREQ confidence map");
 
    /* now save the HIFREQ flat */
 
    enu_flat_save(ERIS_NIX_MASTER_FLAT_SKY_HIFREQ_PRO_CATG,
                  master_flat_hifreq,
                  confidence_hi,
                  cold_bpm,
                  frameset,
                  parlist,
                  "master_flat_hifreq.fits",
                  RECIPE_NAME, NULL);
    enu_check_error_code("Failed to save HIFREQ flat");
 
    if (!lss) {
 
        /* similar for LOFREQ flat */
 
        /* except reject pixels flagged by the master bpm first */
 
        for (cpl_size i = 0; i < hdrl_imagelist_get_size(sky_reduced_data);
             i++) {
            cpl_mask * current_mask = hdrl_image_get_mask(
                                      hdrl_imagelist_get(sky_reduced_data, i));
            cpl_mask_or(current_mask, bpm_mask);
        }
 
        /* get the LO freq flatfield */
 
        master_flat_lofreq = enu_calc_flat(sky_reduced_data,
                                           min_coadds,
                                           collapse_params,
                                           filter_size_x,
                                           filter_size_y,
                                           HDRL_FLAT_FREQ_LOW);
 
        confidence_lo = cpl_image_duplicate(hdrl_image_get_image_const(
                                            master_flat_lofreq));
        cpl_image_fill_rejected(confidence_lo, 0.0);
        cpl_image_accept_all(confidence_lo);
        enu_normalise_confidence(confidence_lo);
        enu_check_error_code("error computing LOFREQ confidence map");
 
        /* QC on FLAT_ON frames not using information from pixels of mbad_pix_map */
        const char* saturation_limit = cpl_propertylist_get_string(gain_lin->plist,
                            "ESO PRO REC1 PARAM25 VALUE");
        double saturation = atof(saturation_limit);
 
        located_imagelist * sky_limlist = enu_limlist_load_from_frameset(frameset,
                          ERIS_NIX_RAW_FLAT_SKY_DO_CATG, used);
 
 
        cpl_propertylist* qclog = enu_raw_flats_qc(sky_limlist, gain_lin->bpm,
                parlist, CONTEXT, saturation, CPL_FALSE, CPL_FALSE);
        enu_located_imagelist_delete(sky_limlist);
 
 
        enu_flat_save(ERIS_NIX_MASTER_FLAT_SKY_LOFREQ_PRO_CATG,
                      master_flat_lofreq,
                      confidence_lo,
                      NULL,
                      frameset,
                      parlist,
                      "master_flat_lofreq.fits",
                      RECIPE_NAME, qclog);
        cpl_propertylist_delete(qclog);
 
 
        enu_check_error_code("Failed to save LOFREQ flat");
      
    }
 
cleanup:
    cpl_mask_delete(bpm_mask);
    cpl_propertylist_delete(bpm_plist);
    hdrl_parameter_delete(bpm_params);
    cpl_mask_delete(cold_bpm);
    hdrl_parameter_delete(collapse_params);
    cpl_image_delete(confidence_hi);
    cpl_image_delete(confidence_lo);
    hdrl_parameter_delete(flat_params);
    cpl_propertylist_delete(flat_plist);
    engl_gain_linearity_delete((gain_linearity *) gain_lin);
    enu_located_imagelist_delete(high_z_limlist);
    enu_located_imagelist_delete(low_z_limlist);
    en_master_bpm_delete(mbad_pix_map);
    en_master_dark_delete(mdark);
    hdrl_image_delete(master_flat_hifreq);
    hdrl_image_delete(master_flat_lofreq);
    hdrl_imagelist_delete(sky_reduced_data);
    cpl_frameset_delete(used);
 
    return (int)cpl_error_get_code();
}