eris_nix_spec_cal_and_stack.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
/*
 * $Author$
 * $Date$
 * $Revision$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include <libgen.h>
#include <math.h>
#include <string.h>
 
#include "casu_utils.h"
#include "casu_mods.h"
 
#include "eris_utils.h"
#include "eris_nix_utils.h"
#include "eris_pfits.h"
#include "eris_dfs.h"
#include "eris_nix_dfs.h"
#include "eris_nix_master_bpm.h"
#include "eris_nix_master_dark.h"
#include "eris_nix_gain_linearity.h"
#include <hdrl.h>
 
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char eris_nix_spec_cal_and_stack_description[] =
"This recipe calibrates a set of OBJECT_SPEC_JITTER frames to \n"
"produce:\n"
"- OBJECT_SPEC_JITTER_CALIB frames.\n"
"- TBD\n"
"The recipes needs:\n"
"- a series of tagged OBJECT_SPEC_JITTER frames.\n"
"- a tagged DETMON coefficients cube file, e.g.\n"
"-- detmon_ir_lg_coeffs_cube.fits "ERIS_NIX_COEFFS_CUBE_PRO_CATG "\n"
"- a tagged MASTER_DARK for matching detector configuration.\n"
"- a tagged MASTER_FLAT_HIFREQ for matching detector configuration.\n"
"The output will be fits files containing:\n"
"- OBJECT_SPEC_JITTER_CALIBRATED frames.\n"
"- TBD.\n"
"\n";
 
#define RECIPE_NAME "eris.eris_nix_spec_cal_and_stack"
 
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
cpl_recipe_define(eris_nix_spec_cal_and_stack, ERIS_BINARY_VERSION,
                  "John Lightfoot",
                  PACKAGE_BUGREPORT, "2017",
                  "Calibrate OBJECT_SPEC_JITTER frames",
                  eris_nix_spec_cal_and_stack_description);
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_spec_cal_and_stack_fill_parameterlist(
  cpl_parameterlist * self) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    cpl_parameter     * p = NULL;
 
    p = cpl_parameter_new_value(RECIPE_NAME".debug_data",
                                CPL_TYPE_BOOL,
                                "write additional data for debugging",
                                RECIPE_NAME, CPL_TRUE);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "debug_data");
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_value(RECIPE_NAME".zero_rejected",
                                CPL_TYPE_BOOL,
                                "set rejected pixels to 0",
                                RECIPE_NAME, CPL_TRUE);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "zero_rejected");
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_value(RECIPE_NAME".premask_objects",
                          CPL_TYPE_BOOL,
                          "create object masks before background estimation",
                          RECIPE_NAME, CPL_TRUE);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "premask_objects");
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_value(RECIPE_NAME".premask_nsigma",
                                CPL_TYPE_DOUBLE,
                                "sigma cutoff for premask object detection",
                                RECIPE_NAME, 5.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "premask_nsigma");
    cpl_parameterlist_append(self, p);
 
    /* tidy up */
 
    return 0;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int eris_nix_spec_cal_and_stack(cpl_frameset * frameset,
                                       const cpl_parameterlist * parlist) {
 
    cpl_table             * astrom_ref_cat = NULL;
    located_imagelist     * cal_sky_jitters1 = NULL; 
    located_imagelist     * cal_sky_jitters2 = NULL; 
    located_imagelist     * cal_jitters1 = NULL; 
    located_imagelist     * cal_jitters2 = NULL; 
/*    located_imagelist     * cal_jitters_final = NULL; */
    hdrl_parameter        * cat_params = NULL;
    const gain_linearity  * gain_lin = NULL;
    master_bpm            * master_bpm = NULL;
    master_dark           * master_dark = NULL;
    master_flat           * master_flat_hifreq = NULL;
    master_flat           * master_flat_lofreq = NULL;
    mef_extension_list    * mefs = NULL;
    hdrl_image            * object_photom = NULL;
    const cpl_parameter   * p = NULL;
    cpl_table             * photom_ref_cat = NULL;
    cpl_vector            * profile = NULL;
    located_imagelist     * raw_sky_jitters = NULL; 
    located_imagelist     * raw_jitters = NULL; 
    located_image         * sky_stack = NULL;
    located_image         * sky_stack2 = NULL;
    cpl_polynomial        * startrace_poly = NULL;
    located_image         * target_stack = NULL;
    located_image         * target_stack2 = NULL;
    cpl_frameset          * used_frameset = NULL;
    cpl_polynomial        * wave_poly = NULL;
 
    enu_check_error_code("%s():%d: An error is already set: %s",
                         cpl_func, __LINE__, cpl_error_get_where());
 
    /* Check input parameters */
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
 
    /* Set the msg verbosity level from environment variable CPL_MSG_LEVEL */
 
    cpl_msg_set_level_from_env();
    cpl_msg_severity severity = cpl_msg_get_level();
    cpl_msg_info(cpl_func, "level %d", (int) severity);
 
    /* Retrieve input parameters */
 
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".debug_data");
    int debug_data = cpl_parameter_get_bool(p);
    cpl_msg_info(cpl_func, "debug_data %d", debug_data);
 
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".zero_rejected");
    int zero_rejected = cpl_parameter_get_bool(p);
 
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".premask_objects");
    int premask_objects = cpl_parameter_get_bool(p);
 
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".premask_nsigma");
    double premask_nsigma = cpl_parameter_get_double(p);
 
    cpl_size x_probe = 1024;
    cpl_size y_probe = 1024;
 
    enu_check_error_code("Could not retrieve input parameters");
 
    /* Identify the RAW and CALIB frames in the input frameset */
 
    eris_nix_dfs_set_groups(frameset);
    enu_check_error_code("Could not identify RAW and CALIB frames");    
    used_frameset = cpl_frameset_new();
 
    /* read the gain and linearity information */
 
    gain_lin = engl_gain_linearity_load_from_frameset(frameset, 
               ERIS_NIX_GAIN_PRO_CATG, ERIS_NIX_COEFFS_CUBE_PRO_CATG,
               ERIS_NIX_NL_BPM_PRO_CATG, 0, used_frameset);
    enu_check_error_code("failed to read gain/linearity information from SoF");
 
    /* read the master_dark */
 
    master_dark = en_master_dark_load_from_frameset(frameset,
                  ERIS_NIX_MASTER_DARK_IMG_PRO_CATG, used_frameset);
    enu_check_error_code("failed to read master dark from SoF");
 
    /* read the master_bpm */
 
    master_bpm = en_master_bpm_load_from_frameset(frameset,
                                                  ERIS_NIX_MASTER_BPM_PRO_CATG,
                                                  used_frameset);
    enu_check_error_code("failed to read master BPM from SoF");
 
    /* read the high-freq flatfield */
 
    master_flat_hifreq = en_master_flat_load_from_frameset(
                         frameset, ERIS_NIX_MASTER_FLAT_HIFREQ_PRO_CATG,
                         used_frameset, CPL_FALSE);
 
    /* get the wavelength cal and dispersion direction polynomials */
    /* p2(x,y) = p0 + p1.x + p2.y + p3.x.y + p4.x^2 + p5.y^2 */
 
    wave_poly = cpl_polynomial_new(2);
    cpl_size pows[] = {0, 1};
    cpl_polynomial_set_coeff(wave_poly, pows, 1.0);    
    cpl_polynomial_dump(wave_poly, stdout);
 
    startrace_poly = cpl_polynomial_new(2);
    pows[0] = 1;
    pows[1] = 0;
    cpl_polynomial_set_coeff(startrace_poly, pows, 1.0);    
    cpl_polynomial_dump(startrace_poly, stdout);
 
    /* Read in OBJECT_SPEC_JITTER data */
 
    raw_jitters = enu_limlist_load_from_frameset(frameset,
                  ERIS_NIX_RAW_OBJECT_LSS_JITTER_DO_CATG, used_frameset);
    cpl_msg_info(cpl_func, "read %d OBJECT_LSS_JITTER frames", 
                 (int) raw_jitters->size);
 
    /* Do 'detector calibration': dark subtraction, linearize, 
       calculate error plane, flatfield, associate with bpm */
 
    int flag_mask = ~(int)0;
    enu_basic_calibrate(raw_jitters, master_dark, gain_lin, master_flat_hifreq,
                        NULL, NULL, flag_mask, zero_rejected, x_probe,
                        y_probe);
    enu_check_error_code("error performing basic calibration of frames");
    enu_debug_limlist_save(debug_data, raw_jitters, "target_basic_cal_", 
                           RECIPE_NAME, frameset, parlist, used_frameset);
    cal_jitters1 = enu_located_imagelist_duplicate(raw_jitters);
 
    /* Use the target calibrated jitters to estimate the sky background. 
 
       This would work for a spread-out jitter pattern. NACO data switches
       between 2 nod positions on the slit, so a straight median does not
       work, instead have to mask out the objects first. */
 
    cpl_msg_info(cpl_func, "Estimating sky background:");
 
    if (premask_objects) {
        enu_opm_spectrum_limlist(cal_jitters1, premask_nsigma);
    }
    enu_sky_subtract_limlist("median", "bracket", 600.0, cal_jitters1,
                             -1, -1, cal_jitters1);
    enu_check_error_code("error in sky background first iteration: sky-subtract");
    cpl_msg_info(cpl_func, "here0 %s", cpl_error_get_message());
 
    /* in debug mode, save the sky-subtracted images */
 
    enu_debug_limlist_save(debug_data, cal_jitters1, "target_skysub_1_", 
                           RECIPE_NAME, frameset, parlist, used_frameset);
 
    /* correct the frames for spectral and startrace distortion */
 
    /* profile is Lanczos2, which works well for images, not sure about
       spectral direction. ISAAC uses default profile, tanh.
 
       Also can use different profiles for spatial and spectral, should
       examine this when we get some real data */
 
    cpl_msg_info(cpl_func, "here0 %s", cpl_error_get_message());
    cal_jitters2 = enu_located_imagelist_duplicate(cal_jitters1);
    cpl_msg_info(cpl_func, "here1 %s", cpl_error_get_message());
 
    cpl_msg_info(cpl_func, "note: check warping profile when have real data");
    cpl_msg_info(cpl_func, "here0 %s", cpl_error_get_message());
    profile = cpl_vector_new(CPL_KERNEL_DEF_SAMPLES);
    cpl_vector_fill_kernel_profile(profile, CPL_KERNEL_LANCZOS,
                                   CPL_KERNEL_DEF_WIDTH);
    for (cpl_size j = 0; j < cal_jitters2->size; j++) {
        cpl_msg_info(cpl_func, "here0 %s", cpl_error_get_message());
        cpl_image_warp_polynomial(hdrl_image_get_image(
                                  cal_jitters2->limages[j]->himage), 
                                  hdrl_image_get_image_const(
                                  cal_jitters1->limages[j]->himage),
                                  startrace_poly, wave_poly, 
                                  profile, CPL_KERNEL_DEF_WIDTH,
                                  profile, CPL_KERNEL_DEF_WIDTH);
    }
 
    /* in debug mode, save the distortion corrected images */
 
    enu_debug_limlist_save(debug_data, cal_jitters2, "target_warp_", 
                           RECIPE_NAME, frameset, parlist, used_frameset);
 
/*
    enu_stack(cal_jitters2, stk_lthr, stk_hthr, stk_method, stk_fast,
              &target_stack);
    enu_check_error_code("error in target reduction: stack");
    target_stack->objects = enu_catalogue_compute(target_stack->himage,
                                                  target_stack->confidence,
                                                  cpl_wcs_new_from_propertylist(
                                                  target_stack->plist),
                                                  cat_params);
    enu_check_error_code("error in target reduction: stack catalogue");
*/
    /* cal_jitters2 holds the final calibrated jitter images */
/*
    cal_jitters_final = cal_jitters2;
*/
    /* save the calibrated jitter images to FITS files */
/*
    for (cpl_size i = 0; i < cal_jitters_final->size; i++) {
*/
        /* the propertylist to be saved with the calibrated jitter 
           image is that of the raw data - NOT with recalibrated wcs.
           The wcs calibration is done by whatever task does the
           stacking into the final image */
/*
        cpl_propertylist_delete(cal_jitters_final->limages[i]->plist);
        cal_jitters_final->limages[i]->plist = cpl_propertylist_duplicate(
                                               raw_jitters->limages[i]->plist);
*/
/*
        cpl_msg_info(cpl_func, "using raw plist");
        const char * tname = cpl_frame_get_filename(cal_jitters_final->
                                                    limages[i]->frame);
        cpl_propertylist * out_plist = cpl_propertylist_load(tname, 0);
*/
/*
        cpl_propertylist_append_string(cal_jitters_final->limages[i]->plist,
                                       CPL_DFS_PRO_CATG,
                                       ERIS_NIX_CALIB_OBJECT_JITTER_PRO_CATG);
*/
        /* Add QC parameters  */
/*
        hdrl_value flat_median = hdrl_image_get_median(master_flat_hifreq);
        cpl_propertylist_append_double(flat_plist, "ESO QC FLAT MED",
                                       (double) flat_median.data);
        hdrl_value flat_mean = hdrl_image_get_mean(master_flat_hifreq);
        cpl_propertylist_append_double(flat_plist, "ESO QC FLAT MEAN", 
          (double) flat_mean.data);
        double flat_rms = hdrl_image_get_stdev(master_flat_hifreq);
        cpl_propertylist_append_double(flat_plist, "ESO QC FLAT RMS", flat_rms);
*/
        /* generate name of output file */
/*
        const char * in_fname = cpl_frame_get_filename(cal_jitters_final->
                                                       limages[i]->frame);
        char * in_fname_copy = cpl_strdup(in_fname);
        char * out_fname = cpl_sprintf("calib.%s", basename(in_fname_copy));
*/
        /* extension to hold confidence array */
/*
        mefs = enu_mef_extension_list_new(1);
        mefs->mef[0] = enu_mef_new_image("CONFIDENCE", 
                                         cal_jitters_final->limages[i]->
                                         confidence);
        cpl_msg_info(cpl_func, "..writing %s", out_fname);
        enu_dfs_save_himage(frameset, parlist, used_frameset, 
                            cal_jitters_final->limages[i]->himage,
                            cal_jitters_final->limages[i]->frame,
                            mefs, RECIPE_NAME, 
                            cal_jitters_final->limages[i]->plist,
                            PACKAGE "/" PACKAGE_VERSION, out_fname);
        cpl_free(in_fname_copy);
        cpl_free(out_fname);
    }
*/
    /* Save the stacked image as an IMG_OBS_COMBINED data product.
       An attempt is made to erase all no longer relevant keywords
       from the propertylist, but some ESO ... keywords remain for some
       unknown reason */
/*
    cpl_propertylist * stack_plist = cpl_propertylist_duplicate(
                                     target_stack->plist);
    cpl_propertylist_erase_regexp(stack_plist,
                                  "SIMPLE|BITPIX|NAXIS|NAXIS1|NAXIS2|"
                                  "EXTEND|DATE|ORIGIN|TELESCOP|"
                                  "INSTRUME|OBJECT|RA|DEC|EQUINOX|"
                                  "RADECSYS|CTYPE1|CTYPE2|CRVAL1|"
                                  "CRVAL2|CRPIX1|CRPIX2|CD1_1|CD1_2|"
                                  "CD2_1|CD2_2|CHECKSUM|DATASUM|"
                                  "ESO OBS DID", 1);
    cpl_propertylist_append_string(stack_plist, CPL_DFS_PRO_CATG,
                                   ERIS_NIX_IMG_OBS_COMBINED_PRO_CATG);
*/
    /* add QC parameters ..tbd */
/*
    const char * ob_name = cpl_propertylist_get_string(cal_jitters_final->
                                                       limages[0]->plist,
                                                       "ESO OBS START");
    char * stack_fname = cpl_sprintf("stack_image_%s.fits", ob_name);
    enu_mef_extension_list_delete(mefs);
    mefs = enu_mef_extension_list_new(2);
    mefs->mef[0] = enu_mef_new_image("CONFIDENCE", target_stack->confidence);
    mefs->mef[1] = enu_mef_new_table("OBJECT_CATALOGUE", 
                                      target_stack->objects->catalogue);
    enu_dfs_save_himage(frameset, parlist, used_frameset, target_stack->himage,
                        raw_jitters->limages[0]->frame, mefs, RECIPE_NAME,
                        stack_plist, PACKAGE "/" PACKAGE_VERSION, stack_fname);
    cpl_free(stack_fname);
    cpl_propertylist_delete(stack_plist);
    enu_check_error_code("Failed to save IMG_OBS_COMBINED product");
*/
/*
    cpl_image_save(hdrl_image_get_image(object_photom), "photom_test.fits",
                   CPL_TYPE_UNSPECIFIED, NULL, CPL_IO_CREATE);
*/
 
cleanup:
    cpl_table_delete(astrom_ref_cat);
    enu_located_imagelist_delete(cal_sky_jitters1);
    enu_located_imagelist_delete(cal_sky_jitters2);
    enu_located_imagelist_delete(cal_jitters1);
    enu_located_imagelist_delete(cal_jitters2);
    hdrl_parameter_delete(cat_params);
    engl_gain_linearity_delete((gain_linearity *) gain_lin);
    en_master_bpm_delete(master_bpm);
    en_master_dark_delete(master_dark);
    en_master_flat_delete(master_flat_hifreq);
    en_master_flat_delete(master_flat_lofreq);
    enu_mef_extension_list_delete(mefs);
    hdrl_image_delete(object_photom);
    cpl_table_delete(photom_ref_cat);
    cpl_vector_delete(profile);
    enu_located_imagelist_delete(raw_sky_jitters);
    enu_located_imagelist_delete(raw_jitters);
    enu_located_image_delete(sky_stack);
    enu_located_image_delete(sky_stack2);
    cpl_polynomial_delete(startrace_poly);
    enu_located_image_delete(target_stack);
    enu_located_image_delete(target_stack2);
    cpl_frameset_delete(used_frameset);
    cpl_polynomial_delete(wave_poly);
 
    return (int) cpl_error_get_code();
}