eris_nix_wave_lamp.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
/*
 * $Author$
 * $Date$
 * $Revision$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include <string.h>
 
#include "eris_utils.h"
#include "eris_nix_utils.h"
#include "eris_nix_wavecal_utils.h"
#include "eris_pfits.h"
#include "eris_dfs.h"
#include "eris_nix_dfs.h"
#include "eris_nix_master_bpm.h"
#include "eris_nix_master_dark.h"
#include "eris_nix_gain_linearity.h"
#include <hdrl.h>
 
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char eris_nix_wave_lamp_description[] =
"This recipe reduces a set of WAVE_LAMP_ON/WAVE_LAMP_OFF frame pairs to \n"
"produce:\n"
"- A MASTER_WAVE.\n"
"The recipes needs:\n"
"- a series of tagged WAVE_LAMP_ON / WAVE_LAMP_OFF frame pairs.\n"
"- a tagged DETMON coefficients cube file, e.g.\n"
"-- detmon_ir_lg_coeffs_cube.fits "ERIS_NIX_COEFFS_CUBE_PRO_CATG "\n"
"- a tagged MASTER_DARK for matching detector configuration.\n"
"The output will be a fits file containing:\n"
"- A MASTER_WAVE.\n"
"\n";
 
#define RECIPE_NAME "eris.eris_nix_wave_lamp"
 
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
cpl_recipe_define(eris_nix_wave_lamp, ERIS_BINARY_VERSION, "John Lightfoot",
                  PACKAGE_BUGREPORT, "2017",
                  "Calculate a MASTER_WAVE",
                  eris_nix_wave_lamp_description);
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_wave_lamp_fill_parameterlist(
                      cpl_parameterlist *self) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    /* the flux threshold - frames with a median above this will be
       deemed 'saturated' and not used */
 
    cpl_parameter * p = NULL;
    p = cpl_parameter_new_value(RECIPE_NAME".saturation_threshold",
      CPL_TYPE_DOUBLE, "frame saturation threshold", RECIPE_NAME, 5000.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "saturation_threshold");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    /* cleanup */
 
    return 0;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int eris_nix_wave_lamp(cpl_frameset * frameset,
                              const cpl_parameterlist * parlist) {
 
    cpl_image            * contrib = NULL;
    hdrl_imagelist       * diff_himlist = NULL;
    const gain_linearity * gain_lin = NULL;
    cpl_vector           * lambda_pos =NULL;
    cpl_matrix           * line_pos = NULL;
    master_bpm           * master_bpm = NULL;
    master_dark          * master_dark = NULL;
    master_flat          * master_flat_hifreq = NULL;
    hdrl_image           * master_wave = NULL;
    mef_extension_list   * mefs = NULL;
    cpl_table            * mef_table = NULL;
    cpl_table            * mef_table2 = NULL;
    located_imagelist    * off_limlist = NULL; 
    cpl_mask             * old_mask = NULL;
    located_imagelist    * on_limlist = NULL; 
    cpl_frameset         * used_frameset = NULL;
    cpl_image            * wave_cal_image = NULL;
    cpl_propertylist     * wave_plist = NULL;
 
    enu_check_error_code("%s():%d: An error is already set: %s",
      cpl_func, __LINE__, cpl_error_get_where());
 
    /* check input parameters */
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
 
    /* set the msg verbosity level from environment variable CPL_MSG_LEVEL */
 
    cpl_msg_set_level_from_env();
    cpl_msg_severity severity = cpl_msg_get_level();
    cpl_msg_info(cpl_func, "level %d", (int) severity);
 
    cpl_msg_info(cpl_func, "fill_rejected parameter hardwired to True");
    int fill_rejected = CPL_TRUE;
 
    cpl_size x_probe = 1024;
    cpl_size y_probe = 1024;
 
    enu_check_error_code("Could not retrieve input parameters");
 
    /* identify the RAW and CALIB frames in the input frameset */
 
    eris_nix_dfs_set_groups(frameset);
    enu_check_error_code("Could not identify RAW and CALIB frames");    
 
    used_frameset = cpl_frameset_new();
 
    /* read the gain and linearity information, if available */
 
    gain_lin = engl_gain_linearity_load_from_frameset(frameset, 
               ERIS_NIX_GAIN_PRO_CATG, ERIS_NIX_COEFFS_CUBE_PRO_CATG,
               ERIS_NIX_NL_BPM_PRO_CATG, CPL_FALSE, used_frameset);
    enu_check_error_code("error trying to read gain/linearity information");
 
    /* read the master_dark if necessary (for linearization) */
 
    if (gain_lin) {
        master_dark = en_master_dark_load_from_frameset(frameset,
                      ERIS_NIX_MASTER_DARK_IMG_PRO_CATG, used_frameset);
        enu_check_error_code("failed to read master dark from SoF");
    }
 
    /* read the master_bpm and high-freq flatfield */
 
    master_bpm = en_master_bpm_load_from_frameset(frameset,
                 ERIS_NIX_MASTER_BPM_SKY_PRO_CATG, used_frameset, CPL_TRUE);
    enu_check_error_code("failed to read master BPM from SoF");
    master_flat_hifreq = en_master_flat_load_from_frameset(
                         frameset, ERIS_NIX_MASTER_FLAT_LSS_HIFREQ_PRO_CATG,
                         used_frameset, CPL_FALSE);
    enu_check_error_code("failed to read master FLAT from SoF");
 
    /* read in the lamp-on and lamp-off data */
 
    on_limlist = enu_limlist_load_from_frameset(frameset,
                 ERIS_NIX_RAW_WAVE_LAMP_ON_DO_CATG, used_frameset);
    off_limlist = enu_limlist_load_from_frameset(frameset,
                  ERIS_NIX_RAW_WAVE_LAMP_OFF_DO_CATG, used_frameset);
 
    enu_check(on_limlist->size > 0, CPL_ERROR_ILLEGAL_INPUT,
              "SoF contains no WAVE_LAMP_ON data");
    enu_check(on_limlist->size == off_limlist->size, CPL_ERROR_ILLEGAL_INPUT,
              "SoF does not contain WAVE_LAMP_ON/WAVE_LAMP_OFF pairs");
 
    /* Do basic calibration of frames */ 
 
    diff_himlist = hdrl_imagelist_new();
    cpl_msg_info(cpl_func, "calculating lamp_on - lamp_off");
    int flag_all_errors = ~(int)0;
    enu_basic_calibrate(on_limlist, master_dark, gain_lin, master_flat_hifreq,
                        NULL, master_bpm, flag_all_errors, fill_rejected,
                        x_probe, y_probe);
    enu_basic_calibrate(off_limlist, master_dark, gain_lin, master_flat_hifreq,
                        NULL, master_bpm, flag_all_errors, fill_rejected,
                        x_probe, y_probe);
    enu_check_error_code("error performing basic calibration of frames");
 
    /* difference the on and off images */
 
    for (int on = 0; on < on_limlist->size; on++) {
        hdrl_image_sub_image(on_limlist->limages[on]->himage,
                             off_limlist->limages[on]->himage);
        hdrl_imagelist_set(diff_himlist,
                           hdrl_image_duplicate(on_limlist->
                           limages[on]->himage),
                           hdrl_imagelist_get_size(diff_himlist));
    }
 
    /* The raw lamp image is the mean of the differences */
 
    hdrl_imagelist_collapse(diff_himlist, HDRL_COLLAPSE_MEAN, &master_wave,
                            &contrib);
 
    /* construct vectors with the wavelengths and rough positions of the
       expected lines. The numbers here are hard-wired for the NaCO test
       data. For NIX the spectrum seen should be pretty consistent so
       should just need to replace these numbers from lab measurements. */
 
    cpl_size nlines = 12;
    cpl_vector * line_guess_pos = cpl_vector_new(nlines);
    cpl_vector * line_lambda = cpl_vector_new(nlines);
    cpl_vector_set(line_guess_pos, 0, (double) 71);
    cpl_vector_set(line_lambda, 0, 20621.9);
    cpl_vector_set(line_guess_pos, 1, (double) 95);
    cpl_vector_set(line_lambda, 1, 20740.0); /*blend*/
    cpl_vector_set(line_guess_pos, 2, (double) 112);
    cpl_vector_set(line_lambda, 2, 20816.7); /*blend*/
    cpl_vector_set(line_guess_pos, 3, (double) 148);
    cpl_vector_set(line_lambda, 3, 20991.8);
    cpl_vector_set(line_guess_pos, 4, (double) 220);
    cpl_vector_set(line_lambda, 4, 21338.7);
    cpl_vector_set(line_guess_pos, 5, (double) 260);
    cpl_vector_set(line_lambda, 5, 21540.1);  
    cpl_vector_set(line_guess_pos, 6, (double) 363);
    cpl_vector_set(line_lambda, 6, 22045.6);
    cpl_vector_set(line_guess_pos, 7, (double) 371);
    cpl_vector_set(line_lambda, 7, 22083.2);
    cpl_vector_set(line_guess_pos, 8, (double) 583);
    cpl_vector_set(line_lambda, 8, 23139.5);
    cpl_vector_set(line_guess_pos, 9, (double) 724);
    cpl_vector_set(line_lambda, 9, 23851.5);
    cpl_vector_set(line_guess_pos, 10, (double) 748);
    cpl_vector_set(line_lambda, 10, 23973.1);
    cpl_vector_set(line_guess_pos, 11, (double) 975);
    cpl_vector_set(line_lambda, 11, 25132.1);
 
    const cpl_size nx = hdrl_image_get_size_x(master_wave);
    const cpl_size ny = hdrl_image_get_size_y(master_wave);
    const cpl_size wave_cal_x = nx / 2;
    line_pos = enwu_linepos_2d(master_wave, wave_cal_x, line_guess_pos);
 
    /* now do 2d calibration fit between x, y pos of peak and lambda */
 
    /* ..first construct the sample pos and wavelength arrays for the fit 
         - these must have no NaNs or the fit routine will fail */
 
    cpl_size nsamples = 0;
    for (cpl_size line_id = 0; line_id < nlines; line_id++) {
        for (cpl_size ix = 0; ix < nx; ix++) {
            double val = cpl_matrix_get(line_pos, ix, line_id);
            if (!isnan(val)) {
                nsamples++;
            }
        }
    }
 
    cpl_matrix * line_pos_fit = cpl_matrix_new(2, nsamples);
    lambda_pos = cpl_vector_new(nsamples);
    cpl_size isample = 0;
    for (cpl_size line_id = 0; line_id < nlines; line_id++) {
        for (cpl_size ix = 0; ix < nx; ix++) {
            double val = cpl_matrix_get(line_pos, ix, line_id);
            if (!isnan(val)) {
                cpl_matrix_set(line_pos_fit, 0, isample, val); 
                cpl_matrix_set(line_pos_fit, 1, isample, (double) ix); 
                cpl_vector_set(lambda_pos, isample, 
                               cpl_vector_get(line_lambda, line_id));
                isample++;
            }
        }
    }
 
    cpl_polynomial * wave_cal = cpl_polynomial_new(2);
    const cpl_size maxdeg2d[] = {4, 2};
    cpl_polynomial_fit(wave_cal, line_pos_fit, NULL, lambda_pos, NULL,
                       CPL_TRUE, NULL, maxdeg2d);
    cpl_msg_info(cpl_func, "The wavelength calibration polynomial:");
    cpl_polynomial_dump(wave_cal, stdout);
 
    /* evaluate the calibration polynomial over the image */
 
    wave_cal_image = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
    for (cpl_size iy = 1; iy < ny+1; iy++) {
        for (cpl_size ix = 1; ix < nx+1; ix++) {
            cpl_vector * pos = cpl_vector_new(2);
            cpl_vector_set(pos, 0, (double) iy);
            cpl_vector_set(pos, 1, (double) ix);
            double val = cpl_polynomial_eval(wave_cal, pos);
            cpl_image_set(wave_cal_image, ix, iy, val); 
            cpl_vector_delete(pos); 
        }
    }
 
    /* evaluate the fit residual over the image */
 
    cpl_vector * wave_residual = cpl_vector_new(cpl_vector_get_size(
                                                lambda_pos));
    cpl_vector_fill_polynomial_fit_residual(wave_residual, lambda_pos,
                                            NULL, wave_cal, line_pos_fit,
                                            NULL);
    const double residual_mad = cpl_vector_get_median_const(wave_residual);
    cpl_msg_info(cpl_func, "MAD of residuals = %4.2f", residual_mad); 
 
    /* construct the propertylist to be saved with the MASTER_WAVE */
 
    wave_plist = cpl_propertylist_new();
    cpl_propertylist_append_string(wave_plist, CPL_DFS_PRO_CATG,
                                   ERIS_NIX_MASTER_WAVE_PRO_CATG);
 
    /* add QC parameters  */
 
    cpl_propertylist_append_double(wave_plist, "ESO QC CAL RESID",
                                   residual_mad);
    enu_check_error_code("error constructing output propertylist");
 
    /* save the MASTER_WAVE to a DFS-compliant MEF file */
 
    mefs = enu_mef_extension_list_new(4);
 
    /* MEF for confidence */
 
    mefs->mef[0] = enu_mef_new_image("CONFIDENCE", cpl_image_duplicate(
                                     master_flat_hifreq->confidence), NULL);
 
    /* MEF for LINE_POSITION - a table with the fitted peak position 
       for each line in each spectrum slice across the array */
 
    mef_table = cpl_table_new(nsamples);
    cpl_table_new_column(mef_table, "x", CPL_TYPE_DOUBLE);
    cpl_table_new_column(mef_table, "ypos", CPL_TYPE_DOUBLE);
    cpl_table_new_column(mef_table, "line_lambda", CPL_TYPE_DOUBLE);
    for (cpl_size isample = 0; isample < nsamples; isample++) {
        cpl_table_set(mef_table, "x", isample, 
                      cpl_matrix_get(line_pos_fit, 1, isample));
        cpl_table_set(mef_table, "ypos", isample,
                      cpl_matrix_get(line_pos_fit, 0, isample));
        cpl_table_set(mef_table, "line_lambda", isample, 
                      cpl_vector_get(lambda_pos, isample));
    }
    mefs->mef[1] = enu_mef_new_table("LINE_POSITION", mef_table, NULL);
 
    /* MEF for WAVE_CAL, containing an image with the calibrated wavelength
       of each pixel in the 2d spectrum */
 
    mefs->mef[2] = enu_mef_new_image("WAVE_CAL", cpl_image_duplicate(
                                     wave_cal_image), NULL);
 
    /* MEF for LINE_RESIDUAL, a table with the fit residual at each line
       position in each spectral slice */
 
    mef_table2 = cpl_table_new(nsamples);
    cpl_table_new_column(mef_table2, "x", CPL_TYPE_DOUBLE);
    cpl_table_new_column(mef_table2, "ypos", CPL_TYPE_DOUBLE);
    cpl_table_new_column(mef_table2, "lambda_residual", CPL_TYPE_DOUBLE);
    for (cpl_size isample = 0; isample < nsamples; isample++) {
        cpl_table_set(mef_table2, "x", isample, 
                      cpl_matrix_get(line_pos_fit, 1, isample));
        cpl_table_set(mef_table2, "ypos", isample,
                      cpl_matrix_get(line_pos_fit, 0, isample));
        cpl_table_set(mef_table2, "lambda_residual", isample, 
                      cpl_vector_get(wave_residual, isample));
    }
    mefs->mef[3] = enu_mef_new_table("LINE_RESIDUAL", mef_table2, NULL);
 
    enu_dfs_save_himage(frameset, parlist, used_frameset, master_wave,
                        on_limlist->limages[0]->frame, mefs,
                        RECIPE_NAME, wave_plist, PACKAGE "/" PACKAGE_VERSION,
                        "master_wave.fits");
 
    enu_check_error_code("Failed to save MASTER_WAVE");
 
cleanup:
    cpl_image_delete(contrib);
    hdrl_imagelist_delete(diff_himlist);
    engl_gain_linearity_delete((gain_linearity *) gain_lin);
    cpl_matrix_delete(line_pos);
    cpl_vector_delete(lambda_pos);
    en_master_bpm_delete(master_bpm);
    en_master_dark_delete(master_dark);
    en_master_flat_delete(master_flat_hifreq);
    enu_mef_extension_list_delete(mefs);
    cpl_table_delete(mef_table);
    cpl_table_delete(mef_table2);
    enu_located_imagelist_delete(off_limlist);
    cpl_mask_delete(old_mask);
    enu_located_imagelist_delete(on_limlist);
    cpl_frameset_delete(used_frameset);
    cpl_image_delete(wave_cal_image);
    cpl_propertylist_delete(wave_plist);
 
    return (int)cpl_error_get_code();
}