eris_nix_lss_stack.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
/*
 * $Author$
 * $Date$
 * $Revision$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include <libgen.h>
#include <math.h>
#include <string.h>
 
#include "casu_utils.h"
#include "casu_mods.h"
 
#include "eris_utils.h"
#include "eris_nix_utils.h"
#include "eris_pfits.h"
#include "eris_dfs.h"
#include "eris_nix_dfs.h"
#include "eris_nix_master_bpm.h"
#include "eris_nix_master_dark.h"
#include "eris_nix_gain_linearity.h"
#include "eris_utils.h"
#include <hdrl.h>
 
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char eris_nix_lss_stack_description[] =
"This recipe uses an HDRL collapse operation to stack a set of already \n"
"registered ERIS/NIX LSS jitter frames.\n"
"\n"
"Input files:\n"
"\n"
"  DO CATG                         Explanation             Req.  #Frames\n"
"  -------                         -----------             ---   -------\n"
"  "ERIS_NIX_CORRECTED_OBJECT_LSS_JITTER_PRO_CATG
                             "     calibrated frames        Y      1-n\n"
"                                  to be stacked.\n"
"             or\n"
"  "ERIS_NIX_CORRECTED_STD_LSS_JITTER_PRO_CATG
                          "        calibrated standard      Y      1-n\n"
"                                  frames to be stacked.\n"
"\n"
"Output files:\n"
"\n"
"  DO CATG                         Explanation \n"
"  -------                         ----------- \n"
"  "ERIS_NIX_LSS_OBS_COMBINED_PRO_CATG
                  "                the stacked 2d-spectrum of the target.\n"
"             or\n"
"  "ERIS_NIX_LSS_STD_COMBINED_PRO_CATG
                  "                the stacked 2d-spectrum of the standard.\n"
"\n"
"  The output image will be in a FITS file named \n"
"  'stack.<first input filename>', with extensions:\n"
"  - DATA, with the stacked target or standard data.\n"
"  - ERR, with the error plane.\n"
"  - DQ, with the stacked image quality\n"
"  - CONFIDENCE, with the target confidence plane.\n"
 
"\n"
"Notes on the method.\n"
" The stacking uses hdrl_image_collapse to combine all jitter\n"
" images such that each output pixel is the median of the inputs.\n"
"\n";
 
 
#define RECIPE_NAME "eris.eris_nix_lss_stack"
 
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
cpl_recipe_define(eris_nix_lss_stack, ERIS_BINARY_VERSION,
                  "John Lightfoot",
                  PACKAGE_BUGREPORT, "2017",
                  "Stack "ERIS_NIX_CORRECTED_OBJECT_LSS_JITTER_PRO_CATG
                  " frames",
                  eris_nix_lss_stack_description);
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_lss_stack_fill_parameterlist(
  cpl_parameterlist * self) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    cpl_parameter * p = NULL;
 
    p = cpl_parameter_new_value(RECIPE_NAME".x_probe", CPL_TYPE_INT,
                                "x coord of diagnostic pixel",
                                RECIPE_NAME, -1);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "x-probe");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_value(RECIPE_NAME".y_probe", CPL_TYPE_INT,
                                "y coord of diagnostic pixel",
                                RECIPE_NAME, -1);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "y-probe");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    return 0;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int eris_nix_lss_stack(cpl_frameset * frameset,
                              const cpl_parameterlist * parlist) {
 
    cpl_image             * contrib = NULL;
    hdrl_imagelist        * himlist = NULL;
    located_imagelist     * jitters = NULL;
    located_imagelist     * object_jitters = NULL;
    const char            * out_catg = NULL;
    located_image         * stack = NULL;
    located_imagelist     * std_jitters = NULL;
    cpl_frameset          * used = NULL;
 
    enu_check_error_code("%s():%d: An error is already set: %s",
                         cpl_func, __LINE__, cpl_error_get_where());
 
    /* Check input parameters */
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
 
    /* Set the msg verbosity level from environment variable CPL_MSG_LEVEL */
 
    cpl_msg_set_level_from_env();
    cpl_msg_severity severity = cpl_msg_get_level();
    cpl_msg_info(cpl_func, "level %d", (int) severity);
 
    /* check for invalid input */
    if(eris_files_dont_exist(frameset) != CPL_ERROR_NONE) {
        return CPL_ERROR_BAD_FILE_FORMAT;
    }
    /* Retrieve input parameters */
 
    const cpl_parameter * p = NULL;
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".x_probe");
    cpl_size x_probe = (cpl_size) cpl_parameter_get_int(p);
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".y_probe");
    cpl_size y_probe = (cpl_size) cpl_parameter_get_int(p);
 
    enu_check_error_code("Could not retrieve input parameters");
 
    /* Identify the RAW and CALIB frames in the input frameset */
 
    eris_nix_dfs_set_groups(frameset);
    enu_check_error_code("Could not identify RAW and CALIB frames");    
    used = cpl_frameset_new();
 
    /* read in the straightened data */
 
    object_jitters = enu_limlist_load_from_frameset(frameset,
                     ERIS_NIX_CORRECTED_OBJECT_LSS_JITTER_PRO_CATG, used);
    std_jitters = enu_limlist_load_from_frameset(frameset,
                  ERIS_NIX_CORRECTED_STD_LSS_JITTER_PRO_CATG, used);
    enu_check_error_code("Error loading frameset");
 
    if (object_jitters->size > 0) {
        enu_check(std_jitters->size == 0, CPL_ERROR_ILLEGAL_INPUT,
                  "SoF contains both object and std data");
        jitters = object_jitters;
        object_jitters = NULL;
        out_catg = ERIS_NIX_LSS_OBS_COMBINED_PRO_CATG;
        cpl_msg_info(cpl_func, "Read in %d "
                     ERIS_NIX_CORRECTED_OBJECT_LSS_JITTER_PRO_CATG" frames",
                     (int)(jitters->size));
    } else if (std_jitters->size > 0) {
        jitters = std_jitters;
        std_jitters = NULL;
        out_catg = ERIS_NIX_LSS_STD_COMBINED_PRO_CATG;
        cpl_msg_info(cpl_func, "Read in %d "
                     ERIS_NIX_CORRECTED_STD_LSS_JITTER_PRO_CATG" frames",
                     (int)(jitters->size));
    }
 
    /* construct an hdrl_imagelist from the located images */
 
    himlist = hdrl_imagelist_new();
    double crval1 = -1.0;
    double crval3 = -1.0;
    for (cpl_size j=0; j < jitters->size; j++) {
        hdrl_imagelist_set(himlist,
                           hdrl_image_duplicate(jitters->limages[j]->himage),
                           j);
 
        crval1 = cpl_propertylist_get_double(jitters->limages[j]->plist, 
                                             "CRVAL1");
        /* LSS data may not have CRVAL3 and other CD matrix keys for 3d axis as they are just images"
        if(cpl_propertylist_has(jitters->limages[j]->plist,"CRVAL3")) {
            crval3 = cpl_propertylist_get_double(jitters->limages[j]->plist,
                    "CRVAL3");
        }
        */
        /* output 'probe' pixel if debugging */
 
        cpl_size nx = hdrl_image_get_size_x(jitters->limages[j]->himage); 
        cpl_size ny = hdrl_image_get_size_y(jitters->limages[j]->himage); 
 
        if (x_probe >= 1 && x_probe <= nx &&
            y_probe >= 1 && y_probe <= ny) {
            int reject = 0;
            hdrl_value val = hdrl_image_get_pixel(jitters->limages[j]->himage,
                                                  x_probe, y_probe, &reject);
            int ignore = 0;
            double confidence = cpl_image_get(jitters->limages[j]->confidence,
                                              x_probe, y_probe, &ignore);
            cpl_msg_info(cpl_func, "j=%d val={%f, %f} reject=%d confidence=%f",
                         (int)j, val.data, val.error, reject, confidence);
        }
    }
 
    /* and stack the data */
 
    hdrl_image * stack_himage = NULL;
    hdrl_imagelist_collapse(himlist, HDRL_COLLAPSE_MEDIAN, &stack_himage,
                            &contrib);
    enu_check_error_code("error set after image stacking");
 
    /* output 'probe' pixel if debugging */
 
    cpl_size nx = hdrl_image_get_size_x(stack_himage); 
    cpl_size ny = hdrl_image_get_size_y(stack_himage); 
 
    if (x_probe >= 1 && x_probe <= nx &&
        y_probe >= 1 && y_probe <= ny) {
 
        int reject = 0;
        hdrl_value val = hdrl_image_get_pixel(stack_himage,
                                              x_probe, y_probe, &reject);
        cpl_msg_info(cpl_func, "val={%f, %f} reject=%d",
                     val.data, val.error, reject);
    }
 
    /* and stack the background */
 
    hdrl_imagelist_delete(himlist); himlist = NULL;
    himlist = hdrl_imagelist_new();
    for (cpl_size j=0; j < jitters->size; j++) {
        hdrl_imagelist_set(himlist,
                           hdrl_image_duplicate(jitters->limages[j]->bkg), j);
    }
 
    hdrl_image * stack_bkg = NULL;
    cpl_image_delete(contrib);
    hdrl_imagelist_collapse(himlist, HDRL_COLLAPSE_MEDIAN, 
                            &stack_bkg,
                            &contrib);
    enu_check_error_code("error set after background stacking");
 
    /* and stack the confidence.
       This assumes the stacking elsewhere was done with HDRL_COLLAPSE_MEDIAN,
       in which case eqn 21 in HDRL Doc section on 'Statistical Esimators'
       says that the sigma on the median is related to that on the mean
       by a constant factor (pi/2), which will drop out when the 
       confidence is normalised.
 
       sigma^2 is 1 / confidence.
    */
 
    cpl_image * stack_confidence = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
    {
        cpl_image * stack_count = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
        double * stack_conf_ptr = cpl_image_get_data_double(stack_confidence);
        double * stack_count_ptr = cpl_image_get_data_double(stack_count);
        for (cpl_size i=0; i < nx * ny; i++) {
            stack_conf_ptr[i] = 0.0;
            stack_count_ptr[i] = 0.0;
        }
 
        /* accumulate jitter confidences */
 
        for (cpl_size j=0; j < jitters->size; j++) {
            for (cpl_size i=1; i < nx+1; i++){
                for (cpl_size jj=1; jj< ny+1; jj++) {
                    // confidence images should have no invalid pixels
                    // BUT the images can and, if they do, those pixels
                    // will not contribute to the median. Nor should
                    // those pixel's confidence contribute to the 
                    // stacked confidence
                    int rejected = 0;
                    hdrl_image_get_pixel(
                                       jitters->limages[j]->himage,
                                       i, jj, &rejected);
                    if (rejected == 0) {
                        int ignore = 0;
                        double jconf = cpl_image_get(
                                       jitters->limages[j]->confidence,
                                       i, jj, &ignore);
                        double sconf = cpl_image_get(stack_confidence,
                                       i, jj, &ignore);
                        if (jconf > 0.0) {
                            if (sconf != CPL_VALUE_PLUSINF) {
                                // valid numbers, convert confidence to variance
                                jconf = 1.0 / jconf;
                                cpl_image_set(stack_confidence, i, jj,
                                       sconf + jconf);
                                double tcount = cpl_image_get(stack_count,
                                       i, jj, &ignore);  
                                cpl_image_set(stack_count, i, jj, tcount + 1);  
                            }  
                        } else {
                            // 0 confidence, infinite variance
                            cpl_image_set(stack_confidence, i, jj,
                                          CPL_VALUE_PLUSINF);
                        }
                    }
                }
            }
        }
 
        /* divide by N^2 (mean is sum/N, mean var = sum_var / N^2 */
 
        for (cpl_size i=1; i < nx+1; i++){
            for (cpl_size j=1; j< ny+1; j++) {
                int ignore = 0;
                double conf = cpl_image_get(stack_confidence,
                                       i, j, &ignore);  
                double count = cpl_image_get(stack_count,
                                       i, j, &ignore);  
                if (conf != CPL_VALUE_PLUSINF && count > 0.0) {
                    cpl_image_set(stack_confidence,
                                  i, j, conf / pow(count, 2));
                } else {  
                    cpl_image_set(stack_confidence,
                                  i, j, CPL_VALUE_PLUSINF);
                }
            }
        }
 
        /* variance -> confidence */              
 
        for (cpl_size i=0; i < nx * ny; i++) {
            if (stack_conf_ptr[i] == CPL_VALUE_PLUSINF) {
                stack_conf_ptr[i] = 0.0;
            } else {
                stack_conf_ptr[i] = 1.0 / stack_conf_ptr[i];
            }
        }
 
        /* normalise confidence */
 
        enu_normalise_confidence(stack_confidence);
 
        cpl_image_delete(stack_count);
    }
    enu_check_error_code("error set after confidence stacking");
 
    /* save the result */
 
    stack = enu_located_image_new(stack_himage,
                                  NULL,
                                  stack_confidence,
                                  stack_bkg,
                                  NULL,
                                  cpl_propertylist_new(),
                                  NULL,
                                  NULL,
                                  NULL,
                                  NULL,
                                  NULL);
 
    /* Set up the wcs to reflect the corrected spectrum.
       Borrows from long-slit example in 
       Calabretta and Greisen 2002 A&A 395, 1077 (example on p.
       1115-16).
 
       ..This scheme, with its degenerate 3rd axis, my be more 'clever'
       ..than we need - maybe better to keep 2 dims [lambda, slit offset]?
 
       In our case the pixel coords are p1,p2 = slit-offset,wavelength
       NB this is different to the example in the paper.
 
       The transformation is to RA, Dec, lambda. Using a CD matrix:
 
       RA        CRVAL1     CD1_1  CD1_2  CD1_3     p1 - CRPIX1
       lambda  = CRVAL2  +  CD2_1  CD2_2  CD2_3  x  p2 - CRPIX2
       Dec       CRVAL3  +  CD3_1  CD3_2  CD3_3     p3 - CRPIX3
 
       The wavelength transformation is 3.045um at 0 to 4.107um at 2047 so:
 
          CRPIX2 = 0.0
          CRVAL2 = 3.045
          CD2_1 = 0
          CD2_2 = 0
          cd2_3 = (4.107 - 3.045) / 2047
 
       The slit offset to RA, Dec transformation combines the RA,Dec of the
       slit centre with the P.A. of the slit rotation (assumed 0 at N, 
       increasing towrds increasing RA), so:
 
          CRPIX1 = 1024
          CRVAL1 = RA centre
          CD1_1 = sin(PA) * pixscale
          RA = CRVAL1 + CD1_1 * (p1 - CRPIX1)
 
           and
 
          CRVAL3 = Dec centre
          CD3_1 = cos(PA) * pixscale
          Dec = CRVAL3 + CD3_1 * (p1 - CRPIX3)
 
       The p3 axis is degenerate so CD entries for it can have
       arbitrary values.
 
          CD1_3 = CD2_3 = CD3_3 = 1
 */
 
    cpl_propertylist * olist = stack->plist;
    cpl_propertylist_update_int(olist, "NAXIS", 3);
    cpl_propertylist_update_string(olist, "CTYPE1", "RA---TAN");
    cpl_propertylist_update_string(olist, "CTYPE2", "WAVE");
    cpl_propertylist_update_string(olist, "CTYPE3", "DEC--TAN");
    cpl_propertylist_update_double(olist, "CRPIX1", 1024.0);
    cpl_propertylist_update_double(olist, "CRPIX2", 0.0);
    cpl_propertylist_update_double(olist, "CRPIX3", 0.0);
    cpl_propertylist_update_double(olist, "CRVAL1", crval1);
    cpl_propertylist_update_double(olist, "CRVAL2", 3.045);
    cpl_propertylist_update_double(olist, "CRVAL3", crval3);
 
    /* the slit p.a. needs to be sorted out */
    double slit_pa = 0.0;
    double pixscale = 0.013;
    cpl_propertylist_update_double(olist, "CD1_1", 
                                   sin(slit_pa) * pixscale / (3600.0));
    cpl_propertylist_update_double(olist, "CD1_2", 0.0);
    cpl_propertylist_update_double(olist, "CD1_3", 1.0);
 
    cpl_propertylist_update_double(olist, "CD2_1", 0.0);
    cpl_propertylist_update_double(olist, "CD2_2",
                                   (4.107 - 3.045) / 2047);
    cpl_propertylist_update_double(olist, "CD2_3", 0.0);
 
    cpl_propertylist_update_double(olist, "CD3_1",
                                   cos(slit_pa) * pixscale / (3600.0));
    cpl_propertylist_update_double(olist, "CD3_2", 0.0);
    cpl_propertylist_update_double(olist, "CD3_3", 1.0);
 
    cpl_propertylist_update_string(olist, "CUNIT1", "DEG");
    cpl_propertylist_update_string(olist, "CUNIT2", "um");
    cpl_propertylist_update_string(olist, "CUNIT3", "DEG");
 
    /* data needs to be converted from dims [nx,ny] to [nx,ny,1] to 
       save in FITS format - FITS required the addition of a degenerate 
       axis */
 
    hdrl_imagelist * himlist1 = hdrl_imagelist_new();
    hdrl_imagelist_set(himlist1, stack->himage, 0);
    stack->himage = NULL;
    stack->himagelist = himlist1;
 
    /* generate name of file */
 
    char * stack_fname = enu_repreface(cpl_frame_get_filename(
                                       jitters->limages[0]->frame),
                                       "stack");
 
    /* construct provenance frameset */
 
    cpl_frameset * provenance = cpl_frameset_new();
    for (cpl_size j=0; j<jitters->size; j++) {
        cpl_frameset_insert(provenance, cpl_frame_duplicate(
                            jitters->limages[j]->frame));        
    }
 
    /* update PRO.CATG and stack info */
 
    cpl_propertylist * applist = cpl_propertylist_new();
    cpl_propertylist_update_string(applist, CPL_DFS_PRO_CATG, out_catg);
    cpl_propertylist_update_string(applist, "PRODCATG",
                                   "ANCILLARY.2DSPECTRUM");
 
    int ncombine = jitters->size;
 
    double total_exptime = 0.0;
    double mjd_start = 1000000.0;
    double mjd_end = 0.0;
 
    cpl_vector * obsid = cpl_vector_new(jitters->size);
    for (cpl_size j=0; j < jitters->size; j++) {
        cpl_vector_set(obsid, j, (double)j);
 
 
        /* .. total integration time */
 
        double dit = enu_get_dit(jitters->limages[j]->plist);
        double exptime = dit * cpl_propertylist_get_int(
                jitters->limages[j]->plist, "ESO DET NDIT");
        total_exptime += exptime;
 
        /* .. MJD of start and end of measurements used */
 
        double jitter_start = cpl_propertylist_get_double(
                jitters->limages[j]->plist, "MJD-OBS");
        if (jitter_start < mjd_start) {
            mjd_start = jitter_start;
        }
        double jitter_end = cpl_propertylist_get_double(
                jitters->limages[j]->plist, "MJD-END");
        if (jitter_end > mjd_end) {
            mjd_end = jitter_end;
        }
        cpl_msg_info(cpl_func, "..combined mjd start: %15.8f  end: %15.8f",
                mjd_start, mjd_end);
 
 
    }
 
 
    cpl_propertylist_update_int(applist, "NCOMBINE", ncombine);
    cpl_propertylist_update_double(applist, "EXPTIME", total_exptime);
    cpl_propertylist_update_double(applist, "TEXPTIME", total_exptime);
    cpl_propertylist_update_double(applist, "MJD-OBS", mjd_start);
    cpl_propertylist_update_double(applist, "MJD-END", mjd_end);
    for (cpl_size j = 0; j < jitters->size; ++j) {
        char * pname = cpl_sprintf("OBID%.0i", (int)(j+1));
        cpl_msg_info(cpl_func, "%d %s", (int)j, pname);
        cpl_propertylist_update_int(applist, pname,
                                    (int)cpl_vector_get(obsid, j));
        cpl_free(pname);
    }
 
    cpl_msg_info(cpl_func, "BUNIT fudged for now");
    cpl_propertylist_update_string(applist, "BUNIT", "adu/s");
 
    /* and save the result */
 
    enu_dfs_save_limage(frameset,
                        parlist,
                        provenance,
                        CPL_FALSE,
                        stack,
                        RECIPE_NAME,
                        stack->frame,
                        applist,
                        PACKAGE "/" PACKAGE_VERSION,
                        stack_fname);
 
    cpl_free(stack_fname);
    cpl_frameset_delete(provenance);
    cpl_propertylist_delete(applist);
 
cleanup:
    cpl_image_delete(contrib);
    hdrl_imagelist_delete(himlist);
    enu_located_imagelist_delete(jitters);
    enu_located_image_delete(stack);
    cpl_frameset_delete(used);
 
    return (int) cpl_error_get_code();
}