eris_nix_lss_skysub.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
/*
 * $Author$
 * $Date$
 * $Revision$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include <libgen.h>
#include <string.h>
 
#include "eris_nix_utils.h"
#include "eris_nix_lss_utils.h"
#include "eris_nix_dfs.h"
#include "eris_utils.h"
#include <hdrl.h>
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char eris_nix_lss_skysub_description[] =
"This recipe estimates and subtracts the sky background from a set of \n"
ERIS_NIX_CAL_DET_OBJECT_LSS_JITTER_PRO_CATG" or "
ERIS_NIX_CAL_DET_STD_LSS_JITTER_PRO_CATG" frames.\n"
"\n"
"Input files:\n"
"\n"
"  The recipe has 2 modes of operation.\n"
"  If --sky-source='target' then the sky background will be estimated \n"
"  from the target frames themselves.\n"
"\n"
"  DO CATG                         Explanation             Req.  #Frames\n"
"  -------                         -----------             ---   -------\n"
"  "ERIS_NIX_CAL_DET_OBJECT_LSS_JITTER_PRO_CATG
                           "       target frames with the   Y      1-n\n"
"             or                   detector signature\n"
"  "ERIS_NIX_CAL_DET_STD_LSS_JITTER_PRO_CATG
                        "          removed.\n"
"\n"
"  If --sky-source='offset' and for "
ERIS_NIX_CAL_DET_OBJECT_LSS_JITTER_PRO_CATG"\n"
"  data only, the sky background will be estimated from a set of \n"
"  frames offset from the target region.\n"
"\n"
"  DO CATG                         Explanation             Req.  #Frames\n"
"  -------                         -----------             ---   -------\n"
"  "ERIS_NIX_CAL_DET_OBJECT_LSS_JITTER_PRO_CATG
                           "       target frames with the   Y      1-n\n"
"                                  detector signature\n"
"                                  removed.\n"
"  "ERIS_NIX_CAL_DET_SKY_LSS_JITTER_PRO_CATG
                        "          sky frames with the      Y      1-m\n"
"                                  detector signature\n"
"                                  removed.\n"
"\n"
"Output files:\n"
"\n"
"  DO CATG                         Explanation \n"
"  -------                         ----------- \n"
"  "ERIS_NIX_SKYSUB_OBJECT_LSS_JITTER_PRO_CATG
                          "        target frames with the sky removed.\n"
"             or\n"
"  "ERIS_NIX_SKYSUB_STD_LSS_JITTER_PRO_CATG"\n"
"\n"
"  The output will be FITS files named 'skysub.<input filename>',\n"
"  containing extensions:\n"
"  - DATA, with the target data.\n"
"  - ERR, with the target error plane.\n"
"  - DQ, with the target data quality plane.\n"
"  - CONFIDENCE, with the target confidence plane.\n"
"  - BKG_DATA, with the estimated sky background.\n"
"  - BKG_ERR, with the error on the sky background.\n"
"  - BKG_CONF, with the confidence of the sky background.\n"
"\n"
"Notes on the method.\n"
"  Sky frames are taken from the pool of target or offset\n"
"  according to param --sky-source.\n"
"\n"
"  The background for each frame is estimated from sky frames\n"
"  taken within sky-bracket-time/2 seconds of it.\n"
"\n"
"  A first estimate of the background is made.\n"
"    If param --sky-method='collapse-median':\n"
"    - The estimate is a median collapse of the sky frames\n"
"      selected. This is prone to problems if the field is\n"
"      very crowded.\n"
"    If --sky-method='median-median':\n"
"    - The median of each sky frame is obtained, and the\n"
"      background frame set at the median of those values.\n"
"      The background is forced to be flat, which is more\n"
"      robust in crowded fields, or where there is\n"
"      nebulosity.\n"
"\n"
"  The estimated backgrounds are subtracted from the sky\n"
"  frames, and a search made for objects in each image.\n"
"  Object masks are constructed which blank out astronomical\n"
"  flux the sky frames. The --catalogue.xxx params listed\n"
"  below are used in this search for objects.\n"
"\n"
"  The masked sky frames are then used to re-estimate the\n"
"  sky backgrounds following the same sky-method.\n";
 
 
#define RECIPE_NAME "eris.eris_nix_lss_skysub"
 
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
cpl_recipe_define(eris_nix_lss_skysub, ERIS_BINARY_VERSION,
                  "John Lightfoot",
                  PACKAGE_BUGREPORT, "2017",
                  "Subtract sky background from LSS frames",
                  eris_nix_lss_skysub_description);
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_lss_skysub_fill_parameterlist(
  cpl_parameterlist * self) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    cpl_parameter     * p = NULL;
 
    /* recipe-specific parameters */
 
    p = cpl_parameter_new_enum(RECIPE_NAME".sky-source", CPL_TYPE_STRING,
                               "data to be used for calculation of sky "
                               "background", RECIPE_NAME,
                               "auto", 3, "auto", "target", "offset");
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "sky-source");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_enum(RECIPE_NAME".sky-selector", CPL_TYPE_STRING,
                               "method for selecting sky frames",
                               RECIPE_NAME, "bracket", 1, "bracket");
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "sky-selector");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_range(RECIPE_NAME".sky-bracket-time", CPL_TYPE_DOUBLE,
                               "2 * max.time between target and sky measurement",
                               RECIPE_NAME, 1800.0, 60.0, 18000.0); 
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "sky-bracket-time");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_value(RECIPE_NAME".premask_objects",
                          CPL_TYPE_BOOL,
                          "create object masks before background estimation",
                          RECIPE_NAME, CPL_TRUE);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "premask-objects");
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_value(RECIPE_NAME".premask_nsigma",
                                CPL_TYPE_DOUBLE,
                                "sigma cutoff for premask object detection",
                                RECIPE_NAME, 5.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "premask-nsigma");
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_value(RECIPE_NAME".debug-data",
                                CPL_TYPE_BOOL, "true to save interim results",
                                RECIPE_NAME, CPL_FALSE);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "debug-data");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    /* coords of pixel to be used for diagnostics during reduction */
 
    p = cpl_parameter_new_value(RECIPE_NAME".x_probe", CPL_TYPE_INT,
                                "x coord of diagnostic pixel",
                                RECIPE_NAME, -1);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "x-probe");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    p = cpl_parameter_new_value(RECIPE_NAME".y_probe", CPL_TYPE_INT,
                                "y coord of diagnostic pixel",
                                RECIPE_NAME, -1);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "y-probe");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    return 0;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int eris_nix_lss_skysub(cpl_frameset * frameset,
                                   const cpl_parameterlist * parlist) {
 
    located_imagelist     * object_jitters = NULL; 
    const char            * out_tag = NULL;
    const cpl_parameter   * p = NULL;
    located_imagelist     * sky_jitters = NULL; 
    located_imagelist     * std_jitters = NULL; 
    located_imagelist     * target_jitters = NULL; 
    cpl_frameset          * used_frameset = NULL;
 
    enu_check_error_code("%s():%d: An error is already set: %s",
                         cpl_func, __LINE__, cpl_error_get_where());
 
    /* Check input parameters */
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
 
    /* Set the msg verbosity level from environment variable CPL_MSG_LEVEL */
 
    cpl_msg_set_level_from_env();
    cpl_msg_severity severity = cpl_msg_get_level();
    cpl_msg_info(cpl_func, "level %d", (int) severity);
 
    /* check for invalid input */
    if(eris_files_dont_exist(frameset) != CPL_ERROR_NONE) {
        return CPL_ERROR_BAD_FILE_FORMAT;
    }
    /* Retrieve input parameters */
 
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".sky-source");
    const char * sky_source = cpl_parameter_get_string(p); 
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".sky-selector");
    const char * sky_selector = cpl_parameter_get_string(p);
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".sky-bracket-time");
    const double sky_time_range = cpl_parameter_get_double(p); 
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".premask_objects");
    int premask_objects = cpl_parameter_get_bool(p);
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".premask_nsigma");
    double premask_nsigma = cpl_parameter_get_double(p);
/*
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".debug-data");
    const int debug_data = cpl_parameter_get_bool(p);
*/
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".x_probe");
    cpl_size x_probe = (cpl_size) cpl_parameter_get_int(p);
    p = cpl_parameterlist_find_const(parlist, RECIPE_NAME".y_probe");
    cpl_size y_probe = (cpl_size) cpl_parameter_get_int(p);
 
    enu_check_error_code("Could not retrieve input parameters");
 
    /* Identify the RAW and CALIB frames in the input frameset */
 
    eris_nix_dfs_set_groups(frameset);
    enu_check_error_code("Could not identify RAW and CALIB frames");    
 
    used_frameset = cpl_frameset_new();
 
    /* Read in CAL_DET_OBJECT_LSS_JITTER data, CAL_DET_SKY_LSS_JITTER, or
       CAL_DET_STD_LSS_JITTER frames if present */
 
    object_jitters = enu_limlist_load_from_frameset(frameset,
                     ERIS_NIX_CAL_DET_OBJECT_LSS_JITTER_PRO_CATG,
                     used_frameset);
    sky_jitters = enu_limlist_load_from_frameset(frameset,
                  ERIS_NIX_CAL_DET_SKY_LSS_JITTER_PRO_CATG, used_frameset);
    std_jitters = enu_limlist_load_from_frameset(frameset,
                  ERIS_NIX_CAL_DET_STD_LSS_JITTER_PRO_CATG, used_frameset);
    enu_check_error_code("Could not load input frames");
 
    cpl_msg_info(cpl_func, "%d "
                 ERIS_NIX_CAL_DET_OBJECT_LSS_JITTER_PRO_CATG" frames read", 
                 (int) object_jitters->size);
    cpl_msg_info(cpl_func, "%d "
                 ERIS_NIX_CAL_DET_SKY_LSS_JITTER_PRO_CATG" frames read", 
                 (int) sky_jitters->size);
    cpl_msg_info(cpl_func, "%d "
                 ERIS_NIX_CAL_DET_STD_LSS_JITTER_PRO_CATG" frames read", 
                 (int) std_jitters->size);
 
    /* check that the SOF collection makes sense */
 
    enu_check(!(object_jitters->size == 0 && std_jitters->size == 0),
              CPL_ERROR_DATA_NOT_FOUND, "SoF contains no "
              ERIS_NIX_CAL_DET_OBJECT_LSS_JITTER_PRO_CATG" or "
              ERIS_NIX_CAL_DET_STD_LSS_JITTER_PRO_CATG" frames");
    enu_check(!(object_jitters->size > 0 && std_jitters->size > 0),
              CPL_ERROR_ILLEGAL_INPUT, "SoF contains both "
              ERIS_NIX_CAL_DET_OBJECT_LSS_JITTER_PRO_CATG" and "
              ERIS_NIX_CAL_DET_STD_LSS_JITTER_PRO_CATG" frames");
    enu_check(!(!strcmp(sky_source,"offset") && sky_jitters->size==0),
              CPL_ERROR_INCOMPATIBLE_INPUT,
              "offset sky position requested but no"
              ERIS_NIX_CAL_DET_SKY_LSS_JITTER_PRO_CATG" in sof");
 
    /* is target an object or std? */
 
    if (object_jitters->size > 0) {
        target_jitters = object_jitters;
        out_tag = ERIS_NIX_SKYSUB_OBJECT_LSS_JITTER_PRO_CATG;
    } else if (std_jitters->size > 0) {
        target_jitters = std_jitters;
        out_tag = ERIS_NIX_SKYSUB_STD_LSS_JITTER_PRO_CATG;
    }
 
    /* do right thing when sky-source=="auto", namely use offset
       sky data if available */
 
    if (!strcmp(sky_source, "auto")) {
        if (sky_jitters->size > 0) {
            sky_source = "offset";
        } else {
            sky_source = "target";
        }
        cpl_msg_info(cpl_func, "sky-source=='auto', resetting to '%s'",
                     sky_source);
    } 
 
    if (!strcmp(sky_source, "offset")) {
 
        if (premask_objects) {
 
            /* Calculate object masks for the 'offset' jitter images
               and attach them the data.*/
 
            enu_opm_lss_limlist(sky_jitters, premask_nsigma);
        }
 
        /* Use the masked images to estimate the sky background and 
           subtract it */
 
        cpl_msg_info(cpl_func, "Estimating sky background with source blanking ...");
        enu_sky_subtract_limlist("collapse-median", sky_selector, sky_time_range,
                                 sky_jitters, x_probe, y_probe,
                                 target_jitters);
        enu_check_error_code("error in sky background subtraction");
 
    } else if (!strcmp(sky_source, "target")) { 
 
        /* Use the target calibrated jitters to estimate the sky background.
 
           This would work for a 'random' jitter pattern. However, NACO 
           data switches between 2 nod positions on the slit, in which
           case a straight median would not - instead need to mask out 
           the objects first. */
 
        cpl_msg_info(cpl_func, "Estimating sky background:");
        if (premask_objects) {
 
           /* Calculate object masks for the target jitter images
              and attach them the data.*/
 
            enu_opm_lss_limlist(target_jitters, premask_nsigma);
        }
        enu_sky_subtract_limlist("collapse-median", sky_selector,
                                 sky_time_range, target_jitters,
                                 x_probe, y_probe, target_jitters);
        enu_check_error_code("error in sky background: sky-subtract");
    }
 
    enu_check_error_code("failed to remove sky background");
 
    /* estimate the slit profile from the sky background and divide by it */
 
    enlu_divide_slit_response(target_jitters);
 
    /* save the sky-subtracted jitter images to FITS files */
 
    for (cpl_size i = 0; i < target_jitters->size; i++) {
        cpl_propertylist * applist = cpl_propertylist_new();
        cpl_propertylist_update_string(applist, CPL_DFS_PRO_CATG, out_tag);
        cpl_propertylist_update_string(applist, "PRODCATG", "ANCILLARY.IMAGE");
 
        /* Generate output file name, write the file */
 
        char * out_fname = enu_repreface(cpl_frame_get_filename(
                                         target_jitters->limages[i]->frame),
                                         "skysub");
        enu_dfs_save_limage(frameset, 
                            parlist,
                            frameset,
                            CPL_TRUE,
                            target_jitters->limages[i],
                            RECIPE_NAME,
                            target_jitters->limages[i]->frame,
                            applist,
                            PACKAGE "/" PACKAGE_VERSION,
                            out_fname);
 
        cpl_free(out_fname);
        cpl_propertylist_delete(applist);
    }
 
cleanup:
    enu_located_imagelist_delete(object_jitters);
    enu_located_imagelist_delete(sky_jitters);
    enu_located_imagelist_delete(std_jitters);
    cpl_frameset_delete(used_frameset);
 
    return (int) cpl_error_get_code();
}