moons_mdark.c

/*
 * This file is part of the MOONS Pipeline
 * Copyright (C) 2002-2016 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include "moo_utils.h"
#include "moo_pfits.h"
#include "moo_dfs.h"
#include "moo_params.h"
#include "moo_drl.h"
#include "moo_products.h"
#include <cpl.h>
 
#include <string.h>
 
/*-----------------------------------------------------------------------------
                              Plugin registration
 -----------------------------------------------------------------------------*/
 
int cpl_plugin_get_info(cpl_pluginlist *list);
 
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
static int _moons_mdark_create(cpl_plugin *plugin);
static int _moons_mdark_exec(cpl_plugin *plugin);
static int _moons_mdark_destroy(cpl_plugin *plugin);
static int
_moons_mdark(cpl_frameset *frameset, const cpl_parameterlist *parlist);
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char *const _moons_mdark_description =
    "This recipe can be use in VIS or NIR mode\n"
    "1) RI detectors (VIS mode)\n"
    "INPUT FRAMES\n"
    "  * RawList n>=3 files                       (RAW) with tag DARK_VIS : "
    "dark files for RI detectors\n"
    "  * [OPTIONAL] ReferenceBadPixMask 1 file    (QUA) with tag BP_MAP_RP : "
    "cosmetic bad pixel map\n"
    "  * [OPTIONAL] NonLinearityBadPixMask 1 file (QUA) with tag BP_MAP_NL : "
    "cosmetic bad pixel map coming from linearity recipe\n"
    "  * MasterBias 1 file                        (DET) with tag MASTER_BIAS : "
    "master bias file\n"
    "PRODUCTS\n"
    "  * MASTER_DARK_VIS.fits                     (DET) with tag "
    "MASTER_DARK_VIS : "
    "the master dark file for RI detectors\n"
    "\n2) YJ, H detectors (NIR mode)\n"
    "INPUT FRAMES\n"
    "  * RawList n>=3 files                       (RAW) with tag DARK_NIR : "
    "dark files for YJ and H detectors\n"
    "  * [OPTIONAL] ReferenceBadPixMask  1 file   (QUA) with tag BP_MAP_RP : "
    "cosmetic bad pixel map\n"
    "  * [OPTIONAL] NonLinearityBadPixMask 1 file (QUA) with tag BP_MAP_NL : "
    "cosmetic bad pixel map coming from linearity recipe\n"
    "PRODUCTS\n"
    "  * MASTER_DARK_NIR.fits                     (DET) with tag "
    "MASTER_DARK_NIR : "
    "the master dark file\n"
    "\n";
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
int
cpl_plugin_get_info(cpl_pluginlist *list)
{
    cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
    cpl_plugin *plugin = &recipe->interface;
 
    if (cpl_plugin_init(plugin, CPL_PLUGIN_API, MOONS_BINARY_VERSION,
                        CPL_PLUGIN_TYPE_RECIPE, "moons_mdark",
                        "Create a master dark product",
                        _moons_mdark_description, "Regis Haigron",
                        PACKAGE_BUGREPORT, moo_get_license(),
                        _moons_mdark_create, _moons_mdark_exec,
                        _moons_mdark_destroy)) {
        cpl_msg_error(cpl_func, "Plugin initialization failed");
        (void)cpl_error_set_where(cpl_func);
        return 1;
    }
 
    if (cpl_pluginlist_append(list, plugin)) {
        cpl_msg_error(cpl_func, "Error adding plugin to list");
        (void)cpl_error_set_where(cpl_func);
        return 1;
    }
 
    return 0;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int
_moons_mdark_create(cpl_plugin *plugin)
{
    cpl_recipe *recipe;
 
    /* Do not create the recipe if an error code is already set */
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
                      cpl_func, __LINE__, cpl_error_get_where());
        return (int)cpl_error_get_code();
    }
 
    if (plugin == NULL) {
        cpl_msg_error(cpl_func, "Null plugin");
        cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
    }
 
    /* Verify plugin type */
    if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
        cpl_msg_error(cpl_func, "Plugin is not a recipe");
        cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
    }
 
    /* Get the recipe */
    recipe = (cpl_recipe *)plugin;
 
    /* Create the parameters list in the cpl_recipe object */
    recipe->parameters = cpl_parameterlist_new();
    if (recipe->parameters == NULL) {
        cpl_msg_error(cpl_func, "Parameter list allocation failed");
        cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT);
    }
 
    moo_params *params = moo_params_new("moons", "moons_mdark");
 
    /* Fill the parameters list */
    moo_params_add_keep_temp(params, recipe->parameters);
    moo_params_add_prepare(params, recipe->parameters);
 
    moo_params_add_correct_bias(params, recipe->parameters,
                                MOO_CORRECT_BIAS_METHOD_MASTER);
    moo_params_add_nos(params, recipe->parameters);
    moo_params_add_hot(params, recipe->parameters);
    moo_params_add_crh(params, recipe->parameters, MOO_CRH_METHOD_SIGCLIP);
 
    moo_params_delete(params);
 
    return 0;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int
_moons_mdark_exec(cpl_plugin *plugin)
{
    cpl_recipe *recipe;
    int recipe_status;
    cpl_errorstate initial_errorstate = cpl_errorstate_get();
 
    /* Return immediately if an error code is already set */
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
                      cpl_func, __LINE__, cpl_error_get_where());
        return (int)cpl_error_get_code();
    }
 
    if (plugin == NULL) {
        cpl_msg_error(cpl_func, "Null plugin");
        cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
    }
 
    /* Verify plugin type */
    if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
        cpl_msg_error(cpl_func, "Plugin is not a recipe");
        cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
    }
 
    /* Get the recipe */
    recipe = (cpl_recipe *)plugin;
 
    /* Verify parameter and frame lists */
    if (recipe->parameters == NULL) {
        cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
        cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
    }
    if (recipe->frames == NULL) {
        cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
        cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
    }
 
    /* Invoke the recipe */
    recipe_status = _moons_mdark(recipe->frames, recipe->parameters);
 
    /* Ensure DFS-compliance of the products */
    if (cpl_dfs_update_product_header(recipe->frames)) {
        if (!recipe_status)
            recipe_status = (int)cpl_error_get_code();
    }
 
    if (!cpl_errorstate_is_equal(initial_errorstate)) {
        /* Dump the error history since recipe execution start.
           At this point the recipe cannot recover from the error */
        cpl_errorstate_dump(initial_errorstate, CPL_FALSE, NULL);
    }
 
    return recipe_status;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int
_moons_mdark_destroy(cpl_plugin *plugin)
{
    cpl_recipe *recipe;
 
    if (plugin == NULL) {
        cpl_msg_error(cpl_func, "Null plugin");
        cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
    }
 
    /* Verify plugin type */
    if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
        cpl_msg_error(cpl_func, "Plugin is not a recipe");
        cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
    }
 
    /* Get the recipe */
    recipe = (cpl_recipe *)plugin;
 
    cpl_parameterlist_delete(recipe->parameters);
 
    return 0;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static cpl_frame *
_moons_prepare(moo_products *products,
               const cpl_frame *frame,
               const char *bpmap_rp_name,
               const char *bpmap_nl_name,
               const cpl_frame *masterbias,
               const cpl_frame *coefs_cube,
               int do_correct_bias,
               moo_prepare_params *prepare_params,
               moo_correct_bias_params *params,
               int i)
{
    cpl_frame *result = NULL;
    cpl_frame *pframe = NULL;
    char *detname1 = NULL;
    moo_det *det = NULL;
 
    cpl_ensure(frame != NULL, CPL_ERROR_NULL_INPUT, NULL);
    cpl_errorstate prestate = cpl_errorstate_get();
 
    moo_try_check(det = moo_prepare(frame, bpmap_rp_name, bpmap_nl_name,
                                    masterbias, coefs_cube, prepare_params),
                  " ");
 
    if (do_correct_bias > 0) {
        moo_try_check(moo_correct_bias(det, masterbias, params), " ");
    }
 
    moo_try_check(detname1 =
                      cpl_sprintf("%s_%d.fits", MOONS_TAG_DARK_CORRECTBIAS, i),
                  " ");
    moo_try_check(pframe = moo_products_add(products, det,
                                            CPL_FRAME_LEVEL_INTERMEDIATE,
                                            MOONS_TAG_DARK_CORRECTBIAS,
                                            detname1, frame),
                  " ");
    moo_try_check(result = cpl_frame_duplicate(pframe), " ");
 
moo_try_cleanup:
    if (!cpl_errorstate_is_equal(prestate)) {
        cpl_frame_delete(result);
        result = NULL;
    }
    moo_det_delete(det);
    cpl_free(detname1);
    return result;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static cpl_frameset *
_moons_prepare_set(cpl_frameset *raw_frames,
                   const char *bpmap_rp_name,
                   const char *bpmap_nl_name,
                   const cpl_frame *masterbias,
                   const cpl_frame *coef_cube,
                   int do_correct_bias,
                   moo_prepare_params *prepare_params,
                   moo_correct_bias_params *params,
                   moo_products *products)
{
    cpl_frameset *detframes = NULL;
    moo_det *det = NULL;
 
    cpl_ensure(raw_frames, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(products, CPL_ERROR_NULL_INPUT, NULL);
 
    cpl_errorstate prestate = cpl_errorstate_get();
 
    moo_try_check(detframes = cpl_frameset_new(), " ");
 
    for (int i = 0; i < cpl_frameset_get_size(raw_frames); ++i) {
        const cpl_frame *current_frame = NULL;
        cpl_frame *frame = NULL;
 
        moo_try_check(current_frame =
                          cpl_frameset_get_position_const(raw_frames, i),
                      " ");
 
        moo_try_check(frame =
                          _moons_prepare(products, current_frame, bpmap_rp_name,
                                         bpmap_nl_name, masterbias, coef_cube,
                                         do_correct_bias, prepare_params,
                                         params, i),
                      " ");
 
        moo_try_check(cpl_frameset_insert(detframes, frame), " ");
    }
moo_try_cleanup:
    if (!cpl_errorstate_is_equal(prestate)) {
        moo_det_delete(det);
        cpl_frameset_delete(detframes);
        detframes = NULL;
    }
    return detframes;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static cpl_error_code
_moons_mdark_check_sof(cpl_frameset *frameset,
                       cpl_frameset **rawframes,
                       const char **bpmap_rp_name,
                       const char **bpmap_nl_name,
                       const cpl_frame **master_bias,
                       const cpl_frame **coefs_cube_frame,
                       int *nb_nir)
{
    cpl_errorstate prestate = cpl_errorstate_get();
 
    cpl_ensure_code(rawframes != NULL, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(frameset != NULL, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(bpmap_rp_name != NULL, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(bpmap_nl_name != NULL, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(master_bias != NULL, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(nb_nir != NULL, CPL_ERROR_NULL_INPUT);
    /* set group */
    cpl_ensure_code(moo_dfs_set_groups(frameset) == CPL_ERROR_NONE,
                    cpl_error_get_code());
 
    cpl_frameset *raw_frames_vis = NULL;
    cpl_frameset *raw_frames_nir = NULL;
    int nraw_vis = 0;
    int nraw_nir = 0;
    int i;
 
    moo_try_check(raw_frames_vis = cpl_frameset_new(), " ");
    moo_try_check(raw_frames_nir = cpl_frameset_new(), " ");
 
    for (i = 0; i < cpl_frameset_get_size(frameset); ++i) {
        const cpl_frame *current_frame = NULL;
        moo_try_check(current_frame =
                          cpl_frameset_get_position_const(frameset, i),
                      " ");
        if (!strcmp(cpl_frame_get_tag(current_frame), MOONS_TAG_DARK_VIS)) {
            cpl_frame *new_frame = cpl_frame_duplicate(current_frame);
            cpl_frameset_insert(raw_frames_vis, new_frame);
            ++nraw_vis;
        }
        else if (!strcmp(cpl_frame_get_tag(current_frame),
                         MOONS_TAG_DARK_NIR)) {
            cpl_frame *new_frame = cpl_frame_duplicate(current_frame);
            cpl_frameset_insert(raw_frames_nir, new_frame);
            ++nraw_nir;
        }
        else if (!strcmp(cpl_frame_get_tag(current_frame),
                         MOONS_TAG_BP_MAP_RP)) {
            *bpmap_rp_name = cpl_frame_get_filename(current_frame);
        }
        else if (!strcmp(cpl_frame_get_tag(current_frame),
                         MOONS_TAG_BP_MAP_NL)) {
            *bpmap_nl_name = cpl_frame_get_filename(current_frame);
        }
        else if (!strcmp(cpl_frame_get_tag(current_frame),
                         MOONS_TAG_MASTER_BIAS)) {
            *master_bias = current_frame;
        }
        else if (!strcmp(cpl_frame_get_tag(current_frame),
                         MOONS_TAG_LINEARITY_COEFF_CUBE)) {
            *coefs_cube_frame = current_frame;
        }
    }
    *nb_nir = nraw_nir;
 
    if (nraw_vis > 0 && nraw_nir > 0) {
        cpl_frameset_delete(raw_frames_vis);
        cpl_frameset_delete(raw_frames_nir);
        return (int)cpl_error_set_message(cpl_func,
                                          CPL_ERROR_INCOMPATIBLE_INPUT,
                                          "You have NIR and VIS raw files");
    }
 
    int nraw = 0;
    const char *tag = NULL;
    if (nraw_vis > 0) {
        nraw = nraw_vis;
        *rawframes = raw_frames_vis;
        cpl_frameset_delete(raw_frames_nir);
        if (*master_bias == NULL) {
            return (
                int)cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND,
                                          "SOF does not have any file tagged "
                                          "with %s",
                                          MOONS_TAG_MASTER_BIAS);
        }
        tag = MOONS_TAG_DARK_VIS;
    }
    else {
        nraw = nraw_nir;
        tag = MOONS_TAG_DARK_NIR;
        *rawframes = raw_frames_nir;
        cpl_frameset_delete(raw_frames_vis);
    }
 
    if (nraw == 0) {
        return (int)cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND,
                                          "SOF does not have any file tagged "
                                          "with %s",
                                          tag);
    }
 
    if (nraw < 3) {
        return (int)cpl_error_set_message(
            cpl_func, CPL_ERROR_DATA_NOT_FOUND,
            "SOF does not have enough files (%d<3) tagged "
            "with %s",
            nraw, tag);
    }
 
moo_try_cleanup:
    if (!cpl_errorstate_is_equal(prestate)) {
        cpl_frameset_delete(raw_frames_vis);
        cpl_frameset_delete(raw_frames_nir);
        *rawframes = NULL;
    }
    return cpl_error_get_code();
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int
_moons_mdark(cpl_frameset *frameset, const cpl_parameterlist *parlist)
{
    /* parameters */
    moo_prepare_params *prepare_params = NULL;
 
    moo_correct_bias_params *correct_bias_params = NULL;
    moo_crh_params *crh_params = NULL;
    moo_nos_params *nos_params = NULL;
    moo_hot_params *hot_params = NULL;
 
    /* input files */
    cpl_frameset *raw_frames = NULL;
    const char *bpmap_rp_name = NULL;
    const char *bpmap_nl_name = NULL;
    const cpl_frame *masterbias = NULL;
    const cpl_frame *coef_cube = NULL;
    int nraw_nir = 0;
    /* structure in use */
    moo_det *medDark = NULL;
    moo_det *medCosmics = NULL;
    moo_detlist *detDarkList = NULL;
    moo_masklist *cosmicsList = NULL;
 
    /* frames used */
    cpl_frameset *detframes = NULL;
 
    moo_products *products = moo_products_new(frameset, parlist, "moons_mdark",
                                              PACKAGE "/" PACKAGE_VERSION);
 
    /* parameters */
    const moo_params *params = moo_products_get_params(products);
    moo_try_check(correct_bias_params =
                      moo_params_get_correct_bias(params, parlist),
                  " ");
    moo_try_check(prepare_params = moo_params_get_prepare(params, parlist),
                  " ");
    moo_try_check(crh_params = moo_params_get_crh(params, parlist), " ");
    moo_try_check(nos_params = moo_params_get_nos(params, parlist), " ");
    moo_try_check(hot_params = moo_params_get_hot(params, parlist), " ");
 
    /* SOF file */
    moo_try_check(_moons_mdark_check_sof(frameset, &raw_frames, &bpmap_rp_name,
                                         &bpmap_nl_name, &masterbias,
                                         &coef_cube, &nraw_nir),
                  " ");
 
    const cpl_frame *ref_frame = cpl_frameset_get_position_const(raw_frames, 0);
 
    moo_try_check(detframes =
                      _moons_prepare_set(raw_frames, bpmap_rp_name,
                                         bpmap_nl_name, masterbias, coef_cube,
                                         (nraw_nir == 0), prepare_params,
                                         correct_bias_params, products),
                  " ");
 
    moo_try_check(detDarkList = moo_detlist_create(detframes), " ");
 
    int nraw = moo_detlist_get_size(detDarkList);
 
    cosmicsList = moo_masklist_create(nraw);
 
    moo_try_check(medDark =
                      moo_remove_CRH(detDarkList, cosmicsList, crh_params),
                  " ");
 
    const char *meddark_filename = MOONS_TAG_DARK_REMOVECRH ".fits";
    moo_try_check(moo_products_add(products, medDark,
                                   CPL_FRAME_LEVEL_INTERMEDIATE,
                                   MOONS_TAG_DARK_REMOVECRH, meddark_filename,
                                   ref_frame),
                  " ");
    moo_try_check(moo_compute_hot_map(medDark, hot_params), " ");
 
    const char *darkhot_filename = MOONS_TAG_DARK_COMPUTEHOT ".fits";
    moo_try_check(moo_products_add(products, medDark,
                                   CPL_FRAME_LEVEL_INTERMEDIATE,
                                   MOONS_TAG_DARK_COMPUTEHOT, darkhot_filename,
                                   ref_frame),
                  " ");
 
    if (nraw_nir > 0) {
        if (strcmp(crh_params->method, MOO_CRH_METHOD_SIGCLIP) != 0) {
            cpl_msg_info(__func__,
                         "Do crh rejection in SIGCLIP mode to detect cosmics");
            crh_params->method = MOO_CRH_METHOD_SIGCLIP;
            moo_try_check(medCosmics = moo_remove_CRH(detDarkList, cosmicsList,
                                                      crh_params),
                          " ");
        }
 
        moo_try_check(moo_compute_noise_map(medDark, detDarkList, cosmicsList,
                                            nos_params),
                      " ");
        moo_try_check(moo_masterdark(medDark, products, 1), " ");
    }
    else {
        moo_try_check(moo_masterdark(medDark, products, 0), " ");
    }
 
moo_try_cleanup:
    // dump error from the state
    moo_det_delete(medDark);
    moo_det_delete(medCosmics);
    moo_detlist_delete(detDarkList);
    moo_masklist_delete(cosmicsList);
    cpl_frameset_delete(detframes);
 
    cpl_frameset_delete(raw_frames);
 
    moo_nos_params_delete(nos_params);
    moo_hot_params_delete(hot_params);
    moo_crh_params_delete(crh_params);
    moo_prepare_params_delete(prepare_params);
 
    moo_correct_bias_params_delete(correct_bias_params);
    moo_products_delete(products);
    return (int)cpl_error_get_code();
}