xsh_mdark.c

Go to the documentation of this file.

/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */
 
/*
 * $Author: amodigli $
 * $Date: 2013-04-29 10:00:50 $
 * $Revision: 1.183 $
*/
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
  Includes
  ----------------------------------------------------------------------------*/
 
 
/* DRL steps */
 
/* Error handling */
#include <xsh_error.h>
/* Utility fonctions */
#include <xsh_utils.h>
#include <xsh_utils_image.h>
#include <xsh_msg.h>
/* DFS functions */
#include <xsh_dfs.h>
/* DRL functions */
#include <xsh_drl.h>
/* Header functions */
#include <xsh_pfits.h>
/* Library */
#include <cpl.h>
/* CRH Remove */
#include <xsh_drl_check.h>
#include <xsh_model_arm_constants.h>
#include <xsh_badpixelmap.h>
 
/*-----------------------------------------------------------------------------
  Defines
  ----------------------------------------------------------------------------*/
 
#define RECIPE_ID "xsh_mdark"
#define RECIPE_AUTHOR "P.Goldoni, L.Guglielmi, R. Haigron, F. Royer, D. Bramich, A. Modigliani"
#define RECIPE_CONTACT "amodigli@eso.org"
 
/*-----------------------------------------------------------------------------
  Functions prototypes
  ----------------------------------------------------------------------------*/
 
/*
 *   Plugin initalization, execute and cleanup handlers
 */
 
static int xsh_mdark_create(cpl_plugin *);
static int xsh_mdark_exec(cpl_plugin *);
static int xsh_mdark_destroy(cpl_plugin *);
 
/* The actual executor function */
static void xsh_mdark(cpl_parameterlist *, cpl_frameset *);
 
/*-----------------------------------------------------------------------------
  Static variables
  ----------------------------------------------------------------------------*/
static char xsh_mdark_description_short[] =
"Create the master dark frame";
 
static char xsh_mdark_description[] =
"This recipe creates a master dark frame\n\
Input Frames : \n\
  - A set of n RAW frames (Format=RAW, n >=3, Tag = DARK_arm)\n\
  - [UVB,VIS] A master bias frame (Format=PRE, Tag = MASTER_BIAS_arm)\n\
  - [OPTIONAL] A map of non linear bad pixels (Format=QUP, Tag = BP_MAP_NL_arm)\n\
  - [OPTIONAL] A map of reference bad pixels (Format = QUP,RAW, Tag = BP_MAP_RP_arm)\n\
Products : \n\
  - A master dark frame (Format=PRE, PRO.CATG = MASTER_DARK_arm)\n\
    A dark frame, (Format=PRE, PRO.CATG = DARK_arm)\n";
 
/*
  - [VIS && bp-output=yes] A badpixel map (Format=QUP, \n\
    PRO.CATG = MASTER_BP_MAP_DARK_arm)\n\
  - If an input master bad pixel map or a map of non linear pixels is provided \n\
 
  - A map frame containing CRHs  (PRO.CATG = CRH_MAP_arm)\n\
*/
/*-----------------------------------------------------------------------------
  Functions code
  ----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
int cpl_plugin_get_info(cpl_pluginlist *list) {
  cpl_recipe *recipe = NULL;
  cpl_plugin *plugin = NULL;
 
  recipe = cpl_calloc(1, sizeof(*recipe));
  if ( recipe == NULL ){
    return -1;
  }
 
  plugin = &recipe->interface ;
 
  cpl_plugin_init(plugin,
                  CPL_PLUGIN_API,                    /* Plugin API */
                  XSH_BINARY_VERSION,             /* Plugin version */
                  CPL_PLUGIN_TYPE_RECIPE,            /* Plugin type */
                  RECIPE_ID,                         /* Plugin name */
                  xsh_mdark_description_short,       /* Short help */
                  xsh_mdark_description,             /* Detailed help */
                  RECIPE_AUTHOR,                     /* Author name */
                  RECIPE_CONTACT,                    /* Contact address */
                  xsh_get_license(),                 /* Copyright */
                  xsh_mdark_create,
                  xsh_mdark_exec,
                  xsh_mdark_destroy);
 
  cpl_pluginlist_append(list, plugin);
 
  return (cpl_error_get_code() != CPL_ERROR_NONE);
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int xsh_mdark_create(cpl_plugin *plugin){
  cpl_recipe *recipe = NULL;
  xsh_clipping_param crh_clip_param = {-1.0, 3, 0.7, 0, 0.3};
  xsh_clipping_param noise_clip_param = {9.0, 5, 0.7, 0, 0.3};
  xsh_fpn_param fpn_param = {-1,-1,-1,-1,4,100};
  xsh_ref_param ref_param = {-1,-1,-1,-1};
  xsh_ron_dark_param ron_param = {-1,-1,-1,-1,4,100};
  xsh_stack_param stack_param = {"median",5.,5.};
  int ival=DECODE_BP_FLAG_DEF;
  /* Reset library state */
  xsh_init();
 
  /* Check input */
  assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin");
 
  /* Get the recipe out of the plugin */
  assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
          CPL_ERROR_TYPE_MISMATCH,
          "Plugin is not a recipe");
 
  recipe = (cpl_recipe *)plugin;
 
  /* Create the parameter list in the cpl_recipe object */
  recipe->parameters = cpl_parameterlist_new();
  assure( recipe->parameters != NULL,
          CPL_ERROR_ILLEGAL_OUTPUT,
          "Memory allocation failed!");
 
  /* Set generic parameters (common to all recipes) */
  check( xsh_parameters_generic( RECIPE_ID, recipe->parameters ) ) ;
  xsh_parameters_decode_bp(RECIPE_ID,recipe->parameters,ival);
  check( xsh_parameters_pre_overscan( RECIPE_ID, recipe->parameters ) ) ;
  check(xsh_parameters_stack_create(RECIPE_ID,recipe->parameters,stack_param));
 
  /* Set bp_output param */
  check(xsh_parameters_new_boolean( recipe->parameters, RECIPE_ID, 
    "bp-output", false,
    "Flag indicating whether the noise bad pixel map is to be computed"));
 
  /* crh clipping params */
  check(xsh_parameters_clipping_crh_create(RECIPE_ID,recipe->parameters,
    crh_clip_param));
 
  /* noise clipping params */
  check(xsh_parameters_clipping_noise_create(RECIPE_ID,
                                             recipe->parameters,
                         noise_clip_param));
 
  check(xsh_parameters_new_double( recipe->parameters,RECIPE_ID, 
    "noise-lower-rejection", 10.,
    "Lower rejection percentile to flag bad pixels via noise"));
 
  check(xsh_parameters_new_double( recipe->parameters, RECIPE_ID, 
    "noise-higher-rejection", 10.,
    "Upper rejection percentile to flag bad pixels via noise"));
 
 
  check(xsh_parameters_ref1_create(RECIPE_ID,recipe->parameters,
                                  ref_param));
 
 
  check(xsh_parameters_ron_dark_create(RECIPE_ID,recipe->parameters,
                                       ron_param));
 
  check(xsh_parameters_fpn_create(RECIPE_ID,recipe->parameters,
                                  fpn_param));
 
 
 
 cleanup:
  if ( cpl_error_get_code() != CPL_ERROR_NONE ){
    xsh_error_dump(CPL_MSG_ERROR);
    return 1;
  }
  else {
    return 0;
  }
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int xsh_mdark_exec(cpl_plugin *plugin) {
  cpl_recipe *recipe = NULL;
 
  /* Check parameter */
 
  assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin" );
 
  /* Get the recipe out of the plugin */
  assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
          CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
 
  recipe = (cpl_recipe *)plugin;
 
  /* Check recipe */
  xsh_mdark(recipe->parameters, recipe->frames);
 
 cleanup:
  if ( cpl_error_get_code() != CPL_ERROR_NONE ) {
    xsh_error_dump(CPL_MSG_ERROR);
    xsh_error_reset();
    return 1;
  }
  else {
    return 0;
  }
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static int xsh_mdark_destroy(cpl_plugin *plugin)
{
  cpl_recipe *recipe = NULL;
 
  /* reset error state before detroying recipe */
  xsh_error_reset(); 
  /* Check parameter */
  assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin" );
 
  /* Get the recipe out of the plugin */
  assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
      CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
  recipe = (cpl_recipe *)plugin;
 
  xsh_free_parameterlist(&recipe->parameters);
 
 cleanup:
  if (cpl_error_get_code() != CPL_ERROR_NONE)
    {
      return 1;
    }
  else
    {
      return 0;
    }
}
 
static cpl_error_code 
xsh_params_set_defaults(cpl_parameterlist* pars,
                        xsh_instrument* inst)
{
   cpl_parameter* p=NULL;
 
   check(p=xsh_parameters_find(pars,RECIPE_ID,"crh-clip-kappa"));
   if(cpl_parameter_get_double(p) <= 0) {
      if (xsh_instrument_get_arm(inst) == XSH_ARM_NIR){
         cpl_parameter_set_double(p,11.);
      }  else {
         cpl_parameter_set_double(p,9.);
      }  
  }
 
 cleanup:
 
  return cpl_error_get_code();
 
}
 
 
 
static cpl_frame*
xsh_get_master_dark_nir_via_noise(cpl_imagelist* dataList, cpl_frame* medFrame,
    cpl_parameterlist* parameters, xsh_clipping_param* noise_clipping,
    xsh_instrument* instrument,cpl_frame** noisyFrame) {
  cpl_frame* nirFrame = NULL;
 
  cpl_parameter* p = NULL;
  int llx = 0;
  int lly = 0;
  int urx = 0;
  int ury = 0;
  int iter = 0;
  int low_rej = 0;
  int high_rej = 0;
  cpl_image* noise_image = NULL;
  cpl_mask* bp_map_noise = NULL;
  cpl_propertylist* plist = NULL;
  const char* bp_map_noise_pro_catg = NULL;
  char bp_map_noise_name[256];
 
 
  xsh_msg_dbg_low(
      "Noise parameters: Sigma %lf, Niteration %d, \
   Fraction %lf, Diff %lf", noise_clipping->sigma, noise_clipping->niter, noise_clipping->frac, noise_clipping->diff);
 
  nirFrame = xsh_compute_noise_map(dataList, medFrame, noise_clipping,
      instrument,noisyFrame);
 
 
  p = cpl_parameterlist_find(parameters, "xsh.xsh_mdark.ref1_llx");
  llx = cpl_parameter_get_int(p);
  p = cpl_parameterlist_find(parameters, "xsh.xsh_mdark.ref1_lly");
  lly = cpl_parameter_get_int(p);
  p = cpl_parameterlist_find(parameters, "xsh.xsh_mdark.ref1_urx");
  urx = cpl_parameter_get_int(p);
  p = cpl_parameterlist_find(parameters, "xsh.xsh_mdark.ref1_ury");
  ury = cpl_parameter_get_int(p);
  p = cpl_parameterlist_find(parameters, "xsh.xsh_mdark.noise-clip-niter");
  iter = cpl_parameter_get_int(p);
  p = cpl_parameterlist_find(parameters, "xsh.xsh_mdark.noise-lower-rejection");
  low_rej = cpl_parameter_get_double(p);
  p = cpl_parameterlist_find(parameters, "xsh.xsh_mdark.noise-higher-rejection");
  high_rej = cpl_parameter_get_double(p);
 
  noise_image = xsh_image_search_bad_pixels_via_noise(dataList, iter, low_rej,
      high_rej, llx, lly, urx, ury);
  bp_map_noise = cpl_mask_threshold_image_create(noise_image,
      QFLAG_HOT_PIXEL - 1, QFLAG_HOT_PIXEL + 1);
  xsh_free_image(&noise_image);
  noise_image = cpl_image_new_from_mask(bp_map_noise);
  cpl_mask_not(bp_map_noise);
  xsh_bpmap_mask_bad_pixel(noise_image, bp_map_noise, QFLAG_HOT_PIXEL);
 
//here good pixels are still flagged as '1'. To have proper code, '0',
//we make a threshold
  cpl_image_threshold(noise_image, 1.1, DBL_MAX, 0, DBL_MAX);
 
  plist = cpl_propertylist_new();
  bp_map_noise_pro_catg = XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_NP, instrument);
  sprintf(bp_map_noise_name, "%s.fits", bp_map_noise_pro_catg);
  xsh_pfits_set_pcatg(plist, bp_map_noise_pro_catg);
  check(cpl_image_save(noise_image,bp_map_noise_name,CPL_BPP_IEEE_FLOAT, plist,CPL_IO_DEFAULT));
  xsh_add_temporary_file(bp_map_noise_name);
 
 
  cleanup:
  xsh_free_mask(&bp_map_noise);
  xsh_free_propertylist(&plist);
  xsh_free_image(&noise_image);
  return nirFrame;
 
}
 
static cpl_frame*
xsh_get_master_dark_opt(cpl_frame* medFrame, cpl_frame* master_bias,
    cpl_parameterlist* parameters,
    cpl_frame* crh_frm, cpl_frame* bp_map_noise_frm, xsh_instrument* instrument,
    const int pre_overscan_corr) {
  cpl_frame* resFrame = NULL;
  cpl_frame* rmbias = NULL;
  if (master_bias != NULL) {
    xsh_msg("Subtract bias");
    check(
        rmbias = xsh_subtract_bias(medFrame,master_bias,instrument,"DARK_",pre_overscan_corr,0));
 
    xsh_msg( "Generates master dark");
    check_msg(
        resFrame=xsh_create_master_dark(rmbias, instrument,parameters, crh_frm,bp_map_noise_frm),
        "Error in create master dark");
  } else {
    check(resFrame=cpl_frame_duplicate(medFrame));
  }
 
  cleanup:
  xsh_free_frame(&rmbias);
  return resFrame;
}
 
static cpl_frame*
xsh_get_crh_frame(cpl_image* crh_ima, xsh_instrument* instrument) {
  char* crh_name = NULL;
  char* crh_pro_catg = NULL;
  cpl_frame* crh_frm = NULL;
  char *prefix = NULL;
 
  crh_name = cpl_sprintf("%s_%s.fits", XSH_CRH_MAP,
      xsh_instrument_arm_tostring(instrument));
 
  if (crh_ima != NULL) {
    /* crh_ima is created only if nraws>=3 */
    crh_pro_catg = XSH_GET_TAG_FROM_ARM(XSH_CRH_MAP,instrument);
    crh_frm = xsh_frame_product(crh_name, crh_pro_catg, CPL_FRAME_TYPE_IMAGE,
        CPL_FRAME_GROUP_PRODUCT, CPL_FRAME_LEVEL_FINAL);
 
    XSH_PREFIX(prefix, "CRH_MAP", instrument);
    cpl_image_save(crh_ima, crh_name, CPL_BPP_32_SIGNED, NULL, CPL_IO_DEFAULT);
     xsh_add_temporary_file(crh_name);
 
  }
  cleanup:
  XSH_FREE(crh_name);
  XSH_FREE(prefix);
  return crh_frm;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static void xsh_mdark(cpl_parameterlist* parameters, cpl_frameset* frameset) {
 
  const char* recipe_tags[1] = { XSH_DARK };
  int recipe_tags_size = 1;
 
  cpl_frameset* raws = NULL;
  cpl_frameset* calib = NULL;
  cpl_frame* bpmap = NULL;
  cpl_frame* master_bias = NULL;
  cpl_frame* rmbias = NULL;
  xsh_instrument* instrument = NULL;
  xsh_clipping_param* crh_clipping = NULL;
  xsh_clipping_param* noise_clipping = NULL;
  cpl_imagelist * dataList = NULL;
 
  /* Intermediate frames */
  cpl_frame * medFrame = NULL;
  cpl_frame * nirFrame = NULL;
  cpl_frame * noisyFrame = NULL;
 
  /* Product frames */
  cpl_frame * resFrame = NULL;
 
  cpl_image* crh_ima = NULL;
  const int mode_or=1;
 
  cpl_frame* crh_frm = NULL;
  xsh_pre* pre=NULL;
  char name[256];
 
  cpl_propertylist* plist = NULL;
  char med_frame_tag[256];
 
 
  cpl_frame* dark_on = NULL;
  cpl_frame* dark_qc = NULL;
 
  int pre_overscan_corr = 0;
  xsh_stack_param* stack_par = NULL;
 
  char pcatg[256];
  cpl_propertylist* qc_log = NULL;
  int nraws;
  int sz=0;
 
  /**************************************************************************/
  /* DFS management */
  /**************************************************************************/
  check( xsh_begin( frameset, parameters, &instrument, &raws, &calib,
          recipe_tags, recipe_tags_size,
          RECIPE_ID, XSH_BINARY_VERSION,
          xsh_mdark_description_short ) );
 
  check(xsh_recipe_params_check(parameters,instrument,RECIPE_ID));
 
  /**************************************************************************/
  /* Recipe frames */
  /**************************************************************************/
  XSH_ASSURE_NOT_ILLEGAL((nraws=cpl_frameset_get_size(raws)) >= 1);
  check(bpmap=xsh_check_load_master_bpmap(calib,instrument,RECIPE_ID));
 
 
  /* In UVB and VIS mode only read master bias if exist */
  if (xsh_instrument_get_arm(instrument) != XSH_ARM_NIR) {
     master_bias = xsh_find_frame_with_tag(calib, XSH_MASTER_BIAS,instrument);
  }
  
  sprintf(med_frame_tag, "DARK_REMOVE_CRH_%s",
          xsh_instrument_arm_tostring(instrument));
 
  /**************************************************************************/
  /* Recipe parameters */
  /**************************************************************************/
  check(xsh_params_set_defaults(parameters,instrument));
  check(
      pre_overscan_corr = xsh_parameters_get_int( parameters, RECIPE_ID, "pre-overscan-corr"));
  stack_par = xsh_stack_frames_get(RECIPE_ID, parameters);
  
  crh_clipping = xsh_parameters_clipping_crh_get(RECIPE_ID, parameters);
  noise_clipping = xsh_parameters_clipping_noise_get(RECIPE_ID, parameters);
 
  /**************************************************************************/
  /* Recipe code */
  /**************************************************************************/
 
  /* prepare RAW frames in XSH format */
  check(xsh_prepare(raws, bpmap, master_bias, XSH_DARK, instrument,pre_overscan_corr,CPL_TRUE));
  /* make sure each input raw frame has the same exp time */
  check(xsh_frameset_check_uniform_exptime(raws,instrument));
 
  /* QC master dark: computes frame median with CRH clip (multi) */
  xsh_msg( "Calling xsh_remove_crh_multiple" );
  check_msg( medFrame = xsh_remove_crh_multiple( raws,med_frame_tag,stack_par,crh_clipping,
                                                 instrument,&dataList,&crh_ima,1 ),"Error in xsh_remove_crh" );
 
  check(crh_frm=xsh_get_crh_frame(crh_ima,instrument));
 
  /* NIR arm */
  if (xsh_instrument_get_arm(instrument) == XSH_ARM_NIR) {
     check(nirFrame=xsh_get_master_dark_nir_via_noise(dataList,medFrame,parameters,noise_clipping,instrument,&noisyFrame));
     xsh_msg( "Create master dark");
     /* create master dark XSH consortium way */
     check_msg(
        resFrame=xsh_create_master_dark(nirFrame,instrument,parameters,crh_frm,bpmap ),
        "Error in create master dark");
  } else {
     // UVB and VIS : no noise map
     check(
        resFrame=xsh_get_master_dark_opt(medFrame,master_bias,parameters,crh_frm,bpmap, instrument,pre_overscan_corr));
  }
 
  /* we extract the QC computed by previous algorithm - CRH multi- 
     and inject in the next one for backward
     * compatibility (to keep same QC as original mdark creation algorithm)
     */
  qc_log = xsh_frame_head_extract_qc(resFrame);
 
  /* end QC */
 
 
  /* create master dark reflex way */
  xsh_free_frame(&resFrame);
  check(resFrame=xsh_create_master_dark2(raws,stack_par,parameters,qc_log,instrument));
 
 
  if (xsh_instrument_get_arm(instrument) == XSH_ARM_NIR) {
     /* only in NIR we may generate a noisy BP map, in which case we bitwise-or 
        combine */
     if(noisyFrame != NULL) {
       cpl_frame* frm=NULL;
        cpl_image* bpm=cpl_image_load(cpl_frame_get_filename(noisyFrame),
                                      CPL_TYPE_INT,0,0);
        pre=xsh_pre_load(resFrame,instrument);
        xsh_badpixelmap_image_coadd(&(pre->qual),bpm,mode_or);
        frm=xsh_pre_save( pre, cpl_frame_get_filename(resFrame),cpl_frame_get_tag(resFrame), 0);
        xsh_free_image(&bpm);
        xsh_pre_free(&pre);
        xsh_free_frame(&frm);
     }
  }
 
  check(xsh_badpixelmap_fill_bp_pattern_holes(resFrame));
  /* saving products */
  xsh_msg( "Save products" );
  check(xsh_add_product_image( resFrame, frameset, parameters,RECIPE_ID, instrument,NULL));
 
  
 
  /* save 1st frame in PRE format for QC */
  dark_on = cpl_frameset_get_frame(raws, 0);
  sprintf(pcatg, "DARK_ON");
  sprintf(name, "%s.fits", pcatg);
  xsh_add_product_image(dark_on,frameset,parameters,RECIPE_ID,instrument,pcatg);
 
 
  /* save last frame in PRE format for QC */
  sz=cpl_frameset_get_size(raws);
  dark_qc = cpl_frameset_get_frame(raws, sz-1);
  sprintf(pcatg, "DARK_QC");
  sprintf(name, "%s.fits", pcatg);
  xsh_add_product_image(dark_qc,frameset,parameters,RECIPE_ID,instrument,pcatg);
 
 
  cleanup: xsh_end(RECIPE_ID, frameset, parameters);
  XSH_FREE(crh_clipping);
  XSH_FREE(noise_clipping);
  XSH_FREE(stack_par);
  xsh_instrument_free(&instrument);
  xsh_free_frameset(&raws);
  xsh_free_frameset(&calib);
 
  xsh_free_imagelist(&dataList);
  xsh_free_frame(&medFrame);
  xsh_free_frame(&nirFrame);
  xsh_free_frame(&resFrame);
  xsh_free_frame(&rmbias);
  xsh_free_frame(&crh_frm);
  xsh_free_frame(&bpmap);
  xsh_free_frame(&noisyFrame);
  xsh_free_image(&crh_ima);
  xsh_free_propertylist(&plist);
  xsh_free_propertylist(&qc_log);
 
  return;
}