xsh_mbias.c

Go to the documentation of this file.

/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */
 
/*
 * $Author: amodigli $
 * $Date: 2013-10-02 15:47:47 $
 * $Revision: 1.133 $
 * $Name: not supported by cvs2svn $
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------
                                Includes
 ----------------------------------------------------------------------------*/
 
#include <hdrl.h>
/* DRL steps */
 
/* Error handling */
#include <xsh_error.h>
/* Utility fonctions */
#include <xsh_utils.h>
#include <xsh_msg.h>
#include <xsh_data_instrument.h>
#include <math.h>
/* DFS functions */
#include <xsh_dfs.h>
/* DRL functions */
#include <xsh_drl.h>
#include <xsh_pfits.h>
#include <xsh_detmon.h>
#include <xsh_paf_save.h>
#include <xsh_utils_image.h>
#include <xsh_parameters.h>
#include <xsh_drl_check.h>
#include <xsh_model_arm_constants.h>
 
/* Library */
#include <cpl.h>
#include <assert.h>
/*----------------------------------------------------------------------------
                                Defines
 ----------------------------------------------------------------------------*/
 
#define RECIPE_ID "xsh_mbias"
#define RECIPE_AUTHOR "P.Goldoni, L.Guglielmi, R. Haigron, F. Royer, D. Bramich, A. Modigliani"
#define RECIPE_CONTACT "amodigli@eso.org"
 
/*----------------------------------------------------------------------------
                            Functions prototypes
 ----------------------------------------------------------------------------*/
 
/*
 *   Plugin initalization, execute and cleanup handlers
 */
 
static int xsh_mbias_create(cpl_plugin *);
static int xsh_mbias_exec(cpl_plugin *);
static int xsh_mbias_destroy(cpl_plugin *);
 
/* The actual executor function */
static void xsh_mbias(cpl_parameterlist *, cpl_frameset *);
 
/*----------------------------------------------------------------------------
                            Static variables
 ----------------------------------------------------------------------------*/
static char xsh_mbias_description_short[] = "Create the master bias frame";
 
static char xsh_mbias_description[] =
    "This recipe creates a master bias frame by computing the median of all \
input bias frames.\n\
Input Frames : \n\
  - A set of n RAW frames (Format=RAW, n >=3, Tag = BIAS_arm)\n\
  - [OPTIONAL] A map of non linear bad pixels (Format=QUP, Tag = BP_MAP_NL_arm)\n\
  - [OPTIONAL] A map of reference bad pixels (Format = QUP,RAW, Tag = BP_MAP_RP_arm)\n\
Products : \n\
  - A master bias frame (Format=PRE, PRO.CATG = MASTER_BIAS_arm)\n";
 
/*----------------------------------------------------------------------------
                              Functions code
 ----------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
 
int cpl_plugin_get_info(cpl_pluginlist * list)
{
    cpl_recipe *recipe = NULL;
    cpl_plugin *plugin = NULL;
 
    recipe = cpl_calloc(1, sizeof(*recipe));
    if (recipe == NULL) {
    return -1;
    }
 
    plugin = &recipe->interface;
 
    cpl_plugin_init(plugin, CPL_PLUGIN_API, /* Plugin API */
            XSH_BINARY_VERSION, /* Plugin version */
            CPL_PLUGIN_TYPE_RECIPE, /* Plugin type */
            RECIPE_ID,  /* Plugin name */
            xsh_mbias_description_short,    /* Short help */
            xsh_mbias_description,  /* Detailed help */
            RECIPE_AUTHOR,  /* Author name */
            RECIPE_CONTACT, /* Contact address */
            xsh_get_license(),  /* Copyright */
            xsh_mbias_create, 
                    xsh_mbias_exec, 
                    xsh_mbias_destroy);
 
    cpl_pluginlist_append(list, plugin);
 
    return (cpl_error_get_code() != CPL_ERROR_NONE);
}
 
/*---------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
 
static int xsh_mbias_create(cpl_plugin * plugin)
{
  cpl_recipe *recipe = NULL;
  xsh_fpn_param fpn_param = {10,10,1024,1024,10,100};
  xsh_ron_param ron_param = {"ALL",
                             10,100,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,3,25};
  xsh_struct_param struct_param = {-1,-1};
  xsh_stack_param stack_param = {"median",5.,5.};
  xsh_pd_noise_param pd_noise_param = {1,1,CPL_FALSE};
  int ival=DECODE_BP_FLAG_DEF;
  /* Reset library state */
  xsh_init();
 
  /* Check input */
  assure(plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin");
 
  /* Get the recipe out of the plugin */
  assure(cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
    CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
 
  recipe = (cpl_recipe *) plugin;
 
  /* Create the parameter list in the cpl_recipe object */
  recipe->parameters = cpl_parameterlist_new();
  assure(recipe->parameters != NULL,
    CPL_ERROR_ILLEGAL_OUTPUT, "Memory allocation failed!");
 
  /* Set generic parameters (common to all recipes) */
  check( xsh_parameters_generic( RECIPE_ID, recipe->parameters ) ) ;
  xsh_parameters_decode_bp(RECIPE_ID,recipe->parameters,ival);
  check(xsh_parameters_stack_create(RECIPE_ID,recipe->parameters,stack_param));
  check(xsh_parameters_fpn_create(RECIPE_ID,recipe->parameters,fpn_param));
  check(xsh_parameters_ron_create(RECIPE_ID,recipe->parameters,ron_param));
  check(xsh_parameters_struct_create(RECIPE_ID,recipe->parameters,struct_param));
  check(xsh_parameters_pd_noise_create(RECIPE_ID,recipe->parameters,
          pd_noise_param));
  cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
    xsh_error_dump(CPL_MSG_ERROR);
    return 1;
    } else {
    return 0;
    }
}
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
 
static int xsh_mbias_exec(cpl_plugin * plugin)
{
    cpl_recipe *recipe = NULL;
 
    /* Check parameter */
    assure(plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin");
 
    /* Get the recipe out of the plugin */
    assure(cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
       CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
 
    recipe = (cpl_recipe *) plugin;
 
    /* Check recipe */
    xsh_mbias(recipe->parameters, recipe->frames);
 
  cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
    xsh_error_dump(CPL_MSG_ERROR);
        cpl_error_reset();
    return 1;
    } else {
    return 0;
    }
}
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
static int xsh_mbias_destroy(cpl_plugin * plugin)
{
    cpl_recipe *recipe = NULL;
 
    xsh_error_reset();
    /* Check parameter */
    assure(plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin");
 
    /* Get the recipe out of the plugin */
    assure(cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
       CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
 
    recipe = (cpl_recipe *) plugin;
 
    xsh_free_parameterlist(&recipe->parameters);
 
  cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
    return 1;
    } else {
    return 0;
    }
}
 
static cpl_error_code
xsh_mbias_fpn(cpl_frameset *frames, const cpl_parameterlist* parameters,
        xsh_instrument* instrument, const char* recipe_id){
    xsh_msg("compute noise");
    int binx=instrument->binx;
    int biny=instrument->biny;
    const char* arm=xsh_instrument_arm_tostring(instrument);
 
    //const char* bias_raw_tag=xsh_get_tag_from_arm(XSH_BIAS, instrument);
 
    const char* bias_raw_tag=XSH_GET_TAG_FROM_ARM_EXT(XSH_BIAS,instrument);
 
    const char* power_spectrum_mask_tag=XSH_GET_TAG_FROM_ARM_EXT(XSH_BIAS_PD_MASK,
            instrument);
    cpl_parameter* p;
 
    p=xsh_parameters_find(parameters,RECIPE_ID,"pd_noise_dc_x");
    int dc_mask_x=cpl_parameter_get_int(p);
 
    p=xsh_parameters_find(parameters,RECIPE_ID,"pd_noise_dc_y");
    int dc_mask_y=cpl_parameter_get_int(p);
 
    /* Create the raw and powerspec_mask framesets (only RAW and POWERSPEC_MASK) */
    cpl_size     nframes  = cpl_frameset_get_size(frames);
    cpl_frameset *fs_raws = cpl_frameset_new();
    cpl_frameset *fs_masks = cpl_frameset_new();
    //xsh_msg("nframes=%ld",nframes);
    for (cpl_size i = 0; i < nframes; i++) {
 
        cpl_frame  *frame    = cpl_frameset_get_position(frames, i);
 
        if (!strcmp(cpl_frame_get_tag(frame), bias_raw_tag)) {
            cpl_frameset_insert(fs_raws, cpl_frame_duplicate(frame));
 
        } else if (!strcmp(cpl_frame_get_tag(frame), power_spectrum_mask_tag)) {
            cpl_frameset_insert(fs_masks, cpl_frame_duplicate(frame));
        }
    }
 
    cpl_size n_masks = cpl_frameset_count_tags(frames, power_spectrum_mask_tag);
    //xsh_msg("n_masks=%ld",n_masks);
    cpl_size n_raws = cpl_frameset_get_size(fs_raws);
    //xsh_msg("n_raws=%ld",n_raws);
 
    
    /* Calculate QC header keywords. We call hdrl_fpn_compute calculations twice, 
       given CPL limitations */ 
    cpl_propertylist *qclist = cpl_propertylist_new();
 
    for (int n_frame = 0; n_frame < n_raws; n_frame++) {
        cpl_frame  *frm_raw      = cpl_frameset_get_position(fs_raws, n_frame);
        const char *filename_raw = cpl_frame_get_filename(frm_raw);
        cpl_msg_info(cpl_func,"Load image, filename=%s ...", filename_raw);
        cpl_image *img_in = cpl_image_load(cpl_frame_get_filename(frm_raw),
                                CPL_TYPE_DOUBLE, 0, 0);
 
        /* Compute Pattern noise */
        cpl_image *power_spectrum  = NULL;
        double std = -1.;
        double std_mad = -1.;
        cpl_mask *mask_in = NULL;
        //cpl_frameset_dump(fs_masks,stdout);
        //if (n_frame < n_masks && n_masks > 0) {
        if (n_masks > 0) {
            /* only one input mask is expected for each arm */
            cpl_frame  *frm_mask = cpl_frameset_get_position(fs_masks, 0);
            const char *filename_mask = cpl_frame_get_filename(frm_mask);
 
            /* Load input mask, extension 0 same mask for any input raw n */
            cpl_msg_info(cpl_func,"Load mask, filename=%s ...", filename_mask);
            mask_in = cpl_mask_load(filename_mask, 0, 0);
        }
 
        if (hdrl_fpn_compute(img_in, mask_in, dc_mask_x, dc_mask_y,
                &power_spectrum, &std, &std_mad) != CPL_ERROR_NONE) {
 
            if (img_in     ) cpl_image_delete(img_in);
            if (mask_in    ) cpl_mask_delete(mask_in);
            if (power_spectrum ) cpl_image_delete(power_spectrum);
 
            return cpl_error_get_code();
        }
        if (!isnan(std) && !isnan(std_mad)) {
            //add to qclist
            const char* key = cpl_sprintf("ESO QC STDRAT%d",n_frame+1);
            cpl_propertylist_update_double(qclist,key, std/std_mad);
        }// else {
            //add junk value?
        //}
    }
 
    cpl_size nqclist = cpl_propertylist_get_size(qclist);
    if(nqclist > 0){
        double max = cpl_property_get_double(cpl_propertylist_get(qclist,0));
        for(int i=1;i<nqclist;i++){
            double val = cpl_property_get_double(cpl_propertylist_get(qclist,i));
            if(val > max){
                max = val;
            }
        }
        cpl_propertylist_update_double(qclist,"ESO QC STD MAX",max);
    }
        
 
    char *out_filename = cpl_sprintf("%s","bias_fpn_power_spectrum.fits");
    char *out_filename_mask = cpl_sprintf("%s","bias_fpn_mask.fits");
    for (int n_frame = 0; n_frame < n_raws; n_frame++) {
        cpl_frame  *frm_raw      = cpl_frameset_get_position(fs_raws, n_frame);
        const char *filename_raw = cpl_frame_get_filename(frm_raw);
        cpl_msg_info(cpl_func,"Load image, filename=%s ...", filename_raw);
        cpl_image *img_in = cpl_image_load(cpl_frame_get_filename(frm_raw),
                                CPL_TYPE_DOUBLE, 0, 0);
 
        /* Compute Pattern noise */
        cpl_image *power_spectrum  = NULL;
        double std = -1.;
        double std_mad = -1.;
        cpl_mask *mask_in = NULL;
        //cpl_frameset_dump(fs_masks,stdout);
        //if (n_frame < n_masks && n_masks > 0) {
        if (n_masks > 0) {
            /* only one input mask is expected for each arm */
            cpl_frame  *frm_mask = cpl_frameset_get_position(fs_masks, 0);
            const char *filename_mask = cpl_frame_get_filename(frm_mask);
 
            /* Load input mask, extension 0 same mask for any input raw n */
            cpl_msg_info(cpl_func,"Load mask, filename=%s ...", filename_mask);
            mask_in = cpl_mask_load(filename_mask, 0, 0);
        }
 
        if (hdrl_fpn_compute(img_in, mask_in, dc_mask_x, dc_mask_y,
                &power_spectrum, &std, &std_mad) != CPL_ERROR_NONE) {
 
            if (img_in     ) cpl_image_delete(img_in);
            if (mask_in    ) cpl_mask_delete(mask_in);
            if (power_spectrum ) cpl_image_delete(power_spectrum);
 
            return cpl_error_get_code();
        }
 
        cpl_propertylist *plist = cpl_propertylist_new();
        cpl_propertylist *xlist = cpl_propertylist_new();
 
        /* Add STD */
        if (!isnan(std)) {
            cpl_propertylist_update_double(plist, "ESO QC FPN STD", std);
            cpl_propertylist_update_double(xlist, "ESO QC FPN STD", std);
 
        } else {
            cpl_propertylist_update_double(plist, "ESO QC FPN STD", -DBL_MAX);
            cpl_propertylist_update_double(xlist, "ESO QC FPN STD", -DBL_MAX);
        }
 
        /* Add Peak value */
        if (!isnan(std_mad)) {
            cpl_propertylist_update_double(plist, "ESO QC FPN STDMAD", std_mad);
            cpl_propertylist_update_double(xlist, "ESO QC FPN STDMAD", std_mad);
        } else {
            cpl_propertylist_update_double(plist, "ESO QC FPN STDMAD", -DBL_MAX);
            cpl_propertylist_update_double(xlist, "ESO QC FPN STDMAD", -DBL_MAX);
        }
 
        /* Set EXTNAME for xlist for QC purposes */
        if(n_frame > 0){
            const char* extname = cpl_sprintf("BIAS_PN_%d",n_frame+1);
            cpl_propertylist_update_string(xlist, "EXTNAME",extname);
        } else {
            cpl_propertylist_update_string(plist, "EXTNAME","BIAS_PN_1");
        }
 
 
        /* Save pattern noise mask image */
        cpl_propertylist_update_string(plist, CPL_DFS_PRO_CATG,
                power_spectrum_mask_tag);
 
 
        /* Create an output frameset */
 
        /* power spectrum mask */
 
        cpl_image *out_mask_image = cpl_image_new_from_mask(cpl_image_get_bpm(power_spectrum));
        if(n_frame == 0) {
            /* same mask for any input raw frame */
            cpl_dfs_save_image(frames, NULL, parameters, frames, frm_raw,
                    out_mask_image, CPL_TYPE_INT, recipe_id, plist, NULL,
                    PACKAGE "/" PACKAGE_VERSION, out_filename_mask);
        }
 
        /* Add QC keywords we calculated earlier */
        cpl_propertylist_append(plist,qclist);
 
        /* power spectrum */
        const char* bias_fpn_tag=xsh_get_tag_from_arm(XSH_BIAS_PD, instrument);
        cpl_propertylist_update_string(plist, CPL_DFS_PRO_CATG, bias_fpn_tag);
 
 
        if(n_frame == 0) {
            cpl_dfs_save_image(frames, NULL, parameters, frames, frm_raw,
                    power_spectrum, CPL_TYPE_DOUBLE, recipe_id, plist, NULL,
                    PACKAGE "/" PACKAGE_VERSION, out_filename);
        } else {
            cpl_image_save(power_spectrum, out_filename, CPL_TYPE_DOUBLE, xlist, CPL_IO_EXTEND);
        }
 
        if(out_mask_image) cpl_image_delete(out_mask_image);
        if(power_spectrum) cpl_image_delete(power_spectrum);
        if (mask_in    ) cpl_mask_delete(mask_in);
 
        cpl_propertylist_delete(plist);
        cpl_propertylist_delete(xlist);
 
    }
    cpl_propertylist_delete(qclist);
    cpl_frameset_delete(fs_raws);
    cpl_frameset_delete(fs_masks);
    cpl_free(out_filename);
    cpl_free(out_filename_mask);
    return cpl_error_get_code();
}
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
static void xsh_mbias(cpl_parameterlist * parameters, cpl_frameset * frameset)
{
  const char* recipe_tags[1] = {XSH_BIAS};
  int recipe_tags_size = 1;
 
  cpl_frameset *raws = NULL;
  cpl_frameset *calib = NULL;
  cpl_frame *bpmap = NULL;
  cpl_frame *master_bias = NULL;
  cpl_frame *product = NULL;
  xsh_instrument* instrument  = NULL;
 
  cpl_frame* bias_frm=NULL;
  cpl_propertylist* plist=NULL;
  char name[256];
  const char* ftag;
 
  int pre_overscan_corr=0;
  xsh_stack_param* stack_par=NULL;
 
  /**************************************************************************/
  /* DFS management */
  /**************************************************************************/
  check( xsh_begin( frameset, parameters, &instrument, &raws, &calib,
                    recipe_tags, recipe_tags_size,
                    RECIPE_ID, XSH_BINARY_VERSION,
                    xsh_mbias_description_short ) ) ;
 
  /* check critical parameter values */
  xsh_recipe_params_check(parameters,instrument,RECIPE_ID);
  /**************************************************************************/
  /* Recipe frames */
  /**************************************************************************/
  //AModigliani: why should be prevent to reduce < 3 frames if it works?
  //XSH_ASSURE_NOT_ILLEGAL(cpl_frameset_get_size(raws) >= 3);
  check(bpmap=xsh_check_load_master_bpmap(calib,instrument,RECIPE_ID));
 
  check(bias_frm=cpl_frameset_get_frame(raws,0));
  check(plist=cpl_propertylist_load(cpl_frame_get_filename(bias_frm),0));
  xsh_free_propertylist(&plist);
 
  ftag=XSH_GET_TAG_FROM_ARM_EXT(XSH_MASTER_BIAS,instrument);
  sprintf(name,"%s.fits",ftag);
  xsh_msg("tag=%s",ftag);
 
  /**************************************************************************/
  /* Recipe parameters */
  /**************************************************************************/
 check( stack_par = xsh_stack_frames_get( RECIPE_ID, parameters));
  /**************************************************************************/
  /* Recipe code */
  /**************************************************************************/
 cpl_parameter* p;
 p=xsh_parameters_find(parameters,RECIPE_ID,"pd_noise_compute");
 int pd_noise_compute=cpl_parameter_get_bool(p);
 
 
  /* prepare RAW frames in XSH format */
  check(xsh_prepare(raws, bpmap, NULL, XSH_BIAS, instrument,pre_overscan_corr,CPL_TRUE));
  /* compute fpn power spectrum */
 
  if(pd_noise_compute) {
    xsh_mbias_fpn(frameset, parameters, instrument,RECIPE_ID );
  }
  /* remove cosmic rays and merge the RAWS */
  
  if(strcmp(stack_par->stack_method,"mean") == 0) {
    check(master_bias = xsh_create_master_bias2(raws,stack_par,instrument,ftag,1));
  } else {
    check(master_bias = xsh_create_master_bias2(raws,stack_par,instrument,ftag,0));
  }
 
 
  /* create master bias */
  check(product=xsh_compute_qc_on_master_bias(raws,master_bias,instrument,parameters));
 
  /**************************************************************************/
  /* Products */
  /**************************************************************************/
 
  xsh_msg("Save products");
  //check( xsh_add_product_pre(product, frameset, parameters, RECIPE_ID,
  //  instrument));
  check(xsh_add_product_image(product, frameset, parameters,RECIPE_ID, instrument,XSH_MASTER_BIAS));
 
  xsh_msg("xsh_mbias success!!"); 
 
  cleanup:
    xsh_end( RECIPE_ID, frameset, parameters );
 
    xsh_free_propertylist(&plist);
    xsh_instrument_free(&instrument);
    xsh_free_frame(&product);
    xsh_free_frameset(&raws);
    xsh_free_frameset(&calib);
    xsh_free_frame(&master_bias);
    xsh_free_frame(&bpmap);
 
    cpl_free(stack_par);
    return;
}