xsh_respon_slit_offset.c

Go to the documentation of this file.

/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */
 
/*
 * $Author: amodigli $
 * $Date: 2013-08-29 10:50:11 $
 * $Revision: 1.105 $
 *
*/
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
  Includes
  ----------------------------------------------------------------------------*/
 
/* DRL steps */
 
/* Error handling */
#include <xsh_error.h>
/* Utility fonctions */
#include <xsh_utils.h>
#include <xsh_utils_table.h>
#include <xsh_utils_scired_slit.h>
#include <xsh_msg.h>
/* DFS functions */
#include <xsh_dfs.h>
#include <xsh_drl_check.h>
#include <xsh_pfits.h>
#include <xsh_data_spectrum1D.h>
#include <xsh_model_arm_constants.h>
 
/* DRL functions */
#include <xsh_drl.h>
/* Library */
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
  Defines
  ---------------------------------------------------------------------------*/
 
#define RECIPE_ID "xsh_respon_slit_offset"
#define RECIPE_AUTHOR "D. Bramich, A.Modigliani"
#define RECIPE_CONTACT "amodigli@eso.org"
 
/*-----------------------------------------------------------------------------
  Functions prototypes
  ----------------------------------------------------------------------------*/
 
/*
 *   Plugin initalization, execute and cleanup handlers
 */
 
static int xsh_respon_slit_offset_create( cpl_plugin *);
static int xsh_respon_slit_offset_exec( cpl_plugin *);
static int xsh_respon_slit_offset_destroy( cpl_plugin *);
 
/* The actual executor function */
static cpl_error_code xsh_respon_slit_offset( cpl_parameterlist *, cpl_frameset *);
 
/*-----------------------------------------------------------------------------
  Static variables
  ----------------------------------------------------------------------------*/
static char xsh_respon_slit_offset_description_short[] =
"Compute the response function in SLIT and on/off mode";
 
static char xsh_respon_slit_offset_description[] =
"This recipe reduces science exposure in SLIT configuration and on/off mode\n\
Input Frames : \n\
  - A set of n Science frames ( n even ), \
      Tag = STD_FLUX_SLIT_OFFSET_arm, SKY_SLIT_arm\n\
  - Spectral format table (Tag = SPECTRAL_FORMAT_TAB_arm)\n\
  - A master flat frame (Tag = MASTER_FLAT_SLIT_arm)\n\
  - An order table frame(Tag = ORDER_TABLE_EDGES_SLIT_arm)\n\
  - A wavelength calibration solution frame(Tag = WAVE_TAB_2D_arm)\n\
  - [OPTIONAL] Table with dispersion coefficients (Tag = DISP_TAB_arm)\n\
  - [OPTIONAL] A telluric model catalog (Tag = TELL_MOD_CAT_arm arm=VIS,NIR)\n\
  - A standard star fluxes catalog (Tag = FLUX_STD_CATALOG_arm Type = FLX)\n\
  - A table to set response sampling points (Tag = RESP_FIT_POINTS_CAT_arm)\n\
  - An atmospheric extinction table (Tag = ATMOS_EXT_arm)\n\
    if provided this is the one used to flux calibrate the spectra\n\
Products : \n\
Products : \n\
  - [If STD is in catal] The response ord-by-ord function (Tag = RESPONSE_ORDER1D_SLIT_arm)\n\
  - [If STD is in catal] The response merged function (Tag = RESPONSE_MERGE1D_SLIT_arm)\n\
  - PREFIX_ORDER2D_arm extracted spectrum, order-by-order, 2D\n\
  - PREFIX_ORDER1D_arm extracted spectrum, order-by-order, 1D\n\
  - PREFIX_MERGE2D_arm merged spectrum, 2D\n\
  - PREFIX_MERGE1D_arm merged spectrum, 1D\n\
 - [If STD is in catal] Flux calibrated order-by-order 2D spectrum (Tag = PREFIX_FLUX_ORDER2D_arm)\n\
 - [If STD is in catal] Flux calibrated order-by-order 1D spectrum (Tag = PREFIX_FLUX_ORDER1D_arm)\n\
 - [If STD is in catal] Flux calibrated merged 2D spectrum (Tag = PREFIX_FLUX_MERGE2D_arm)\n\
 - [If STD is in catal] Flux calibrated merged 1D spectrum (Tag = PREFIX_FLUX_MERGE1D_arm)\n\
 - PREFIX_SKY_arm, 2D sky frame\n\
 - SKY_SLIT_ORDER2D_arm, 2D sky image (order-by-order)\n\
 - SKY_SLIT_MERGE2D_arm, 2D sky image (merged)\n\
 - [If STD is in catal] The efficiency (Tag = EFFICIENCY_arm)\n\
 - PREFIX_WAVE_MAP_arm, wave map image\n\
 - PREFIX_SLIT_MAP_arm, slit map image\n\
 - where PREFIX is SCI, FLUX, TELL if input raw DPR.TYPE contains OBJECT or FLUX or TELLURIC";
 
/*-----------------------------------------------------------------------------
  Functions code
  ----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
int cpl_plugin_get_info(cpl_pluginlist *list) {
  cpl_recipe *recipe = NULL;
  cpl_plugin *plugin = NULL;
 
  recipe = cpl_calloc(1, sizeof(*recipe));
  if ( recipe == NULL ){
    return -1;
  }
 
  plugin = &recipe->interface ;
 
  cpl_plugin_init(plugin,
                  CPL_PLUGIN_API,                    /* Plugin API */
                  XSH_BINARY_VERSION,             /* Plugin version */
                  CPL_PLUGIN_TYPE_RECIPE,            /* Plugin type */
                  RECIPE_ID,                         /* Plugin name */
                  xsh_respon_slit_offset_description_short,   /* Short help */
                  xsh_respon_slit_offset_description,         /* Detailed help */
                  RECIPE_AUTHOR,                     /* Author name */
                  RECIPE_CONTACT,                    /* Contact address */
                  xsh_get_license(),                 /* Copyright */
                  xsh_respon_slit_offset_create,
                  xsh_respon_slit_offset_exec,
                  xsh_respon_slit_offset_destroy);
 
  cpl_pluginlist_append(list, plugin);
 
  return (cpl_error_get_code() != CPL_ERROR_NONE);
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int xsh_respon_slit_offset_create(cpl_plugin *plugin){
  cpl_recipe *recipe = NULL;
 
  /* First param (conv_kernel) should be initialized correctly ! */
  xsh_remove_crh_single_param crh_single = { 0.1, 20, 2.0, 4} ;
  xsh_rectify_param rectify = { "tanh",
                                CPL_KERNEL_DEFAULT, 
                                2,
                                -1.0, 
                                -1.0,
                                1,0,0.};
  /* 2nd and 3rd params should be initialized correctly */
  xsh_localize_obj_param loc_obj = 
     {10, 0.1, 0, 0, LOC_MANUAL_METHOD, 0, 2.0,3,3,FALSE};
  //xsh_extract_param extract_par = { LOCALIZATION_METHOD};
  xsh_interpolate_bp_param ipol_par = { 30};
  xsh_combine_nod_param nod_param = { 5, TRUE, 5, 2, 0.1, "throwlist.asc",TRUE } ;
 
  /* Reset library state */
  xsh_init();
 
  /* Check input */
  assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin");
  /* Get the recipe out of the plugin */
  assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
          CPL_ERROR_TYPE_MISMATCH,
          "Plugin is not a recipe");
 
  recipe = (cpl_recipe *)plugin;
 
  /* Create the parameter list in the cpl_recipe object */
  recipe->parameters = cpl_parameterlist_new();
  assure( recipe->parameters != NULL,
          CPL_ERROR_ILLEGAL_OUTPUT,
          "Memory allocation failed!");
 
  /* Set generic parameters (common to all recipes) */
  check( xsh_parameters_generic( RECIPE_ID, recipe->parameters ) ) ;
  xsh_parameters_decode_bp(RECIPE_ID,recipe->parameters,-1);
  check( xsh_parameters_pre_overscan( RECIPE_ID, recipe->parameters ) ) ;
 
  /* remove_crh_single */
  check(xsh_parameters_remove_crh_single_create(RECIPE_ID,recipe->parameters,
                        crh_single )) ;
  /* xsh_rectify */
  check(xsh_parameters_rectify_create(RECIPE_ID,recipe->parameters,
                        rectify )) ;
  /* xsh_localize_object */
  check(xsh_parameters_localize_obj_create(RECIPE_ID,recipe->parameters,
                       loc_obj )) ;
  /* xsh_optimal_extract
  check(xsh_parameters_optimal_extract_create(RECIPE_ID,
                          recipe->parameters,-1. )) ;
                          */
  /* trivial extract (Just temporary)
  check(xsh_parameters_extract_create(RECIPE_ID,
                      recipe->parameters,
                      extract_par,LOCALIZATION_METHOD )) ;
  */
  check(xsh_parameters_interpolate_bp_create(RECIPE_ID,
                recipe->parameters,ipol_par)) ;
 
  check(xsh_parameters_combine_nod_create(RECIPE_ID,
                      recipe->parameters,
                      nod_param )) ;
  /*
  check( xsh_parameters_new_boolean( recipe->parameters, RECIPE_ID,
    "do-flatfield", TRUE, 
    "TRUE if we do the flatfielding"));
*/
 
  check( xsh_parameters_new_boolean( recipe->parameters, RECIPE_ID,
    "gen-sky", TRUE, 
    "if TRUE a 2D sky frame, a 2D rectified, a 2D merged sky are generated"));
 
    check( xsh_parameters_new_boolean( recipe->parameters, RECIPE_ID,
                       "correct-tellurics", TRUE,
    "TRUE if during response computation we apply telluric correction"));
 
 
  cleanup:
    if ( cpl_error_get_code() != CPL_ERROR_NONE ){
      xsh_error_dump(CPL_MSG_ERROR);
      return 1;
    }
    else {
      return 0;
    }
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int xsh_respon_slit_offset_exec(cpl_plugin *plugin) {
  cpl_recipe *recipe = NULL;
 
  /* Check parameter */
 
  assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin");
 
  /* Get the recipe out of the plugin */
  assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
          CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
 
  recipe = (cpl_recipe *)plugin;
 
  /* Check recipe */
  xsh_respon_slit_offset(recipe->parameters, recipe->frames);
 
  cleanup:
    if ( cpl_error_get_code() != CPL_ERROR_NONE ) {
      xsh_error_dump(CPL_MSG_ERROR);
      xsh_error_reset();
      return 1;
    }
    else {
      return 0;
    }
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static int xsh_respon_slit_offset_destroy(cpl_plugin *plugin)
{
  cpl_recipe *recipe = NULL;
 
  /* reset error state before detroying recipe */
  xsh_error_reset(); 
  /* Check parameter */
  assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin" );
 
  /* Get the recipe out of the plugin */
  assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
    CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
 
  recipe = (cpl_recipe *)plugin;
 
  xsh_free_parameterlist(&recipe->parameters);
 
 cleanup:
  if (cpl_error_get_code() != CPL_ERROR_NONE)
    {
      return 1;
    }
  else
    {
      return 0;
    }
}
 
 
static cpl_error_code 
xsh_params_monitor(xsh_rectify_param * rectify_par,
                   xsh_localize_obj_param * loc_obj_par)
{
 
  xsh_msg_dbg_low("rectify params: radius=%g bin_lambda=%g bin_space=%g",
      rectify_par->rectif_radius,rectify_par->rectif_bin_lambda,
      rectify_par->rectif_bin_space);
 
  xsh_msg_dbg_low("localize params: chunk_nb=%d nod_step=%g",
      loc_obj_par->loc_chunk_nb,loc_obj_par->nod_step);
 
  return cpl_error_get_code();
 
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static cpl_error_code
 xsh_respon_slit_offset( cpl_parameterlist* parameters, 
  cpl_frameset* frameset)
{
   const char* recipe_tags[2] = {XSH_STD_FLUX_SLIT_OFFSET, XSH_SKY_SLIT};
  int recipe_tags_size = 2;
 
  /* Input frames */
  cpl_frameset* raws = NULL;
 
  cpl_frameset * raw_object = NULL;
  cpl_frameset * raw_sky = NULL;
  cpl_frameset * calib = NULL;
  int nb_sub_frames = 0;
  /* Beware, do not "free" the following input frames, they are part
     of the input frameset */
  cpl_frame* bpmap = NULL;
  cpl_frame* master_bias = NULL;
  cpl_frame* master_dark = NULL;
  cpl_frame* master_flat = NULL;
  cpl_frame* order_tab_edges = NULL;
  cpl_frame * wave_tab = NULL ;
  cpl_frame * model_config_frame = NULL ;
  cpl_frame * slitmap = NULL ;
  cpl_frame *disp_tab_frame = NULL;
  cpl_frame * spectral_format = NULL ;
 
  /* Parameters */
  xsh_remove_crh_single_param * crh_single_par = NULL ;
  xsh_rectify_param * rectify_par = NULL ;
  xsh_localize_obj_param * loc_obj_par = NULL ;
 
  xsh_extract_param * extract_par = NULL ;
  xsh_combine_nod_param * combine_nod_param = NULL ;
 
  xsh_instrument* instrument = NULL;
 
  /* Intermediate frames */
 
  cpl_frame * loc_table_frame = NULL ; 
  cpl_frame * clean_frame = NULL ; 
  cpl_frameset * sub_frameset = NULL ;  
  cpl_frameset * clean_frameset = NULL ;    
  cpl_frameset * rect_frameset = NULL ; 
  cpl_frameset * rect2_frameset_tables = NULL ; 
  /* Output Frames (results)*/
  cpl_frame * res_1D_frame = NULL ; 
  cpl_frame * res_2D_frame = NULL ; 
  cpl_frame * ext_frame = NULL ;
  cpl_frame * ext_frame_eso = NULL ;
  cpl_frame * comb_frame = NULL ;
  cpl_frame * comb_frame_eso = NULL ;
 
  char  file_name[256];
  char  arm_str[16] ;
  char  file_tag[40];
 
  int i ;
 
  int do_flatfield=1;
  int gen_sky=0;
  int generate_sdp_format=0;
  char * prefix=NULL;
  char rec_prefix[256];
  char sky_prefix[256];
  char sky_tag[256];
  cpl_frame* avg_sky=NULL;
  cpl_frame* rec_sky=NULL;
  cpl_frame* rec_sky_eso=NULL;
  cpl_frame* mer_sky=NULL;
  cpl_frame * rectif_tab = NULL ;
  cpl_frame * sky_divided = NULL ;
  cpl_frameset* sky_bias=NULL;
  cpl_frameset* sky_dark=NULL;
  cpl_frame * wavemap = NULL ;
 
 
  cpl_frame* std_flux_frame=NULL;
  double exptime=1.;
  cpl_frame* response_frame=NULL;
  cpl_frame* response_ord_frame=NULL;
 
  //int do_compute_eff = 0 ;
  cpl_frame* frm_atmext=NULL;
  cpl_frame* frm_std_cat=NULL;
  cpl_frame* eff_frame=NULL;
  cpl_frame* rect_eff_frame=NULL;
  cpl_frame* ext_eff_frame=NULL;
  cpl_frame* ext_eff_frame2=NULL;
  cpl_frame* frm_eff=NULL;
 
  cpl_frame * rect_eff_frame_eso = NULL ; 
  cpl_frame * rect_eff_frame_tab = NULL ; 
  cpl_frame * div_clean_frame = NULL ;
  cpl_frame * fluxcal_rect_1D_frame = NULL ;    
  cpl_frame * fluxcal_rect_2D_frame = NULL ;    
  cpl_frame * fluxcal_1D_frame = NULL ; 
  cpl_frame * fluxcal_2D_frame = NULL ; 
  cpl_frame * nrm_1D_frame = NULL ;    
  cpl_frame * nrm_2D_frame = NULL ;
  cpl_frame* high_abs_win=NULL; 
  int pre_overscan_corr=0;
  int merge_par=0;
  xsh_interpolate_bp_param *ipol_bp=NULL;
  cpl_frame* tell_mod_cat=NULL;
  cpl_frameset* crh_clean_obj = NULL;
  cpl_frameset* crh_clean_sky = NULL;
  cpl_frame* resp_fit_points_cat_frame=NULL;
  cpl_frame* ext_sky_frame=NULL;
  cpl_frame* ext_sky_frame_eso=NULL;
  cpl_frame* res_1D_sky_frame=NULL;
  cpl_frame* sky_map_frm =NULL;
  int do_computemap=1;
  int use_model=0;
  int corr_tell=0;
  int scale_nod=0;
  cpl_propertylist* qclist  = cpl_propertylist_new();
  cpl_frame* reference_response = NULL;
  
 
 
  /**************************************************************************/
  /* DFS management */
  /**************************************************************************/
  check( xsh_begin( frameset, parameters, &instrument, &raws, &calib,
                    recipe_tags, recipe_tags_size,
                    RECIPE_ID, XSH_BINARY_VERSION,
                    xsh_respon_slit_offset_description_short));
 
  //get average airm and iwv from raws; add keywords to qclist
  get_average_qc_from_raws(raws,instrument,qclist);
 
  check(xsh_ensure_raws_input_offset_recipe_is_proper(&raws,instrument));
  assure( instrument->mode == XSH_MODE_SLIT, CPL_ERROR_ILLEGAL_INPUT,
      "Instrument NOT in Slit Mode" ) ;
 
  check(xsh_recipe_params_check(parameters,instrument,RECIPE_ID));
  if(instrument->arm == XSH_ARM_NIR) {
    xsh_calib_nir_corr_if_JH(calib,instrument,RECIPE_ID);
    xsh_instrument_nir_corr_if_JH(raws,instrument);
    xsh_calib_nir_respon_corr_if_JH(calib,instrument);
  }
 
  /* prepare RAW frames in XSH format: NB we moved this piece of REDUCTION here
    * because we need to have the calib frameset with input frames already with correct binning
    * that is done by xsh_set_recipe_file_prefix (which needs frames in PRE format)
    */
  /* But we also need to have pre-overscan-corr properly set */
  check( pre_overscan_corr = xsh_parameters_get_int( parameters, RECIPE_ID,
                                                            "pre-overscan-corr"));
 
 
 
 
  /* prepare RAW frames in PRE format */
   check(xsh_prepare(raws, bpmap, master_bias, XSH_STD_FLUX_SLIT_OFFSET,
         instrument,pre_overscan_corr,CPL_TRUE));
 
   /* in case the input master flat/bias do not match in bin size with the raw data,
     * correct the binning so that :
     *
     * 1x1 std 2x2 mflat,mbias --> correct bin of std
     * 2x2 std 1x1 mflat,mbias --> correct bin of mflat,mbias
     *
     */
    //check(xsh_frameset_uniform_bin(&raws, &calib,instrument));
 
 
  /**************************************************************************/
  /* Recipe frames */
  /**************************************************************************/
    /*
  check(xsh_slit_offset_get_calibs(calib,instrument,&bpmap,&master_bias,
                                   &master_dark,&order_tab_edges,
                                   &model_config_frame,&wave_tab,&master_flat,
                                   &wavemap,&slitmap,&spectral_format,RECIPE_ID));
*/
    {
 
 
        xsh_get_normal_calibs(calib, instrument, RECIPE_ID, pre_overscan_corr,
                &spectral_format, &master_bias, &bpmap, &order_tab_edges,
                &master_flat);
 
        /* TODO: Is mdark input really needed? */
        if((master_dark=xsh_find_frame_with_tag(calib,XSH_MASTER_DARK,
                instrument))==NULL){
            xsh_error_reset();
        }
 
        //The following need to be defined
        int do_computemap=1;
        xsh_get_dispersion_calibs(calib, instrument, do_computemap,
                &model_config_frame,&wave_tab, &wavemap);
        //The following is also present in xsh_get_dispersion_calibs
        check( wavemap = xsh_find_wavemap( calib, instrument));
        //The following should be removed to xsh_get_dispersion_calibs
        check( slitmap = xsh_find_slitmap( calib, instrument));
 
 
 
    }
 
  /* QC extra frames */
  resp_fit_points_cat_frame=xsh_find_frame_with_tag(calib,
          XSH_RESP_FIT_POINTS_CAT,instrument);
  if(resp_fit_points_cat_frame==NULL) {
     xsh_msg_warning("Missing input %s_%s response will not be computed",
                     XSH_RESP_FIT_POINTS_CAT,
                     xsh_instrument_arm_tostring( instrument ));
     xsh_msg_warning("and data will not be flux calibrated");
  }
  /* response specific frames */
  frm_atmext=xsh_find_frame_with_tag(calib,XSH_ATMOS_EXT,instrument);
  if(frm_atmext==NULL) {
     xsh_msg_error("Provide atmospheric extinction frame");
     return CPL_ERROR_DATA_NOT_FOUND;
  }
 
  frm_std_cat=xsh_find_frame_with_tag(calib,XSH_FLUX_STD_CAT,instrument);
  if(frm_std_cat==NULL) {
     xsh_msg_error("Provide std star catalog frame");
     return CPL_ERROR_DATA_NOT_FOUND;
  }
  tell_mod_cat=xsh_find_frame_with_tag(calib,XSH_TELL_MOD_CAT,instrument);
 
  /* to compute efficiency */
  if(NULL == (disp_tab_frame = xsh_find_disp_tab( calib, instrument))) {
     xsh_msg("To compute efficiency, you must give a DISP_TAB_ARM input");
  }
 
 
 
  /**************************************************************************/
  /* Recipe parameters */
  /**************************************************************************/
  check(xsh_slit_offset_get_params(parameters,RECIPE_ID,
                                  &loc_obj_par,&rectify_par,&crh_single_par,
                                  &extract_par,
                                  &combine_nod_param,&do_flatfield,&gen_sky,
                                  &generate_sdp_format));
  /*
  check( do_computemap = xsh_parameters_get_boolean( parameters, RECIPE_ID,
    "compute-map"));
  */
  check(ipol_bp = xsh_parameters_interpolate_bp_get(RECIPE_ID,parameters));
  check(xsh_rectify_params_set_defaults(parameters,RECIPE_ID,instrument,rectify_par));
  rectify_par->conserve_flux=FALSE;
  check(xsh_params_monitor(rectify_par,loc_obj_par));
  check( corr_tell = xsh_parameters_get_boolean( parameters,RECIPE_ID,
                                                      "correct-tellurics"));
  reference_response=xsh_find_frame_with_tag(calib,"RRESPONSE_MERGE1D_SLIT",instrument);
 
 
  /**************************************************************************/
  /* Recipe code */
  /**************************************************************************/
 
  /* make sure each input raw frame has the same exp time */
  check(xsh_frameset_check_uniform_exptime(raws,instrument));
 
  /* TODO: No dark subrtracted: this is ok only is off frame and on frame have same exposure time */
  /* Separate OBJECT and SKY frames */
  check( nb_sub_frames = xsh_dfs_split_offset( raws, XSH_STD_FLUX_SLIT_OFFSET,
                           XSH_SKY_SLIT, &raw_object,
                           &raw_sky ) ) ;
 
  prefix=xsh_set_recipe_file_prefix(raw_object,"xsh_respon_slit_offset");
  check(strcpy(rec_prefix,prefix));
  XSH_FREE(prefix);
 
  xsh_msg("cmap=%d wavemap=%p slitmap=%p",do_computemap,wavemap,slitmap);
  if( (do_computemap == 1) && (wavemap ==NULL || slitmap ==NULL ) ) {
    if (model_config_frame != NULL) {
       use_model=1;
    }
  check( xsh_check_get_map( disp_tab_frame, order_tab_edges,
                master_flat, model_config_frame, calib, instrument,
                do_computemap, use_model, rec_prefix,
                &wavemap, &slitmap));
  }
 
 
  /* remove crh on each obj or sky frame */
    xsh_msg("rec_prefix=%s",rec_prefix);
    sky_map_frm = xsh_find_frame_with_tag(calib,XSH_SKY_MAP, instrument);
    crh_clean_obj = xsh_frameset_crh_single(raw_object, crh_single_par,
                    sky_map_frm,instrument, rec_prefix,"OBJ");
    crh_clean_sky = xsh_frameset_crh_single(raw_sky, crh_single_par,
                    sky_map_frm, instrument,rec_prefix,"SKY");
 
 
    check(
        sub_frameset = xsh_subtract_sky_offset( crh_clean_obj, crh_clean_sky, nb_sub_frames,instrument));
    xsh_free_frameset(&crh_clean_obj);
    xsh_free_frameset(&crh_clean_sky);
 
    xsh_msg_dbg_low(
        "Nb of Subtracted Frames: %" CPL_SIZE_FORMAT "", cpl_frameset_get_size( sub_frameset ));
 
    sprintf(arm_str, "%s", xsh_instrument_arm_tostring(instrument));
 
 
  if(gen_sky) {
     sprintf(sky_prefix,xsh_set_recipe_sky_file_prefix(rec_prefix));
  }
  if(nb_sub_frames==0) {
     xsh_msg_error("nb_sub_frames=%d something wrong check your input raws",
                   nb_sub_frames);
     goto cleanup;
 
  }
 
 
 
  xsh_msg_dbg_low( "Nb of Subtracted Frames: %" CPL_SIZE_FORMAT "",
       cpl_frameset_get_size( sub_frameset ) ) ;
 
  //check( clean_frameset = cpl_frameset_new() ) ;
 
  sprintf( arm_str, "%s",  xsh_instrument_arm_tostring(instrument) ) ;
 
  if(gen_sky) {
    /* Not yet working.. */
     if(master_bias!= NULL && pre_overscan_corr ==0 ) {
        check(sky_bias=xsh_pre_frameset_subtract_frame(raw_sky,master_bias,"MBIAS",instrument));
     } else {
        sky_bias=cpl_frameset_duplicate(raw_sky);
     }
     
     if(master_dark!= NULL) {
        check(sky_dark=xsh_pre_frameset_subtract_frame(sky_bias,master_dark,"MDARK",instrument));
     } else {
        sky_dark=cpl_frameset_duplicate(sky_bias);
     }
 
     sprintf(sky_tag,"%s%s%s",rec_prefix,"_SKY_",arm_str);
     check(xsh_frameset_uniform_bin(&sky_dark, &calib,instrument));
     check(avg_sky=xsh_frameset_average_pre(sky_dark,instrument,sky_tag));
     if(do_flatfield==1) {
        sprintf(sky_tag,"%s%s%s",rec_prefix,"_FF_SKY_",arm_str);
        sprintf(file_name,"%s.fits",sky_tag);
  
    check( sky_divided = xsh_divide_flat( avg_sky,master_flat, sky_tag,
                          instrument ) ) ;
        xsh_add_temporary_file(file_name);
 
     } else {
        sky_divided=cpl_frame_duplicate(avg_sky);
     }
  }
 
 
 
  /* Get Exptime */
  {
    xsh_pre * xpre = NULL ;
    std_flux_frame= cpl_frameset_get_frame( sub_frameset, 0 ) ;
    check( xpre = xsh_pre_load( std_flux_frame, instrument ) ) ;
    exptime = xpre->exptime ;
    xsh_msg_dbg_medium( "EXPTIME: %lf", exptime ) ;
 
    xsh_pre_free( &xpre ) ;
  }
  check(xsh_frameset_uniform_bin(&sub_frameset, &calib,instrument));
  if (do_flatfield == 1) {
    clean_frameset=xsh_frameset_mflat_divide(sub_frameset,master_flat,instrument);
  } else {
    clean_frameset = cpl_frameset_duplicate(sub_frameset);
  }
  /* as we copied results on clean_frameset we do not need the following
   * frameset anymore
   * */
  xsh_free_frameset(&sub_frameset);
 
  check(div_clean_frame = cpl_frameset_get_frame(clean_frameset,0));
  /* Now rectify each clean frame */
  check( rect_frameset = cpl_frameset_new() ) ;
  for( i = 0 ; i < nb_sub_frames ; i++ ) {
    cpl_frame * rectif = NULL ;
    cpl_frame * rectif_eso = NULL ;
    cpl_frame * clean = NULL ;
    char str[16] ;
 
    sprintf( str, "%d", i ) ;
    check( clean = cpl_frameset_get_frame( clean_frameset, i ) ) ;
    xsh_msg( "Rectifying Frame '%s'", cpl_frame_get_filename( clean ) ) ;
    sprintf(file_name,"RECTIFIED_SLIT_OFFSET_%s_%s.fits",arm_str,str) ;
    check( rectif = xsh_rectify( clean, order_tab_edges, wave_tab,
                 model_config_frame, instrument,
                 rectify_par, spectral_format,
                                 disp_tab_frame,
                 file_name,&rectif_eso,&rectif_tab,
                 rec_prefix) ) ;
    xsh_add_temporary_file(file_name);
 
    check( cpl_frameset_insert( rect_frameset, cpl_frame_duplicate(rectif) ) ) ;
    check( cpl_frameset_insert( rect_frameset, cpl_frame_duplicate(rectif_eso) ) ) ;
    xsh_free_frame(&rectif);
    xsh_free_frame(&rectif_tab);
    xsh_free_frame(&rectif_eso);
  }
 
 
 
 
  /* Combine rectified Frames */
  sprintf(file_tag,"%s_%s_%s",rec_prefix,XSH_ORDER2D, arm_str) ;
  check(rect2_frameset_tables=xsh_frameset_ext_table_frames(rect_frameset));
 
  check( comb_frame = xsh_combine_nod(rect2_frameset_tables, combine_nod_param,
                       file_tag,instrument,&comb_frame_eso,scale_nod));
  xsh_free_frameset(&rect2_frameset_tables);
 
  if ( extract_par->method == LOCALIZATION_METHOD ||
       extract_par->method == CLEAN_METHOD ) {
    xsh_msg( "Localize before extraction" ) ;
    sprintf(file_name,"LOCALIZE_%s_ALL.fits",arm_str) ;
    check( loc_table_frame = xsh_localize_obj( comb_frame, NULL,instrument,
                           loc_obj_par, NULL,
                           file_name) ) ;
    xsh_add_temporary_file(file_name);
  }
 
  if(gen_sky) {
     xsh_msg("rectify sky frame");
     sprintf(file_name,"%s_RECTIFIED_SKY_%s.fits",sky_prefix,arm_str) ;
     check( rec_sky = xsh_rectify(sky_divided, order_tab_edges, wave_tab,
                                  model_config_frame, instrument,
                                  rectify_par, spectral_format,
                                  disp_tab_frame,
                                  file_name,&rec_sky_eso,&rectif_tab,
                                  sky_prefix) ) ;
     xsh_free_frame(&rectif_tab);
     xsh_add_temporary_file(file_name);
 
     xsh_msg("Merge 2D sky frame");
     check( mer_sky = xsh_merge_ord( rec_sky, instrument,
                                     merge_par,sky_prefix));
 
     check(ext_sky_frame = xsh_extract_clean(rec_sky, loc_table_frame,
                         instrument, extract_par,ipol_bp,
                         &ext_sky_frame_eso,rec_prefix )) ;
   
     check( res_1D_sky_frame = xsh_merge_ord( ext_sky_frame, instrument,
                          merge_par,sky_prefix));
  
  
  }
 
  xsh_msg( "Calling xsh_extract" ) ;
  
  check(ext_frame = xsh_extract_clean(comb_frame, loc_table_frame,
                instrument, extract_par,ipol_bp,&ext_frame_eso,
                rec_prefix )) ;
  
  xsh_msg( "Calling xsh_merge_ord with 1D frame" ) ;
  check( res_1D_frame = xsh_merge_ord( ext_frame, instrument,
                      merge_par,rec_prefix));
  /* More todo here
     mark_telluric
     calibrate_flux
  */
 
  xsh_msg( "Calling xsh_merge_ord with 2D frame" ) ;
  check( res_2D_frame = xsh_merge_ord( comb_frame, instrument, merge_par,
                      rec_prefix) ) ;
 
 
  /* Response computation */
  if(frm_std_cat!=NULL && frm_atmext!=NULL && 
     resp_fit_points_cat_frame != NULL) {
     xsh_msg( "Calling xsh_compute_response" ) ;
     if( (response_ord_frame = xsh_compute_response_ord( ext_frame_eso,
                                                         frm_std_cat,
                                                         frm_atmext,
                                                         high_abs_win,
                                                         instrument,
                                                         exptime )) == NULL) {
       xsh_msg_warning("Some error occurred during response computation");
       xsh_print_rec_status(0);
        cpl_error_reset();
     }
     
     if( (  response_frame = xsh_compute_response2( res_1D_frame,
                                                   frm_std_cat,
                                                   frm_atmext,
                           high_abs_win,
                           resp_fit_points_cat_frame,
                                                   tell_mod_cat,
                                                   instrument,
                           exptime,
                                                   corr_tell )) == NULL) {
       xsh_msg_warning("Some error occurred during response computation");
       xsh_print_rec_status(0);
       cpl_error_reset();
     } else {
 
        check(xsh_frame_table_monitor_flux_qc(response_frame,"LAMBDA",
                                              "RESPONSE","RESP",instrument));
     }
 
  }
 
 
  if( response_frame != NULL) {
 
    check(xsh_flux_calibrate(comb_frame_eso,ext_frame_eso,frm_atmext,
                 response_frame,merge_par,instrument,rec_prefix,
                 &fluxcal_rect_2D_frame,&fluxcal_rect_1D_frame,
                 &fluxcal_2D_frame,&fluxcal_1D_frame));
      if(reference_response != NULL){
        //if an optional ref response has been added to the sof, try to load the table...
        //TODO: Set error handling here to be robust (no errors introduced by this routine)
        cpl_table* ref_resp = cpl_table_load(cpl_frame_get_filename(reference_response),1,0);
        cpl_table* master_resp = cpl_table_load(cpl_frame_get_filename(response_frame),1,0);
        if(ref_resp != NULL && master_resp != NULL){
            if(instrument->arm == XSH_ARM_UVB) {
                calc_resp_qc(master_resp,ref_resp,instrument,4741,13408,"QTH",qclist);
                calc_resp_qc(master_resp,ref_resp,instrument,408,4074,"D2",qclist);
            } else {
                calc_resp_qc(master_resp,ref_resp,instrument,0,0,NULL,qclist);
            }
        }
        cpl_error_reset();
      }
 
  }
 
 /* efficiency part */
  if(response_ord_frame != NULL && disp_tab_frame != NULL) {
     int conserve_flux=rectify_par->conserve_flux;
     xsh_msg( "Calling xsh_multiply_flat" ) ;
 
     sprintf(file_tag,"SLIT_STARE_NOCRH_NOT_FF_%s",arm_str) ;
     if(do_flatfield) {
        check( eff_frame = xsh_multiply_flat( div_clean_frame, master_flat, 
                                              file_tag, instrument ) ) ;
     } else {
        check(eff_frame=cpl_frame_duplicate(div_clean_frame));
     }
     sprintf(file_name,"%s_EFF_%s_%s.fits",rec_prefix,XSH_ORDER2D,
             xsh_instrument_arm_tostring(instrument));
     xsh_add_temporary_file(file_name);
 
     /* force flux conservation for efficiency computation */
     rectify_par->conserve_flux=1;
     check( rect_eff_frame = xsh_rectify( eff_frame, 
                                          order_tab_edges,
                                          wave_tab,
                                          model_config_frame,
                                          instrument,
                                          rectify_par, 
                                          spectral_format,
                                          disp_tab_frame,
                                          file_name,
                                          &rect_eff_frame_eso,
                                          &rect_eff_frame_tab,
                                          rec_prefix));
 
     /*
       xsh_msg( "Calling xsh_localize_obj" ) ;
       check( loc_table_frame = xsh_localize_obj( rect_frame, instrument,
       loc_obj_par, NULL,NULL));
     */
 
     xsh_msg( "Calling xsh_extract" ) ;
     check( ext_eff_frame = xsh_extract_clean(rect_eff_frame,loc_table_frame,
                                        instrument,extract_par, ipol_bp,
                                        &ext_eff_frame2,rec_prefix)) ;
 
 
     frm_eff=xsh_compute_efficiency(ext_eff_frame2,frm_std_cat,
                      frm_atmext,high_abs_win,instrument);
     /* reset value to original default */
     rectify_par->conserve_flux=conserve_flux;
 
  }
 
 
  if(model_config_frame!=NULL && wavemap != NULL&& slitmap != NULL) {
 
     check(xsh_compute_resampling_accuracy(wavemap,slitmap,order_tab_edges,model_config_frame,comb_frame_eso,instrument));
     check(xsh_compute_resampling_accuracy(wavemap,slitmap,order_tab_edges,model_config_frame,res_2D_frame,instrument));
 
     check(xsh_compute_wavelength_resampling_accuracy(wavemap,order_tab_edges,model_config_frame,res_1D_frame,instrument));
     check(xsh_compute_wavelength_resampling_accuracy(wavemap,order_tab_edges,model_config_frame,ext_frame_eso,instrument));
 
     xsh_add_afc_info(model_config_frame,wavemap);
     xsh_add_afc_info(model_config_frame,slitmap);
 
     if(fluxcal_rect_2D_frame != NULL) {
 
        check(xsh_compute_resampling_accuracy(wavemap,slitmap,order_tab_edges,model_config_frame,fluxcal_rect_2D_frame,instrument));
        check(xsh_compute_resampling_accuracy(wavemap,slitmap,order_tab_edges,model_config_frame,fluxcal_2D_frame,instrument));
 
        check(xsh_compute_wavelength_resampling_accuracy(wavemap,order_tab_edges,model_config_frame,fluxcal_rect_1D_frame,instrument));
        check(xsh_compute_wavelength_resampling_accuracy(wavemap,order_tab_edges,model_config_frame,fluxcal_1D_frame,instrument));
 
     }
 
     if(gen_sky) {
       check(xsh_compute_resampling_accuracy(wavemap,slitmap,order_tab_edges,model_config_frame,rec_sky_eso,instrument));
       check(xsh_compute_resampling_accuracy(wavemap,slitmap,order_tab_edges,model_config_frame,mer_sky,instrument));
     }
 
  }
 
  /* Save products */
  xsh_msg( "Saving products") ;
  if(response_ord_frame!=NULL) {
    check( xsh_add_product_table(response_ord_frame, frameset, parameters,
                                       RECIPE_ID, instrument,NULL));
  }
  if(response_frame!=NULL) {
    check( xsh_add_product_table( response_frame, frameset, parameters,
                  RECIPE_ID, instrument,NULL));
  }
 
 
  /* Removed as format not easy to read
   check( xsh_add_product_table( comb_frame, frameset,
                  parameters, RECIPE_ID, instrument));
   *
   */
 
  check( xsh_add_product_image( comb_frame_eso, frameset, 
                parameters, RECIPE_ID, instrument,NULL));
 
  /* Removed as format not easy to read
  check( xsh_add_product_table( ext_frame, frameset,
                      parameters, RECIPE_ID, instrument));
  */
 
  check( xsh_add_product_orders_spectrum( ext_frame_eso, frameset,
                parameters, RECIPE_ID, instrument,NULL));
 
  /* Calculate curvature QC params early */
  cpl_image* itmp = cpl_image_load(cpl_frame_get_filename(res_2D_frame),CPL_TYPE_DOUBLE,0,0);
  calc_curve_qc(itmp,instrument,qclist);
  cpl_image_delete(itmp);
 
  check( xsh_add_product_spectrum( res_2D_frame, frameset, parameters,
                RECIPE_ID, instrument, NULL,qclist));
 
  check(xsh_monitor_spectrum1D_flux(res_1D_frame,instrument));
 
  check(get_chromatic_eff(frm_eff,qclist));
  check(xsh_monitor_spectrum1D_extra_qc(res_1D_frame,instrument,qclist));
  check( xsh_add_product_spectrum( res_1D_frame, frameset, parameters,
                  RECIPE_ID, instrument, NULL,qclist));
 
 
  if(fluxcal_2D_frame != NULL) {
     check( xsh_add_product_image(fluxcal_rect_2D_frame,frameset,parameters, 
                                  RECIPE_ID, instrument,NULL));
     check( xsh_add_product_orders_spectrum(fluxcal_rect_1D_frame,frameset,parameters,
                                  RECIPE_ID, instrument,NULL));
     check( xsh_add_product_spectrum( fluxcal_2D_frame, frameset, parameters, 
                                      RECIPE_ID, instrument, NULL,qclist));
     check( xsh_add_product_spectrum( fluxcal_1D_frame, frameset, parameters, 
                                      RECIPE_ID, instrument, NULL,qclist));
  }
 
  if(gen_sky) {
     check( xsh_add_product_image( avg_sky, frameset, parameters,
                                   RECIPE_ID, instrument,NULL));
 
     check( xsh_add_product_image( rec_sky_eso, frameset, parameters,
                                   RECIPE_ID, instrument,NULL));
 
     check( xsh_add_product_image( mer_sky, frameset, parameters,
                                   RECIPE_ID, instrument,NULL));
 
     check( xsh_add_product_spectrum( res_1D_sky_frame, frameset, parameters,
                      RECIPE_ID, instrument, NULL,NULL));
  }
 
  if(frm_eff!=NULL) {
     check(xsh_add_product_table(frm_eff, frameset,parameters, 
                                       RECIPE_ID, instrument,NULL));
  }
 
 
  if (do_computemap){
     //sprintf(prefix,"%s_WAVE_MAP",rec_prefix);
 
 
    check(xsh_add_product_image( wavemap, frameset,
      parameters, RECIPE_ID, instrument, NULL));
 
    //check(sprintf(prefix,"%s_SLIT_MAP",rec_prefix));
 
    check(xsh_add_product_image( slitmap, frameset,
      parameters, RECIPE_ID, instrument,NULL));
  }
 
  cleanup:
 
    xsh_end( RECIPE_ID, frameset, parameters );
    XSH_FREE( crh_single_par ) ;
    XSH_FREE( rectify_par ) ;
    XSH_FREE( loc_obj_par ) ;
    XSH_FREE(extract_par);
 
 
    XSH_FREE(combine_nod_param);
    XSH_FREE(ipol_bp);
 
    xsh_instrument_free(&instrument );
    xsh_free_frameset(&rect2_frameset_tables);
    xsh_free_frameset(&raws);
 
    /* the following two frees are dangerous in case of failure (e.g. nsky=0) */
    xsh_free_frameset( &raw_object);
    xsh_free_frameset( &raw_sky);
 
    xsh_free_frameset(&calib);
    xsh_free_frameset(&clean_frameset);
 
    xsh_free_frameset(&sky_bias);
    xsh_free_frameset(&sky_dark);
    xsh_free_frameset(&rect_frameset);
 
    xsh_free_frame(&loc_table_frame) ;
    xsh_free_frame(&clean_frame) ;
    xsh_free_frame(&res_1D_frame) ;
    xsh_free_frame(&res_2D_frame) ;
    xsh_free_frame(&fluxcal_rect_1D_frame) ;
    xsh_free_frame(&fluxcal_rect_2D_frame) ;
    xsh_free_frame(&fluxcal_1D_frame) ;
    xsh_free_frame(&fluxcal_2D_frame) ;
    xsh_free_frame(&nrm_1D_frame) ;
    xsh_free_frame(&nrm_2D_frame) ;
 
    xsh_free_frame( &comb_frame ) ;
    xsh_free_frame( &comb_frame_eso ) ;
    xsh_free_frame( &ext_frame ) ;
    xsh_free_frame( &ext_frame_eso ) ;
    xsh_free_frame( &eff_frame ) ;
    xsh_free_frame( &ext_sky_frame ) ;
    xsh_free_frame( &ext_sky_frame_eso ) ;
    xsh_free_frame( &res_1D_sky_frame ) ;
    xsh_free_frame( &rect_eff_frame ) ;
    xsh_free_frame( &rect_eff_frame_eso ) ;
    xsh_free_frame( &rect_eff_frame_tab ) ;
    xsh_free_frame( &bpmap );
    xsh_free_frame( &avg_sky ) ;
    xsh_free_frame( &wavemap ) ;
    xsh_free_frame( &slitmap ) ;
    xsh_free_frame( &avg_sky ) ;
    xsh_free_frame( &sky_divided ) ;
    xsh_free_frame( &rec_sky ) ;
    xsh_free_frame( &rec_sky_eso ) ;
    xsh_free_frame( &mer_sky ) ;
    xsh_free_frame( &ext_eff_frame ) ;
    xsh_free_frame( &ext_eff_frame2 ) ;
    xsh_free_frame( &rec_sky ) ;
    xsh_free_frame( &rec_sky_eso ) ;
    xsh_free_frame( &mer_sky ) ;
    xsh_free_frame( &frm_eff ) ;
    xsh_free_frame( &response_frame ) ;
    xsh_free_frame( &response_ord_frame ) ;
    return CPL_ERROR_NONE;
}