xsh_2dmap.c

Go to the documentation of this file.

/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */
 
/*
 * $Author: amodigli $
 * $Date: 2013-03-18 11:40:23 $
 * $Revision: 1.154 $
 * $Name: not supported by cvs2svn $
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------
                                Includes
 ---------------------------------------------------------------------------*/
 
 
/* DRL steps */
 
/* Error handling */
#include <xsh_error.h>
/* Utility fonctions */
#include <xsh_utils.h>
#include <xsh_utils_table.h>
#include <xsh_pfits.h>
#include <xsh_msg.h>
/* DFS functions */
#include <xsh_dfs.h>
/* DRL functions */
#include <xsh_data_resid_tab.h>
 
#include <xsh_data_order.h>
#include <xsh_drl.h>
#include <xsh_drl_check.h>
#include <xsh_model_kernel.h>
#include <xsh_model_arm_constants.h>
 
/* Library */
#include <cpl.h>
 
/*---------------------------------------------------------------------------
                                Defines
 ---------------------------------------------------------------------------*/
 
#define RECIPE_ID "xsh_2dmap"
#define RECIPE_AUTHOR "R. Haigron, P. Bristow, D. Bramich, A. Modigliani"
#define RECIPE_CONTACT "amodigli@eso.org"
 
/*---------------------------------------------------------------------------
                            Functions prototypes
 ---------------------------------------------------------------------------*/
 
/*
 *   Plugin initalization, execute and cleanup handlers
 */
 
static int xsh_2dmap_create(cpl_plugin *);
static int xsh_2dmap_exec(cpl_plugin *);
static int xsh_2dmap_destroy(cpl_plugin *);
 
/* The actual executor function */
static void xsh_2dmap(cpl_parameterlist *, cpl_frameset *);
 
/*---------------------------------------------------------------------------
                            Static variables
 ---------------------------------------------------------------------------*/
static char xsh_2dmap_description_short[] =
"Creates a wavelength and spatial resampling solution, a clean arc line list";
 
static char xsh_2dmap_description[] =
"This recipe creates a wavelength and spatial resampling solution, a clean arc line list.\n\
 a residual map and a wave map.\n\
Input Frames:\n\
  Raw Frame (Tag = WAVE_arm)\n\
  Arc Line List (Tag = ARC_LINE_LIST_arm)\n\
  Order Table (Tag = ORDER_TAB_EDGES_arm)\n\
  Master Bias (Tag = MASTER_BIAS_arm)\n\
  [poly mode] Wave Solution (Tag = WAVE_TAB_GUESS_arm)\n\
  [poly mode] Theoretical Map (Tag = THEO_TAB_MULT_arm)\n\
  [physical model mode] model cfg table (Tag = XSH_MOD_CFG_OPT_FMT_arm)\n\
  [OPTIONAL] Master Dark (Tag = MASTER_DARK_arm)\n\
  [OPTIONAL] Non-linear Bad Pixel Map (Tag = BP_MAP_NL_arm)\n\
  [OPTIONAL] Reference Bad Pixel Map (Tag = BP_MAP_RP_arm)\n\
  [OPTIONAL] Reference list to monitor line intensity (Tag = ARC_LINE_LIST_INTMON_arm)\n \
Prepare the frames.\n\
For UVB,VIS:\n\
   Subtract Master Bias.\n\
   Subtract Master Dark.\n\
For NIR:\n\
  Subtract ON OFF \n\
  Compute Wavelength Solution, clean arc line list, residual table, wave map\n\
  Products:\n\
   Wavelength table solution, PRO.CATG = WAVE_TAB_2D_arm [poly mode]\n\
   A Residual tab, PRO.CATG=WAVE_RESID_TAB_SLIT_arm\n\
   A Wavelelength image map, PRO.CATG=WAVE_MAP_arm. [if model-wavemap-compute=TRUE]\n\
   A Slit image map, PRO.CATG=SLIT_MAP_NIR [if model-wavemap-compute=TRUE]\n\
   A Dispersion solution table, PRO.CATG=DISP_TAB_NIR\n\
   An Arc frame in pre format bias subtracted, \n\
   PRO.CATG=WAVE_ON_arm\n\
   The optimized model cfg frame, PRO.CATG=XSH_MOD_CFG_OPT_2D_arm [if physical model mode].\n";
 
 
 
/*---------------------------------------------------------------------------
                              Functions code
 ---------------------------------------------------------------------------*/
 
/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
 
int cpl_plugin_get_info(cpl_pluginlist *list) {
  cpl_recipe *recipe = NULL;
  cpl_plugin *plugin = NULL;
 
  recipe = cpl_calloc(1, sizeof(*recipe));
  if ( recipe == NULL ){
    return -1;
  }
 
  plugin = &recipe->interface ;
 
  cpl_plugin_init(plugin,
                  CPL_PLUGIN_API,                   /* Plugin API */
                  XSH_BINARY_VERSION,            /* Plugin version */
                  CPL_PLUGIN_TYPE_RECIPE,           /* Plugin type */
                  RECIPE_ID,                        /* Plugin name */
                  xsh_2dmap_description_short,      /* Short help */
                  xsh_2dmap_description,            /* Detailed help */
                  RECIPE_AUTHOR,                    /* Author name */
                  RECIPE_CONTACT,                   /* Contact address */
                  xsh_get_license(),                /* Copyright */
                  xsh_2dmap_create,
                  xsh_2dmap_exec,
                  xsh_2dmap_destroy);
 
  cpl_pluginlist_append(list, plugin);
 
  return (cpl_error_get_code() != CPL_ERROR_NONE);
 }
 
/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
 
static int xsh_2dmap_create(cpl_plugin *plugin){
  cpl_recipe *recipe = NULL;
  xsh_clipping_param detarc_clip_param =  {2.0, 0, 0.7, 0, 0.3};
  xsh_detect_arclines_param detarc_param = 
  {6, 3, 0, 5, 4, 1, 5, 5.0,
    XSH_GAUSSIAN_METHOD, FALSE};  
  xsh_dispersol_param dispsol_param = { 4, 5 } ;
  char paramname[256];
  cpl_parameter* p=NULL;
  int ival=DECODE_BP_FLAG_DEF;
 
  /* Reset library state */
  xsh_init();
 
  /* Check input */
  assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin");
 
  /* Get the recipe out of the plugin */
  assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
          CPL_ERROR_TYPE_MISMATCH,
          "Plugin is not a recipe");
 
  recipe = (cpl_recipe *)plugin;
 
  /* Create the parameter list in the cpl_recipe object */
  recipe->parameters = cpl_parameterlist_new();
  assure( recipe->parameters != NULL,
          CPL_ERROR_ILLEGAL_OUTPUT,
          "Memory allocation failed!");
 
  /* Set generic parameters (common to all recipes) */
  check( xsh_parameters_generic( RECIPE_ID, recipe->parameters ) ) ;
  xsh_parameters_decode_bp(RECIPE_ID,recipe->parameters,ival);
  check( xsh_parameters_pre_overscan( RECIPE_ID, recipe->parameters ) ) ;
 
  /* General 2dmap parameters */
 
  /* detect arclines params */
  check(xsh_parameters_detect_arclines_create(RECIPE_ID, recipe->parameters,
    detarc_param));
  check(xsh_parameters_clipping_detect_arclines_create(RECIPE_ID, 
    recipe->parameters, detarc_clip_param));
  check(xsh_parameters_dispersol_create(RECIPE_ID,
                 recipe->parameters,
                 dispsol_param ) ) ;
 
  /* Annealing parameter */
  check(  xsh_parameters_new_int( recipe->parameters, RECIPE_ID, 
                                      "model-maxit",500,
                                  "Number/10 of annealing iterations "
                                  "(physical model mode)."));
 
 
  check(  xsh_parameters_new_double( recipe->parameters, RECIPE_ID, 
                                      "model-anneal-factor",1.0,
                                  "Multiplier applied to the automatic "
                                  "parameter ranges (i.e. when scenario!=0). "
                  "For routine operations should be 1.0. "
                                  "(physical model mode)."));
 
  
  sprintf(paramname,"xsh.%s.%s",RECIPE_ID,"model-scenario");
 
  /*Scenario descriptions for development use removed so as not to confuse users*/
  check(p=cpl_parameter_new_enum(paramname,CPL_TYPE_INT,
                 "selects preset flag and range combinations "
                                 "appropriate to common scenarios: \n"
/*               "-1- Only the position across the slit and" 
                 " camera focal length are open\n" */
                 " 0 - No scenario, input cfg flags and limits"
                 "used.\n"
                 " 1 - scenario appropriate for the startup"
                 "recipe (large ranges for parameters "
                 "affecting single ph exposures, dist "
                 "coeff fixed)\n"
                 " 2 - Like 1, but includes parameters "
                 "affecting all ph positions\n"
                 " 3 - Scenario for use in fine tuning cfg "
                 "to match routine single pinhole exposures. "
                 "All parameters affecting 1ph exposures "
                 "except dist coeffs are included and "
                 "parameter ranges are small. (For use by "
                 "predict in 1ph case).\n"
                 " 4 - Like 3 but includes parameters "
                 "affecting all ph positions (Standard for "
                 "use by predict in 9ph case and 2dmap).\n"
/*               " 5 - Like 4 but includes also dist coeffs\n" 
                 " 6 - Just dist coeffs (and chipx, chipy)\n" */
                 ,RECIPE_ID,4,9,-1,0,1,2,3,4,5,6,8));
 
  check(cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
                "model-scenario"));
  check(cpl_parameterlist_append(recipe->parameters,p));
 
  cleanup:
    if ( cpl_error_get_code() != CPL_ERROR_NONE ){
      xsh_error_dump(CPL_MSG_ERROR);
      return 1;
    }
    else {
      return 0;
    }
}
 
/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
 
static int xsh_2dmap_exec(cpl_plugin *plugin) {
  cpl_recipe *recipe = NULL;
 
  /* Check parameter */
  assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin" );
 
  /* Get the recipe out of the plugin */
  assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
          CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
 
  recipe = (cpl_recipe *)plugin;
  /* Check recipe */
  xsh_2dmap(recipe->parameters, recipe->frames);
 
  cleanup:
    if ( cpl_error_get_code() != CPL_ERROR_NONE ) {
      xsh_error_dump(CPL_MSG_ERROR);
      cpl_error_reset();
      return 1;
    }
    else {
      return 0;
    }
}
 
/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
static int xsh_2dmap_destroy(cpl_plugin *plugin)
{
    cpl_recipe *recipe = NULL;
 
    /* Check parameter */
    assure( plugin != NULL, CPL_ERROR_NULL_INPUT, "Null plugin" );
 
    /* Get the recipe out of the plugin */
    assure( cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE,
            CPL_ERROR_TYPE_MISMATCH, "Plugin is not a recipe");
 
    recipe = (cpl_recipe *)plugin;
 
    xsh_free_parameterlist(&recipe->parameters);
 
  cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE)
        {
            return 1;
        }
    else
        {
            return 0;
        }
}
 
 
 
static cpl_error_code
xsh_verify_2dmap_poly_input(cpl_frame* order_tab_edges_frame,
                       cpl_frame* spectralformat_frame,
                       cpl_frame* theo_tab_frame)
{
 
   cpl_table* tab_edges=NULL;
   cpl_table* tab_spectral_format=NULL;
   cpl_table* theo_tab=NULL;
   const char* name=NULL;
   int ord_min_ref=0;
   int ord_max_ref=0;
   int ord_min=0;
   int ord_max=0;
 
   check(name=cpl_frame_get_filename(order_tab_edges_frame));
   check(tab_edges=cpl_table_load(name,1,0));
   check(ord_min_ref=cpl_table_get_column_min(tab_edges,"ABSORDER"));
   check(ord_max_ref=cpl_table_get_column_max(tab_edges,"ABSORDER"));
 
   check(name=cpl_frame_get_filename(spectralformat_frame));
   check(tab_spectral_format=cpl_table_load(name,1,0));
   check(ord_min=cpl_table_get_column_min(tab_spectral_format,"ORDER"));
   check(ord_max=cpl_table_get_column_max(tab_spectral_format,"ORDER"));
 
   if(ord_min != ord_min_ref) {
      xsh_msg_error("Edge order table's ord_min != spectral format table's ord_min");
      return CPL_ERROR_INCOMPATIBLE_INPUT;
   }
 
 
   if(ord_max != ord_max_ref) {
      xsh_msg_error("Edge order table's ord_max != spectral format table's ord_max");
      return CPL_ERROR_INCOMPATIBLE_INPUT;
   }
 
 
   if(theo_tab_frame != NULL) {
      check(name=cpl_frame_get_filename(theo_tab_frame));
      check(theo_tab=cpl_table_load(name,1,0));
      check(ord_min=cpl_table_get_column_min(theo_tab,"Order"));
      check(ord_max=cpl_table_get_column_max(theo_tab,"Order"));
 
      if(ord_min != ord_min_ref) {
         xsh_msg_error("Theo table's ord_min != spectral format table's ord_min");
         return CPL_ERROR_INCOMPATIBLE_INPUT;
      }
 
 
      if(ord_max != ord_max_ref) {
         xsh_msg_error("Theo table's ord_max != spectral format table's ord_max");
         return CPL_ERROR_INCOMPATIBLE_INPUT;
      }
 
 
   }
 
 
 
 cleanup:
   xsh_free_table(&tab_edges);
   xsh_free_table(&tab_spectral_format);
   xsh_free_table(&theo_tab);
 
   return cpl_error_get_code();
 
}
 
static cpl_error_code 
xsh_params_set_defaults(cpl_parameterlist* pars,
                        xsh_detect_arclines_param* arclines_p,
                        xsh_instrument* inst)
{
   cpl_parameter* p=NULL;
   check(p=xsh_parameters_find(pars,RECIPE_ID,"detectarclines-min-sn"));
   if(cpl_parameter_get_double(p) <= 0) {
      if (xsh_instrument_get_arm(inst) == XSH_ARM_UVB){
         arclines_p->min_sn=3;
      } else if (xsh_instrument_get_arm(inst) == XSH_ARM_VIS) {
         arclines_p->min_sn=6;
      } else {
         arclines_p->min_sn=10;
      }
   }
 
   check(p=xsh_parameters_find(pars,RECIPE_ID,"detectarclines-fit-win-hsize"));
   if(cpl_parameter_get_int(p) <= 0) {
      if (xsh_instrument_get_arm(inst) == XSH_ARM_NIR){
         arclines_p->fit_window_hsize=3;
      } else {
         arclines_p->fit_window_hsize=4;
      }
   }
  cleanup:
 
   return cpl_error_get_code();
 
}
 
 
/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
static void xsh_2dmap(cpl_parameterlist* parameters, cpl_frameset* frameset)
{
  const char* recipe_tags[1] = {XSH_WAVE};
  int recipe_tags_size = 1;
 
  /* RECIPES parameters */
  xsh_clipping_param* detect_arclines_clipping = NULL;
  xsh_detect_arclines_param* detect_arclines_p = NULL;
  xsh_dispersol_param *dispsol_param = NULL ;
  int model_map_compute=CPL_TRUE;
 
  /* ALLOCATED locally */
  cpl_frameset* raws = NULL;
  cpl_frameset* calib = NULL;
  cpl_frameset* on = NULL;
  cpl_frameset* off = NULL;
  cpl_frameset* on_off = NULL;
  cpl_frame* dmap_rmbias = NULL;
  cpl_frame* dmap_rmdark = NULL; 
  xsh_instrument* instrument = NULL;
  /* Others */
  cpl_frame* model_config_frame = NULL;
  cpl_frame* opt_model_config_frame = NULL;
  cpl_frame* spectralformat_frame = NULL;
  cpl_frame* bpmap = NULL;
  cpl_frame* master_bias = NULL;
  cpl_frame* master_dark = NULL;
  cpl_frame* theo_tab_mult = NULL;
  cpl_frame* resid_map = NULL;
  cpl_frame* resid_dan = NULL;
  cpl_frame* arclines = NULL;
  cpl_frame* clean_arclines = NULL;
  cpl_frame* wave_tab_guess = NULL;
  cpl_frame* wave_tab_2d = NULL;
  cpl_frame * order_tab_edges = NULL ;
  cpl_frame * wave_map = NULL ;
  cpl_frame * line_intmon = NULL ;
  char wave_map_tag[256];
  char slit_map_tag[256];
 
  char *prefix = NULL ;
  char filename[256];
  int solution_type=XSH_DETECT_ARCLINES_TYPE_MODEL;
  char paramname[256];
  cpl_parameter * p =NULL;
  cpl_frame *dispersol_frame = NULL;
  cpl_frame *slitmap_frame = NULL;
  int pre_overscan_corr=0;
  cpl_propertylist* plist=NULL;
  double exptime=0;
  cpl_boolean mode_phys;
  char tag[256];
  int resid_name_sw=0;
  /**************************************************************************/
  /* DFS management */
  /**************************************************************************/
  check( xsh_begin( frameset, parameters, &instrument, &raws, &calib, recipe_tags,
             recipe_tags_size,
         RECIPE_ID, XSH_BINARY_VERSION,
         xsh_2dmap_description_short ));
 
  xsh_recipe_params_check(parameters,instrument,RECIPE_ID);
  /**************************************************************************/
  /* Recipe frames */
  /**************************************************************************/
  mode_phys=xsh_mode_is_physmod(calib,instrument);
 
  check(bpmap=xsh_check_load_master_bpmap(calib,instrument,RECIPE_ID));
  //check(arclines = xsh_find_arc_line_list_clean( calib, instrument));
  cknull_msg(arclines = xsh_find_frame_with_tag(calib,
                       XSH_ARC_LINE_LIST,
                        instrument),
         "Missing %s frame",XSH_ARC_LINE_LIST);
  check(order_tab_edges = xsh_find_order_tab_edges(calib,instrument));
 
  if (!mode_phys) {
    if((theo_tab_mult = xsh_find_frame_with_tag( calib,XSH_THEO_TAB_MULT, 
                         instrument)) == NULL) {
 
    }
    solution_type=XSH_DETECT_ARCLINES_TYPE_POLY;
  } else {
    if((model_config_frame = xsh_find_frame_with_tag(calib,
                             XSH_MOD_CFG_OPT_FMT,
                             instrument)) == NULL) {
 
      xsh_error_reset();
 
      if ((model_config_frame = xsh_find_frame_with_tag(calib,XSH_MOD_CFG_TAB, 
                            instrument)) == NULL) {
    xsh_error_reset();
      }
 
    }
    solution_type=XSH_DETECT_ARCLINES_TYPE_MODEL;
  }
  if( (model_config_frame!=NULL) && (theo_tab_mult != NULL) ) {
 
     xsh_msg_error("You cannot provide both a %s and a %s frame. Decide if you are in poly or physical model mode. We exit",
                   XSH_THEO_TAB_MULT,XSH_MOD_CFG_TAB);
     goto cleanup;
  }
 
  if((model_config_frame==NULL) && (theo_tab_mult == NULL) ) {
     xsh_msg_error("You must provide either a %s or a %s frame",
                   XSH_THEO_TAB_MULT,XSH_MOD_CFG_TAB);
     goto cleanup;
 
  }
 
  /* Could be in sof */
  check( spectralformat_frame = xsh_find_frame_with_tag( calib, 
    XSH_SPECTRAL_FORMAT, instrument));
 
  /* TODO: remove the following that looks like duplication of same task done previous line */
  if( (spectralformat_frame = xsh_find_frame_with_tag( calib, 
                                                       XSH_SPECTRAL_FORMAT, 
                                                       instrument)) == NULL) {
     xsh_msg("Frame %s not provided",
                    XSH_SPECTRAL_FORMAT); 
    xsh_error_reset();
 }
 
  
  if((line_intmon = xsh_find_frame_with_tag( calib, XSH_ARC_LINE_LIST_INTMON, 
                                             instrument)) == NULL) {
    xsh_error_reset();
  }
  
 
/*
  AMO: To be implemented, small function to verify that those frame 
       are compativble one with another, if provided in input
*/ 
  if(theo_tab_mult!=NULL) {
     check(xsh_verify_2dmap_poly_input(order_tab_edges, spectralformat_frame,
       theo_tab_mult));
  }
 
 
  /* In UVB and VIS mode */
  if ( xsh_instrument_get_arm(instrument) != XSH_ARM_NIR){
    /* RAWS frameset must have only one file */
    XSH_ASSURE_NOT_ILLEGAL(cpl_frameset_get_size(raws) == 1);
    if((master_bias = xsh_find_frame_with_tag(calib,XSH_MASTER_BIAS,
                          instrument)) == NULL) {
 
      xsh_msg_warning("Frame %s not provided",XSH_MASTER_BIAS);
      xsh_error_reset();
    }
 
    //check(master_dark =  xsh_find_master_dark(calib,instrument));
    if((master_dark = xsh_find_frame_with_tag(calib,XSH_MASTER_DARK,
                          instrument)) == NULL){
      xsh_msg_warning("Frame %s not provided",XSH_MASTER_DARK);
      xsh_error_reset();
    }
  }
  /* IN NIR mode */
  else {
    /* RAWS frameset must have only two files */
    check(xsh_dfs_split_nir(raws,&on,&off));
    XSH_ASSURE_NOT_ILLEGAL(cpl_frameset_get_size(on) == 1);
    XSH_ASSURE_NOT_ILLEGAL(cpl_frameset_get_size(off) == 1);
  }
  check( xsh_instrument_update_from_spectralformat( instrument,
    spectralformat_frame));
 
  /**************************************************************************/
  /* Recipe parameters */
  /**************************************************************************/
  check( pre_overscan_corr = xsh_parameters_get_int( parameters, RECIPE_ID,
                             "pre-overscan-corr"));
 
  check( detect_arclines_clipping = 
    xsh_parameters_clipping_detect_arclines_get( RECIPE_ID, parameters));
  check( detect_arclines_p = xsh_parameters_detect_arclines_get( RECIPE_ID,
    parameters));
  check( dispsol_param = xsh_parameters_dispersol_get( RECIPE_ID,parameters)) ;
 
  check(xsh_params_set_defaults(parameters,detect_arclines_p,instrument));
 
  /**************************************************************************/
  /* Recipe code */
  /**************************************************************************/
  /* In UVB and VIS mode */
  if ( xsh_instrument_get_arm(instrument) != XSH_ARM_NIR){
    cpl_frame* multi_pin_hole = NULL;
 
    /* prepare RAW frames in XSH format */
    check(xsh_prepare(raws,bpmap, master_bias, XSH_WAVE, instrument,
              pre_overscan_corr,CPL_TRUE));
    check(multi_pin_hole = cpl_frameset_get_frame(raws,0));
    if(master_bias != NULL) {
    /* subtract bias */
    check(dmap_rmbias = xsh_subtract_bias(multi_pin_hole,master_bias, 
                       instrument,"WAVE_",pre_overscan_corr,0));
    } else {
      dmap_rmbias=cpl_frame_duplicate(multi_pin_hole);
    }
    if(master_dark != NULL) {
    /* subtract dark */
      strcpy(filename,xsh_stringcat_any( "WAVE_DARK_",
                  xsh_instrument_arm_tostring( instrument ),
                     ".fits", (void*)NULL )) ;
 
    check(dmap_rmdark = xsh_subtract_dark(dmap_rmbias, master_dark,
                      filename, instrument));
    } else {
      dmap_rmdark=cpl_frame_duplicate(dmap_rmbias);
    }
  }
  /* in NIR mode */
  else{
    /* prepare ON frames in XSH format */
    check(xsh_prepare(on,bpmap, NULL, "ON", instrument,pre_overscan_corr,CPL_TRUE));
    /* prepare OFF frames in XSH format */
    check(xsh_prepare(off,bpmap, NULL, "OFF", instrument,pre_overscan_corr,CPL_TRUE));
 
    /* subtract dark */
 
    check(on_off = xsh_subtract_nir_on_off(on, off, instrument));
    check(dmap_rmdark = cpl_frame_duplicate(
      cpl_frameset_get_frame(on_off,0)));
  }
  /* check on binning */
  if ( xsh_instrument_get_arm(instrument) != XSH_ARM_NIR){
     check(xsh_check_input_is_unbinned(dmap_rmdark));
  }
  /* detect arclines */
  /* TODO: the exptime is used only in qc function to monitor line intensity
   * we could move this within that function
   */
  plist=cpl_propertylist_load(cpl_frame_get_filename(dmap_rmdark),0);
  exptime=xsh_pfits_get_exptime(plist);
  check(xsh_free_propertylist(&plist));
 
  /* commented out: not yet robust on some data */
  if(model_config_frame!=NULL) {
    resid_name_sw=1;
 check( xsh_detect_arclines_dan(dmap_rmdark, theo_tab_mult,
                 arclines, wave_tab_guess, NULL, 
                 model_config_frame, spectralformat_frame, NULL,
                 &clean_arclines, &wave_tab_2d,
                 &resid_dan, XSH_SOLUTION_ABSOLUTE,
                 detect_arclines_p, detect_arclines_clipping, 
                instrument,RECIPE_ID,0,resid_name_sw));
 
 //xsh_free_frame(&clean_arclines);
  xsh_free_frame(&wave_tab_2d);
 
 
  check(resid_map = cpl_frame_duplicate(resid_dan));
  cpl_table* resid = NULL;
  cpl_table* ext = NULL;
  const char* fname = NULL;
  char rtag[256];
  char fout[256];
  cpl_propertylist* phead = NULL;
  cpl_propertylist* xhead = NULL;
 
  fname = cpl_frame_get_filename(resid_dan);
  resid = cpl_table_load(fname, 1, 0);
  phead = cpl_propertylist_load(fname, 0);
 
  sprintf(rtag, "WAVE_RESID_TAB_LINES_GFIT_%s",
      xsh_instrument_arm_tostring(instrument));
  sprintf(fout, "%s.fits", rtag);
 
  check(cpl_table_and_selected_int(resid, "Flag", CPL_EQUAL_TO, 0));
  ext = cpl_table_extract_selected(resid);
  cpl_table_save(ext, phead, xhead, fout, CPL_IO_DEFAULT);
  xsh_free_table(&ext);
  xsh_free_table(&resid);
  xsh_free_propertylist(&phead);
 
  cpl_frame_set_filename(resid_map, fout);
  check(cpl_frame_set_tag(resid_map, rtag));
  xsh_add_temporary_file(fout);
 
  } else {
  /* TODO: remove this second call as we did in xsh_predict and make sure that
   * the table input of annealing uses only flag==0 values */
  
  check( xsh_detect_arclines(dmap_rmdark, theo_tab_mult,
                 arclines, wave_tab_guess, NULL, 
                 model_config_frame, spectralformat_frame, NULL,
                 &clean_arclines, &wave_tab_2d,
                 &resid_map, XSH_SOLUTION_ABSOLUTE,
                 detect_arclines_p, detect_arclines_clipping, 
                             instrument,RECIPE_ID,0,resid_name_sw));
 
  }
  /* Now call xsh_create_wavemap */
 
  sprintf(slit_map_tag,"SLIT_MAP_%s",
      xsh_instrument_arm_tostring( instrument )) ;
 
  sprintf(wave_map_tag,"WAVE_MAP_%s",
      xsh_instrument_arm_tostring( instrument )) ;
 
 
  check( xsh_data_check_spectralformat( spectralformat_frame,
    order_tab_edges, wave_tab_2d, model_config_frame, instrument));
 
  /* force scenario to a given value */
  int maxit=200;
  double ann_fac=1.0;
  int scenario=4;
  if ( model_config_frame == NULL){
 
    check( wave_map = xsh_create_poly_wavemap( dmap_rmdark, wave_tab_2d,
                           order_tab_edges,
                           spectralformat_frame,
                           dispsol_param, instrument,
                           wave_map_tag, &dispersol_frame,
      &slitmap_frame));
  }
  else { 
     /* Number of iterations is maxit*10 */
     sprintf(paramname,"xsh.%s.%s",RECIPE_ID, "model-maxit");
     check(p = cpl_parameterlist_find(parameters,paramname));
     check(maxit=cpl_parameter_get_int(p));
 
     sprintf(paramname,"xsh.%s.%s",RECIPE_ID, "model-anneal-factor");
     check(p = cpl_parameterlist_find(parameters,paramname));
     check(ann_fac=cpl_parameter_get_double(p));
 
     sprintf(paramname,"xsh.%s.%s",RECIPE_ID, "model-scenario");
     check(p = cpl_parameterlist_find(parameters,paramname));
     check(scenario=cpl_parameter_get_int(p));
     xsh_msg_dbg_low("maxit=%d ann_fac=%g scenario=%d",maxit,ann_fac,scenario);
 
     check(opt_model_config_frame=xsh_model_pipe_anneal(model_config_frame, 
                                                        resid_map,
                                                        maxit,ann_fac,
                                                        scenario,2));
 
 
    if (model_map_compute){
      check( xsh_create_model_map( opt_model_config_frame, instrument,
                                   wave_map_tag,slit_map_tag, 
                                   &wave_map, &slitmap_frame,0));
 
 
 
      check(dispersol_frame=xsh_create_dispersol_physmod(dmap_rmdark,
                             order_tab_edges,
                             opt_model_config_frame,
                             wave_map,
                             slitmap_frame,
                             dispsol_param,
                             spectralformat_frame,
                             instrument,1));
    }
  }
  if(wave_map!=NULL) {
       check(xsh_wavemap_qc(wave_map,order_tab_edges));
  }
 
  xsh_msg_dbg_low("QC wavetab");
  if ( model_config_frame == NULL){
    check(xsh_wavetab_qc(resid_map,true));
  } else {
    check(xsh_wavetab_qc(resid_map,false));
  }
 
  if(line_intmon) {
     check(xsh_wavecal_qclog_intmon(resid_map,line_intmon,exptime,instrument));
  }
  check(xsh_table_merge_clean_and_resid_tabs(resid_map,clean_arclines));
  /**************************************************************************/
  /* Products */
  /**************************************************************************/
  xsh_msg("Saving products");
 
  if ( wave_tab_2d != NULL){ 
    check(xsh_add_product_table(wave_tab_2d, frameset,
                                      parameters, RECIPE_ID, instrument,NULL));
  }
 
  if(model_config_frame!=NULL) {
   
  check(xsh_frame_table_resid_merge(resid_dan,resid_map,solution_type));
  check(xsh_add_product_table( resid_dan, frameset,
                                     parameters, RECIPE_ID, instrument,NULL));
  /*
    check(xsh_add_product_table( resid_map, frameset,parameters, RECIPE_ID, 
                 instrument,"WAVE_RESID_TAB_LINES"));
  */
  } else {
    check(xsh_add_product_table( resid_map, frameset,parameters, RECIPE_ID, 
                 instrument,"WAVE_RESID_TAB_LINES"));
  }
  if ( wave_map != NULL){
    check(xsh_add_product_image( wave_map, frameset,
      parameters, RECIPE_ID, instrument, XSH_WAVE_MAP ));
  }
  if (slitmap_frame != NULL){
    check(xsh_add_product_image( slitmap_frame, frameset,
      parameters, RECIPE_ID, instrument, XSH_SLIT_MAP ));
  }
  if ( dispersol_frame != NULL){
    check(xsh_add_product_table( dispersol_frame, frameset,parameters, 
                                 RECIPE_ID, instrument,XSH_DISP_TAB));
  }
  sprintf(tag,"WAVE_ON");
  if ( xsh_instrument_get_arm(instrument) != XSH_ARM_NIR){
    check(xsh_add_product_image(dmap_rmbias,frameset,parameters,
                                RECIPE_ID,instrument,tag));
  } else {
 
    check(xsh_add_product_image(dmap_rmdark,frameset,parameters,
                  RECIPE_ID,instrument,tag));
 
  }
 
  if(opt_model_config_frame!=NULL) {
     check(xsh_add_product_table(opt_model_config_frame, frameset,
                                       parameters, RECIPE_ID, instrument,NULL));
  }
 
  xsh_msg("xsh_2dmap success!!");
 
  cleanup:
    xsh_end( RECIPE_ID, frameset, parameters );
    /* free parameters */
    XSH_FREE( detect_arclines_clipping);
    XSH_FREE( detect_arclines_p);
    XSH_FREE( dispsol_param);
 
    xsh_free_frameset(&raws);
    xsh_free_frameset(&calib);
    xsh_free_frameset(&on);
    xsh_free_frameset(&off);
    xsh_free_frameset(&on_off);
    xsh_free_frame(&clean_arclines);
    xsh_free_frame(&resid_map);
    xsh_free_frame(&resid_dan);
    xsh_free_frame(&bpmap);
    xsh_free_frame(&wave_tab_2d);
    xsh_free_frame(&dmap_rmbias);
    xsh_free_frame(&dmap_rmdark); 
    xsh_free_frame(&dispersol_frame); 
    xsh_free_frame(&slitmap_frame); 
    xsh_free_frame(&opt_model_config_frame); 
    xsh_free_propertylist(&plist);
 
    xsh_instrument_free(&instrument); 
    xsh_free_frame( &wave_map ) ;
    XSH_FREE( prefix ) ;
    return;
}