xsh_subtract.c

Go to the documentation of this file.

/*                                                                           *
 *   This file is part of the ESO X-shooter Pipeline                         *
 *   Copyright (C) 2006 European Southern Observatory                        *
 *                                                                           *
 *   This library is free software; you can redistribute it and/or modify    *
 *   it under the terms of the GNU General Public License as published by    *
 *   the Free Software Foundation; either version 2 of the License, or       *
 *   (at your option) any later version.                                     *
 *                                                                           *
 *   This program is distributed in the hope that it will be useful,         *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of          *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
 *   GNU General Public License for more details.                            *
 *                                                                           *
 *   You should have received a copy of the GNU General Public License       *
 *   along with this program; if not, write to the Free Software             *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA    *
 *                                                                           */
 
/*
 * $Author: amodigli $
 * $Date: 2013-04-25 14:06:28 $
 * $Revision: 1.159 $
 * $Name: not supported by cvs2svn $
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <xsh_cpl_size.h>
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------
                                Includes
 ----------------------------------------------------------------------------*/
#include <math.h>
#include <xsh_utils.h>
#include <xsh_drl.h>
#include <xsh_pfits.h>
#include <xsh_error.h>
#include <xsh_msg.h>
#include <xsh_badpixelmap.h>
#include <xsh_utils_image.h>
#include <xsh_data_order.h>
#include <xsh_data_grid.h>
 
/*---------------------------------------------------------------------------
                            Typedefs
  ---------------------------------------------------------------------------*/
int xsh_bkg_yskip_lo_uvb[XSH_ORDERS_UVB]={0,0,0,0,
                                          0,0,0,0,
                                          0,0,0,0}; //XSH_ORDERS_UVB=12
 
int xsh_bkg_yskip_up_uvb[XSH_ORDERS_UVB]={0,0,0,0,
                                          0,0,0,0,
                                          0,0,0,0}; //XSH_ORDERS_UVB=12
 
 
int xsh_bkg_yskip_lo_vis[XSH_ORDERS_VIS]={0,0,0,0,0,
                                          0,0,0,0,0,
                                          0,0,0,0,0}; //XSH_ORDERS_VIS=15
 
int xsh_bkg_yskip_up_vis[XSH_ORDERS_VIS]={0,0,0,0,0,
                                          50,50,70,50,50,
                                          50,50,50,50,50}; //XSH_ORDERS_VIS=15
 
 
int xsh_bkg_yskip_lo_nir[XSH_ORDERS_NIR]={0,0,0,0,
                                           0,50,50,80,
                                           70,50,50,50,
                                           50,50,50,50}; //XSH_ORDERS_NIR=16
 
 
int xsh_bkg_yskip_up_nir[XSH_ORDERS_NIR]={0,0,0,0,
                                          0,50,100,60,
                                          90,70,50,50,
                                          50,50,50,50}; //XSH_ORDERS_NIR=16
 
/*---------------------------------------------------------------------------
                            Functions prototypes
  ---------------------------------------------------------------------------*/
static cpl_image*
xsh_image_generate_background(const int sx, const int sy, const polynomial *background_pol);
 
static cpl_image*
xsh_background_poly(xsh_pre* image, cpl_table** grid_tbl,xsh_background_param* back_par);
 
/*---------------------------------------------------------------------------
                              Implementation
  ---------------------------------------------------------------------------*/
 
 
cpl_frame* 
xsh_subtract_bias( cpl_frame *frame, 
                   cpl_frame *bias,
           xsh_instrument* instr, 
                   const char * type, 
                   const int pre_overscan_corr,
                   const int save_tmp)
{
  /* MUST BE DEALLOCATED in caller */
  cpl_frame * result = NULL;
  /* ALLOCATED locally */
  xsh_pre* xframe = NULL;
  xsh_pre* xbias = NULL;
  /* Others */
  char resultname[256];
  //int binx=0;
  //int biny=0;
  char tag[256];
  double avg=0;
 
  /* check input */  
  XSH_ASSURE_NOT_NULL( frame);
  XSH_ASSURE_NOT_NULL( bias);
  XSH_ASSURE_NOT_NULL( instr);
 
  /* load the frames */
  check( xframe = xsh_pre_load( frame, instr));
  check( xbias = xsh_pre_load( bias, instr)); 
 
  //check(binx=xsh_pfits_get_binx(xbias->data_header));
  //check(biny=xsh_pfits_get_biny(xbias->data_header));
  /* do the subtract operation */
  if(pre_overscan_corr==0) {
    check( xsh_pre_subtract( xframe, xbias));
  } else {
    /* The master bias level is set to 0 
       before subtraction if the overscan was already set by the user
       in order to correct for the bias structures */
    check(avg=cpl_image_get_mean(xbias->data));
    check(xsh_pre_subtract_scalar(xbias,avg));
    check( xsh_pre_subtract( xframe, xbias));
  }
  /* save result */
  sprintf(tag,"%sON_%s",type,xsh_instrument_arm_tostring (instr));
 
  /* Sabine does not like binning in file name
  sprintf(resultname,"%s_%dx%d.fits",tag,binx,biny);
  */
  sprintf(resultname,"%s.fits",tag);
 
  check( xsh_pfits_set_pcatg( xframe->data_header, tag));
  check( result = xsh_pre_save ( xframe, resultname, tag,save_tmp));
  check( cpl_frame_set_tag( result,tag));
 
  cleanup:
    if ( cpl_error_get_code() != CPL_ERROR_NONE) {
      xsh_free_frame( &result);
    }
    xsh_pre_free( &xframe);
    xsh_pre_free( &xbias);
    return result;
}
 
 
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
cpl_frameset* xsh_subtract_nir_on_off(cpl_frameset* on, cpl_frameset* off, 
  xsh_instrument* instr){
  /* MUST BE DEALLOCATED in caller */
  cpl_frameset * result = NULL;
  /* ALLOCATED locally */
  cpl_frame* res = NULL;
  /* Others */
  char resultname[256];
  int i = 0, size_on = 0, size_off = 0;
 
  /* check input parameters */
  XSH_ASSURE_NOT_NULL(on);
  XSH_ASSURE_NOT_NULL(off);
  XSH_ASSURE_NOT_NULL(instr);
 
  check(size_on = cpl_frameset_get_size(on));
  check(size_off = cpl_frameset_get_size(off));
 
  XSH_ASSURE_NOT_ILLEGAL(size_on == size_off); 
 
  XSH_NEW_FRAMESET(result);
 
  /* Loop on frames */
  for ( i=0; i< size_on; i++) { 
    cpl_frame* on_f = NULL;
    cpl_frame* off_f = NULL;
 
    check( on_f = cpl_frameset_get_frame(on,i));
    check( off_f = cpl_frameset_get_frame(off,i));
    sprintf(resultname, "ON-OFF_%d.fits",i);
 
    check( res = xsh_subtract_dark(on_f, off_f, resultname, instr));
    check(cpl_frameset_insert(result, res));
    xsh_add_temporary_file(resultname);
  }
  
  cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
      xsh_free_frameset(&result);
      xsh_free_frame(&res);
    }
    return result; 
}
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
cpl_frame * xsh_subtract_dark(cpl_frame * frame,cpl_frame * dark,
  const char* filename, xsh_instrument* instr)
{
  /* MUST BE DEALLOCATED in caller */
  cpl_frame * result = NULL;
  /* ALLOCATED locally */
  xsh_pre* xframe = NULL;
  xsh_pre* xdark = NULL;
  /* Others */
  double exptime = 0.0;  
  double dit_raw=0;
  double dit_dark=0;
  double dit_tol=0.001;
  const char* tag=NULL;
 
  /* check input parameters */
  XSH_ASSURE_NOT_NULL(frame);
  XSH_ASSURE_NOT_NULL(dark);
  XSH_ASSURE_NOT_NULL(filename);
  XSH_ASSURE_NOT_NULL(instr);
 
  /* load the frames */
  check(xframe = xsh_pre_load(frame, instr));
  check(xdark = xsh_pre_load(dark, instr));
 
  /* compute the master to the same exptime */
  if (xsh_instrument_get_arm(instr) != XSH_ARM_NIR) {
    exptime = xframe->exptime;
    assure(exptime > 0,CPL_ERROR_ILLEGAL_INPUT,
      "EXPTIME must be greater than 0 : %f",exptime);
    check(xsh_pre_multiply_scalar(xdark,exptime));
  } else {
    dit_raw=xsh_pfits_get_dit(xframe->data_header);
    dit_dark=xsh_pfits_get_dit(xdark->data_header);
    XSH_ASSURE_NOT_ILLEGAL_MSG( fabs(dit_raw-dit_dark) < dit_tol,"Make sure dark has same DIT as raw data");
  }
  /* do the subtract operation */
  check(xsh_pre_subtract(xframe,xdark));
  /* save result */
  tag=cpl_frame_get_tag(frame);
  check(result = xsh_pre_save (xframe, filename, tag,0));
  check(cpl_frame_set_tag (result,tag));
  
  cleanup:
    if (cpl_error_get_code () != CPL_ERROR_NONE) {
      xsh_free_frame(&result);
    }
    xsh_pre_free(&xframe);
    xsh_pre_free(&xdark);
  return result;
}
 
 
inline static void xsh_fill_bkg_mask_range(const int y, const int nx, const int xmin,
                       const int xmax, const int x_hbox, 
                       const int decode_bp, int* qual, 
                       int* pmask, int* size) {
  int x = 0;
  int xcen=0;
  int xs=0;
  int xe=0;
  int offset = nx*y;
  if((xmax-xmin+1) >= 2*x_hbox+1) {
    xcen = 0.5*(xmin+xmax);
    xs = xcen-x_hbox;
    xe = xcen+x_hbox;
    xs = (xs > 0) ? xs: 0;
    xe = (xe < nx) ? xe: nx-1;
 
    for (x = xs; x <= xe; x++) {
      if ( (qual[offset + x] & decode_bp) == 0) {
    pmask[offset + x] = 1;
    (*size)++;
      }
    }
 
  } else {
    //xsh_msg("skip range [%d-%d,%d]",xmin,xmax,offset/2048);
  }
  return;
}
 
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
static xsh_grid*
xsh_crea_grid_from_mask(xsh_pre *pre, xsh_order_list *orderlist,
    const int x_hbox , xsh_instrument *instr,const char* prefix) {
  xsh_grid *grid = NULL;
 
  int ord = 0;
  int x = 0;
  int y = 0;
  int xmin = 0;
  int xmax = 0;
  //int xcen = 0;
 
  float x1 = 0, x2 = 0;
  int* pmask = NULL;
  int* pqual = NULL;
  cpl_image* mask = NULL;
  int nx = pre->nx;
  int ny = pre->ny;
  int lost = 0;
  float* pdata = NULL;
 
  int size = 0;
   /* margin: UVB=4 VIS=7, NIR=2 */
  int pix = 0;
  float sx_illum = 0;
  float sx_backg = 0;
  int decode_bp=0;
  int* qual=NULL;
  //float* errs=NULL;
  float* perrs=NULL;
  int y_skip_up=0;
  int y_skip_lo=0;
  int x_margin1=0;
  int x_margin2=0;
  //int x_margin3=0;
  if (xsh_instrument_get_arm(instr) == XSH_ARM_NIR) {
     y_skip_lo=20;
     y_skip_up=20;
     x_margin1=40;
     x_margin2=80;
     //x_margin3=62;
  } else if (xsh_instrument_get_arm(instr) == XSH_ARM_UVB) {
    x_margin1=21;
    x_margin2=135;
    if ( strcmp(prefix,"MFLAT_D2") == 0 ) {
    y_skip_lo=0;
    y_skip_up=0;
    }
 
  } else if (xsh_instrument_get_arm(instr) == XSH_ARM_VIS) {
    x_margin1=30;
    x_margin2=134;
    //x_margin3=62;
  }
  decode_bp=instr->decode_bp;
 
  qual=cpl_image_get_data_int(pre->qual);
  //errs=cpl_image_get_data_float(pre->errs);
 
  mask = cpl_image_new(nx, ny, CPL_TYPE_INT);
  pmask = cpl_image_get_data_int(mask);
  xsh_msg("prepare mask data");
  for (y = 1; y < ny; y++) {
 
    /* special case for NIR: some orders cover only part of the detector
     if (xsh_instrument_get_arm(instr) == XSH_ARM_NIR) {
     if ((y <= orderlist->list[orderlist->size - 2].starty)
     || y >= orderlist->list[orderlist->size - 2].endy) {
     // add to grid
     x1=1;
     x2=
     for (x = x1; x <= x2; x++) {
     pmask[y * nx + x] = 1;
     }
     }
     }
     */
    for (ord = orderlist->size - 1; ord > 0; ord--) {
      int starty, endy;
      starty = orderlist->list[ord - 1].starty;
      endy = orderlist->list[ord - 1].endy;
      if (xsh_instrument_get_arm(instr) == XSH_ARM_NIR) {
       y_skip_lo=xsh_bkg_yskip_lo_nir[ord];
       y_skip_up=xsh_bkg_yskip_up_nir[ord];
      }
      if (xsh_instrument_get_arm(instr) == XSH_ARM_UVB) {
            y_skip_lo=xsh_bkg_yskip_lo_uvb[ord];
            y_skip_up=xsh_bkg_yskip_up_uvb[ord];
      }
      starty += y_skip_lo;
      endy -= y_skip_up;
 
      /* Special case: NIR: If outside order, then skip */
      if (xsh_instrument_get_arm(instr) == XSH_ARM_NIR) {
        if (y <= starty || y >= endy) {
          lost++;
          continue;
        }
      } else {
        if (ord < (orderlist->size - 1)) {
          if (y <= starty || y >= endy) {
            lost++;
            continue;
          }
        }
      }
 
 
      if( ord >0 ) {
      /* as we scan the order list from greater to lower order number, x1 < x2 */
    x1 = xsh_order_list_eval(orderlist, orderlist->list[ord].edguppoly, y);
    x2 = xsh_order_list_eval(orderlist, orderlist->list[ord - 1].edglopoly,
                 y);
    xmin = x1;
    xmax = x2;
        //xsh_msg("ord=%d absord=%d x1=%g x2=%g",ord,orderlist->list[ord].absorder,x1,x2);
    xsh_fill_bkg_mask_range(y,nx,xmin,xmax,x_hbox,decode_bp,qual,pmask,&size);
      
      }
 
 
      if (ord == orderlist->size - 1) {
        if ( strcmp(prefix,"MFLAT_D2") == 0 ) {
          /* special case: last order, on the left hand side of detector:
           * as the pipeline skips two orders we first estimate what is on the latest
           * detected order the size of the illuminated region and the one of the not
           * illuminated one so that we can set properly mask points only in the not illuminated */
 
 
     
        if (y <= starty || y >= endy) {
          lost++;
          continue;
        }
      
          x1 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edglopoly, y);
          x2 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edguppoly, y);
 
          sx_illum = x2 - x1;
          x1 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edguppoly, y);
          x2 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 2].edglopoly, y);
 
          sx_backg = x2 - x1;
          x2 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edglopoly, y);
          x1 = x2 - sx_backg;
 
          xmin = x1;
          xmax = x2;
        
          xsh_fill_bkg_mask_range(y , nx, xmin, xmax, x_hbox, decode_bp,qual, pmask,
                                                &size);
 
       // For D2 left side we trace the bkg in inverse orderr: 1st we compute positions then we check on y
          x2 = x1 - sx_illum;
          x1 = x2-sx_backg;
          xmin = x1+x_hbox-20;
          xmax = x2-x_hbox-10;
          int xmin1=xmin;
          int xmax1=xmax;
 
          if (y <= 650 || y >= endy) {
                                   lost++;
                                   continue;
                    }
 
          xsh_fill_bkg_mask_range(y , nx, xmin1, xmax1, x_hbox, decode_bp,qual, pmask,
                                       &size);
 
 
          x2 = x1 - sx_illum; /* note that in the latest x2 definition x_hbox war already taken out */
          x1 = x2-sx_backg;
          xmin = x1+x_hbox-50;
          xmax = x2-x_hbox-40;
            int xmin2=xmin;
            int xmax2=xmax;
 
 
 
            // this is the last order traced on the left
              if (y <= 1600 || y >= endy) {
                             lost++;
                             continue;
              }
 
 
          xsh_fill_bkg_mask_range(y , nx, xmin2, xmax2, x_hbox, decode_bp,qual, pmask,
              &size);
 
 
 
 
          /*
 
          x2 = x1 - sx_illum; // note that in the latest x2 definition x_hbox war already taken out
          xmin = 1;
          xmax = x2-x_hbox-60;
     
 
 
          xsh_fill_bkg_mask_range(y , nx, xmin, xmax, x_hbox, decode_bp,qual, pmask,
              &size);
            */
 
          continue;
            /* end special case for UVB-D2 flats */
        } else {
 
          /* special case: last order, on the left hand side of detector:
           * as the pipeline skips one order we first estimate what is on the latest
           * detected order the size of the illuminated region and the one of the not
           * illuminated one so that we can set properly mask points only in the not illuminated */
 
          if (xsh_instrument_get_arm(instr) != XSH_ARM_UVB) {
          x1 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edglopoly, y);
          x2 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edguppoly, y);
 
          sx_illum = x2 - x1;
          x1 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edguppoly, y);
          x2 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 2].edglopoly, y);
 
          sx_backg = x2 - x1;
          x2 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edglopoly, y);
          x1 = x2 - sx_backg + x_hbox;
          x2 -= x_hbox;
 
          xmin = x1;
          xmax = x2;
 
          xsh_fill_bkg_mask_range(y , nx, xmin, xmax, x_hbox, decode_bp,qual, pmask,
              &size);
 
          }
           /* extreme left between edge and last traced order:
          x2 = xsh_order_list_eval(orderlist,
              orderlist->list[orderlist->size - 1].edglopoly, y);
          xmin = 1;
          //note that in the latest x2 definition x_hbox war already taken out 
          x2 = x2 - sx_backg - sx_illum; 
          xmax = x2;
          xmax -= x_hbox;
 
          xsh_fill_bkg_mask_range(y , nx, xmin, xmax, x_hbox, decode_bp,qual, pmask,
              &size);
          */
          continue;
        }
      } else {
 
        if ( strcmp(prefix,"MFLAT_D2") == 0 ) {
 
      if (ord == orderlist->size - 2) {
        if (y <= starty || y >= endy) {
          xsh_msg("starty=%d endy=%d yskip=%d %d",starty,endy,y_skip_lo,y_skip_up);
           lost++;
           continue;
        }
      }
    }
 
      }
 
      if (ord == 1) {
 
          x1 = xsh_order_list_eval(orderlist, orderlist->list[0].edguppoly, y);
          xmin = x1;
          xmax = x1+x_margin1;
          xsh_fill_bkg_mask_range(y,nx,xmin,xmax,x_hbox,decode_bp,qual,pmask,&size);
         if( xsh_instrument_get_arm(instr) != XSH_ARM_NIR ) {
 
          xmin = xmax;
          xmax = xmin+x_margin2;
          xsh_fill_bkg_mask_range(y,nx,xmin,xmax,x_hbox,decode_bp,qual,pmask,&size);
 
         }    
         /*
          if( xsh_instrument_get_arm(instr) == XSH_ARM_VIS ) {
            xmin = xmax;
            xmax = xmin+x_margin3;
            xsh_fill_bkg_mask_range(y,nx,xmin,xmax,x_hbox,decode_bp,qual,pmask,&size);
 
           }
           */
      
          continue;
        //}
 
      } /* end case ord = 1 */
 
    } /* end loop over orders */
 
  } /* end loop over y */
 
 
  check( grid = xsh_grid_create(size));
  char name[256];
  sprintf(name,"bkg_mask_%s.fits",prefix);
  //cpl_image_save(mask, name, XSH_PRE_DATA_BPP, NULL,CPL_IO_DEFAULT);
 
  pdata = cpl_image_get_data_float(pre->data);
  perrs = cpl_image_get_data_float(pre->errs);
  pqual = cpl_image_get_data_int(pre->qual);
 
  for (y = 1; y < ny; y++) {
    for (x = 1; x < nx; x++) {
      pix = y * nx + x;
      if ((pmask[pix] == 1) && ((pqual[pix] & instr->decode_bp) == 0)) {
        check( xsh_grid_add(grid, x, y, pdata[pix],perrs[pix],pqual[pix]));
      }
    }
  }
 
 
 
  check( xsh_grid_sort(grid));
 
  cleanup: 
  xsh_free_image(&mask);
  return grid;
}
#if 0
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
static xsh_grid*
xsh_crea_grid_from_samples(xsh_pre *pre, xsh_order_list *orderlist,
    xsh_background_param *background_par, xsh_instrument *instr) {
 
 
  xsh_grid *grid = NULL;
 
  double *tab = NULL;
 
  int grid_size_x = 0, grid_size_y = 0;
  int incr = 0, i = 0, y = 0, lost = 0;
  int status = 0;
  int x = 0;
  float x1 = 0, x2 = 0, dx = 0;
  double medflux = 0.0;
 
  /* create the grid */
  grid_size_x = orderlist->size + 5;
  grid_size_y = background_par->sampley + 2;
 
  assure(background_par->sampley > 0, CPL_ERROR_ILLEGAL_INPUT,
      "parameter background-nb-y=%d "
      "must be set to a positive value", background_par->sampley);
  XSH_ASSURE_NOT_ILLEGAL(background_par->radius_x >= 0);
  XSH_ASSURE_NOT_ILLEGAL(background_par->radius_y >= 0);
 
  assure(
      grid_size_y > 0 && grid_size_y < pre->ny,
      CPL_ERROR_ILLEGAL_INPUT,
      "grid_size_y=%d must be in (0,%d) range "
      "parameter background-nb-y=%d "
      "may have been set to a too large or too small value", grid_size_y, pre->ny, background_par->sampley);
 
  check( grid = xsh_grid_create( grid_size_x*grid_size_y));
 
  /* allocate share memory for medflux */
  assure(
      background_par->radius_x<pre->nx,
      CPL_ERROR_ILLEGAL_INPUT,
      "parameter -background-radius-x (%d) must be > 0 and < %d", background_par->radius_x, pre->nx);
  assure(
      background_par->radius_y<pre->ny,
      CPL_ERROR_ILLEGAL_INPUT,
      "parameter -background-radius-y (%d) must be > 0 and < %d", background_par->radius_y, pre->ny);
 
  int box_sizex = 2 * background_par->radius_x + 1;
  int box_sizey = 2 * background_par->radius_y + 1;
  XSH_MALLOC(tab, double, box_sizex*box_sizey);
 
  /* compute incr along Y axis */
incr  = (int) ceil((float) pre->ny / (float) background_par->sampley);
  int sizey = 0;
  int oy = 0;
  /* Step 1 : Fill the grid */
  for (y = 1; y <= (pre->ny - 1 + incr); y += incr) {
 
    /* force to do the last line */
    if ((y + incr) > pre->ny) {
      y = pre->ny;
    }
 
    /* special cases in Y */
    if ((y == 1) || (y == pre->ny)) {
      sizey = background_par->radius_y + 1;
      oy = 0;
    } else {
      sizey = box_sizey;
      oy = -background_par->radius_y;
    }
 
    /* Special case: X = 1, estimate flux at left hand side detector edge */
    /* computes flux level in window box using a median method,
     considering only good pixel codes.*/
 
    check(
        medflux = xsh_pre_data_window_median_flux_pa(pre, 1, y+oy, box_sizex, sizey, tab,&status));
 
    if (status == 0) {
      /* special case for NIR: some orders cover only part of the detector */
      if (xsh_instrument_get_arm(instr) == XSH_ARM_NIR) {
        if ((y <= orderlist->list[orderlist->size - 2].starty)
            || y >= orderlist->list[orderlist->size - 2].endy) {
          /* add to grid */
          check( xsh_grid_add(grid, 1, y, medflux,1,0));
        } else {
          lost++;
        }
      } else {
        check( xsh_grid_add(grid, 1, y, medflux,1,0));
      }
    } else {
      lost++;
    }
 
    /* Points in [orderN->edgeup, order0->edgeup] */
    for (i = orderlist->size - 1; i > 0; i--) {
      int starty, endy;
      starty = orderlist->list[i - 1].starty;
      endy = orderlist->list[i - 1].endy;
 
      /* Special case: NIR: If outside order, then skip */
      if (xsh_instrument_get_arm(instr) == XSH_ARM_NIR) {
        if (y <= starty || y >= endy) {
          lost++;
          continue;
        }
      }
 
      /* TODO: drop this special case */
      if (starty == -999) {
        /* this condition can happen only if qc_mode=TRUE and some order
         has edges with S/N less than the user defined threshold
         it is used for QC in order to make recipe more robust and
         generate all QC. Normal users should set qc_mode=FALSE.
         */
        xsh_msg("PROBLEMS");
        check(xsh_grid_add(grid, 0, y, 0.,1,0));
        lost++;
        break;
      }
 
      /* calculate the center of the inter-order region */
      check(
          x1 = xsh_order_list_eval( orderlist, orderlist->list[i].edguppoly, y));
      check(
          x2 = xsh_order_list_eval( orderlist, orderlist->list[i-1].edglopoly, y));
      /* compute x */
      x = floor(0.5 * (x1 + x2 + 1));
      dx = 0.5 * (x2 - x1);
      xsh_msg_dbg_high("x %d dx %f", x, dx);
 
      if ((x > (background_par->radius_x + 1)) && (box_sizex < dx)) {
        /* In most of the cases we are far enough from the detector left
         hand side, and the inter-order space between two adjacent edges
         is large enough to be able to evaluate the inter-order background
         flux level. In this case we estimate the median flux in
         window box */
        check(
            medflux = xsh_pre_data_window_median_flux_pa(pre,x-background_par->radius_x, y+oy, box_sizex,sizey,tab,&status));
        if (status == 0) {
          check( xsh_grid_add(grid, x, y, medflux,1,0));
        } else {
          lost++;
        }
      } else {
        /* there are a few cases where previous condition is not verifyed.
         These pixels are lost. We count them. */
        lost++;
      }
 
      /* special case X in ]0, orderN->edgelow] */
      if (i == (orderlist->size - 1)) {
        check(
            x2 = xsh_order_list_eval( orderlist, orderlist->list[i].edglopoly, y));
        x = x2 - dx;
        if (dx < 0) {
          xsh_msg("Monitor dx=%d x=%g y=%d", x, dx, y);
        }
        if (x > (background_par->radius_x + 1)) {
          check(
              medflux = xsh_pre_data_window_median_flux_pa(pre, x-background_par->radius_x, y+oy, box_sizex, sizey, tab,&status));
          if (status == 0) {
            check(xsh_grid_add(grid, x, y, medflux,1,0));
          } else {
            lost++;
          }
        } else {
          lost++;
        }
      }
 
      /* special case X in ]order0->edgeup, pre->nx] */
      if (i == 1) {
        check(
            x1 = xsh_order_list_eval( orderlist, orderlist->list[0].edguppoly, y));
        /* special case in VIS */
        if (xsh_instrument_get_arm(instr) == XSH_ARM_VIS) {
          check(
              x2 = xsh_order_list_eval( orderlist, orderlist->list[0].edglopoly, y));
          x = x1 + (x1 - x2) * 1.2;
        } else {
          x = x1 + dx;
        }
        if ((x > (background_par->radius_x + 1))
            && (x + background_par->radius_x < pre->nx)) {
          check(
              medflux = xsh_pre_data_window_median_flux_pa(pre, x-background_par->radius_x, y+oy, box_sizex, sizey, tab,&status));
          if (status == 0) {
            check(xsh_grid_add(grid, x, y, medflux,1,0));
          } else {
            lost++;
          }
        } else {
          lost++;
        }
 
      } /* if i==1 */
 
    }/* end loop over i (orders) */
 
    /* Special case of: X=pre->nx (right hand size of detector, in PRE format) */
    check(
        medflux = xsh_pre_data_window_median_flux_pa(pre, pre->nx-background_par->radius_x, y+oy, background_par->radius_x+1, sizey, tab,&status));
 
    if (status == 0) {
      /* special case for NIR: some orders cover only part of the detector */
      if (xsh_instrument_get_arm(instr) == XSH_ARM_NIR) {
        if ((y <= orderlist->list[0].starty) || y >= orderlist->list[0].endy) {
          /* add to grid */
          check( xsh_grid_add(grid, x, y, medflux,1,0));
        } else {
          lost++;
        }
      } else {
        check( xsh_grid_add(grid, x, y, medflux,1,0));
      }
    } else {
      lost++;
    }
 
  } /* end loop over Y */
 
  xsh_msg_dbg_low("Nb of lost points %d ", lost);
  xsh_msg_dbg_low("Nb of grid points %d ", xsh_grid_get_index(grid));
 
  /* sort the grid points */
  check( xsh_grid_sort(grid));
 
  cleanup: return grid;
 
}
#endif
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------*/
cpl_frame* xsh_subtract_background( cpl_frame *frame, 
                    cpl_frame *edges_order_tab_frame,
                    xsh_background_param *background_par,
                    xsh_instrument *instr,
                    const char *prefix,
                                    cpl_frame **grid_frame,
                                    cpl_frame **backg_frame,
                                    const int save_bkg,
                                    const int save_grid,
                              const int save_sub_bkg)
{
  /* MUST BE DEALLOCATED in caller */
  cpl_frame *result = NULL;
  /* ALLOCATED locally */
  xsh_order_list *orderlist = NULL;
  xsh_pre *pre = NULL;
  xsh_grid *grid = NULL;
  cpl_image *background_img = NULL;
  /* Others */
  //float *data = NULL;
 
  char *fname = NULL;
  char *pcatg = NULL;
 
  cpl_propertylist *plist = NULL;
  cpl_table *grid_tbl = NULL;
 
  /* check parameters */
  XSH_ASSURE_NOT_NULL_MSG( instr, "Instrument structure setting");
  XSH_ASSURE_NOT_NULL_MSG( frame,"Input frame");
  XSH_ASSURE_NOT_NULL_MSG( edges_order_tab_frame,"Edge table");
  XSH_ASSURE_NOT_NULL_MSG( background_par,"Background params");
  XSH_ASSURE_NOT_NULL_MSG( grid_frame,"Grid frame");
  XSH_ASSURE_NOT_NULL_MSG( backg_frame,"Background frame");
 
  /* load data */
  check( pre = xsh_pre_load(frame, instr));
  check( orderlist = xsh_order_list_load( edges_order_tab_frame, instr));
  xsh_order_list_set_bin_x( orderlist, pre->binx);
  xsh_order_list_set_bin_y( orderlist, pre->biny);
  //check( data = cpl_image_get_data_float(pre->data));
 
  /* create grid */
 
  /* grid need to to be erased */
  check(grid=xsh_crea_grid_from_mask(pre, orderlist,background_par->edges_margin/pre->binx,instr,prefix));
  //check(grid=xsh_crea_grid_from_samples(pre,orderlist,background_par,instr));
 
  /* generate background image: only poly method (UVES-like) offered: 2D poly fit to data points */
  //check( result = xsh_pre_save( pre, "test.fits", "test", 1));
 
  xsh_msg("generate background image: poly");
  grid_tbl = xsh_grid2table(grid);
  xsh_grid_free( &grid);
 
  //check( cpl_table_save( grid_tbl, plist, NULL, "test.fits", CPL_IO_DEFAULT));
 
 
  check( background_img = xsh_background_poly(pre, &grid_tbl,background_par));
  xsh_msg("Prepare final products");
  /* prepare final products */
  XSH_NAME_PREFIX_LAMP_MODE_ARM( pcatg, prefix, "_BACK", "", instr);
  XSH_NAME_PREFIX_LAMP_MODE_ARM( fname, prefix, "_BACK", ".fits", instr);
  plist=cpl_propertylist_new();
  double crpix1=1.;
  double crval1=1.;
  double cdelt1=xsh_instrument_get_binx(instr);
 
  double crpix2=1.;
  double crval2=1.;
  double cdelt2=xsh_instrument_get_biny(instr);
 
  check( xsh_pfits_set_pcatg( plist, pcatg));
  xsh_pfits_set_wcs(plist,crpix1,crval1,cdelt1,crpix2,crval2,cdelt2);
  check( cpl_image_save( background_img, fname, CPL_BPP_IEEE_FLOAT,
    plist, CPL_IO_DEFAULT));
  if(!save_bkg) {
    xsh_add_temporary_file(fname);
  }
 
  xsh_free_propertylist( &plist);
 
  check( *backg_frame = xsh_frame_product( fname, pcatg, 
    CPL_FRAME_TYPE_IMAGE, CPL_FRAME_GROUP_CALIB,
    CPL_FRAME_LEVEL_FINAL));
 
 
  check( cpl_image_subtract( pre->data, background_img));
 
  XSH_FREE( fname);
  XSH_FREE( pcatg);
  XSH_NAME_PREFIX_LAMP_MODE_ARM( fname, prefix, "_SUB_BACK", ".fits", instr);
  XSH_NAME_PREFIX_LAMP_MODE_ARM( pcatg, prefix, "_SUB_BACK", "", instr);
 
  check( xsh_pfits_set_pcatg( pre->data_header, pcatg));
 
  if(save_sub_bkg==0) {
  check( result = xsh_pre_save( pre, fname, pcatg, 1));
  } else {
  check( result = xsh_pre_save( pre, fname, pcatg, 0));
  }
  //xsh_add_temporary_file(fname);
 
  check(cpl_frame_set_tag (result, pcatg));
  check(cpl_frame_set_type (result, CPL_FRAME_TYPE_IMAGE));
  check(cpl_frame_set_group (result, CPL_FRAME_GROUP_CALIB));
  check(cpl_frame_set_level (result, CPL_FRAME_LEVEL_FINAL));
 
  /*
  if(!save_tmp) {
    xsh_add_temporary_file(fname);
  }
  */
  XSH_FREE( fname);
  XSH_FREE( pcatg);
  XSH_NAME_PREFIX_LAMP_MODE_ARM( fname, prefix, "_GRID_BACK", ".fits", instr);
  XSH_NAME_PREFIX_LAMP_MODE_ARM( pcatg, prefix, "_GRID_BACK", "", instr);
 
  plist = cpl_propertylist_new();
  check( xsh_pfits_set_pcatg( plist, pcatg));
  check( cpl_table_save( grid_tbl, plist, NULL, fname, CPL_IO_DEFAULT));
  xsh_free_propertylist( &plist);
  xsh_free_table( &grid_tbl);
 
  if(save_grid==0) {
    xsh_add_temporary_file(fname);
  } 
 
 
  *grid_frame =xsh_frame_product(fname,pcatg,CPL_FRAME_TYPE_TABLE, 
                CPL_FRAME_GROUP_CALIB,CPL_FRAME_LEVEL_FINAL);
 
 
  cleanup:
 
    if (cpl_error_get_code () != CPL_ERROR_NONE) {
      xsh_free_frame(&result);
    }
 
    xsh_free_propertylist(&plist);
     xsh_order_list_free( &orderlist);
     XSH_FREE( fname);
     XSH_FREE( pcatg);
     //xsh_grid_free( &grid);
     xsh_pre_free( &pre);
     xsh_free_image( &background_img);
 
    return result;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
    
static cpl_image*
xsh_image_generate_background(const int sx, const int sy, const polynomial *background_pol)
{
    int x, y;
    cpl_image* image_bkg=NULL;
    float *background_data = NULL;
    int offset=0;
 
    image_bkg=cpl_image_new(sx,sy,XSH_PRE_DATA_TYPE);
    background_data = cpl_image_get_data_float(image_bkg);
 
    for (y = 1; y <= sy; y++)
    {
      offset=(y-1)*sx;
        for (x = 1; x <= sx; x++)
        {
            background_data[(x-1) + offset] = xsh_polynomial_evaluate_2d(background_pol,x,y);
        }
    }/* for each pixel... */
 
 
    return image_bkg;
}
 
static cpl_image*
xsh_background_poly(xsh_pre* pre, cpl_table** grid_tab,xsh_background_param* background_par) {
 
  /* Sampling done. Fit poly. */
  polynomial* background = NULL;
  double mse = 0, rmse = 0; /* mse, rms of fit */
  cpl_size total_clipped = 0;
  cpl_image* bkg_img = NULL;
 
  int degx = background_par->poly_deg_x;
  int degy = background_par->poly_deg_y;
  double kappa = background_par->poly_kappa;
  xsh_msg("degx=%d degy=%d kappa=%g", degx, degy, kappa);
  //int nrow=0;
  //int i=0;
 
  //double* pres=NULL;
  //double* pfit=NULL;
  //double* px=NULL;
  //double* py=NULL;
 
 
  //check(cpl_table_save(t,NULL,NULL,"grid_table.fits",CPL_IO_DEFAULT));
 
  {
    cpl_size n_clipped;
    do {
      cpl_size deg_xy = (degx + 1) * (degy + 1);
      assure(
          cpl_table_get_nrow(*grid_tab) > (degx + 1)*(degy + 1),
          CPL_ERROR_ILLEGAL_OUTPUT,
          "Too few sample points available (%" CPL_SIZE_FORMAT " point(s)) to make the fit "
          "(more than %" CPL_SIZE_FORMAT " points needed). "
          "Increase number of sample points or increase kappa", cpl_table_get_nrow(*grid_tab), deg_xy);
 
      /* Fit, calculate Zfit */
      xsh_polynomial_delete(&background);
      check_msg(
          background = xsh_polynomial_regression_2d( *grid_tab, "X", "Y", "INT", "ERR", degx, degy, "INTfit", NULL, NULL, &mse, NULL, NULL, kappa, -1),
          "Error fitting polynomial");
 
      /* Residual := INT - INTfit */
      cpl_table_duplicate_column(*grid_tab, "Residual", *grid_tab, "INT");
      cpl_table_subtract_columns(*grid_tab, "Residual", "INTfit");
 
      /* Calculate standard eviation of residuals */
      rmse = cpl_table_get_column_stdev(*grid_tab, "Residual");
 
      xsh_msg_dbg_medium("rmse=%g", rmse);
      /* Kappa-sigma clipping */
      if (kappa > 0) {
        check_msg(
            n_clipped = xsh_select_table_rows( *grid_tab, "Residual", CPL_GREATER_THAN, kappa * rmse),
            "Error selecting rows");
        /*
        check_msg(
            n_clipped += xsh_select_table_rows( *grd_tab, "Residual", CPL_LESS_THAN, -kappa * rmse),
            "Error selecting rows");
            */
      } else {
        n_clipped = 0;
      }
 
      total_clipped += n_clipped;
       /*
      xsh_msg_dbg_medium(
          "RMS = %f. %" CPL_SIZE_FORMAT " of %" CPL_SIZE_FORMAT " points rejected in kappa-sigma clipping", rmse, n_clipped, cpl_table_get_nrow(t));
       */
      xsh_msg(
           "RMS = %f. %" CPL_SIZE_FORMAT " of %" CPL_SIZE_FORMAT " points rejected in kappa-sigma clipping", rmse, n_clipped, cpl_table_get_nrow(*grid_tab));
 
      cpl_table_erase_selected(*grid_tab);
 
      if (n_clipped > 0) {
        cpl_table_erase_column(*grid_tab, "INTfit");
        cpl_table_erase_column(*grid_tab, "Residual");
      }
 
    } while (n_clipped > 0);
  }
 
  /*
  nrow=cpl_table_get_nrow(t);
  pres=cpl_table_get_data_double(t,"Residual");
  pfit=cpl_table_get_data_double(t,"INTfit");
  px=cpl_table_get_data_double(t,"X");
  py=cpl_table_get_data_double(t,"Y");
 
  for(i=0;i<nrow;i++) {
    (grid->list[i])->fit=pfit[i];
    (grid->list[i])->res=pres[i];
  }
  */
  
  //cpl_table_save(*grid_tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);
  /* Try to do some quality checking of the background subtraction.
   The number of rejected points (the signal)
   is often around 10-20 %  */
  {
    double percentage = 100.0 * ((double) total_clipped)
        / (total_clipped + cpl_table_get_nrow(*grid_tab));
 
    if (kappa > 0) {
      xsh_msg_dbg_medium(
          "%" CPL_SIZE_FORMAT " of %" CPL_SIZE_FORMAT " points (%.2f %%) were rejected in "
          "kappa-sigma clipping. RMS = %.2f ADU", total_clipped, cpl_table_get_nrow(*grid_tab) + total_clipped, percentage, sqrt(mse));
    }
 
 
  }
 
  check_msg( bkg_img=xsh_image_generate_background(pre->nx,pre->ny, background),
      "Error generating background polynomial");
 
  cleanup:
 
  xsh_polynomial_delete(&background);
  return bkg_img;
 
}