xsh_compute_linearity.c

Go to the documentation of this file.

/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */
 
/*
 * $Author: amodigli $
 * $Date: 2012-12-01 18:42:28 $
 * $Revision: 1.50 $
 * $Name: not supported by cvs2svn $
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
  Includes
  -----------------------------------------------------------------------------*/
 
#include <math.h>
#include <xsh_drl.h>
 
#include <xsh_badpixelmap.h>
#include <xsh_data_pre.h>
#include <xsh_dfs.h>
#include <xsh_pfits.h>
#include <xsh_error.h>
#include <xsh_msg.h>
#include <xsh_fit.h>
 
#include <xsh_compute_linearity.h>
 
#include <cpl.h>
 
#define USE_CPL_POLYNOMIAL 
#define OPTIMIZE_IMAGE_HANDLING
 
/*-----------------------------------------------------------------------------
  Functions prototypes
  -----------------------------------------------------------------------------*/
 
/*-----------------------------------------------------------------------------
  Implementation
  -----------------------------------------------------------------------------*/
 
static int
orderCompare (const void *one, const void *two)
{
  TIME_FRAME *un = (TIME_FRAME *) one;
  TIME_FRAME *deux = (TIME_FRAME *) two;
 
  if (un->exptime <= deux->exptime)
    return -1;
  else
    return 1;
}
 
static int
onoffCompare (const void *one, const void *two)
{
  TIME_FRAME *un = (TIME_FRAME *) one;
  TIME_FRAME *deux = (TIME_FRAME *) two;
 
  if (un->on_off <= deux->on_off)
    return 1;
  else
    return -1;
}
 
cpl_frameset * xsh_subtract_on_off( cpl_frameset *set,
                    xsh_instrument *instrument )
{
  cpl_frame *currOn = NULL, *currOff = NULL;
  cpl_frameset * subSet = NULL ;
  int nframes, i ;
  xsh_pre * subtracted = NULL ;
  xsh_pre *preOn = NULL, * preOff = NULL ;
 
  /*
    Subtraction: 1-2, 3-4, ...
  */
  xsh_msg( "===> Subtract on_off" ) ;
 
  nframes = cpl_frameset_get_size( set ) ;
  subSet = cpl_frameset_new() ;
  assure( subSet != NULL, cpl_error_get_code(),
      "Cant create new frameset" ) ;
 
  for( i = 0 ; i<nframes ; i+=2 ) {
    const char *filename = NULL;
    currOn = cpl_frameset_get_frame(set,i);
    assure( currOn != NULL, cpl_error_get_code(),
        "Cant get frame of frameset" ) ;
    filename = cpl_frame_get_filename (currOn);
    xsh_msg_dbg_low( " Subtracting ON : %s", filename ) ;
    check_msg( preOn = xsh_pre_load( currOn, instrument ),
           "Cant load PRE ON" ) ;
 
    currOff = cpl_frameset_get_frame (set,i+1);
    assure( currOff != NULL, cpl_error_get_code(),
        "Cant get frame of frameset" ) ;
    filename = cpl_frame_get_filename (currOff);
    xsh_msg_dbg_low( "             OFF: %s", filename ) ;
    check_msg( preOff = xsh_pre_load( currOff, instrument ),
           "Cant load PRE frame OFF" ) ;
 
    /* Now substract images, save and add to frameset */
    subtracted = xsh_pre_duplicate(preOn);
    xsh_pre_subtract( subtracted, preOff ) ;
    assure( subtracted != NULL, cpl_error_get_code(),
        "Cant subtract images" ) ;
    /* save subtracted  */
    {
      cpl_frame * saved = NULL ;
      char outname[128] ;
      const char *tag = XSH_LINEARITY_NIR ;
 
      sprintf( outname, "linear_sub_set_%d.fits", i ) ;
      saved = xsh_pre_save( subtracted, outname, tag,1 ) ;
      assure( saved != NULL, cpl_error_get_code(),
          "Cant save subtracted frame" ) ;
      /* Add a valid TAG (missing in the function xsh_pre_save) */
      check_msg( cpl_frame_set_tag( saved, tag ),
         "Cant set frame tag" ) ;
      /* insert into frameset */
      check_msg( cpl_frameset_insert( subSet, saved ),
         "Cant insert frame %d into subSet", i ) ;
    }
    /* Cleanup memory */
    xsh_pre_free( &preOn ) ;
    xsh_pre_free( &preOff ) ;
    xsh_pre_free( &subtracted ) ;
  }
 
 cleanup:
  if ( cpl_error_get_code() != CPL_ERROR_NONE ) {
    xsh_free_frameset( &subSet ) ;
    subSet = NULL ;
  }
  xsh_pre_free( &preOn ) ;
  xsh_pre_free( &preOff ) ;
  xsh_pre_free( &subtracted ) ;
  return subSet ;
}
 
int
xsh_linear_group_by_exptime (cpl_frameset * raws, xsh_instrument* instrument,
                 double exp_toler, cpl_frameset ** groupSet)
{
  /* for all frames in the frame set, get the EXPTIME or DIT (if NIR)
     group by same EXPTIME (or DIT if NIR) (+- exp_toler)
     and by ON/OFF flags (if NIR)
     returns the number of groups and the list of framesets
  */
  int i, ngroups = 0;
  int nframes = 0;
  TIME_FRAME *expOrdered = NULL;
  TIME_FRAME *porder = NULL;
  cpl_frame *current = NULL;
  double temps = 0.0;
  int onoff = 0;
  cpl_frameset **pgrpset = NULL;
 
  nframes = cpl_frameset_get_size (raws);
  xsh_msg( "Nb of Frames = %d", nframes ) ;
 
  expOrdered = cpl_malloc (nframes * sizeof (TIME_FRAME));
  assure (expOrdered != NULL, cpl_error_get_code (),
      "Cant create tempory time+fframe array");
 
  porder = expOrdered;
 
  for ( i = 0 ; i<nframes ; i++, porder++) {
    const char *filename;
    cpl_propertylist *header;
    current = NULL ;
 
    current = cpl_frameset_get_frame (raws,i);
 
    assure( current != NULL, CPL_ERROR_ILLEGAL_INPUT,
        "Cant get Frame #%d from frameset", i ) ;
    /* Load propertylist of current frame */
    filename = cpl_frame_get_filename (current);
    xsh_msg_dbg_low ("load propertylist file %s\n", filename);
    check_msg (header = cpl_propertylist_load (filename, 0),
           "Could not load header of file '%s'", filename);
 
    /* Get EXPTIME (or DIT in case of NIR) and store */
    if ( instrument->arm == XSH_ARM_NIR )
      porder->exptime = xsh_pfits_get_dit (header);
    else porder->exptime = xsh_pfits_get_exptime( header ) ;
 
    porder->frame = current;
    if ( instrument->arm == XSH_ARM_NIR ) {
      bool onoffs = false;
      check(onoffs = xsh_pfits_get_lamp_on_off( header )) ;
      if ( onoffs ) porder->on_off = 1;
      else porder->on_off = 0 ;
    }
    xsh_msg_dbg_low ("Exptime[%d] = %lf", i, porder->exptime);
    xsh_free_propertylist( &header ) ;
  }
 
  /* Order the array by increasing EXPTIME/DIT */
  xsh_msg_dbg_low ("==== Before qsort");
  qsort (expOrdered, nframes, sizeof (TIME_FRAME), orderCompare);
 
  /* In addition, if NIR, separate ON and OFF frames */
  {
    TIME_FRAME *first = expOrdered;
    //TIME_FRAME *last ;
    int nb = 0 ;
 
    temps = expOrdered->exptime ;
 
    for (i = 0, porder = expOrdered; i < nframes; i++, porder++) {
      if ( porder->exptime == temps ) {
    //last = porder ;
    nb++ ;
      }
      else {
    xsh_msg_dbg_low( "Call ON/OFF compare (%d frames)", nb ) ;
    qsort( first, nb, sizeof (TIME_FRAME), onoffCompare ) ;
    first = porder ;
    nb = 1 ;
    temps = porder->exptime ;
      }
    }
    /* Do the last group */
    qsort( first, nb, sizeof (TIME_FRAME), onoffCompare ) ;
  }
    
  /*  if ( cpl_msg_get_level() == CPL_MSG_DEBUG ) */ {
    xsh_msg_dbg_low ("==== After qsort");
    for (i = 0, porder = expOrdered; i < nframes; i++, porder++)
      xsh_msg_dbg_low("    DIT[%d] = %lf (ON/OFF = %d)", i, porder->exptime,
          porder->on_off);
  }
 
  /* Now create groups */
  porder = expOrdered;
  temps = porder->exptime;
  onoff = porder->on_off ;
 
  pgrpset = groupSet;
 
  xsh_msg_dbg_low ("New frameset 0 starting  at Frame 0" );
  *pgrpset = cpl_frameset_new ();
  check_msg (cpl_frameset_insert (*pgrpset, porder->frame),
         "Cant insert frame %d in group %d", i, ngroups);
  porder++;
  ngroups = 1;
  for (i = 1; i < nframes; i++, porder++) {
    if (fabs (porder->exptime - temps) > exp_toler || 
    porder->on_off != onoff ) {
      xsh_msg_dbg_low ("New frameset %d starting  at Frame %d", ngroups, i);
      ngroups++;
      temps = porder->exptime;
      onoff = porder->on_off ;
      pgrpset++;
      *pgrpset = cpl_frameset_new ();
      assure( *pgrpset != NULL, cpl_error_get_code(),
          "Cant create new framest" ) ;
    }
    else
      xsh_msg_dbg_low ("Same group frame %d - %lf vs %lf", i,
             porder->exptime, temps);
    xsh_msg_dbg_low ("Add frame %d to frameset group %d", i, ngroups);
    check_msg (cpl_frameset_insert (*pgrpset, porder->frame),
           "Cant insert frame %d in group %d", i, ngroups);
  }
 
 cleanup:
  if ( expOrdered != NULL ) cpl_free( expOrdered ) ;
  return ngroups;
}
 
#if !defined(USE_CPL_POLYNOMIAL)
 
static cpl_imagelist *xsh_imagelist_fit_polynomial( cpl_imagelist * medlist,
                         double *dit, int nframes,
                         int nx, int ny,
                         int degree )
{
  int ll, ix, iy ;
  cpl_vector * pixvect = NULL;
  cpl_vector * temps = NULL;
  cpl_polynomial *poly = NULL;
  cpl_imagelist *poly_list = NULL, 
    *final_list = NULL ;    
  float **pixbuf = NULL ;   
  cpl_binary **maskbuf = NULL ; 
  assure( medlist != NULL, CPL_ERROR_NULL_INPUT,
      "Input list of median images is NUL !" ) ;
 
  poly_list = cpl_imagelist_new() ;
  assure( poly_list != NULL, cpl_error_get_code(),
      "Cant create imagelist" ) ;
  /*
    Create the "data cube" (image list) to store the polynomial coefficients
  */
  for ( ll = 0 ; ll<= degree ; ll++ ) {
    cpl_image *newone = NULL ;
 
    newone = cpl_image_new( nx, ny, CPL_TYPE_DOUBLE ) ;
    assure( newone != NULL, cpl_error_get_code(),
        "Cant create new image" ) ;
    cpl_imagelist_set( poly_list, newone, ll ) ;
  }
 
  /* 
     Loop over pixels.
     Populate a data vector with the content of same pixel of all
       the images
     Fit polynomial
     Populate the resulting imagelist with the polynome coefficients
  */
  xsh_msg( "Polynomial Fit - XSH version - nb of frames: %d", nframes ) ;
#if 1
  /*
    For better efficiency, one could/should use the pointer to pixel data
    in the image instead of using cpl_image_get function.
    Use cpl_image_get_data (input frames are PRE format ==> Float data)
  */
 
  pixbuf = cpl_malloc( nframes * sizeof(double *) ) ;
  assure( pixbuf != NULL, cpl_error_get_code(),
      "Cant allocate memory for pixel data" ) ;
  maskbuf = cpl_malloc( nframes * sizeof(cpl_binary *) ) ;
  assure( maskbuf != NULL, cpl_error_get_code(),
      "Cant allocate memory for BPM data" ) ;
  /* Creating list of data and masks */
  for( ll = 0 ; ll < nframes ; ll++ ) {
    cpl_image *pcur = cpl_imagelist_get( medlist, ll ) ;
 
    *(pixbuf+ll) = cpl_image_get_data( pcur ) ;
    *(maskbuf+ll) = cpl_mask_get_data( cpl_image_get_bpm( pcur ) ) ;
  }
 
  /* Loop over pixels */
  for ( iy = 0 ; iy < ny ; iy++ )
    for( ix = 0 ; ix < nx ; ix++ ) {
      cpl_binary ** pmask = maskbuf ;
      float ** cur = pixbuf ;
      int npix = 0 ;
 
      pixvect = cpl_vector_new( nframes ) ;
      assure( pixvect != NULL, cpl_error_get_code(),
          "Cant create PixVect Vector" ) ;
      temps = cpl_vector_new( nframes ) ;
      assure( temps != NULL, cpl_error_get_code(),
          "Cant create Time Vector" ) ;
      xsh_msg_dbg_low( "++ Fit one pixel: %d,%d", ix, iy ) ;
 
      for( ll = 0 ; ll<nframes ; ll++ , pmask++, cur++ ) {
    double pixval ;
    cpl_binary rej ;
    float *pcur = *cur ;
    cpl_binary *mask = *pmask ;
 
    pixval = *(pcur + (iy*nx) + ix ) ;
    rej = *(mask + (iy*nx) + ix ) ;
    if ( rej == 0 ) {
      cpl_vector_set( temps, ll, *(dit+ll) ) ;
      cpl_vector_set( pixvect, ll, pixval ) ;
      npix++ ;
    }
      }
      xsh_msg_dbg_low( "  Nb of pixels: %d", npix ) ;
      /* Adjust the size of the vector to npix */
      if ( npix != nframes ) {
    cpl_vector_set_size( temps, npix ) ;
    cpl_vector_set_size( pixvect, npix ) ;
      }
      /* Now fit */
      poly = xsh_polynomial_fit_1d_create( temps, pixvect,
                       degree, NULL ) ;
      xsh_free_vector( &temps ) ;
      xsh_free_vector( &pixvect ) ;
      assure( poly != NULL, CPL_ERROR_ILLEGAL_INPUT,
          "Error calling xsh_polynomial_fit_1d_create" ) ;
      /* And fill the resulting coefficients frames */
      for( ll = 0 ; ll <= degree ; ll++ ) {
    cpl_image_set( cpl_imagelist_get( poly_list, ll ),
               ix+1, iy+1, cpl_polynomial_get_coeff( poly, &ll ) ) ;
      }
      xsh_free_polynomial( &poly ) ;
    }
#else
  for ( iy = 1 ; iy <= ny ; iy++ )
    for( ix = 1 ; ix <= nx ; ix++ ) {
      int npix = 0 ;
 
      pixvect = cpl_vector_new( nframes ) ;
      assure( pixvect != NULL, cpl_error_get_code(),
          "Cant create PixVect Vector" ) ;
      temps = cpl_vector_new( nframes ) ;
      assure( temps != NULL, cpl_error_get_code(),
          "Cant create Time Vector" ) ;
      xsh_msg_dbg_low( "++ Fit one pixel: %d,%d", ix, iy ) ;
      for( ll = 0 ; ll < nframes ; ll++ ) {
    double pixval ;
    int rej ;
    cpl_image *cur = NULL ;
    cur = cpl_imagelist_get( medlist, ll ) ;
    assure( cur != NULL, cpl_error_get_code(),
        "Cant get image #%d from liste" ) ;
    /* Add pixel value to vector */
    pixval = cpl_image_get( cur, ix, iy, &rej ) ;
    xsh_msg_dbg_low( " Pixval: %lf", pixval ) ;
    if ( rej == 0 ) {
      cpl_vector_set( temps, ll, *(dit+ll) ) ;
      cpl_vector_set( pixvect, ll, pixval ) ;
      npix++ ;
    }
      }
      xsh_msg_dbg_low( "  Nb of pixels: %d", npix ) ;
      /* Adjust the size of the vector to npix */
      if ( npix != nframes ) {
    cpl_vector_set_size( temps, npix ) ;
    cpl_vector_set_size( pixvect, npix ) ;
      }
      /* Now fit */
      poly = xsh_polynomial_fit_1d_create( temps, pixvect,
                       degree, NULL ) ;
      assure( poly != NULL, cpl_error_get_code(),
          "Error calling xsh_polynomial_fit_1d_create" ) ;
      xsh_free_vector( &temps ) ;
      xsh_free_vector( &pixvect ) ;
      /* And fill the resulting coefficients frames */
      //xsh_msg( "  Pixel %d,%d:", ix, iy ) ;
      for( ll = 0 ; ll <= degree ; ll++ ) {
    cpl_image_set( cpl_imagelist_get( poly_list, ll ),
               ix, iy, cpl_polynomial_get_coeff( poly, &ll ) ) ;
      }
      xsh_free_polynomial( &poly ) ;
    }
#endif
 
  final_list = cpl_imagelist_duplicate( poly_list ) ;
 
 cleanup:
  xsh_free_imagelist( &poly_list ) ;
  xsh_free_polynomial( &poly ) ;
  xsh_free_vector( &temps ) ;
  xsh_free_vector( &pixvect ) ;
  xsh_free_imagelist( &poly_list ) ;
  cpl_free( pixbuf ) ;
  cpl_free( maskbuf ) ;
 
  return final_list ;
}
#endif
 
cpl_frame *
xsh_compute_linearity (cpl_frameset * medSet, xsh_instrument *instrument,
               xsh_clipping_param * lin_clipping,const int decode_bp)
{
  cpl_image * resBpmap = NULL ; 
  int nframes = 0 ;
  cpl_frame * resFrame = NULL ; 
  int i = 0, ix = 0, iy = 0, nx = 0, ny = 0;
  int nbad, totbad = 0 ;
  cpl_imagelist *medList = NULL ; 
  cpl_imagelist *bpmapList = NULL ; 
  cpl_frame *current = NULL ;
  cpl_propertylist *bpmap_header = NULL ; 
  double *Dit = NULL;
#if defined(USE_CPL_POLYNOMIAL)
  cpl_vector * v_dit = NULL ;
  //  cpl_image * img_err = NULL ;
#endif
  cpl_imagelist * polyList = NULL ; 
  xsh_msg( "==> xsh_compute_linearity" ) ;
  assure (medSet != NULL, CPL_ERROR_ILLEGAL_INPUT, "Frameset is NULL");
  nframes = cpl_frameset_get_size (medSet);
 
 
  /* Create an imagelist from the medSet frameset
     Create the imagelist of bad pixel maps (to build the general one)
  */
  medList = cpl_imagelist_new() ;
  assure( medList != NULL, cpl_error_get_code(),
      "Cant create medList" ) ;
  bpmapList = cpl_imagelist_new() ; 
  assure( bpmapList != NULL, cpl_error_get_code(),
      "Cant create bpmapList" ) ;
 
  current = cpl_frameset_get_frame( medSet,0 ) ;
  assure( current != NULL, cpl_error_get_code(),
      "Cant get current frame" ) ;
 
  /* get the Detector Integration Time */
  Dit = cpl_malloc( nframes * sizeof(double) ) ;
  assure( Dit != NULL, cpl_error_get_code(),
      "Cant allocate Dit array" ) ;
#if defined(USE_CPL_POLYNOMIAL)
  v_dit = cpl_vector_wrap( nframes, Dit ) ;
  assure( v_dit != NULL, cpl_error_get_code(),
      "Cant create v_dit vector" ) ;
#endif
  xsh_msg( "Loop over all frames" ) ;
 
  for( i = 0 ; i<nframes ; i++ ) {
    const char *filename ;
    cpl_propertylist *header = NULL ;
    cpl_image * curimage = NULL ;
    cpl_image *curbpmap = NULL ;    
 
    filename = cpl_frame_get_filename(current);
    assure( filename != NULL, cpl_error_get_code(),
        "Cant get filename from frame %d", i ) ;
 
    curimage = cpl_image_load( filename, XSH_PRE_DATA_TYPE, 0, 0 ) ;
    assure( curimage != NULL, cpl_error_get_code(),
        "Cant get curimage from frame %d", i ) ;
 
    cpl_imagelist_set( medList, curimage, i ) ;
    header = cpl_propertylist_load( filename, 0 ) ;
    assure( header != NULL, cpl_error_get_code(),
        "Cant get header of frame %d", i ) ;
 
    xsh_msg_dbg_low( "  Getting EXPTIME/DIT[%d]", i ) ;
    /* Get EXPTIME (or DIT in case of NIR) and store */
    if ( instrument->arm == XSH_ARM_NIR )
      *(Dit+i) = xsh_pfits_get_dit( header ) ;
    else *(Dit+i) = xsh_pfits_get_exptime( header ) ;
    cpl_propertylist_delete( header ) ;
 
    if ( i == 0 ) {
      nx = cpl_image_get_size_x( curimage ) ;
      ny = cpl_image_get_size_y( curimage ) ;
    }
 
    /* Now get the bad pixel map from this frame (extension Nb 2) */
    xsh_msg_dbg_low( "  Loading BpMap[%d]", i ) ;
 
    curbpmap = cpl_image_load( filename, CPL_TYPE_INT, 0, 2 ) ;
    assure( curbpmap != NULL, cpl_error_get_code(),
        "Cant load bpmap frame #%d", i ) ;
    cpl_imagelist_set( bpmapList, curbpmap, i ) ;
    if ( i == 0 ) {
      bpmap_header = cpl_propertylist_load( filename, 2 ) ;
      assure( bpmap_header != NULL, cpl_error_get_code(),
          "Cant get bpmap header of frame %d", i ) ;
    }
    current = cpl_frameset_get_frame( medSet,i+1 ) ;
    xsh_msg_dbg_low( "-- Next Frame" ) ;
  }
 
  /* Collapse all bad pixel maps from the frameset into a single one */
  xsh_msg( "Collapsing BpMaps's" ) ;
 
  resBpmap = xsh_bpmap_collapse_bpmap_create ( bpmapList,decode_bp ) ;
  assure( resBpmap != NULL, cpl_error_get_code(),
      "Cant create collapsed bpmap" ) ;
  /* memory cleanup */
  xsh_msg_dbg_low( "Freeing bpmapList" ) ;
  xsh_free_imagelist( &bpmapList ) ;
 
  /* Insert bad pixels into the images of the list before fitting */
  for( i = 0 ; i<nframes ; i++ ) {
    xsh_set_image_cpl_bpmap ( cpl_imagelist_get( medList, i ),
                  resBpmap, decode_bp ) ;
  }
 
#if defined(USE_CPL_POLYNOMIAL)
  /* CPL imagelist Polynomial fit */
  xsh_msg( "Now fit polynomial - CPL version" ) ;
 
  /* Use the function provided by J.Larsen (not the "real" CPL one) */
  polyList = xsh_fit_imagelist_polynomial( v_dit, medList, 0,
                       LINEAR_POLYNOMIAL_DEGREE,
                       CPL_FALSE, NULL ) ;
  //  polyList = cpl_imagelist_fit_polynomial( medList, Dit,
  //                   LINEAR_POLYNOMIAL_DEGREE ) ;
  assure( polyList != NULL, cpl_error_get_code(),
      "Cant fit polynomial - CPL version" ) ;
  xsh_msg( "Polynomial Fit Finished - CPL Version" ) ;
#else
  /*
    The cpl function cpl_imagelist_fit_polynomial does not work
    as expected. Hence we have to create our own fit function,
    based on the CPL function xsh_polynomial_fit_1d_create
  */
  xsh_msg( "Now fit polynomial - XSH version" ) ;
  polyList = xsh_imagelist_fit_polynomial( medList, Dit, nframes, nx, ny,
                       LINEAR_POLYNOMIAL_DEGREE ) ;
  assure( polyList != NULL, cpl_error_get_code(),
      "Cant fit polynomial - XSH version" ) ;
  xsh_msg( "Polynomial Fit Finished - XSH Version" ) ;
#endif
  /* The cpl bad pixels of polyList are not set according to the bad pixels
     of the input image list ==> do it now
  */
  for( i = 0 ; i<=LINEAR_POLYNOMIAL_DEGREE ; i++ ) {
    check(xsh_set_image_cpl_bpmap ( cpl_imagelist_get( polyList, i ),
                    resBpmap, decode_bp )) ;
  }
 
  /* Check linearity per pixel */
  xsh_msg_dbg_low( "Clipping sigma: %f", lin_clipping->sigma ) ;
 
  for( i=0; i <= LINEAR_POLYNOMIAL_DEGREE ; i++ ) {
    /* Check linearity for each polynome parameter */
    cpl_image *pplane = NULL ;
    int iter = 0 ;
    double mean = 0.0, stdev = 0.0, med = 0.0;
 
    pplane = cpl_imagelist_get( polyList, i ) ;
    assure( pplane != NULL, cpl_error_get_code(),
        "Gant get plane Nb %d", i ) ;
    /* Iteration:
       Calculate average and sigma (taking into account bad pixels)
       mark bad pixels (bad pixel map is empty for the 1st iteration)
    */
 
    xsh_msg( "Sigma clipping on coefficient %d", i ) ;
    for( iter = 0 ; iter<lin_clipping->niter ; iter++ ) {
#if defined(OPTIMIZE_IMAGE_HANDLING)
      double *pdata = NULL ;
      cpl_binary *pbpm = NULL ;
#endif
      xsh_msg( "   Iteration Nb %d/%d", iter+1, lin_clipping->niter ) ;
      /* Calculate mean and stdev of current parameter image pplane */
      mean = cpl_image_get_mean( pplane ) ;
      stdev = cpl_image_get_stdev( pplane ) ;
      med = cpl_image_get_median( pplane ) ;
      xsh_msg( "      Mean: %lf, stdev: %lf, Median: %lf",
           mean, stdev, med ) ;
      /* Loop over good pixels
     compare current pplane value with sigma*lin_clipping->sigma
     Mark bad pixels
      */
      nbad = 0 ;
#if defined(OPTIMIZE_IMAGE_HANDLING)
      pdata = cpl_image_get_data_double( pplane ) ;
      assure( pdata != NULL, cpl_error_get_code(),
          "Cant get pplane[%d] data", i ) ;
      pbpm = cpl_mask_get_data( cpl_image_get_bpm( pplane ) ) ;
      assure( pdata != NULL, cpl_error_get_code(),
          "Cant get pplane[%d] bpm", i ) ;
 
      for( iy = 0; iy < ny ; iy++ )
    for( ix = 0 ; ix < nx ; ix++ ) {
      double coeff = *(pdata + ix + (iy*nx)) ;
      cpl_binary rej = *(pbpm + ix + (iy*nx)) ;
      if ( rej != 0 ) {
        xsh_msg_dbg_low( "  **** rejected %d,%d", ix+1, iy+1 ) ;
        continue ;
      }
#else
      for( iy = 1 ; iy <= ny ; iy++ )
    for( ix = 1 ; ix<=nx ; ix++ ) {
      int rej ;
      double coeff = cpl_image_get( pplane, ix, iy, &rej ) ;
      if ( rej != 0 ) {
        xsh_msg_dbg_low( "  **** rejected %d,%d", ix, iy ) ;
        continue ;
      }
#endif
      //xsh_msg( "   **** %d,%d -> %lf", ix, iy, coeff ) ;
      if ( fabs( coeff - mean) > stdev*lin_clipping->sigma ) {
        xsh_msg_dbg_low( "          BAD PIXEL at %d,%d - %lf vs %lf",
             ix, iy, mean, coeff ) ;
        /* add bad pixel to this pseudo image for next iteration*/
#if defined(OPTIMIZE_IMAGE_HANDLING)
        cpl_image_reject( pplane, ix+1, iy+1 ) ;
        /* add to the global bad pixel map */
        xsh_bpmap_set_bad_pixel( resBpmap, ix+1, iy+1,
                     QFLAG_NON_LINEAR_PIXEL ) ;
#else
        cpl_image_reject( pplane, ix, iy ) ;
        /* add to the global bad pixel map */
        xsh_bpmap_set_bad_pixel( resBpmap, ix, iy,
                     QFLAG_NON_LINEAR_PIXEL ) ;
#endif
        nbad++ ;
      }
    }
      xsh_msg( "      Nbad: %d", nbad ) ;
      if ( nbad == 0 ) break ; // stop iteration if no bad detected
      totbad += nbad ;
    }
    /* Now add mean, rms and med of each coefficient to bpmap header */
    xsh_pfits_set_qc_multi( bpmap_header, (void *)&mean,
                XSH_QC_BP_MAP_LINi_MEAN, instrument, i ) ;
    xsh_msg_dbg_low( "%s (%d) = %lf", XSH_QC_BP_MAP_LINi_MEAN, i, mean ) ;
    xsh_pfits_set_qc_multi( bpmap_header, (void *)&med,
                XSH_QC_BP_MAP_LINi_MED, instrument, i ) ;
    xsh_msg_dbg_low( "%s (%d) = %lf", XSH_QC_BP_MAP_LINi_MED, i, med ) ;
    xsh_pfits_set_qc_multi( bpmap_header, (void *)&stdev,
                XSH_QC_BP_MAP_LINi_RMS, instrument, i ) ;
    xsh_msg_dbg_low( "%s (%d) = %lf", XSH_QC_BP_MAP_LINi_RMS, i, stdev ) ;
  }
  xsh_msg( "====> Total Non linear pixels: %d", totbad ) ;
  /* Add the total nb of bad pixels to the bpmap header */
  {
    int nbadpix = cpl_image_count_rejected( resBpmap ) ;
    xsh_pfits_set_qc( bpmap_header, (void *)&nbadpix,
              XSH_QC_BP_MAP_NBADPIX, instrument ) ;
    xsh_msg_dbg_low( "%s = %d", XSH_QC_BP_MAP_NBADPIX, nbadpix ) ;
  }
 
 
  /* Finished */
  /* Save bpmap image file */
  {
    char bpmapFname[132] ;
    const char *sarm = xsh_instrument_arm_tostring( instrument ) ;
 
    sprintf( bpmapFname, "LINEARITY_BAD_PIXEL_MAP_%s.fits", sarm ) ;
    check_msg( cpl_image_save( resBpmap, bpmapFname, CPL_BPP_32_SIGNED,
                   bpmap_header, CPL_IO_DEFAULT ),
           "Cant save resulting BpMap image" ) ;
    //xsh_add_temporary_file(bpmapFname);
 
    /* insert bpmap into resFrame */
 
    check(resFrame=xsh_frame_product(bpmapFname,
                                     XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_LIN,
                              instrument),
                     CPL_FRAME_TYPE_IMAGE,
                     CPL_FRAME_GROUP_PRODUCT,
                     CPL_FRAME_LEVEL_FINAL));
 
  }
 
 cleanup:
  if ( Dit != NULL ) cpl_free( Dit ) ;
  xsh_free_imagelist( &polyList ) ;
  xsh_free_imagelist( &bpmapList ) ;
  xsh_free_imagelist( &medList ) ;
  xsh_free_propertylist( &bpmap_header ) ;
  xsh_free_image( &resBpmap ) ;
 
  return resFrame ;
 
}
 
/*----------------------------------------------------------------------------*/