xsh_remove_crh_multi.c

Go to the documentation of this file.

/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */
 
/*
 * $Author: amodigli $
 * $Date: 2013-01-31 08:56:43 $
 * $Revision: 1.133 $
 * $Name: not supported by cvs2svn $
 *
 */
 
#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
  Includes
-----------------------------------------------------------------------------*/
 
#include <math.h>
 
#include <xsh_drl.h>
#include <xsh_data_pre.h>
#include <xsh_dfs.h>
#include <xsh_pfits.h>
#include <xsh_error.h>
#include <xsh_msg.h>
#include <cpl.h>
#include <xsh_badpixelmap.h>
#include <xsh_irplib_mkmaster.h>
#include <xsh_utils_image.h>
#include <xsh_utils_imagelist.h>
 
/*-----------------------------------------------------------------------------
  Functions prototypes
 -----------------------------------------------------------------------------*/
 
/*-----------------------------------------------------------------------------
  Implementation
 -----------------------------------------------------------------------------*/
 
 
static int
xsh_remove_cr (cpl_imagelist * datalist, cpl_imagelist * errslist,
           cpl_image * median,
           cpl_image * errs, cpl_mask * mask,
           xsh_clipping_param * crh_clipping)
{
  int nx, ny, nimg, i;
  cpl_image **pimg = NULL;
  cpl_image **perrs = NULL;
  int ix = 0,iy = 0;
  int crcount = 0;  
  float* medvals = NULL;
  float* mederrs = NULL;
  cpl_binary *bpmdata = NULL; 
 
 
 
  check(nimg = cpl_imagelist_get_size (datalist));
  
  /* create the array of pointers to images in list */
  pimg = cpl_malloc (nimg * sizeof (cpl_image *));
  assure (pimg != NULL, cpl_error_get_code (),
      "Cant allocate memory for image pointers");
  perrs = cpl_malloc (nimg * sizeof (cpl_image *));
  assure (pimg != NULL, cpl_error_get_code (),
          "Cant allocate memory for image pointers");
 
  xsh_msg_dbg_low ("Nb of images: %d\n", nimg);
 
  /* Populate images pointer array */
  for (i = 0; i < nimg; i++){
    *(pimg + i) = cpl_imagelist_get (datalist, i);
    *(perrs + i) = cpl_imagelist_get (errslist, i);
  }
  
  /* get size from first image */
  check(nx = cpl_image_get_size_x (*pimg));
  check(ny = cpl_image_get_size_y (*pimg));
 
  /* Loop over all pixels */
  check(medvals = cpl_image_get_data_float(median));
  check(mederrs = cpl_image_get_data_float(errs));
  check(bpmdata = cpl_mask_get_data(mask));
 
  for (iy = 1; iy <= ny; iy++){
    for (ix = 1; ix <= nx; ix++) {
      int k;
      double medval;
      double error;
      int badpix = 0;
      /* crh_clipping->sigma is what is usually called 'kappa' */
      error = mederrs[ix-1+(iy-1)*nx] * crh_clipping->sigma;
      //xsh_msg("errs=%g kappa=%g",mederrs[ix-1+(iy-1)*nx],crh_clipping->sigma);
      medval = medvals[ix-1+(iy-1)*nx];
      badpix = 0;
      
 
      for (k = 0; k < nimg; k++) {
        /* if good pixel, check if still good */
    if (!bpmdata[ix-1+(iy-1)*nx]) {
      double delta = 0.0, kvalue = 0.0;
          int rej = 0;
      kvalue = cpl_image_get (*(pimg + k), ix, iy, &rej);
      if (rej != 0) {
        xsh_msg_dbg_high ("Pixel %d,%d already bad in image %d", ix, iy, k);
        continue;
      }
      delta = fabs(kvalue - medval);
      /*      
          xsh_msg("error=%g kvalue=%g medval=%g delta=%g",
          error,kvalue,medval,delta);
      */
      if (delta > error) {
        cpl_image_reject (*(pimg + k), ix, iy);
            cpl_image_reject (*(perrs + k), ix, iy);
                if ( crcount < 10 )
            xsh_msg ("CR Rejected for Pixel %d,%d [%d] (%lf vs %lf)",
            ix, iy, k, medval, kvalue);
        
        crcount++;
            badpix = 1;
      }
    }
    else {
      xsh_msg_dbg_high("Pixel %d,%d already known as bad pixel in image %d", ix, iy,
            k+1);
      break;
    }
      }
      /*TODO: simply bitwise or combine with code for CRH pix */
      /* mask good pixel */
      if (!badpix){
        /*xsh_msg("add to mask %d %d",ix,iy);*/
        bpmdata[ix-1+(iy-1)*nx] = CPL_BINARY_1;
      }
 
    }
  }
 
  cpl_free (pimg);
  cpl_free(perrs);
 
  cleanup:
    return crcount;
}
 
static void
add_qc_crh (xsh_pre* pre, int nbcrh, int nframes)
{
  double qc_crrate = 0.0;
  float nbcr_avg ;      
  XSH_ASSURE_NOT_NULL(pre);
  XSH_ASSURE_NOT_ILLEGAL(pre->pszx >0. && pre->pszy > 0); 
  XSH_ASSURE_NOT_ILLEGAL(pre->exptime > 0); 
  /*
    QC.CRRATE = avg_nb_of_crh_per_frame/(exposure_time*size_of_detector_in_cm2)
    Nb of Cosmics per cm2 per second.
    The whole size of the image in cm2 is given by pszx and pszy. These
    values are in MICRONS.
  */
  xsh_msg_dbg_medium( "add_qc_crh - Exptime = %f", pre->exptime ) ;
  qc_crrate =
    (double) nbcrh / (((double) nframes) * (pre->exptime * 
    (pre->pszx / 10000.0) * (pre->pszy /10000.0) * pre->nx * pre->ny)) ;
  nbcr_avg = (float)nbcrh/nframes ;
 
  /*
    Now set QC KW
  */
  check( xsh_pfits_set_qc_crrate( pre->data_header,qc_crrate) ) ;
  check( xsh_pfits_set_qc_ncrh( pre->data_header, nbcrh) ) ;
  check( xsh_pfits_set_qc_ncrh_mean( pre->data_header,nbcr_avg) ) ;
  check( xsh_pfits_set_qc_ncrh_tot( pre->data_header,nbcrh) ) ;
  /* Same thing for the qual header */
  check( xsh_pfits_set_qc_crrate( pre->qual_header,qc_crrate)) ;
  check( xsh_pfits_set_qc_ncrh( pre->qual_header,nbcrh)) ;
  check( xsh_pfits_set_qc_ncrh_mean( pre->qual_header,nbcr_avg) ) ;
  cleanup:
    return ;
}
 
static cpl_error_code
xsh_find_cosmics(cpl_frameset* rawFrames,xsh_pre* pre,xsh_clipping_param* crh_clipping,
    cpl_imagelist* dataList,
                 cpl_imagelist* errsList,cpl_mask* badpix_mask,xsh_instrument* instr,cpl_mask** crh_mask)
{
  long int nbpixcosmic=-1;
  int nb_crh_total=0;
  int iter=0;
  int size=0;
  /* TODO: we could drop rawFrames argument */
  check( size = cpl_frameset_get_size( rawFrames));
 
  /* loop to remove cosmics */
  long nbpixtotal = cpl_frameset_get_size(rawFrames) * pre->nx * pre->ny;
  double frac_accepted = 1;
 
  while(iter < crh_clipping->niter && frac_accepted >= crh_clipping->frac
      && (nbpixcosmic!=0) ) {
 
 
    /* try to remove cosmics */
    check(nbpixcosmic = xsh_remove_cr (dataList, errsList,pre->data,
        pre->errs, badpix_mask, crh_clipping));
    if(iter==0) {
      *crh_mask=cpl_mask_duplicate(badpix_mask);
    }
 
    //AModigliani: here one have to use an and (or better an or) to coadd the
    //bad pixels found on each iteration. Instead the actual bad pixel
    //image saved on the QUAL part is the one coming from the last iteration
    //which usually does not find any bad pixel. This is wrong. I propose
    //to use the one which I save as crh_mask, even bettter if coming from
    //an or coaddition: cpl_mask_or(...)
    check(cpl_mask_and(*crh_mask,badpix_mask));
    //check(crh_img=cpl_image_new_from_mask(badpix_mask));
    //sprintf(name,"%s_ITER_%d%s","crh_ima_",iter,".fits");
    //cpl_image_save (crh_img,name,CPL_BPP_32_SIGNED,NULL, CPL_IO_DEFAULT);
    //xsh_free_image(&crh_img);
 
    xsh_msg(" Iteration Nb %d/%d, new cosmic found %ld",
        iter+1, crh_clipping->niter, nbpixcosmic);
 
    /* computes fraction */
    frac_accepted -= ((double)nbpixcosmic / (double)nbpixtotal);
    xsh_msg_dbg_medium("accepted pixels / total pixels = %f)",frac_accepted);
 
    /* recreate median image */
    xsh_image_flag_bp(pre->data,pre->qual,instr);
    check(xsh_bpmap_collapse_median(pre->data,dataList,*crh_mask));
    //pre->data=xsh_irplib_mkmaster_median(dataList,5,5,1.e-5);
    /* recreate errs image */
    xsh_image_flag_bp(pre->errs,pre->qual,instr);
    check(xsh_collapse_errs(pre->errs,errsList,0));
    iter++;
 
    /* Total nb of cosmics */
    nb_crh_total += nbpixcosmic ;
  }
 
 
 
  //check( pre->data=xsh_imagelist_collapse_sigclip_iter_create(dataList,3,3,2));
 
 
  xsh_msg("Nb of cosmics removed by frame stacking: %d ", nb_crh_total);
 
  /* Add QC Keywords */
  if (pre->exptime > 0){
    check(add_qc_crh( pre, nb_crh_total, size));
  }
 
  cleanup:
  return cpl_error_get_code();
}
 
cpl_error_code
xsh_flag_cosmic_debug(xsh_pre* pre,cpl_imagelist* dataList)
{
 
  cpl_mask* cosmic_mask = NULL;
  int imask=0;
  int jmask=0;
  int i=0;
  cpl_image* img=NULL;
  cpl_mask* img_mask=NULL;
  int size=cpl_imagelist_get_size(dataList);
 
  cosmic_mask = cpl_mask_new(pre->nx, pre->ny);
  for(i=0;i< size; i++){
    img = cpl_imagelist_get( dataList, i);
    img_mask = cpl_image_get_bpm( img);
    for(jmask=1; jmask<=pre->ny; jmask++){
      for(imask=1; imask <= pre->nx; imask++){
        if (cpl_mask_get(img_mask,imask, jmask) == CPL_BINARY_1){
          cpl_mask_set(cosmic_mask, imask, jmask,CPL_BINARY_1);
        }
      }
    }
  }
  //xsh_bpmap_mask_bad_pixel( pre->qual, cosmic_mask,
  //QFLAG_COSMIC_RAY_REMOVED);
  //AModigliani: I think here is the place one need to modify the BP map.
  //xsh_bpmap_mask_bad_pixel( pre->qual, cpl_mask_threshold_image_create(crh_ima,-0.5,0.5),
  //  QFLAG_COSMIC_RAY_REMOVED);
  cpl_mask_delete( cosmic_mask);
 
  return cpl_error_get_code();
}
/*---------------------------------------------------------------------------*/
static cpl_frame* 
xsh_remove_cosmics (cpl_frameset * rawFrames, 
            const char *result_tag, 
            xsh_stack_param* stack_par,
            xsh_clipping_param * crh_clipping,
            xsh_instrument* instr, 
            cpl_imagelist ** list,
            cpl_image** crh_ima,
            const int save_tmp )
{
  /* MUST BE DEALLOCATED in caller */
  cpl_frame* medFrame = NULL;
  cpl_imagelist *dataList = NULL;
  cpl_image* qual_comb = NULL;
 
  /* ALLOCATED locally */
  cpl_imagelist *errsList = NULL;
  xsh_pre* pre = NULL;
  cpl_mask* badpix_mask = NULL;
  cpl_mask* crh_mask = NULL;
  /* Others */
  cpl_frame *current = NULL;
  int i=0, size = 0;
 
  //cpl_image* crh_img=NULL;
  char name[256];
  char result_name[256];
  const int mode_and = 0;
  int ncrh_tot = 0;
 
  /* Check parameters */
  XSH_ASSURE_NOT_NULL( instr);
  XSH_ASSURE_NOT_NULL( rawFrames);
  XSH_ASSURE_NOT_NULL( result_tag);
  check( size = cpl_frameset_get_size( rawFrames));
  xsh_msg("size=%d",size);
 
  XSH_ASSURE_NOT_ILLEGAL( size > 0);  
 
  /* Create the image lists */
  check( dataList = cpl_imagelist_new());
  check( errsList = cpl_imagelist_new());
 
  /* Populate image lists with images in frame set */
  cpl_mask* mask_comb;;
  cpl_mask* mask_tmp;
  for (i=0;i < size;i++){
 
    check(current = cpl_frameset_get_frame(rawFrames,i));
    /* Load pre file */
    check(pre = xsh_pre_load(current,instr));
 
    /* inject bad pixels to change statistics */
    xsh_image_flag_bp(pre->data, pre->qual, instr);
    xsh_image_flag_bp(pre->errs, pre->qual, instr);
    mask_tmp=cpl_image_get_bpm(pre->data);
 
    check_msg( cpl_imagelist_set(dataList,cpl_image_duplicate(pre->data), 
      i),"Can't add Data Image %d to imagelist", i) ;
    check_msg( cpl_imagelist_set(errsList,cpl_image_duplicate(pre->errs), 
      i),"Can't add Data Image %d to imagelist", i) ;
 
    if(cpl_propertylist_has(pre->data_header,XSH_QC_CRH_NUMBER)) {
       ncrh_tot += xsh_pfits_get_qc_ncrh(pre->data_header);
    }
 
    /* combining bad pixels */
    if (i == 0) {
      mask_comb=cpl_mask_duplicate(mask_tmp);
      qual_comb = cpl_image_duplicate(pre->qual);
    } else {
      xsh_badpixelmap_image_coadd(&qual_comb, pre->qual, mode_and);
      cpl_mask_or(mask_comb,mask_tmp);
    }
 
    /* Free memory */
    if (i < (size-1)) {
      xsh_pre_free(&pre);
    }
  }
  //xsh_bpmap_mask_bad_pixel(qual_comb,mask_comb,QFLAG_INCOMPLETE_DATA);
  /*free memory on product extensions that will be re-allocated later on (errs is only modified) */
    xsh_free_image(&pre->data);
    xsh_free_image(&pre->qual);
    cpl_mask_delete(mask_comb);
 
 
  
  /* computes the median or sigma-clipped mean on data and errs */
    if(strcmp(stack_par->stack_method,"median") == 0) {
     pre->data = xsh_imagelist_collapse_median_create(dataList);
 
     check( xsh_collapse_errs(pre->errs,errsList,0));
 
   } else {
     pre->data=cpl_imagelist_collapse_sigclip_create(dataList,
         stack_par->klow,stack_par->khigh,.1,CPL_COLLAPSE_MEAN,NULL);
     check( xsh_collapse_errs(pre->errs,errsList,1));
 
   }
   int nrej=cpl_image_count_rejected(pre->data);
   if(nrej>0) {
     xsh_badpixel_flag_rejected(pre->data,qual_comb);
   }
 
   /* store combined bp map in qual extension of result */
   pre->qual = cpl_image_duplicate(qual_comb);
   xsh_free_image(&qual_comb);
 
    /* loop to remove cosmics */
    if ( size >= 3 && crh_clipping != NULL) {
       check(badpix_mask = cpl_mask_duplicate(xsh_pre_get_bpmap(pre)));
       check(xsh_find_cosmics(rawFrames,pre,crh_clipping,dataList,errsList,badpix_mask,instr,&crh_mask));
       xsh_free_mask(&badpix_mask);
    }
 
  /* Add PRO KW */
  //check( xsh_pfits_set_datancom( pre->data_header, size));
 
  /* remove irrelevant KW */
  check( cpl_propertylist_erase_regexp( pre->data_header, "PRO BIAS *", 0));
 
  /* Now save the relevant frame of PRE
   With: median image, errs image and bad pixel map
   */
  sprintf(result_name, "%s.fits", result_tag);
  xsh_msg_dbg_high("save frame %s tag=%s", result_name, result_tag);
 
  if(strstr(result_tag,"DARK") != NULL) {
     if(!cpl_propertylist_has(pre->data_header,XSH_QC_CRH_NUMBER_TOT)) {
         xsh_pfits_set_qc_ncrh_tot(pre->data_header,ncrh_tot);
     }
     if(!cpl_propertylist_has(pre->data_header,XSH_QC_CRH_NUMBER_MEAN)) {
         xsh_pfits_set_qc_ncrh_mean(pre->data_header,(float)ncrh_tot/size);
     }
  } else {
     //if(!cpl_propertylist_has(pre->data_header,XSH_QC_CRH_NUMBER_TOT)) {
         xsh_pfits_set_qc_ncrh_tot(pre->data_header,ncrh_tot);
         //}
         //if(!cpl_propertylist_has(pre->data_header,XSH_QC_CRH_NUMBER_MEAN)) {
         xsh_pfits_set_qc_ncrh_mean(pre->data_header,(float)ncrh_tot/size);
         //}
  }
  check(medFrame = xsh_pre_save( pre, result_name, result_tag,save_tmp));
  check(cpl_frame_set_tag(medFrame, result_tag));
 
  //AMo: add crh map save for QC
  if ((crh_mask != NULL) && ((crh_ima) != NULL)) {
    xsh_free_image(crh_ima);
    check(*crh_ima=cpl_image_new_from_mask(crh_mask));
    cpl_mask_not(crh_mask);
    xsh_bpmap_mask_bad_pixel(*crh_ima, crh_mask, QFLAG_COSMIC_RAY_REMOVED);
    //here good pixels are still flagged as '1'. To have proper code, '0',
    //we make a threshold
    cpl_image_threshold(*crh_ima, 1.1, DBL_MAX, 0, DBL_MAX);
    check(sprintf(name,"%s.fits",XSH_GET_TAG_FROM_ARM(XSH_CRH_MAP,instr)));
    //cpl_image_save (*crh_ima,name,CPL_BPP_32_SIGNED,NULL, CPL_IO_DEFAULT);
  }
 
  /* At this point, the images in dataList contain the "cpl bad pixel map"
   with the locally found CRh ==> should be used in compute_noise
   How should we return this list to the main recipe ==> in list
   */
 
  cleanup:
    if (cpl_error_get_code () != CPL_ERROR_NONE) {
      xsh_free_imagelist(&dataList);
      xsh_free_frame(&medFrame);
      if (list != NULL){
        *list = NULL;
      }
    }
    if (list != NULL){
      *list = dataList;
    }
    else{
      xsh_free_imagelist(&dataList);
    }
    //xsh_free_image(crh_ima);
    xsh_free_mask(&crh_mask);
    xsh_free_mask(&badpix_mask);
    xsh_pre_free(&pre);
    xsh_free_imagelist(&errsList) ;
    return medFrame ;
}
 
cpl_frame* 
xsh_remove_crh_multiple (cpl_frameset * rawFrames, 
                         const char *result_tag, 
                         xsh_stack_param* stack_par,
                         xsh_clipping_param * crh_clipping,
                               xsh_instrument* instrument,
                         cpl_imagelist ** list,
                         cpl_image** crh_ima,
                         const int save_tmp)
{
  cpl_frame * result = NULL ;
 
  XSH_ASSURE_NOT_NULL( rawFrames ) ;
  XSH_ASSURE_NOT_NULL( result_tag ) ;
  XSH_ASSURE_NOT_NULL( stack_par ) ;
  XSH_ASSURE_NOT_NULL( instrument ) ;
  check( result = xsh_remove_cosmics( rawFrames, result_tag, stack_par,crh_clipping,
                      instrument, list, crh_ima,save_tmp ) ) ;
 cleanup:
  return result ;
}
 
cpl_frame*
xsh_combine_offset(cpl_frameset * rawFrames, const char * result_tag,
    xsh_stack_param* stack_par,
    xsh_instrument * instrument, cpl_imagelist ** list, cpl_image** crh_ima,
    const int save_tmp)
{
  cpl_frame * result = NULL ;
 
  XSH_ASSURE_NOT_NULL( rawFrames ) ;
  XSH_ASSURE_NOT_NULL( result_tag ) ;
  XSH_ASSURE_NOT_NULL( stack_par ) ;
  XSH_ASSURE_NOT_NULL( instrument ) ;
 
  check( result = xsh_remove_cosmics( rawFrames, result_tag, stack_par,NULL,
                      instrument, list, crh_ima,save_tmp ) ) ;
 cleanup:
  return result ;
}