xsh_compute_noise_map.c

Go to the documentation of this file.

/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */
 
/*
 * $Author: amodigli $
 * $Date: 2012-06-13 09:24:33 $
 * $Revision: 1.57 $
 * $Name: not supported by cvs2svn $
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
#include <math.h>
 
#include <xsh_drl.h>
#include <xsh_data_pre.h>
#include <xsh_dfs.h>
#include <xsh_pfits.h>
#include <xsh_error.h>
#include <xsh_msg.h>
#include <xsh_badpixelmap.h>
 
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Typedefs
 -----------------------------------------------------------------------------*/
 
/*-----------------------------------------------------------------------------
                            Functions prototypes
 -----------------------------------------------------------------------------*/
 
/*-----------------------------------------------------------------------------
                              Implementation
 -----------------------------------------------------------------------------*/
 
static void add_noisy_pixel_to_ascii_file( int x, int y )
{
  static FILE *fout ;
  const char * fname = "noisy_pixels.dat" ;
 
  if ( fout == NULL )
    fout = fopen( fname, "w" ) ;
  fprintf( fout, "%d %d\n", x, y ) ;
  if ( fout ) fclose( fout ) ;
 
}
 
static int
flag_noisy_pixels (cpl_imagelist * raws,
           cpl_image * bpmap,
           cpl_image * noisymap,
           int nx, int ny,
           xsh_clipping_param * noise_clipping,xsh_instrument* inst)
{
  int ix, iy, i;
  cpl_image **pimg = NULL; 
  int nimg, nbad = 0, npix = 0;
  double medStackv = 0., 
  aveStackv = 0., 
  sumStackv = 0., errorStackv = 0.;
  double *Stackv = NULL, 
  *pStackv = NULL;
  double *stack = NULL;
  double *bisStackv = NULL, 
  *pbis = NULL;
  double diff = 0.0, diff2 = 0.0;
  double min = 999999., max = -9999999.;
  double maxdelta = 0.0;
  cpl_binary *bpmap_mask = NULL;
  int idpix = 0;
  float ** pixdata = NULL;
  cpl_binary **pixmask = NULL;
  int nhot = 0;
  int totpix = nx * ny;
  nimg = cpl_imagelist_get_size(raws);
 
  XSH_MALLOC(pimg, cpl_image*, nimg);
 
  /* Populate images pointer array */
for(  i = 0; i < nimg; i++) {
    *(pimg + i) = cpl_imagelist_get (raws, i);
  }
  /* Create array of sigmas */
  XSH_MALLOC(Stackv, double, nx * ny);
 
pStackv  = Stackv;
 
  xsh_msg_dbg_low( "Rejected in bpmap: %" CPL_SIZE_FORMAT "", cpl_image_count_rejected( bpmap ));
 
  /* allocate a temporary array for 1 pixel stack */
  XSH_MALLOC(stack, double, nimg);
 
  /* Get bad pixel mask */
bpmap_mask  = cpl_mask_get_data(cpl_image_get_bpm(bpmap));
  assure( bpmap_mask != NULL, cpl_error_get_code(),
      "Cant get bpmap Mask" );
 
  XSH_MALLOC(bisStackv, double, totpix);
pbis  = bisStackv;
 
  /* Get pixel data and masks of images */
  {
    int j;
    XSH_MALLOC(pixdata, float *, nimg);
XSH_MALLOC    (pixmask,cpl_binary *,nimg);
 
    for (j = 0; j < nimg; j++) {
      *(pixdata + j) = cpl_image_get_data_float(*(pimg + j));
      *(pixmask + j) = cpl_mask_get_data(cpl_image_get_bpm(*(pimg + j)));
    }
  }
  idpix = 0;
 
  for (iy = 1; iy <= ny; iy++) {
    for (ix = 1; ix <= nx; ix++) {
      int j, count = 0;
      double sum = 0.;
 
      /* Loop over all images (1 pixel) */
      if ((*(bpmap_mask + idpix) & inst->decode_bp) > 0) {
        nbad++;
        xsh_msg_dbg_medium("Pixel %d,%d globally bad", ix, iy);
        *pStackv = 0.;
      } else {
        for (j = 0; j < nimg; j++) {
          int rej;
          double fval = *(*(pixdata + j) + idpix);
          rej = (*(*(pixmask + j) + idpix ) & inst->decode_bp);
          //xsh_msg( "pixel[%d,%d] (%d) = %lf", ix, iy, idpix, fval ) ;
          if (rej > 0) {
            /* bad pixel */
            xsh_msg_dbg_medium("Don't use pixel %d,%d [%d]", ix, iy, j);
            continue;
          }
          sum += fval;
 
          *(stack + count) = fval;
          count++;
        }
      }
 
      if (count > 1) {
        /* Calculate sigma of this stack */
        double avg = sum / (double) count;
        double dif, dif2 = 0., sigma = 0.;
        int k;
 
        xsh_msg_dbg_medium(
            "          [%d,%d] Count=%d - sum = %lf, avg = %lf", ix, iy, count, sum, avg);
        //TEST
        for (k = 0; k < count; k++) {
          xsh_msg_dbg_medium( "             stack[%d] = %lf", k, *(stack+k));
          dif = *(stack + k) - avg;
          dif2 += dif * dif;
        }
        sigma = sqrt(dif2 / (double) (count - 1));
        *pStackv = sigma;
        *pbis++ = sigma;
        sumStackv += sigma;
        xsh_msg_dbg_medium(
            "++ dif2 = %lf, count= %d, sigma=%lf, Stackv=%lf, sum=%lf", dif2, count, sigma, *pStackv, sumStackv);
        if ((npix % 500) == 0)
          xsh_msg_dbg_medium(
              "+++++ (%d,%d) sumStackv = %lf", ix, iy, sumStackv);
        npix++;
      } else {
        xsh_msg_dbg_medium( "Not enough good pixels (%d)", count);
        *pStackv = 0.;
      }
      pStackv++;
      idpix++;
    } /* end loop on x */
  } /* end loop on y */
  xsh_msg_dbg_medium( "+++++ sumStackv = %lf", sumStackv);
  /* Calculate medstackv
   ATTENTION: La fonction cpl_tools_get_median_double modifie le
   vecteur !
   */
  medStackv = xsh_tools_get_median_double(bisStackv, npix);
 
  /* Calculate aveStackv */
  aveStackv = sumStackv / (double) npix;
  xsh_msg_dbg_low(
      "--- Npix=%d - sumStackv=%lf - aveStackv=%lf", npix, sumStackv, aveStackv);
 
  /* Calculate errorStackv */
  //for (pStackv = Stackv, i = 0; i < npix; i++, pStackv++) {
  for (pStackv = bisStackv, i = 0; i < npix; i++, pStackv++) {
    diff = *pStackv - aveStackv;
    if (diff < min)
      min = diff;
    if (diff > max)
      max = diff;
    diff2 += diff * diff;
  }
  cpl_free(bisStackv);
  bisStackv = NULL;
 
  xsh_msg_dbg_low("Diff Min: %lf, Max: %lf", min, max);
  errorStackv = sqrt(diff2 / (double) (npix - 1));
  xsh_msg_dbg_low( "errorStackv=%lf", errorStackv);
 
  /* Now sigma clipping */
  maxdelta = noise_clipping->sigma * errorStackv;
 
  xsh_msg_dbg_low(
      "Npix:%d/%d, sum: %lf, med:%lf, avg:%lf, err:%lf, delta: %lf", npix, nx * ny, sumStackv, medStackv, aveStackv, errorStackv, maxdelta);
 
  pStackv = Stackv;
  idpix = 0;
 
  for (iy = 1; iy <= ny; iy++) {
    for (ix = 1; ix <= nx; ix++) {
      if ( (*(bpmap_mask + idpix) & inst->decode_bp) == 0 ) {
        double delta = *pStackv - medStackv;
        if (fabs(delta) > maxdelta) {
          nhot++;
          /* Flag this pixel in bpmap */
          xsh_bpmap_set_bad_pixel(bpmap, ix, iy, QFLAG_ELECTRONIC_PICKUP);
          if (xsh_debug_level_get() >= XSH_DEBUG_LEVEL_LOW)
            add_noisy_pixel_to_ascii_file(ix, iy);
          // And in the noisymap as well !
          xsh_bpmap_set_bad_pixel(noisymap, ix, iy, QFLAG_ELECTRONIC_PICKUP);
 
        }
 
      }
      pStackv++;
      idpix++;
    }
  }
  xsh_msg( "Found %d Electronic Pickup Noise Hot Pixels", nhot );
 
  cleanup:
  XSH_FREE(stack);
  XSH_FREE(Stackv);
  XSH_FREE(bisStackv);
  XSH_FREE(pimg);
  XSH_FREE(pixdata);
  XSH_FREE(pixmask);
  return nhot;
}
 
static void
set_pickup_noise_pixels_qc (cpl_propertylist * header, int nbad,
             xsh_instrument * instrument)
{
    xsh_pfits_set_qc( header, (void *)&nbad,
              XSH_QC_BP_MAP_PICKUP_NOISE_PIX, instrument ) ;  
}
 
cpl_frame *
xsh_compute_noise_map (cpl_imagelist * dataList,
               cpl_frame * medFrame,
               xsh_clipping_param * noise_clipping,
               xsh_instrument* instr,
                cpl_frame ** noisyFrame
               )
{
  int nx, ny, nframes = 0;
  cpl_image *resBpMap = NULL;
  cpl_propertylist *bpmapHeader = NULL;
  cpl_frame *resFrame = NULL;
  int prevnbad;
  const char *bpmapFile = cpl_frame_get_filename (medFrame);
  xsh_pre *medPre = NULL;
  int iter = 0,curnbad = 0;
  char  result_name[256] ;
  const char* tag = XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,instr);
  char noisy_name[256];
  cpl_image * noisyMap = NULL ;
  cpl_propertylist * noisymapHeader = NULL ;
 
  int binx=0;
  int biny=0;
  cpl_image* ima_aux=NULL;
 
  /*
     Compute noise for each pixel stack
     Compare with average noise
     Declare pixel bad is difference exceeds clip_sigma
   */
 
  cpl_msg_indent_more ();
  xsh_msg ("*** Removing Noisy Pixels (%s)", 
          xsh_instrument_arm_tostring(instr));
 
  /* Get bad pixel map (from medFrame) */
  xsh_msg ("Bpmap file = \"%s\"", bpmapFile);
 
  /* Loader PRE structure */
  medPre = xsh_pre_load (medFrame,instr);
  XSH_ASSURE_NOT_NULL(medPre);
 
  resBpMap = xsh_pre_get_qual (medPre);
  bpmapHeader = medPre->qual_header;
 
  nx = medPre->nx;
  ny = medPre->ny;
 
  check( noisyMap = cpl_image_new( nx, ny, CPL_TYPE_INT ) ) ;
  check( noisymapHeader = cpl_propertylist_duplicate( bpmapHeader ) ) ;
  xsh_msg( "    Image size: %d,%d", nx, ny ) ;
 
  xsh_set_image_cpl_bpmap (resBpMap, resBpMap, instr->decode_bp);
  prevnbad = cpl_image_count_rejected (resBpMap);
 
  nframes = cpl_imagelist_get_size (dataList);
  xsh_msg ("%d dark images in image list", nframes);
 
  /* Iterate */
  for (iter = 0; iter < noise_clipping->niter; iter++) {
    int nhot=0 ;
 
    xsh_msg (">>>> Iteration Nb %d/%d", iter+1, noise_clipping->niter);
    cpl_msg_indent_more ();
 
    if(nframes>1) {
    /* Suppress noisy pixels and add to bad pixel map as ???  */
    nhot = flag_noisy_pixels (dataList, resBpMap, noisyMap,
                  nx, ny, noise_clipping,instr);
    }
    cpl_msg_indent_less ();
    if ( nhot == 0 ) break ;
  }
 
  /* Print total number of bad pixels */
  curnbad = cpl_image_count_rejected (resBpMap);
  xsh_msg ("End of noisy pixels, total bad pixels: %d", curnbad);
 
  xsh_msg ("  Nb of noisy pixels: %d", curnbad - prevnbad);
 
  /* Populate resFrame from medFrame + badpixelmap */
  /* Now save the relevant frame of PRE
     With: median image, errs image and bad pixel map
   */
  //strcpy(result_name,xsh_stringcat_any( tag, ".fits", "" )) ;
  strcpy(result_name,bpmapFile) ;
  xsh_msg ("save frame %s\n", result_name);
 
  /* Calculate QC parameters */
  /* QC.DARKMED.AVE
     QC.DARKMED.STDEV
     QC.BP-MAP.NBADPIX = current_nb_badpix - previous_nb_badpix
   */
  check(set_pickup_noise_pixels_qc (medPre->data_header, curnbad, instr ));
  /* Add QC.NHPIX parameters */
  check(xsh_pfits_set_qc_nhpix( medPre->data_header, curnbad - prevnbad )) ;
  if ( xsh_instrument_get_arm(instr) == XSH_ARM_NIR ) {
     sprintf(noisy_name,"%s.fits",XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,instr));
  } else {
    check(binx=xsh_pfits_get_binx(medPre->data_header));
    check(biny=xsh_pfits_get_biny(medPre->data_header));
    sprintf(noisy_name,"%s_%dx%d.fits",XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,
                                                            instr),
            binx,biny);
  }
  ima_aux=cpl_image_cast(medPre->data,CPL_TYPE_FLOAT);
  check(cpl_image_save (ima_aux,result_name, CPL_BPP_IEEE_FLOAT,
            medPre->data_header, CPL_IO_DEFAULT));
 
  xsh_free_image(&ima_aux);
 
  xsh_pfits_set_extname (medPre->errs_header, "ERRS");
  ima_aux=cpl_image_cast(medPre->errs,CPL_TYPE_FLOAT);
  check(cpl_image_save (ima_aux,result_name, CPL_BPP_IEEE_FLOAT,
            medPre->errs_header, CPL_IO_EXTEND));
  xsh_free_image(&ima_aux);
 
 
  xsh_pfits_set_extname (bpmapHeader, "QUAL");
  check(cpl_image_save (resBpMap, result_name, XSH_PRE_DATA_BPP, bpmapHeader,
            CPL_IO_EXTEND));
 
 
  check(resFrame=xsh_frame_product(result_name,tag,
                                   CPL_FRAME_TYPE_IMAGE,
                                   CPL_FRAME_GROUP_PRODUCT,
                                   CPL_FRAME_LEVEL_FINAL));
 
  // save hot pixel image
  /* Add QC.NHPIX parameters */
  set_pickup_noise_pixels_qc (noisymapHeader, curnbad, instr );
  xsh_pfits_set_qc_nhpix( noisymapHeader, (curnbad - prevnbad) ) ;
  cpl_image_save (noisyMap, noisy_name, XSH_PRE_DATA_BPP,
          noisymapHeader,
          CPL_IO_DEFAULT );
  xsh_add_temporary_file(noisy_name);
  // Create Frame
  check(*noisyFrame=xsh_frame_product(noisy_name,tag,
                                   CPL_FRAME_TYPE_IMAGE,
                                   CPL_FRAME_GROUP_PRODUCT,
                                   CPL_FRAME_LEVEL_TEMPORARY));
  //xsh_add_temporary_file(noisy_name);
 
cleanup:
 
  xsh_free_propertylist(&noisymapHeader);
  xsh_free_image(&noisyMap);
  xsh_free_image(&ima_aux);
  xsh_pre_free( &medPre ) ;
 
  return resFrame ;
}
 
/*----------------------------------------------------------------------------*/