xsh_merge_ord.c

Go to the documentation of this file.

/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */
 
/*
 * $Author: amodigli $
 * $Date: 2012-12-18 14:15:45 $
 * $Revision: 1.71 $
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
  Includes
  -----------------------------------------------------------------------------*/
 
#include <math.h>
#include <xsh_drl.h>
 
#include <xsh_utils_table.h>
#include <xsh_badpixelmap.h>
#include <xsh_data_pre.h>
#include <xsh_dfs.h>
#include <xsh_pfits.h>
#include <xsh_error.h>
#include <xsh_msg.h>
#include <xsh_fit.h>
#include <xsh_data_instrument.h>
#include <xsh_data_rec.h>
#include <xsh_data_spectrum.h>
#include <xsh_ifu_defs.h>
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
  Functions prototypes
  ----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
  Implementation
  ----------------------------------------------------------------------------*/
 
/*****************************************************************************/
/*****************************************************************************/
static void xsh_merge_point( double flux_a, double weight_a,
  double flux_b, double weight_b, double *flux_res,
  double *err_res)
{
  XSH_ASSURE_NOT_NULL( flux_res);
  XSH_ASSURE_NOT_NULL( err_res);
 
  double tmp_val = 1.0/(weight_a+weight_b);
  *flux_res = (weight_a*flux_a+weight_b*flux_b) * tmp_val;
  *err_res =  sqrt(tmp_val);
 
  cleanup:
    return;
}
#if 0
static void
xsh_spectrum_clean(xsh_spectrum *spectrum,const int decode_bp){
 
     //double* spectrum_flux = xsh_spectrum_get_flux( spectrum);
     //double* spectrum_errs = xsh_spectrum_get_errs( spectrum);
     //int* spectrum_qual = xsh_spectrum_get_qual( spectrum);
 
     cpl_mask* mask = xsh_qual_to_cpl_mask(spectrum->qual,decode_bp);
 
     cpl_mask * filtered = xsh_bpm_filter(mask,5,5,CPL_FILTER_CLOSING);
     xsh_free_mask(&mask);
     cpl_image_save(spectrum->flux, "before.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
     cpl_image_reject_from_mask(spectrum->flux,filtered);
 
     cpl_image_reject_from_mask(spectrum->errs,filtered);
     cpl_detector_interpolate_rejected(spectrum->flux);
     cpl_detector_interpolate_rejected(spectrum->errs);
     cpl_image_save(spectrum->flux, "after.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
     xsh_free_mask(&filtered);
 
     return;
 
}
#endif
/*****************************************************************************/
/*****************************************************************************/
static cpl_frame * xsh_merge_ord_with_tag(cpl_frame *rec_frame,
    xsh_instrument *instrument, int merge_par, const char *tag) {
  cpl_frame *res_frame = NULL;
  xsh_rec_list *rec_list = NULL;
  xsh_spectrum *spectrum = NULL;
  //int i;
  int iorder = 0, islit = 0;
  int spectrum_size_lambda = 0;
  int spectrum_size_slit = 0;
  double lambda_min = 0.0, lambda_max = 0.0, lambda_step = 0.0;
  double *spectrum_flux = NULL;
  double *spectrum_errs = NULL;
  int *spectrum_qual = NULL;
  int spectrum_flux_index = 0;
  int *spectrum_by_lambda = NULL;
  const char* rec_pcatg = NULL;
  char* spectrum_name = NULL;
  const char* name = NULL;
  cpl_propertylist* header = NULL;
  int naxis = 0;
  int eso_mode = 0;
  int decode_bp = instrument->decode_bp;
  /* Check parameters */
  XSH_ASSURE_NOT_NULL( rec_frame);
  XSH_ASSURE_NOT_NULL( instrument);
  XSH_ASSURE_NOT_NULL( tag);
 
 
  /* Load rectified table */
  name = cpl_frame_get_filename(rec_frame);
  header = cpl_propertylist_load(name, 0);
  naxis = xsh_pfits_get_naxis(header);
  xsh_free_propertylist(&header);
  if (naxis == 2) {
    check( rec_list = xsh_rec_list_load_eso( rec_frame, instrument));
    eso_mode = 1;
  } else if (naxis == 1) {
    check( rec_list = xsh_rec_list_load_eso_1d( rec_frame, instrument));
    eso_mode = 1;
  } else {
    check( rec_list = xsh_rec_list_load( rec_frame, instrument));
  }
 
  check(lambda_min = xsh_pfits_get_rectify_lambda_min( rec_list->header));
  check( lambda_max = xsh_pfits_get_rectify_lambda_max( rec_list->header));
  check( lambda_step = xsh_pfits_get_rectify_bin_lambda( rec_list->header));
 
  /* Product spectrum */
  check( rec_pcatg = xsh_pfits_get_pcatg( rec_list->header));
 
  /* Check if 1D or 2D spectrum */
  if (strstr(rec_pcatg, XSH_ORDER2D) != NULL
      || strstr(rec_pcatg, XSH_MERGE2D) != NULL
      || strstr(rec_pcatg, XSH_COMBINED_OFFSET_2D_SLIT) != NULL
      || strstr(rec_pcatg, XSH_COMBINED_OFFSET_2D_IFU) != NULL) {
 
    double slit_min = 0.0, slit_max = 0.0, slit_step = 0.0;
 
    check( slit_min = xsh_pfits_get_rectify_space_min( rec_list->header));
    check( slit_max = xsh_pfits_get_rectify_space_max( rec_list->header));
    check( slit_step = xsh_pfits_get_rectify_bin_space( rec_list->header));
 
    xsh_msg_dbg_high(
        "Construct 2D spectrum %f %f %f", slit_min, slit_max, slit_step);
    xsh_msg_dbg_medium(
        "lambda min %f max %f step %f", lambda_min, lambda_max, lambda_step);
    check(
        spectrum = xsh_spectrum_2D_create( lambda_min, lambda_max, lambda_step, slit_min, slit_max, slit_step));
  } else {
    xsh_msg_dbg_high("Construct 1D spectrum");
    xsh_msg_dbg_medium(
        "lambda min %f max %f step %f", lambda_min, lambda_max, lambda_step);
    check(
        spectrum = xsh_spectrum_1D_create( lambda_min, lambda_max, lambda_step));
  }
 
  spectrum_name = xsh_stringcat_any(tag, ".fits", (void*)NULL);
 
  check( spectrum_size_lambda = xsh_spectrum_get_size_lambda( spectrum));
  check( spectrum_size_slit = xsh_spectrum_get_size_slit( spectrum));
  check( spectrum_flux = xsh_spectrum_get_flux( spectrum));
  check( spectrum_errs = xsh_spectrum_get_errs( spectrum));
  check( spectrum_qual = xsh_spectrum_get_qual( spectrum));
  XSH_CALLOC( spectrum_by_lambda, int, spectrum->size);
  //xsh_msg("spectrum_size_slit=%d",spectrum_size_slit);
  xsh_msg_dbg_medium  ( "Spectrum size (Lambda X slit) = (%d X %d) \n"
      "Lambda %f to %f", spectrum_size_lambda, spectrum_size_slit,
      lambda_min, lambda_min+lambda_step*(spectrum_size_lambda-1));
  /*
   xsh_msg("ord_max=%d spectrum_size_slit %d",
   rec_list->size-1,spectrum_size_slit);
   */
  int problem_slit_size = 0;
  /* loop over spatial direction: we merge each spectrum along the slit */
  for (islit = 0; islit < spectrum_size_slit; islit++) {
    for (iorder = rec_list->size - 1; iorder >= 0; iorder--) {
      int nlambda = xsh_rec_list_get_nlambda(rec_list, iorder);
      double* lambda = xsh_rec_list_get_lambda(rec_list, iorder);
      float *flux = xsh_rec_list_get_data1(rec_list, iorder);
      float *errs = xsh_rec_list_get_errs1(rec_list, iorder);
      int *qual = xsh_rec_list_get_qual1(rec_list, iorder);
      int nslit = xsh_rec_list_get_nslit(rec_list, iorder);
      int ilambda = 0;
      double n;
      /*
       xsh_msg("iorder=%d islit %d nslit=%d nlambda=%d",
       iorder,islit,nslit,nlambda);
       */
      if (spectrum_size_slit != nslit) {
        /* AMO: REFLEX: move this check out of the loop to decrease log size
         xsh_msg_warning("spectrum size slit %d is different from order size %d",
         spectrum_size_slit, nslit);
         */
        problem_slit_size = 1;
      }
 
      xsh_msg_dbg_high(
          "slit %d lambda %f %f", islit, lambda[0], lambda[nlambda-1]);
      /*
       xsh_msg( "slit %d lambda %f %f", islit,
       lambda[0], lambda[nlambda-1]);
       */
      n = (lambda[0] - lambda_min) / lambda_step;
      spectrum_flux_index = (int) xsh_round_double(n);
 
      xsh_msg_dbg_high(
          "iorder %d begin at index %f --> %d", iorder, n, spectrum_flux_index);
 
      if (islit < nslit) {
        int ilambda_off=spectrum_flux_index+ islit*spectrum_size_lambda;
        int islit_nlambda = islit * nlambda;
        /* look over wavelength */
        for (ilambda = 0; ilambda < nlambda; ilambda++) {
 
 
          int i_rec_order = ilambda + islit_nlambda;
          int i_spectrum = ilambda_off + ilambda;
          /* skip merging if error associated to given point is zero.
           * this may happen if extraction has not found enough slices
           * with good points to get the cross-order spectrum extraction profile
           * In this case the merged flux and associated error os 0, but the
           * qualifier is the same as the one of the original spectrum
           */
          if (errs[i_rec_order] == 0) {
            spectrum_qual[i_spectrum] = qual[i_rec_order];
            continue;
          }
 
          /* The following setting may be useful for debug purpose to check
           * at which wavelength we are
           * double wav=ilambda*lambda_step+lambda[0];
           */
          /* initialise the merged spectrum value with the corresponding values
           * from the extracted profile. Uses the array spectrum_by_lambda to
           * check if such event already occurred for a If this is the case at
           * a given wavelength, it means that there is overlap between the
           * current order spectrum data point and the one of the previous
           * order. In that case we need to merge the intensity of the two data
           * points. Correspondingly we distinguish three cases:
           * A good, B bad: use only A contribute
           * A bad, B good: use only B contribute
           * A, B good: use weighted mean from A and B contributes
           * Finally we set the result to the proper value.
           */
          if (spectrum_by_lambda[i_spectrum] == 0) {
            spectrum_by_lambda[i_spectrum] = 1;
            spectrum_flux[i_spectrum] = flux[i_rec_order];
            spectrum_errs[i_spectrum] = errs[i_rec_order];
            spectrum_qual[i_spectrum] = qual[i_rec_order];
          } else {
            double flux_a, err_a, weight_a;
            double flux_b, err_b, weight_b;
            double flux_res = 0.0, err_res = 1.0;
            int qual_a, qual_b, qual_res;
 
            flux_a = spectrum_flux[i_spectrum];
            err_a = spectrum_errs[i_spectrum];
            qual_a = spectrum_qual[i_spectrum];
 
            flux_b = flux[i_rec_order];
            err_b = errs[i_rec_order];
            qual_b = qual[i_rec_order];
 
            /* Predefine outputs as from first A spectra */
            flux_res = flux_a;
            err_res = err_a;
            qual_res = qual_a;
            //xsh_msg("wave=%g",lambda[ilambda]);
            /* A good, B bad */
            if (((qual_a & decode_bp) == 0) && ((qual_b & decode_bp) > 0)) {
              flux_res = flux_a;
              err_res = err_a;
              qual_res = qual_a;
 
            /* A bad, B good */
            } else if (((qual_a & decode_bp) > 0) && ((qual_b & decode_bp) == 0)) {
              flux_res = flux_b;
              err_res = err_b;
              qual_res = qual_b;
 
            /* A and B good, or A and B bad */
            } else {
 
              
              weight_a = 1.0 / (err_a * err_a);
              weight_b = 1.0 / (err_b * err_b);
              
              check(xsh_merge_point( flux_a, weight_a, flux_b, weight_b, &flux_res, &err_res));
              qual_res = qual_a | qual_b;
 
            }
 
            /* Save values */
            spectrum_flux[i_spectrum] = flux_res;
            spectrum_errs[i_spectrum] = err_res;
            spectrum_qual[i_spectrum] = qual_res;
            spectrum_by_lambda[i_spectrum] = 1;
          }
          //xsh_msg("loop lambda ok");
        }
        //xsh_msg("check slit ok");
      }
    }/* end order*/
 
  }/*end slit*/
  if (problem_slit_size == 1) {
    xsh_msg_warning("spectrum size slit is different from order size");
  }
  /* Set the header from input frame */
  if (eso_mode) {
    cpl_propertylist_erase_regexp(rec_list->header, "^CRVAL1", 0);
    cpl_propertylist_erase_regexp(rec_list->header, "^CRVAL2", 0);
    cpl_propertylist_erase_regexp(rec_list->header, "^CRPIX1", 0);
    cpl_propertylist_erase_regexp(rec_list->header, "^CRPIX2", 0);
    cpl_propertylist_erase_regexp(rec_list->header, "^CDELT1", 0);
    cpl_propertylist_erase_regexp(rec_list->header, "^CDELT2", 0);
  }
  check( cpl_propertylist_append( spectrum->flux_header, rec_list->header));
 
  const char* bunit = xsh_pfits_get_bunit(spectrum->flux_header);
  xsh_pfits_set_bunit(spectrum->errs_header, bunit);
  /* AMO to verify quality of extraction */
  //xsh_spectrum_clean(spectrum,decode_bp);
 
  check( res_frame = xsh_spectrum_save(spectrum, spectrum_name,tag));
  check( cpl_frame_set_tag( res_frame, tag));
 
  cleanup: xsh_spectrum_free(&spectrum);
  xsh_rec_list_free(&rec_list);
  XSH_FREE( spectrum_by_lambda);
  XSH_FREE( spectrum_name);
  return res_frame;
}
/*****************************************************************************/
/*****************************************************************************/
/*****************************************************************************/
cpl_frame * xsh_merge_ord( cpl_frame * rec_frame, xsh_instrument* instrument,
               int merge_par,const char* rec_prefix)
{
  cpl_frame *res_frame = NULL;
  xsh_msg("Merge slit orders");
  check( res_frame = xsh_merge_ord_slitlet( rec_frame, instrument, merge_par, 
                        CENTER_SLIT,rec_prefix));
 
  cleanup:
    return res_frame; 
}
/*****************************************************************************/
/*****************************************************************************/
cpl_frame * xsh_merge_ord_slitlet( cpl_frame * rec_frame, 
                   xsh_instrument* instrument, 
                   int merge_par, int slitlet,
                   const char* rec_prefix)
{
  cpl_frame *res_frame = NULL;
  cpl_propertylist *header = NULL;
  const char *rec_pcatg = NULL;
  const char *tag_suf = NULL;
  const char *filename = NULL;
  char tag[256];
 
  /*Check parameters */
  XSH_ASSURE_NOT_NULL( rec_frame);
  XSH_ASSURE_NOT_NULL( instrument);
 
  /* Check 1D or 2D spectrum */
  check( filename = cpl_frame_get_filename( rec_frame));
  check( header = cpl_propertylist_load( filename, 0));
  check( rec_pcatg = xsh_pfits_get_pcatg( header));
  check(tag_suf = cpl_frame_get_tag(rec_frame));
 
  if ( strstr( rec_pcatg, XSH_FLUX_ORDER2D ) != NULL ) {
    tag_suf = XSH_GET_TAG_FROM_ARM( XSH_FLUX_MERGE2D, instrument);
  }
  else if ( strstr( rec_pcatg, XSH_NORM_ORDER2D ) != NULL ) {
    tag_suf = XSH_GET_TAG_FROM_ARM( XSH_NORM_MERGE2D, instrument);
  }
//  else if ( strstr( rec_pcatg, XSH_SKY_ORDER2D ) != NULL ) {
//    tag_suf = XSH_GET_TAG_FROM_SLITLET( XSH_SKY_MERGE2D, slitlet,
//      instrument);
//  }
  else if ( strstr( rec_pcatg, XSH_ORDER2D ) != NULL ) {
    tag_suf = XSH_GET_TAG_FROM_SLITLET( XSH_MERGE2D, slitlet,
      instrument);
  }
  else if ( strstr( rec_pcatg, XSH_COMBINED_OFFSET_2D_SLIT ) != NULL){
    /* No IFU */
    tag_suf = XSH_GET_TAG_FROM_ARM( XSH_MERGE2D,
      instrument);
  }
  else if ( strstr( rec_pcatg, XSH_ORDER_EXT1D ) != NULL){
    /* No IFU */
    tag_suf = XSH_GET_TAG_FROM_ARM( XSH_MERGE_EXT1D,
      instrument);
  }
  else if ( strstr( rec_pcatg, XSH_ORDER_OXT1D ) != NULL){
    /* No IFU */
    tag_suf = XSH_GET_TAG_FROM_ARM( XSH_MERGE_OXT1D,
      instrument);
  }
  else if ( strstr( rec_pcatg, XSH_FLUX_OXT1D ) != NULL){
    tag_suf = XSH_GET_TAG_FROM_ARM( XSH_FLUX_MOXT1D, instrument);
  }
  else if ( strstr( rec_pcatg, XSH_FLUX_ORDER1D ) != NULL){
    tag_suf = XSH_GET_TAG_FROM_ARM( XSH_FLUX_MERGE1D, instrument);
  }
  else if ( strstr( rec_pcatg, XSH_NORM_ORDER1D ) != NULL){
    tag_suf = XSH_GET_TAG_FROM_ARM( XSH_NORM_MERGE1D, instrument);
  }
  else if ( strstr( rec_pcatg, XSH_ORDER1D ) != NULL){
    tag_suf = XSH_GET_TAG_FROM_SLITLET( XSH_MERGE1D, slitlet,
      instrument);
  }
  else{
    tag_suf="";
    xsh_msg_error("Invalid PRO.CATG %s for file %s", rec_pcatg, filename);
  }
  sprintf(tag,"%s_%s",rec_prefix,tag_suf);
  check( res_frame = xsh_merge_ord_with_tag( rec_frame, instrument,
    merge_par, tag));
 
  cleanup:
    xsh_free_propertylist( &header);
    return res_frame ;
}
 
/*****************************************************************************/
static void xsh_frame_set_shift_ref( cpl_frame *rec_frame, cpl_frame *shift_frame)
{
  cpl_propertylist *rec_header = NULL;
  cpl_propertylist *shift_header = NULL;
  const char *rec_name = NULL;
  const char *shift_name = NULL;  
  double waveref, slitref;
  
  XSH_ASSURE_NOT_NULL( rec_frame);
  XSH_ASSURE_NOT_NULL( shift_frame);
  
  check( rec_name = cpl_frame_get_filename( rec_frame));
  check( shift_name = cpl_frame_get_filename( shift_frame));
  
  check( rec_header = cpl_propertylist_load( rec_name, 0));
  check( shift_header = cpl_propertylist_load( shift_name, 0));
  
  waveref = xsh_pfits_get_shiftifu_lambdaref( shift_header);
  slitref = xsh_pfits_get_shiftifu_slitref( shift_header);
  
  if (cpl_error_get_code() == CPL_ERROR_NONE){
    check( xsh_pfits_set_shiftifu_lambdaref( rec_header, waveref));
    check( xsh_pfits_set_shiftifu_slitref( rec_header, slitref));
    check( cpl_propertylist_save( rec_header, rec_name, CPL_IO_EXTEND));
  }
  xsh_error_reset();
  
  cleanup:
    xsh_free_propertylist( &rec_header);
    xsh_free_propertylist( &shift_header);
    return;  
}
/*****************************************************************************/
 
/*****************************************************************************/
/*****************************************************************************/
cpl_frameset* xsh_merge_ord_ifu(cpl_frameset* rec_frameset,
                  xsh_instrument* instrument, 
                int merge_par,
                const char* rec_prefix)
{
  cpl_frameset *result_set = NULL;
  cpl_frameset *drl_frameset = NULL;
  int slitlet;
  int i=0;
 
  XSH_ASSURE_NOT_NULL( rec_frameset);
  XSH_ASSURE_NOT_NULL( instrument);
 
  xsh_msg("Merge IFU orders");
  check( result_set = cpl_frameset_new());
  check( drl_frameset = xsh_frameset_drl_frames( rec_frameset));
 
  /* Loop over the 3 IFU slitlets */
  for( slitlet = LOWER_IFU_SLITLET; slitlet <= UPPER_IFU_SLITLET; slitlet++){
    cpl_frame * rec_frame = NULL ;
    cpl_frame * ord_frame = NULL ;
 
    check( rec_frame = cpl_frameset_get_frame( drl_frameset, i));
    
    /* Now merge order for this slitlet */
    check( ord_frame = xsh_merge_ord_slitlet( rec_frame, instrument,
                          merge_par,slitlet,rec_prefix));
                          
    check( xsh_frame_set_shift_ref( rec_frame, ord_frame));
    xsh_msg( "Merge for Slitlet %s, %s", SlitletName[slitlet],
      cpl_frame_get_filename( ord_frame)) ;
    check( cpl_frameset_insert( result_set, ord_frame));
    i++;
  }
  
  cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE){
      xsh_free_frameset( &result_set);
    }
    xsh_free_frameset( &drl_frameset);
    return result_set ;
}
/*--------------------------------------------------------------------------*/