xsh_order_table_from_fmtchk.c

Go to the documentation of this file.

/*                                                                           *
 *   This file is part of the ESO X-shooter Pipeline                         *
 *   Copyright (C) 2006 European Southern Observatory                        *
 *                                                                           *
 *   This library is free software; you can redistribute it and/or modify    *
 *   it under the terms of the GNU General Public License as published by    *
 *   the Free Software Foundation; either version 2 of the License, or       *
 *   (at your option) any later version.                                     *
 *                                                                           *
 *   This program is distributed in the hope that it will be useful,         *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of          *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
 *   GNU General Public License for more details.                            *
 *                                                                           *
 *   You should have received a copy of the GNU General Public License       *
 *   along with this program; if not, write to the Free Software             *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA    *
 *                                                                           */
 
/*
 * $Author: amodigli $
 * $Date: 2009-09-27 10:40:48 $
 * $Revision: 1.19 $
*/
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*---------------------------------------------------------------------------*/
/*---------------------------------------------------------------------------
  Includes
  ----------------------------------------------------------------------------*/
 
#include <math.h>
#include <xsh_drl.h>
#include <xsh_data_order.h>
#include <xsh_data_spectralformat.h>
#include <xsh_error.h>
#include <xsh_utils.h>
#include <xsh_msg.h>
#include <xsh_data_pre.h>
#include <xsh_pfits.h>
#include <xsh_parameters.h>
 
#include <cpl.h>
 
typedef struct {
  int order ;           
  int pos_x ;
  int pos_y ;
  double flux ;
} CENTER_ORDER ;
 
typedef struct {
  int absorder ;
  int start_x ;
} START_ORDER ;
 
/*
  In the spectral format table:
    DIST_ORDER (In header keyword OR recipe parameter)
    ABS_ORDER   In the table
    XDISP_CEN   .... The X position of the center of the order
 
  Starting from each possible measured order, calculate
  the delta between theoretical (from spectral format table) and measured
   orders.
  Evaluate the validity (mean value of deltas)
  Keep the best as the starting abs order.
 
  The theoretical distance i scalculated from the XDISP_CEN column of the
  spectral format table and the DIST ORDER keyword from the header of the
  spectral format frame.
*/
 
static void compute_dist_pixel( int * xdisp, int * pdist, int dist_order,
                int first, int last )
{
  int i, nb ;
 
  nb = last - first ;
  memset( pdist, 0, nb*sizeof(int) ) ;
 
  for( i = 0 ; (i+dist_order) < nb ; i++, pdist++ ) {
    *pdist = xdisp[i] - xdisp[i+dist_order] ;
    xsh_msg( " Order: %d, Distance: %d\n", first+i, *pdist ) ;
  }
 
}
 
static int compute_abs_order( START_ORDER * start_x, int nb,
                  cpl_frame * spectral_frame,
                  xsh_detect_continuum_param * detect_param,
                  xsh_instrument * instrument )
{
  int dist_order ;
  xsh_spectralformat_list * spectral_list = NULL ;
  int * abs_orders = NULL ; 
  int * dist_pixel = NULL ; 
  int * xdisp_cen = NULL ;
  int spec_size=0 ;     
  int i=0 ;
  int * distances = NULL, * delta = NULL, * the_absorder = NULL ;
  int ndist=0;
  int good_order=0;
  int ngood=0 ;
  int first_order=0;        
  int  last_order=0 ;
  int min_idx=0 ;
  XSH_ARM the_arm ;
 
  xsh_msg( ">>>> Entering compute_abs_order" ) ;
  the_arm = xsh_instrument_get_arm( instrument ) ;
 
  /* Load spectral format list */
  check( spectral_list = xsh_spectralformat_list_load(spectral_frame,
                              instrument)) ;
  spec_size = spectral_list->size ;
  xsh_msg( "  Nb of Orders in Spectral Format: %d", spec_size ) ;
 
  /* and get all relevant data needed */
  XSH_CALLOC(dist_pixel, int, spec_size ) ;
  XSH_CALLOC(xdisp_cen, int, spec_size ) ;
  XSH_CALLOC(abs_orders, int, spec_size ) ;
  check(xsh_spectralformat_list_get_xdisp_cen( spectral_list,
                        abs_orders, xdisp_cen));
  if ( detect_param->dist_order == 0 ) dist_order = spectral_list->dist_order ;
  else dist_order = detect_param->dist_order ;
  xsh_msg( "    DIST_ORDER = %d", dist_order ) ;
  first_order = abs_orders[0] ;
  last_order = abs_orders[spec_size-1]+1 ;
  compute_dist_pixel( xdisp_cen, dist_pixel, dist_order, first_order,
              last_order ) ;
 
  xsh_msg( "First Order: %d, Last Order: %d", first_order, last_order ) ;
  /* Calculate distances berween orders */
  XSH_CALLOC( distances, int, nb ) ;
  xsh_msg( "Calcul Distances" ) ;
  for( ndist = 0, i = 0 ; i<nb ; i++, ndist++ ) {
    int j ;
 
    j = i+dist_order ;
    if ( j >= nb ) break ;
    distances[i] = abs((start_x+i)->start_x - (start_x+j)->start_x ) ;
    xsh_msg( "distances[%d] = %d", i, distances[i] ) ;
  }
 
  /*
    Loop over orders from spectral format:
      compute avg of deltas
      and get the minimum
  */
  {
    float min_avg = 9999999. ;
 
    for( i = 0 ; i<ndist ; i++ ) {
      int j, k ;
      float sdelta = 0., avg = 0. ;
 
      if ( distances[i] <= 0 ) break ;
      xsh_msg( "    Start order %d", i ) ;
      for( k = i, j = 0 ; j<spec_size; j++, k++ ) {
    if ( dist_pixel[j] <= 0 ) break ;
    sdelta += abs(distances[k] - dist_pixel[j]) ;
    xsh_msg( "      found: %d, spec: %d, delta: %d (%d,%d)", k, j,
         abs(distances[k] - dist_pixel[j]), distances[k],
         dist_pixel[j] ) ;
      }
      if ( j == 0 ) continue ;
      avg = sdelta/(float)j ;
      xsh_msg( "          ===> avg: %.3lf", avg ) ;
 
      if ( avg < min_avg ) {
    min_avg = avg ;
    min_idx = i ;
      }
    }
    xsh_msg( "  Best first absolute order found: %d (%d)", min_idx,
         abs_orders[0] ) ;
  }
 
  good_order = abs_orders[0] ;
  xsh_msg( "First good order: index %d, absorder %d, Position %d",
       min_idx, good_order, (start_x+min_idx)->start_x ) ;
 
  /* Set absolute orders TEST !*/
  for( ngood=0, i=min_idx ; i<nb && good_order<last_order ; i++, ngood++ ) {
    (start_x+i)->absorder = good_order++ ;
    xsh_msg( "Center[%d]: Absorder %d, Position %d", i, (start_x+i)->absorder,
          (start_x+i)->start_x) ;
  }
 
  /* if min_idx is not 0, suppress spurious orders */
  if ( min_idx != 0 ) {
    xsh_msg( "Remove spurious orders" ) ;
    memmove( start_x, start_x+min_idx, sizeof(START_ORDER)*ngood ) ;
  }
 
  {
    int expected ;
    expected = instrument->config->orders ;
    xsh_msg( " =====  Orders Found: %d (should be %d)", ngood, expected ) ;
    XSH_ASSURE_NOT_ILLEGAL_MSG( ngood == expected,
                  "\n***** NOT ENOUGH ORDERS FOUND. STOP *****\n" ) ;
  }
 
 cleanup:
  XSH_FREE( distances ) ;
  XSH_FREE( dist_pixel ) ;
  XSH_FREE( abs_orders ) ;
  XSH_FREE( delta ) ;
  XSH_FREE( the_absorder ) ;
  xsh_spectralformat_list_free( &spectral_list ) ;
 
  return ngood ;
}
 
static void invert_startx( START_ORDER * pstart, int nb )
{
  int i, j ;
  START_ORDER * nstart = NULL; 
 
  XSH_CALLOC( nstart, START_ORDER, nb ) ;
 
  for( j = 0, i = (nb-1) ; i>=0 ; i--, j++ )
    *(nstart+j) = *(pstart+i) ;
 
  memcpy( pstart, nstart, nb*sizeof(START_ORDER) ) ;
 
 cleanup:
  return ;
}
 
static int comp_center( const void * one, const void * two )
{
  CENTER_ORDER * first = (CENTER_ORDER *)one ;
  CENTER_ORDER * secnd = (CENTER_ORDER *)two ;
 
  if ( first->order < secnd->order ) return -1 ;
  else if (  first->order > secnd->order ) return 1 ;
  else return 0 ;
}
 
static int find_maximum( int * found, double * pos_x, int ix0, int max,
             double thresh )
{
  int ix ;
  double * ppos ;
 
  xsh_msg_dbg_high( "Find_maximum: Start at %d, thresh=%lf", ix0, thresh ) ;
  if ( thresh < 0 ) thresh = 0. ;
  for( ppos = pos_x+ix0, ix = ix0 ; ix < max ; ix++, ppos++ ) {
    double flux0 ;
 
    if ( *ppos < thresh ) continue ;
    /* Got a pixel with flux > threshold
       now search maximum
    */
    for( flux0 = 0., ppos++, ix++ ; ix < max ; ix++, ppos++ ) {
      if ( *ppos > flux0 ) {
    flux0 = *ppos ;
    *found = ix ;
    continue ;
      }
      else {
    /*
      continue till flux < threshold
    */
    xsh_msg_dbg_high( "   Max X: %d, %lf", *found, flux0 ) ;
    for( ; ix < max ; ix++, ppos++ ) {
      if ( *ppos < thresh ) {
        /* finished ! */
        return ix ;
      }
      xsh_msg_dbg_high( "      X: %d, %lf", ix, *ppos ) ;
    }
      }
    }
  }
  xsh_msg( "   NOT FOUND !" ) ;
  return -1 ;
}
 
static int search_max( xsh_pre * pre, int iy, int ix0, int ix1 )
{
  int ix, ixmax ;
  double max = -9999999. ;
 
  for( ixmax = 0, ix = ix0 ; ix<ix1 ; ix++ ) {
    int rej ;
    double flux ;
 
    flux = cpl_image_get( pre->data, ix, iy, &rej ) ;
 
    if ( rej != 0 ) continue ;
    if ( flux > max ) {
      ixmax = ix ;
      max = flux ;
    }
  }
 
  return ixmax ;
}
 
static void save_centers(CENTER_ORDER * pcent, int fpos, int npos,
             const char * arm )
{
  static FILE * fgnu = NULL, * freg = NULL ;
  static const char * gnuname  ;
  int first_call ;
  FILE *fout ;
  char fname[32] ;
  int i, order ;
  CENTER_ORDER * ppcent ;
 
  if ( fgnu == NULL ) {
    gnuname = xsh_stringcat_any( "all_orders_", arm, ".gnu", (void*)NULL ) ;
    fgnu = fopen( gnuname, "w" ) ;
    fprintf( fgnu, "set term x11\nplot " ) ;
    first_call = 1 ;
  }
  else first_call = 0 ;
  ppcent = pcent + fpos ;
  order = ppcent->order ;
  sprintf( fname, "order_%s_%02d.dat", arm, order ) ;
  fout = fopen( fname, "w" ) ;
 
  if ( !first_call ) fprintf( fgnu, "," ) ;
  fprintf( fgnu, "'%s' u 1:2 w points pt 5 t''", fname ) ;
 
  for( i = fpos ; i<npos; i++, ppcent++ )
    fprintf( fout, "%d %d %.3lf\n",
         ppcent->pos_x, ppcent->pos_y, ppcent->flux ) ;
 
  fclose( fout ) ;
  if ( order == 0 ) {
    fprintf( fgnu, "\n" ) ;
    fclose( fgnu ) ;
  }
 
  /* Now create a .reg file */
  if ( freg == NULL )
    freg = fopen( "order_create.reg", "w" ) ;
  fprintf( freg, "# Region file format: DS9 version 4.0\n" ) ;
  fprintf( freg, "global color=red font=\"helvetica 4 normal\"select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\n" ) ;
  fprintf( freg, "image\n# RED center_x center_y (pixels)\n" ) ;
  for( ppcent = pcent+fpos, i = fpos ; i <npos ; i++, ppcent++ ) {
    fprintf( freg, "point(%d.,%d.) #point=cross color=red font=\"helvetica 4 normal\"\n", ppcent->pos_x, ppcent->pos_y ) ;
  }
  if ( order == 0 ) fclose( freg ) ;
}
 
static void save_pos_x( double * posx, int size, const char * arm )
{
  FILE * fout ;
  char fname[32] ;
  int i ;
 
  sprintf( fname, "histo_%s.dat", arm ) ;
  fout = fopen( fname, "w" ) ;
 
  for( i = 0 ; i<size ; i++ )
    fprintf( fout, "%d %lf\n", i, *(posx+i) ) ;
 
  fclose( fout ) ; 
}
 
static void fit_order_list( xsh_order_list * list,
                int npos, CENTER_ORDER * pcent, int degree)
{
  double * vorder = NULL, * po ;
  double * pos_x = NULL, * px ;
  double * pos_y = NULL, * py ;
  int i ;
 
  XSH_CALLOC( vorder, double, npos ) ;
  XSH_CALLOC( pos_x, double, npos ) ;
  XSH_CALLOC( pos_y, double, npos ) ;
 
  po = vorder ;
  px = pos_x ;
  py = pos_y ;
  for( i = 0 ; i<npos ; i++, po++, px++, py++, pcent++ ) {
    *po = pcent->order ;
    *px = pcent->pos_x ;
    *py = pcent->pos_y ;
  }
 
  xsh_order_list_fit(list, npos, vorder, pos_x, pos_y, degree) ;
 
 cleanup:
  XSH_FREE( vorder ) ;
  XSH_FREE( pos_x ) ;
  XSH_FREE( pos_y ) ;
  return ;
}
 
/*
  Starting at the center Y=NY/2
  Detect local maxima with an S/N ratio 'fmtchk_search_sn'
  ==> gives us a list of center of orders
  For each order:
  + start form position of the previous center --> high Y
  shift in Y by step_y pixels
  search the maximum in a window 'fmtchk_window' around new y center
  ==> the new center added to list of centers for this order
  + start form position of the previous center --> low Y
  shift in Y by step_y pixels
  search the maximum in a window 'fmtchk_window' around new y center
  ==> the new center added to list of centers for this order
 
  Fit the polynomials (xsh_order_list_fit)
 
  TBD:
  How do i know the absolute orders ? (necessary for the order_table)
 
*/
 
 
cpl_frame *
xsh_order_table_from_fmtchk( xsh_pre * pre,
                 cpl_frame * spectral_frame,
                 xsh_detect_continuum_param * detect_param,
                 xsh_instrument * instrument )
{
  xsh_order_list * res_list = NULL ;
  cpl_frame * result = NULL ;
  int nx, ny, iy0, iy=0, ix, ix0, ix1, step_y ;
  int degree ;
  int norders ;
  int imax=0, found=0;
  double * pos_x = NULL ;
  double med, stdev ;
  int nfound, iord, npos, fpos ;
  CENTER_ORDER * centers = NULL ;
  START_ORDER * order_startx = NULL ;
  const char* tag=NULL;
  const char* fname=NULL;
 
  xsh_msg( " ======= xsh_order_table_from_fmtchk" ) ;
  xsh_msg( "   Window: %d, Step: %d, Search S/N; %.1lf, Follow S/N: %.1lf",
       detect_param->fmtchk_window, detect_param->fmtchk_step,
       detect_param->fmtchk_search_sn,
       detect_param->fmtchk_follow_sn ) ;
 
  XSH_ASSURE_NOT_NULL( pre ) ;
  XSH_ASSURE_NOT_NULL( spectral_frame ) ;
  XSH_ASSURE_NOT_NULL( detect_param ) ;
  XSH_ASSURE_NOT_NULL( instrument ) ;
 
  /*
    set the expected nb of orders 
    should be available in instrument structure
  */
  if ( instrument->config == NULL )
    xsh_instrument_get_config( instrument ) ;
 
  norders = instrument->config->orders ;
  xsh_msg( "   Expected Orders: %d", norders ) ;
  /* We reserve room for extra orders. Multipliying by 2 is certainly too much
     but is conservative. */
  XSH_CALLOC( order_startx, START_ORDER, norders*2 ) ;
 
  nx = xsh_pre_get_nx( pre ) ;
  ny = xsh_pre_get_ny( pre ) ;
 
  if ( instrument->arm == XSH_ARM_VIS ) iy0 = 0.75*ny ;
  else iy0 = ny/2 ;
 
  degree = detect_param->poly_degree ;
  step_y = detect_param->fmtchk_step ;
 
  XSH_CALLOC( centers, CENTER_ORDER, ((norders*2)*(ny/step_y))+10 ) ;
 
  /*
    starting from the middle of the frame
    make a projection along X axis.
    calculate maxima
    move to the end of the frame (x = nx) by step pixels
    Same by moving to the beginning (x = 0)
 
    Compute the polynomial coefficients
 
    That's it
  */
 
  XSH_CALLOC( pos_x, double, nx ) ;
 
  /* Create empty order list */
  check( res_list=xsh_order_list_create( instrument ) ) ;
 
  /* Search maxima at y = ny/2 */
  // get median value in the window
 
  med = cpl_image_get_median_window( pre->data, 1, iy0, nx, iy0 ) ;
  stdev = cpl_image_get_stdev_window( pre->data, 1, iy0, nx, iy0 ) ;
  xsh_msg( "   ===> Y=%d, median = %lf, stdev = %lf", iy, med, stdev ) ;
 
  for( ix = 1; ix <= nx ; ix++ ) {
    double flux ;
    int rej ;
 
    flux = cpl_image_get( pre->data, ix, iy0, &rej ) ;
    *(pos_x+ix-1) = flux ;
  }
  if ( xsh_debug_level_get() >= XSH_DEBUG_LEVEL_LOW )
    save_pos_x( pos_x, nx,
        xsh_instrument_arm_tostring( instrument ) ) ;
 
  /* Search maxima in pos_x array */
  for( ix = 0, imax = 0, nfound = 0 ; nfound < (norders*2) ; ) {
    ix = find_maximum( &found, pos_x, ix, nx-1,
               med*detect_param->fmtchk_search_sn ) ;
    if ( ix == -1 ) break ;
    xsh_msg( "Found Order %d at %d,%d, value: %lf", nfound, found, iy0,
         *(pos_x+found) ) ;
    (order_startx+nfound)->start_x = found+1 ; 
    nfound++ ;
  }
  if (nfound ==0) {
   xsh_msg("Found only 0 orders!");
   xsh_msg("Try to decrease drecoverformat-search-min-sn parameter value");
  }
 
  /* Invert the order of the orders (high X/high lambda first) */
  check(invert_startx( order_startx, nfound ) );
 
  /* Now we should
     a) find the first "correct" order
     b) assign at each order an absolute order number
     c) keep only the "good" orders ie as many orders as possible up to
        the nb of orders for this arm
  */
  check(norders = compute_abs_order( order_startx, nfound,
                     spectral_frame, detect_param,
                     instrument )) ;
 
  /* For each order found, find maxima for y --> ny and
     for y --> 1 */
  for( fpos = 0, npos = 0, iord = 0 ; iord < norders ; iord++ ) {
    double max ;
    int dum ;
    int low_y = 0, up_y = ny-1 ;
    int the_ord ;
 
    xsh_msg( " Seaching for Order %d (window: %d)", iord,
         detect_param->fmtchk_window ) ;
    xsh_msg( "   Starting position: %d,%d",
         (order_startx+iord)->start_x, iy0 );
 
    ix = (order_startx+iord)->start_x ;
    the_ord = (order_startx+iord)->absorder ;
    centers[npos].order = the_ord ; /* TEST !!! */
    centers[npos].pos_x = ix ;
    centers[npos].pos_y = iy0 ;
    centers[npos].flux = *(pos_x+ix) ;
    npos++ ;
    for( iy = (iy0+step_y) ; iy <= ny ; iy+= step_y, npos++ ) {
      // ix, iy in [1,n] space
      ix0 = ix - detect_param->fmtchk_window ;
      ix1 = ix + detect_param->fmtchk_window ;
      if ( ix0 < 1 ) ix0 = 1 ;
      if ( ix1 >= nx ) ix1 = nx ;
      /* get max in a window (in X) 2*search_window + 1 */
      ix = search_max( pre, iy, ix0, ix1 ) ;
      max = cpl_image_get( pre->data, ix, iy, &dum ) ;
      if ( dum != 0 ) continue ;
      if ( max <= 0 || max < med*detect_param->fmtchk_follow_sn ) break ;
      up_y = iy ;
      centers[npos].order = the_ord ; /* TEST !!! */
      centers[npos].pos_x = ix ;
      centers[npos].pos_y = iy ;
      centers[npos].flux = max ;
      xsh_msg_dbg_high( "      Max at %d,%d: %lf", ix, iy, max ) ;
    }
    ix = (order_startx+iord)->start_x ;
    for( iy = (iy0-step_y) ; iy > 0 ; iy -= step_y, npos++ ) {
      ix0 = ix - detect_param->fmtchk_window ;
      ix1 =ix + detect_param->fmtchk_window ;
      if ( ix0 < 1 ) break ;
      if ( ix1 > nx ) break ;
      /* get max in a window (in X) 2*detect_param->fmtchk_window + 1 */
      ix = search_max( pre, iy, ix0, ix1 ) ;
      max = cpl_image_get( pre->data, ix, iy, &dum ) ;
      if ( max <= 0 || max < med*detect_param->fmtchk_follow_sn ) {
    xsh_msg( "   Stop Following at %d,%d (value = %.2lf)", ix, iy, max ) ;
    break ;
      }
      low_y = iy ;
      centers[npos].order = the_ord ; /* TEST !!! */
      centers[npos].pos_x = ix ;
      centers[npos].pos_y = iy ;
      centers[npos].flux = max ;
      xsh_msg_dbg_high( "      Max at %d,%d: %lf", ix, iy, max ) ;
    }
    if ( xsh_debug_level_get() >= XSH_DEBUG_LEVEL_LOW )
      save_centers( centers, fpos, npos,
            xsh_instrument_arm_tostring( instrument ) ) ;
    fpos = npos ;
    res_list->list[iord].starty = low_y ;
    res_list->list[iord].endy = up_y ;
    res_list->list[iord].absorder = the_ord ; /* TEST !!! */
  }
 
  /*Fit order list */
  qsort( centers, npos, sizeof( CENTER_ORDER ), comp_center ) ;
  fit_order_list(res_list, npos, centers, degree);
  //xsh_order_list_dump( res_list, NULL ) ;
 
  /*
    Save order table frame
  */
  tag=XSH_GET_TAG_FROM_ARM( XSH_ORDER_TAB_GUESS,instrument);
  fname="ORDER_TAB_FROM_FMTCHK.fits";
  check(result=xsh_order_list_save(res_list,instrument,fname,tag,pre->ny)) ;
 
 cleanup:
  XSH_FREE( pos_x ) ;
  XSH_FREE( centers ) ;
  XSH_FREE( order_startx ) ;
 
  return result ;
}