eris_nix_lss_startrace.c

/* $Id$
 *
 * This file is part of the ERIS Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
/*
 * $Author$
 * $Date$
 * $Revision$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
 
#include <libgen.h>
#include <string.h>
 
#include "eris_utils.h"
#include "eris_nix_utils.h"
#include "eris_nix_lss_utils.h"
#include "eris_pfits.h"
#include "eris_dfs.h"
#include "eris_nix_dfs.h"
#include "eris_utils.h"
#include <hdrl.h>
 
#include <cpl.h>
 
/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/
 
static const char eris_nix_lss_startrace_description[] =
"This recipe reduces a set of "ERIS_NIX_SKYSUB_OBJECT_LSS_JITTER_PRO_CATG" or\n"
ERIS_NIX_SKYSUB_STD_LSS_JITTER_PRO_CATG" frames to produce a "
ERIS_NIX_MASTER_STARTRACE_PRO_CATG" file.\n"
"\n"
"This file can be used to rectify the 2d spectra so that the slit axis\n"
"is horizontal and the dispersion axis vertical and wavelength\n"
"calibrated.\n"
"\n"
"Input files:\n"
"\n"
"  DO CATG                         Explanation             Req.  #Frames\n"
"  -------                         -----------             ---   -------\n"
"  "ERIS_NIX_SKYSUB_OBJECT_LSS_JITTER_PRO_CATG
                          "        sky-subtracted           Y      >5\n"
"                                  target frames.\n"
"             or\n"
"  "ERIS_NIX_SKYSUB_OBJECT_LSS_JITTER_PRO_CATG
                          "        sky-subtracted           Y      >5\n"
"                                  standard frames.\n"
"\n"
"Output files:\n"
"\n"
"  DO CATG                         Explanation \n"
"  -------                         ----------- \n"
"  "ERIS_NIX_MASTER_STARTRACE_PRO_CATG
                  "                file containing the\n"
"                                  warp polynomials\n"
"                                  for correcting the\n"
"                                  LSS spectrum.\n"
"  "ERIS_NIX_LSS_TRACE_FIT
               "                   debug files containing\n"
"                                  the startrace image\n"
"                                  and the polynomial fit\n"
"                                  to the trace. This can\n"
"                                  be plotted with the\n"
"                                  'plot_trace' method\n"
"                                  of 'startrace_plot.py'\n"
"  "ERIS_NIX_LSS_STARTRACE_SKY
                   "               debug file containing\n"
"                                  the median 2d sky\n"
"                                  background spectrum\n"
"                                  of the observations.\n"
"                                  This can be plotted\n"
"                                  with the 'plot_lines'\n"
"                                  method of\n"
"                                  'startrace_plot.py'\n"
"\n"
"  The "ERIS_NIX_MASTER_STARTRACE_PRO_CATG
 " result will be in a FITS file named \n"
"  'master_wave.<first input filename>', with extensions:\n"
"  - I_POLYNOMIAL, a table with polynomial coefficients for\n"
"    mapping x,lambda => x_actual.\n"
"  - J_POLYNOMIAL, a table with polynomial coefficients for\n"
"    mapping x,lambda => y_actual.\n"
"\n"
"  There will be one "ERIS_NIX_LSS_TRACE_FIT" debug files \n"
"  for each input image, named 'startrace.<input filename>.\n"
"\n"
"  The "ERIS_NIX_LSS_STARTRACE_SKY" file will be named\n"
"  'startrace_sky.<first input filename>.fits'.\n"
"\n"
"  Notes on the method.\n"
"    It is assumed that the jitter pattern moves the object up and\n"
"    down a line along the slit direction. The reference position\n"
"    and the offset of each jitter from it are calculated as\n"
"    follows:\n"
"       1/ The reference is jitter 0. Often this lies near the \n"
"          middle of the pattern and the object is manually \n"
"          positioned near the middle of the slit.\n"
"       2/ Derive the slit-line PA (0 = N, increasing E) from the\n"
"          extrema of the jitter centres.\n"
"       3/ Report how far each jitter pointing lies from the\n"
"          slit-line.\n"
"       4/ Estimate the offset of each jitter relative to the\n"
"          reference jitter along the slit-line.\n"
"\n"
"    Divide the slit response into the jitter images. The slit\n"
"    response is calculated by collapsing the background sky\n"
"    spectrum in the dispersion direction then normalising by\n"
"    its median value.\n"
"\n"
"    Measure the star trace in each frame. The trace is the \n"
"    spectrum of the star dispersed over the detector. For each\n"
"    frame proceed as follows:\n"
"       1/ Get a rough position for the trace. This is done\n" 
"          by locating the maximum signal across a slice of the\n"
"          detector that has few bad pixels and is well\n"
"          illuminated.\n"
"       2/ Cycle up and down from the 'start' slice, fitting\n"
"          a parabola to the points straddling the rough centre\n"
"          of the trace to get a more accurate position. The\n"
"          rough trace position for the next slice is set to\n"
"          the result for the current slice, so that the process\n"
"          can follow the trace as it slowly slants across the\n"
"          detector columns.\n"
"       3/ 'Capture' the trace by fitting a polynomial to the line\n"
"          of measured peaks for each slice.\n"
"\n" 
"    Calculate the median of the sky backgrounds of all frames.\n"
"\n"
"    Measure several atmospheric features/lines across the 2d\n"
"    background sky spectrum. The process is similar to that\n"
"    for measuring the traces:\n"
"       1/ Use a library of lines (currently hardwired into the\n"
"          the code) identifying the line wavelength, y coord\n"
"          at x=1024, and line width.\n"
"       2/ For each line, cycle left and right from the 'start'\n"
"          column, fitting a parabola to the points straddling\n"
"          the rough centre of the feature to get a more accurate\n"
"          position. Unlike following a trace the rough position\n"
"          for the next column is always set to that for x=1024.\n"
"          This is because some of the spectral feaures are weak\n"
"          and the fit unreliable, and the lines do not wander as\n"
"          much as the traces.\n"
"       3/ Fit a polynomial to the line of measured peaks along\n"
"          each feature.\n"
"\n"
"    Do 2d calibration fit between ideal position and actual:\n"
"              x,lambda => x_actual\n"
"              x,lambda => y_actual\n"
"\n"
"    The calibrated 2d spectrum will have  2048 pixels running\n"
"    from 3.045um[0] to 4.107[2047] in dispersion [y], and 2048\n"
"    pixels with 0.013arcsec pixels, centred at 1024 in slit\n"
"    offset [x].\n"
"\n"
"    With trace behaviour described by the polynomial fits\n"
"    derived earlier and dispersive behaviour by the polynomial\n"
"    fits to known atmospheric features:\n"
"       1/ Construct the slit offset, wavelength => x_actual and\n"
"          offset,wavelength => y_actual sample arrays to be fit.\n"
"          Do this by populating the sample arrays with the\n"
"          intersection x,y of the fits to each trace and\n"
"          atmospheric line.\n"
"       2/ Do a 2d polynomial fit to x,lambda => x_actual.\n"
"          Do a 2d polynomial fit to x,lambda => y_actual.\n"
"\n"
"    Save the results to FITS.\n"
"\n";
 
#define RECIPE_NAME "eris.eris_nix_lss_startrace"
 
/*-----------------------------------------------------------------------------
                            Private function prototypes
 -----------------------------------------------------------------------------*/
 
cpl_recipe_define(eris_nix_lss_startrace, ERIS_BINARY_VERSION,
                  "John Lightfoot",
                  PACKAGE_BUGREPORT, "2017",
                  "Calculate a MASTER_WAVE",
                  eris_nix_lss_startrace_description);
 
/*-----------------------------------------------------------------------------
                                Function code
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static cpl_error_code eris_nix_lss_startrace_fill_parameterlist(
                      cpl_parameterlist *self) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
 
    /* the flux threshold - frames with a median above this will be
       deemed 'saturated' and not used */
 
    cpl_parameter * p = NULL;
    p = cpl_parameter_new_value(RECIPE_NAME".saturation_threshold",
      CPL_TYPE_DOUBLE, "frame saturation threshold", RECIPE_NAME, 5000.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "saturation_threshold");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(self, p);
 
    /* cleanup */
 
    return 0;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
static int eris_nix_lss_startrace(cpl_frameset * frameset,
                                  const cpl_parameterlist * parlist) {
 
    cpl_propertylist     * applist = NULL;
    cpl_image            * contrib = NULL;
    cpl_vector           * crval1_vector = NULL;
    cpl_vector           * crval2_vector = NULL;
    located_imagelist    * jitters = NULL;
    cpl_polynomial      ** line_polys = NULL;
    mef_extension_list   * mefs = NULL;
    located_imagelist    * object_jitters = NULL;
    hdrl_imagelist       * skylist = NULL;
    hdrl_image           * sky_background = NULL;
    cpl_vector           * slit_offset = NULL;
    cpl_matrix           * startrace_pos = NULL;
    located_imagelist    * std_jitters = NULL;
    cpl_polynomial      ** trace_polys = NULL;
    cpl_frameset         * used = NULL;
 
    /* check input parameters */
 
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
 
    /* set the msg verbosity level from environment variable CPL_MSG_LEVEL */
 
    cpl_msg_set_level_from_env();
    cpl_msg_severity severity = cpl_msg_get_level();
    cpl_msg_info(cpl_func, "level %d", (int) severity);
 
    /* check for invalid input */
    if(eris_files_dont_exist(frameset) != CPL_ERROR_NONE) {
        return CPL_ERROR_BAD_FILE_FORMAT;
    }
    enu_check_error_code("Could not retrieve input parameters");
 
    /* identify the RAW and CALIB frames in the input frameset */
 
    eris_nix_dfs_set_groups(frameset);
    enu_check_error_code("Could not identify RAW and CALIB frames");    
 
    used = cpl_frameset_new();
 
    /* read in the sky-subtracted data */
 
    object_jitters = enu_limlist_load_from_frameset(frameset,
                             ERIS_NIX_SKYSUB_OBJECT_LSS_JITTER_PRO_CATG,
                             used);
    std_jitters = enu_limlist_load_from_frameset(frameset,
                             ERIS_NIX_SKYSUB_STD_LSS_JITTER_PRO_CATG,
                             used);
    enu_check_error_code("Error loading frameset");    
 
    if (object_jitters->size > 0) {
        enu_check(std_jitters->size == 0, CPL_ERROR_ILLEGAL_INPUT,
                  "SoF contains both object and std data");
        jitters = object_jitters;
        enu_located_imagelist_delete(std_jitters);
        std_jitters = NULL;
    } else if (std_jitters->size > 0) {
        jitters = std_jitters;
        enu_located_imagelist_delete(object_jitters);
        object_jitters = NULL;
    }
    enu_check(jitters!=NULL, CPL_ERROR_ILLEGAL_INPUT,
              "SoF contains no valid data");
 
    /* NB jitter centering code may be largely duplicated in straighten recipe */
    /* estimate the centre of the jitter pattern
       .. the jitter centres should lie along a line on the sky
       .. as the object is move up and down the slit.
       .. Consequently, find the ends of this line by looking at
       .. extreme points. */
 
    crval1_vector = cpl_vector_new(jitters->size);
    crval2_vector = cpl_vector_new(jitters->size);
    for (cpl_size i = 0; i < jitters->size; i++) {
        double crval1 = cpl_propertylist_get_double(
                                       jitters->limages[i]->plist,
                                       "CRVAL1");
        cpl_vector_set(crval1_vector, i, crval1);
        double crval2 = cpl_propertylist_get_double(
                                       jitters->limages[i]->plist,
                                       "CRVAL2");
        cpl_vector_set(crval2_vector, i, crval2);
        cpl_msg_info(cpl_func, "Jitter %d at RA=%10.7f Dec=%10.7f",
                     (int)i, crval1, crval2);
    }
 
    double lat = cpl_vector_get_mean(crval2_vector);
    double coslat = cos(lat / 57.2958);
    cpl_msg_info(cpl_func, "lat %f coslat %f", lat, coslat);
 
    cpl_size crval1_maxpos = cpl_vector_get_maxpos(crval1_vector);
    double crval1_max = cpl_vector_get(crval1_vector, crval1_maxpos);
    cpl_size crval1_minpos = cpl_vector_get_minpos(crval1_vector);
    double crval1_min = cpl_vector_get(crval1_vector, crval1_minpos);
 
    cpl_size crval2_maxpos = cpl_vector_get_maxpos(crval2_vector);
    double crval2_max = cpl_vector_get(crval2_vector, crval2_maxpos);
    cpl_size crval2_minpos = cpl_vector_get_minpos(crval2_vector);
    double crval2_min = cpl_vector_get(crval2_vector, crval2_minpos);
 
    double enda[2] = {0.0, 0.0};
    double endb[2] = {0.0, 0.0};
    if (fabs(crval1_max - crval1_min) > fabs(crval2_max - crval2_min)) {
        enda[0] = crval1_min;
        enda[1] = cpl_vector_get(crval2_vector, crval1_minpos);
        endb[0] = crval1_max;
        endb[1] = cpl_vector_get(crval2_vector, crval2_maxpos);
    } else {
        enda[0] = cpl_vector_get(crval1_vector, crval2_minpos);
        enda[1] = crval2_min;
        endb[0] = cpl_vector_get(crval1_vector, crval2_maxpos);
        endb[1] = crval2_max;
    }
    cpl_msg_info(cpl_func, "Jitter line end points (RA, DEC): "
                 "(%10.7f, %10.7f) and (%10.7f, %10.7f)",
                 enda[0], enda[1], endb[0], endb[1]);
 
    /* define the reference position as a point halfway along the
       slit line pattern */
 
    double centre[2] = {(enda[0] + endb[0]) / 2.0,
                        (enda[1] + endb[1]) / 2.0};
 
    /* get direction of slit line, assume axis increases to left */
 
    double slit_line[2] = {(endb[0]-enda[0]) * coslat, endb[1]-enda[1]};
    double slit_norm = sqrt(pow(slit_line[0],2) + pow(slit_line[1],2));
    slit_line[0] /= slit_norm;
    slit_line[1] /= slit_norm;
    enu_check_error_code("error finding line of slit");
    cpl_msg_info(cpl_func, "..slit direction vector (long, lat) = (%f, %f)",
                 slit_line[0], slit_line[1]);
    
    /* calculate offset of each jitter along slit line, not sure how best
       to do this but vectors seem simplest */
 
    double cd1_1 = cpl_propertylist_get_double(jitters->limages[0]->
                                               plist, "CD1_1");
    double cd2_1 = cpl_propertylist_get_double(jitters->limages[0]->
                                               plist, "CD2_1");
    double pixsize = sqrt(cd1_1 * cd1_1 + cd2_1 * cd2_1);
 
    slit_offset = cpl_vector_new(jitters->size);
    for (cpl_size i = 0; i < jitters->size; i++) {
        double offset = -1000.0;
        /* be wary of 0 components in slit_line direction vector */
        if (fabs(slit_line[0]) > 1e-6) {
            offset = (cpl_vector_get(crval1_vector, i) - centre[0]) * coslat /
                     slit_line[0];
        } else if (fabs(slit_line[1]) > 1e-6) {
            offset = (cpl_vector_get(crval2_vector, i) - centre[1]) / 
                     slit_line[1];
        }
 
        /* check how far each jitter is from the notional slit line
           ..are the numbers reasonable? */
        double remainder[2] = {0.0,0.0};
        remainder[0] = ((cpl_vector_get(crval1_vector, i) - centre[0]) * coslat - 
                       (offset * slit_line[0])) / pixsize;
        remainder[1] = ((cpl_vector_get(crval2_vector, i) - centre[1]) - 
                       (offset * slit_line[1])) / pixsize;
        if (fabs(remainder[0]) > 1 || fabs(remainder[1]) > 1) {
            cpl_msg_warning(cpl_func, "...offset remainder (%f, %f) pixels",
                            remainder[0], remainder[1]);
            cpl_msg_warning(cpl_func, "...slit offset does not fall on line");
        }
 
        cpl_vector_set(slit_offset, i, offset);
        cpl_msg_info(cpl_func, "..slit offset %d %f", (int)i, offset);
    }
    enu_check_error_code("error calculating slit offsets");
 
    /* calculate nominal slit-offset for each jitter */
    /* 0 offset would put object at column 1024 */
 
    cpl_vector * nominal_offset = cpl_vector_new(jitters->size);
    for (cpl_size i = 0; i < jitters->size; i++) {
        cpl_vector_set(nominal_offset, i, 1024.0 - cpl_vector_get(
                       slit_offset, i) / pixsize);
    }
    cpl_msg_info(cpl_func, "Nominal offset:");
    cpl_vector_dump(nominal_offset, NULL);
 
    /* calculate slit response and divide into jitter images */
 
    enlu_divide_slit_response(jitters);
 
    /* measure the star trace in each frame */
 
    cpl_size ntrace = jitters->size;
    trace_polys = cpl_calloc(ntrace, sizeof(cpl_polynomial *));
 
    const cpl_size nx = hdrl_image_get_size_x(jitters->limages[0]->himage);
    const cpl_size ny = hdrl_image_get_size_y(jitters->limages[0]->himage);
 
    /* horizontal start slice index chosen to avoid big patches of bad
       pixels */ 
 
    const cpl_size start_slice = 1800;
    startrace_pos = cpl_matrix_new(ny, ntrace);
    skylist = hdrl_imagelist_new();
 
    /* loop through the jitters, fitting the trace in each, and 
       accumulating data for the median sky background */
 
    for (cpl_size j = 0; j < ntrace; j++) {
 
        /* accumulate the sky backgrounds as we go for use later */
 
        hdrl_image * imcopy = hdrl_image_duplicate(
                                       jitters->limages[j]->bkg);
        hdrl_imagelist_set(skylist, imcopy,
                           hdrl_imagelist_get_size(skylist));
 
        /* get a rough position for the trace at start_slice. This 
           should be within 4 pixels or so of the correct answer to 
           cover the trace peak or the refined fit will not work.
           Try to be robust - the choice of start_slice should help 
           by being on a well-behaved, well illuminated part of the
           detector */
 
        cpl_msg_info(cpl_func, "jitter %d", (int)j);
        char * name_copy = cpl_strdup(cpl_frame_get_filename(
                                      jitters->limages[j]->frame));
        cpl_msg_info(cpl_func, "%s", basename(name_copy));
        cpl_free(name_copy);
        cpl_msg_info(cpl_func, "nominal offset %7.2f",
                     cpl_vector_get(nominal_offset, j));
 
        cpl_vector * maximum_pos = cpl_vector_new(10);
        cpl_vector * maximum_val = cpl_vector_new(10);
        for (cpl_size iy = start_slice-5; iy < start_slice+5; iy++) {
            cpl_vector * slice1d = cpl_vector_new_from_image_row(
                                       hdrl_image_get_image_const(
                                       jitters->limages[j]->himage),
                                       iy+1);
            cpl_vector_set(maximum_pos, iy-(start_slice-5),
                           cpl_vector_get_maxpos(slice1d)); 
            cpl_vector_set(maximum_val, iy-(start_slice-5),
                           cpl_vector_get_max(slice1d)); 
 
            cpl_vector_delete(slice1d);
        }
        //cpl_vector_dump(maximum_pos, NULL);
        //cpl_vector_dump(maximum_val, NULL);
        double start_guess_pos = cpl_vector_get_median(maximum_pos);
        double trace_val = cpl_vector_get_median(maximum_val);
    
        cpl_vector_delete(maximum_pos);
        cpl_vector_delete(maximum_val);
    
        /* now go for a more accurate value with a fit for each
           y-slice - moving up and down from start_slice */
 
        double guess_pos = start_guess_pos;
 
        /* ..working up from start_slice */
 
        for (cpl_size iy = start_slice; iy < ny; iy++) {
            cpl_vector * slice1d = cpl_vector_new_from_image_row(
                                       hdrl_image_get_image_const(
                                       jitters->limages[j]->himage),
                                       iy+1);
 
            double fitted_pos = enlu_linepos_1d(slice1d, guess_pos, 4);
            if (!isnan(fitted_pos) && 
                (cpl_vector_get(slice1d, (int)fitted_pos) > 0.1 * trace_val)) {
 
                /* fit didn't fail and also fit position looks to 
                   contain some flux */
 
                /* set the fit peak as the measured position of the line,
                   set next_guess_pos to the actual fitted_pos so that
                   it can follow slow variations with iy */
 
                cpl_matrix_set(startrace_pos, iy, j, fitted_pos);
                guess_pos = fitted_pos;
                cpl_msg_info(cpl_func, "fitted pos %d %7.2f", 
                             (int)iy, fitted_pos);
            } else {
                cpl_matrix_set(startrace_pos, iy, j, fitted_pos);
            }
            cpl_vector_delete(slice1d);
        }
        enu_check_error_code("error following trace up");
 
        /* ..and working down from start_slice */
 
        guess_pos = start_guess_pos;
 
        for (cpl_size iy = start_slice; iy >= 0; iy--) {
 
            cpl_vector * slice1d = cpl_vector_new_from_image_row(
                                       hdrl_image_get_image_const(
                                       jitters->limages[j]->himage),
                                       iy+1);
 
            double fitted_pos = enlu_linepos_1d(slice1d, guess_pos, 4);
            cpl_vector_delete(slice1d);
 
            if (!isnan(fitted_pos) && 
                (cpl_vector_get(slice1d, (int)fitted_pos) > 0.1 * trace_val)) {
                cpl_matrix_set(startrace_pos, iy, j, fitted_pos);
                guess_pos = fitted_pos;
                cpl_msg_info(cpl_func, "fitted pos %d %7.2f", 
                             (int)iy, fitted_pos);
            } else {
                cpl_matrix_set(startrace_pos, iy, j, fitted_pos);
            }
        }
        enu_check_error_code("error following trace down");
 
        /*  now capture trace by fitting poly to peak through y */
 
        trace_polys[j] = cpl_polynomial_new(1);
        cpl_vector * spectrum   = cpl_vector_new(ny);
        {
            cpl_matrix * samppos1d = cpl_matrix_new(1, ny);
            cpl_vector * fitvals   = cpl_vector_new(ny);
            const cpl_boolean sampsym = CPL_FALSE;
            const cpl_size maxdeg1d = 3;
 
            cpl_size isamp = 0;
 
            for (cpl_size iy = 0; iy < ny; iy++) {
                double fitted_pos = cpl_matrix_get(startrace_pos, iy, j);
                if (!isnan(fitted_pos)) {
                    cpl_matrix_set(samppos1d, 0, isamp, (double)iy);
                    cpl_vector_set(fitvals, isamp, fitted_pos);
                    isamp++;
                }
            }
 
            cpl_matrix_set_size(samppos1d, 1, isamp);
            cpl_vector_set_size(fitvals, isamp);
            //for (cpl_size i=0; i<isamp; i++) {
            //    cpl_msg_info(cpl_func, "%3.1f %7.2f",
            //                 cpl_matrix_get(samppos1d, 0, i),
            //                 cpl_vector_get(fitvals, i));
            //}
 
            cpl_polynomial_fit(trace_polys[j], samppos1d, &sampsym,
                               fitvals, NULL, CPL_FALSE, NULL, &maxdeg1d);
 
            cpl_polynomial_dump(trace_polys[j], stdout);
        }
        enu_check_error_code("error fitting trace");
 
        /* calculate the sky bkg spectrum along the trace */
 
        {
            cpl_image * double_image = cpl_image_cast(
                                       hdrl_image_get_image_const(
//                                       jitters->limages[j]->himage),
                                       jitters->limages[j]->bkg),
                                       CPL_TYPE_DOUBLE);
            double * data = cpl_image_get_data(double_image);
 
            cpl_image * double_confidence = cpl_image_cast(
                                       jitters->limages[j]->confidence,
                                       CPL_TYPE_DOUBLE);
            double * confidence = cpl_image_get_data(double_confidence);
 
            for (cpl_size iy = 0; iy < ny; iy++) {
                double pos =  cpl_polynomial_eval_1d(trace_polys[j], (double)iy, NULL);
                double val = 0.0;
                double sumwt = 0.0;
 
//                /* Lanczos with a = 2. wt = sinc(x) * sinc(x/a) */
//
//                for (cpl_size dy=-2; dy <= 2; dy++) {
//                    for (cpl_size dx=-2; dx <= 2; dx++) {
//                        cpl_size jdatum = iy + dy;
//                        cpl_size idatum = (int)pos + dx;
 
//                        if (jdatum >= 0 && jdatum < ny &&
//                            idatum >= 0 && idatum < nx) {
 
//                            /* point to interpolate to is (pos, iy) */
 
//                            double r = sqrt(pow(pos-idatum, 2) + pow(dy, 2));
//                            double wt = 0.0;
//                            if (r < 1.0e-10) {
//                                wt = 1.0;
//                            } else if (r < 2.0) {
//                                wt = 2.0 * sin(M_PI * r) * sin(M_PI * r / 2.0) /
//                                     pow(M_PI * r, 2);
//                            }
 
//                            val+= (wt * (data[jdatum * nx + idatum] *
//                                   confidence[jdatum * nx + idatum]));
//                            sumwt += (wt * confidence[jdatum * nx + idatum]);
 
//                        }
//                    }
//                }
 
                /* average along row centred on pos */
                for (cpl_size dx=-20; dx <= 20; dx++) {
                    cpl_size idatum = (int)pos + dx;
 
                    if (idatum >= 0 && idatum < nx) {
                        double wt = 1.0;
                        val+= (wt * (data[iy * nx + idatum] *
                               confidence[iy * nx + idatum]));
                        sumwt += (wt * confidence[idatum * nx + idatum]);
 
//                       if (iy==914) {
//                           cpl_msg_info(cpl_func, "%f %d %d %f %f %f %f %d %d %f %f", 
//                                        pos, (int)dx, (int)dy, r, wt, val, sumwt, 
//                                        (int)idatum, (int)jdatum,
//                                        data[jdatum * nx + idatum],
//                                        confidence[jdatum * nx + idatum]);
//                       }
                    }
                }
 
                if (fabs(sumwt) > 0) {
                    val /= sumwt;
                }
                cpl_vector_set(spectrum, iy, val);
            }
 
            cpl_image_delete(double_image);
 
            /* save the trace info */
 
            char * out_fname = enu_repreface(cpl_frame_get_filename(
                                             jitters->limages[j]->frame),
                                             "startrace");
            enlu_trace_save(ERIS_NIX_LSS_TRACE_FIT,
                            jitters->limages[j]->himage,
                            jitters->limages[j]->confidence,
                            1, (const cpl_polynomial **) &(trace_polys[j]),
                            1, (const cpl_vector **) &spectrum,
                            0, NULL,
                            frameset,
                            parlist,
                            out_fname,
                            RECIPE_NAME);
            cpl_free(out_fname);
        }
        enu_check_error_code("error calculating trace spectrum / saving trace");
    }
 
    /* Find the median of the sky backgrounds */
 
    hdrl_imagelist_collapse(skylist, HDRL_COLLAPSE_MEDIAN, &sky_background,
                            &contrib);
    enu_check_error_code("error collapsing sky backgrounds");
 
    /* now trace atmospheric features/lines */
 
    // used wavelengths.py to try and cross match lines with ESO skycalc spectrum
    #define NLINES 6
    double wavelength[NLINES] = {3.262,
                                 3.316,
                                 3.392,
                                 3.428,
                                 3.456,
                                 3.901};
    double expected_iy[NLINES] = {364,
                                  471,
                                  617,
                                  688,
                                  738,
                                  1600};
    double half_width[NLINES] = {10.0,
                                 10.0,
                                 10.0,
                                 10.0,
                                 10.0,
                                 50.0};
 
    line_polys = cpl_calloc(NLINES, sizeof(cpl_polynomial *));
 
    {
        /* subtract a slope from the sky spectrum to flatten it - this makes
           it easier for the fitting code to 'follow' absorption features */ 
 
        for (cpl_size iy = 1; iy <= ny; iy++) {
            for (cpl_size ix = 1; ix <= nx; ix++) {
                int reject = 0;
                hdrl_value value = hdrl_image_get_pixel(sky_background,
                                                        ix, iy, 
                                                        &reject);
                if (reject==0) {
                    value.data = value.data - (10.0 + 71.0 * (iy-207.0) / 1626);
                    hdrl_image_set_pixel(sky_background, ix, iy, value);
                }
            }
        }
 
        /* Loop through the absorption features expected in the sky spectrum.
           ..start in the middle of the slit length where we think we know 
             roughly where the feature is
           ..loop up and down the slit and follow the line position by fitting
             it at each position
           ..whether to the expected position or that from the last fit as the
             guess for next line fit is uncertain. Using the last result as the
             guess for next is unstable in patchy parts of the detector - for
             now stick with the expected position as more robust */
 
        cpl_matrix * line_pos = cpl_matrix_new(nx, NLINES);
 
        for (cpl_size line = 0; line < NLINES; line++) {
            double guess_pos = expected_iy[line];
            cpl_size xstart = 1024;
 
            /* now go for a more accurate value with a fit for each
               slice - moving away from the start_slice in each
               direction */
 
            /* ..working up from start_slice */
 
            for (cpl_size ix = xstart; ix < nx; ix++) {
 
                cpl_vector * slice1d = cpl_vector_new_from_image_column(
                                       hdrl_image_get_image_const(
                                       sky_background),
                                       ix+1);
 
                double fitted_pos = enlu_linepos_1d(slice1d, guess_pos,
                                                    half_width[line]);
                cpl_matrix_set(line_pos, ix, line, fitted_pos);
 
                if (!isnan(fitted_pos)) {
                    cpl_msg_debug(cpl_func, "line %d %e %e", (int)ix,
                                  guess_pos, fitted_pos);
 
                    /* NO, stick with the initial guess as more robust */
 
                    /* set the fit peak as the measured position of the line,
                       set next_guess_pos to the actual fitted_pos so that
                       it can follow slow variations with iy */
                    //guess_pos = fitted_pos;
                }
                cpl_vector_delete(slice1d);
            }
 
            /* ..and working down from start_slice */
 
            for (cpl_size ix = xstart; ix >=0; ix--) {
 
                cpl_vector * slice1d = cpl_vector_new_from_image_column(
                                       hdrl_image_get_image_const(
                                       sky_background),
                                       ix+1);
 
                double fitted_pos = enlu_linepos_1d(slice1d, guess_pos,
                                                    half_width[line]);
                cpl_matrix_set(line_pos, ix, line, fitted_pos);
                if (!isnan(fitted_pos)) {
                    cpl_msg_debug(cpl_func, "line %d %e %e", (int)ix, 
                                  guess_pos, fitted_pos);
                    //guess_pos = fitted_pos;
                }
                cpl_vector_delete(slice1d);
            }
 
            /* now fit a curve to the line of fitted peaks along the feature */
 
            line_polys[line] = cpl_polynomial_new(1);
            {
                cpl_matrix * samppos1d = cpl_matrix_new(1, nx);
                cpl_vector * fitvals   = cpl_vector_new(nx);
                const cpl_boolean sampsym = CPL_FALSE;
                const cpl_size maxdeg1d = 2;
 
                cpl_size isamp = 0;
 
                for (cpl_size ix = 0; ix < nx; ix++) {
                    double fitted_pos = cpl_matrix_get(line_pos, ix, line);
                    if (!isnan(fitted_pos)) {
                        cpl_matrix_set(samppos1d, 0, isamp, (double)ix);
                        cpl_vector_set(fitvals, isamp, fitted_pos);
                        isamp++;
                    }
                }
 
                cpl_matrix_set_size(samppos1d, 1, isamp);
                cpl_vector_set_size(fitvals, isamp);
                cpl_polynomial_fit(line_polys[line], samppos1d, &sampsym, fitvals, NULL,
                                   CPL_FALSE, NULL, &maxdeg1d);
 
                cpl_msg_info(cpl_func, "line %d poly", (int)line);
                cpl_polynomial_dump(line_polys[line], stdout);
            }
        }
        enu_check_error_code("error following line features");
 
        /* save the spectral features info */
 
        char * out_fname = enu_repreface(cpl_frame_get_filename(
                                         jitters->limages[0]->frame),
                                         "startrace_sky");
        enlu_trace_save(ERIS_NIX_LSS_STARTRACE_SKY,
                        sky_background,
                        jitters->limages[0]->confidence,
                        0, NULL,
                        0, NULL,
                        NLINES, (const cpl_polynomial **) line_polys,
                        frameset,
                        parlist,
                        out_fname,
                        RECIPE_NAME);
        cpl_free(out_fname);
        enu_check_error_code("error saving line features");
    }
 
    /* now do 2d calibration fit between ideal position and actual:
              x,lambda => x_actual
              x,lambda => y_actual
       This direction of mapping is required by cpl_image_warp_polynomial
       which is the routine used to do the actual correction. */
 
    /* The ideal detector has 2048 pixels running from 3.045um(0)
       to 4.107(2047) in dispersion (y), and 2048 pixels with
       pixel size 0.013arcsec with centred at 1024 in slit 
       offset (x) */
 
    /* trace_polys are polynomials describing x_actual along traces */
 
    /* construct the sample position and wavelength arrays 
       for the fit - these must contain no NaNs or the fit routine 
       will fail */
 
    cpl_size nsamples = NLINES * ntrace;
    cpl_size isample = 0;
 
    cpl_matrix * ij_nominal = cpl_matrix_new(2, nsamples);
    cpl_vector * i_actual = cpl_vector_new(nsamples);
    cpl_vector * j_actual = cpl_vector_new(nsamples);
 
    for (cpl_size trace = 0; trace < ntrace; trace++) {
 
        /* nominal ix of trace for jitter i */
 
        double i_nominal = cpl_vector_get(nominal_offset, trace);
 
        /* loop through the calibration lines */
 
        for (cpl_size line = 0; line < NLINES; line++) {
            /* nominal position of line from desired wcs */
            double lambda = wavelength[line];
            double j_nominal = ny * (lambda - 3.045) / (4.107 - 3.045); 
 
            /* actual i,j is where this trace intersects this line */
 
            cpl_size jtrace_min = -1;
            double mindist = DBL_MAX;
            for (cpl_size jtrace=0; jtrace < ny; jtrace++) {
                double itrace = cpl_polynomial_eval_1d(trace_polys[trace],
                                                       (double)jtrace, NULL);
                double jline = cpl_polynomial_eval_1d(line_polys[line],
                                                      itrace, NULL);
                double dist = fabs(jline - jtrace);
                if (dist < mindist) {
                    jtrace_min = jtrace;
                    mindist = dist;
                }
            }
 
            double i_actual_val = 0.0;
            double j_actual_val = 0.0;
            if (jtrace_min != -1) {
                j_actual_val = jtrace_min;
                i_actual_val = cpl_polynomial_eval_1d(trace_polys[trace],
                                                      j_actual_val, NULL);
            }
   
            //cpl_msg_info(cpl_func, "%d %d %4.2f %4.1f %4.1f %4.1f", 
            //             (int)trace, (int)line,
            //             j_nominal, j_actual_val,
            //             i_nominal, i_actual_val);
 
            /*  construct i_nominal, j_nominal -> i_actual */
 
            cpl_matrix_set(ij_nominal, 0, isample, i_nominal); 
            cpl_matrix_set(ij_nominal, 1, isample, j_nominal); 
            cpl_vector_set(i_actual, isample, i_actual_val); 
 
            /* and i_nominal, j_nominal -> j_actual */
 
            cpl_vector_set(j_actual, isample, j_actual_val);
 
            isample++;
        }
    }
 
    /* solve the i polynomial */
 
    cpl_polynomial * i_cal = cpl_polynomial_new(2);
    const cpl_size maxdeg2d[] = {2, 2};
    cpl_polynomial_fit(i_cal, ij_nominal, NULL, i_actual, NULL,
                       CPL_TRUE, NULL, maxdeg2d);
 
    /* report results and save to table */
 
    cpl_msg_info(cpl_func, "The i calibration polynomial:");
    cpl_polynomial_dump(i_cal, stdout);
    cpl_table * i_table = enlu_warp_poly_save_to_table(i_cal);
 
    /* likewise for the j polynomial */
 
    cpl_polynomial * j_cal = cpl_polynomial_new(2);
    cpl_polynomial_fit(j_cal, ij_nominal, NULL, j_actual, NULL,
                       CPL_TRUE, NULL, maxdeg2d);
    cpl_msg_info(cpl_func, "The j calibration polynomial:");
    cpl_polynomial_dump(j_cal, stdout);
    cpl_table * j_table = enlu_warp_poly_save_to_table(j_cal);
 
    /* evaluate the calibration polynomial over the image */
 
    //wave_cal_image = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
    //for (cpl_size iy = 1; iy < ny+1; iy++) {
    //    for (cpl_size ix = 1; ix < nx+1; ix++) {
    //        cpl_vector * pos = cpl_vector_new(2);
    //        cpl_vector_set(pos, 0, (double) iy);
    //        cpl_vector_set(pos, 1, (double) ix);
    //        double val = cpl_polynomial_eval(wave_cal, pos);
    //        cpl_image_set(wave_cal_image, ix, iy, val); 
    //        cpl_vector_delete(pos); 
    //    }
    //}
 
    /* evaluate the fit residual over the image */
 
    //cpl_vector * wave_residual = cpl_vector_new(cpl_vector_get_size(
    //                                            lambda_pos));
    //cpl_vector_fill_polynomial_fit_residual(wave_residual, lambda_pos,
    //                                        NULL, wave_cal, line_pos_fit,
    //                                        NULL);
    //const double residual_mad = cpl_vector_get_median_const(wave_residual);
    //cpl_msg_info(cpl_func, "MAD of residuals = %4.2f", residual_mad); 
 
    /* construct the propertylist to be saved with the MASTER_STARTRACE */
 
    applist = cpl_propertylist_new();
    cpl_propertylist_append_string(applist, CPL_DFS_PRO_CATG,
                                   ERIS_NIX_MASTER_STARTRACE_PRO_CATG);
 
    /* add QC parameters  */
 
    //cpl_propertylist_append_double(applist, "ESO QC CAL RESID",
    //                               residual_mad);
    //enu_check_error_code("error constructing output propertylist");
 
 
    /* save the MASTER_STARTRACE to a DFS-compliant MEF file */
 
    mefs = enu_mef_extension_list_new(2);
 
    /* MEF for i and j polynomials */ 
 
    mefs->mef[0] = enu_mef_new_table("I_POLYNOMIAL", i_table, NULL);
    mefs->mef[1] = enu_mef_new_table("J_POLYNOMIAL", j_table, NULL);
 
    /* MEF for LINE_RESIDUAL, a table with the fit residual at each line
       position in each spectral slice */
 
    //mef_table2 = cpl_table_new(nsamples);
    //cpl_table_new_column(mef_table2, "x", CPL_TYPE_DOUBLE);
    //cpl_table_new_column(mef_table2, "ypos", CPL_TYPE_DOUBLE);
    //cpl_table_new_column(mef_table2, "lambda_residual", CPL_TYPE_DOUBLE);
    //for (cpl_size isample = 0; isample < nsamples; isample++) {
    //    cpl_table_set(mef_table2, "x", isample, 
    //                  cpl_matrix_get(line_pos_fit, 1, isample));
    //    cpl_table_set(mef_table2, "ypos", isample,
    //                  cpl_matrix_get(line_pos_fit, 0, isample));
    //    cpl_table_set(mef_table2, "lambda_residual", isample, 
    //                  cpl_vector_get(wave_residual, isample));
    //}
    //mefs->mef[3] = enu_mef_new_table("LINE_RESIDUAL", mef_table2, NULL);
 
    /* generate name of output file */
 
    char * out_fname = enu_repreface(cpl_frame_get_filename(
                                     jitters->limages[0]->frame),
                                     "master_wave");
    cpl_frameset * provenance = cpl_frameset_new();
    for (cpl_size j = 0; j<jitters->size; j++) {
        cpl_frameset_insert(provenance, cpl_frame_duplicate(
                            jitters->limages[j]->frame));
    }
 
    cpl_msg_info(cpl_func, "..writing %s", out_fname);
    enu_dfs_save_himage(used,
                        parlist,
                        provenance,
                        CPL_TRUE,
                        NULL,
                        NULL,
                        mefs,
                        RECIPE_NAME,
                        NULL,
                        applist,
                        NULL,
                        PACKAGE "/" PACKAGE_VERSION,
                        out_fname);
 
    cpl_frameset_delete(provenance);
    cpl_free(out_fname);
    enu_check_error_code("Failed to save MASTER_WAVE");
 
cleanup:
    if (trace_polys != NULL) {
        for (cpl_size j=0; j < jitters->size; j++) {
            cpl_polynomial_delete(trace_polys[j]);
        }
        cpl_free(trace_polys);
    }
    if (line_polys != NULL) {
        for (cpl_size line=0; line < NLINES; line++) {
            cpl_polynomial_delete(line_polys[line]);
        }
        cpl_free(line_polys);
    }
    cpl_propertylist_delete(applist);
    cpl_image_delete(contrib);
    cpl_vector_delete(crval1_vector);
    cpl_vector_delete(crval2_vector);
    enu_located_imagelist_delete(object_jitters);
    enu_located_imagelist_delete(std_jitters);
    if (ij_nominal != NULL) {
      cpl_matrix_delete(ij_nominal);
    }
    enu_mef_extension_list_delete(mefs);
    hdrl_imagelist_delete(skylist);
    hdrl_image_delete(sky_background);
    cpl_vector_delete(slit_offset);
    cpl_matrix_delete(startrace_pos);
    cpl_frameset_delete(used);
 
    return (int)cpl_error_get_code();
}