eris_nix_lss_utils.c

/* $Id$
 *
 * This file is part of the ERIS/NIX Pipeline
 * Copyright (C) 2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
 /*
 * $Author$
 * $Date$
 * $Rev$
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                   Includes
 -----------------------------------------------------------------------------*/
 
#include "eris_nix_lss_utils.h"
#include "eris_nix_dfs.h"
#include "eris_nix_utils.h"
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
cpl_error_code enlu_divide_slit_response(located_imagelist * jitters) {
 
    /* check inputs */
   
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
    cpl_ensure_code(jitters, CPL_ERROR_NULL_INPUT);
 
    cpl_msg_info(cpl_func, "..dividing by slit response");
 
    for (cpl_size j = 0; j < jitters->size; j++) {
 
        /* Get copy of background, remove dodgy values */
 
        cpl_image * bkg_copy = cpl_image_duplicate(
                                       hdrl_image_get_image(
                                       jitters->limages[j]->bkg));
        cpl_image_reject_value(bkg_copy, CPL_VALUE_NOTFINITE);
 
        /* collapse the background along the dispersion axis to get
           the slit response profile (sky spectrum should be the same 
           for all slit offsets) */
        /* we drop 300 pix at the left and at the right as those are anyway not illuminated */
        cpl_size drop_ll = 300;
        cpl_size drop_ur = 300;
        cpl_image * bkg_collapse = cpl_image_collapse_median_create(bkg_copy, 0, drop_ll, drop_ur);
 
      
 
        /* normalise, bit fiddly as spectrum covers less than half of image
           ..mask out unilluminated pixels */
        cpl_size sx = cpl_image_get_size_x(bkg_collapse);
        double maxval = 0;
        double medval = 0;
        double stdval = 0;
        cpl_size niter = 3;
        double kappa = 2.0;
 
        for(cpl_size kk = 0; kk < niter; kk++) {
            maxval = cpl_image_get_max_window(bkg_collapse,drop_ll,1,sx - drop_ur,1);
            medval = cpl_image_get_median_window(bkg_collapse,drop_ll,1,sx - drop_ur,1);
            stdval = cpl_image_get_stdev_window(bkg_collapse,drop_ll,1,sx - drop_ur,1);
            cpl_msg_warning(cpl_func,"maxval: %g", maxval);
            cpl_msg_warning(cpl_func,"medval: %g", medval);
            cpl_msg_warning(cpl_func,"stdval: %g", stdval);
            cpl_mask_threshold_image(cpl_image_get_bpm(bkg_collapse),
                    bkg_collapse,
                    medval - kappa * stdval,
                    DBL_MAX,
                    CPL_BINARY_0);
            cpl_image_fill_rejected(bkg_collapse, 0.0);
        }
 
        /* ..normalise by median. Do both image and sky background but
             leave confidence alone - the slit profile is assumed
             noiseless so confidence unaffected (?) */
 
        medval = cpl_image_get_median(bkg_collapse);
        cpl_msg_warning(cpl_func,"medval: %g", medval);
 
        cpl_image_divide_scalar(bkg_collapse, medval);
 
        enlu_divide_slit_response_worker(jitters->limages[j]->himage,
                                        bkg_collapse);
        enlu_divide_slit_response_worker(jitters->limages[j]->bkg,
                                        bkg_collapse);
 
     
        /* sanity check, plotted slit profile should be flat */
/*
        {
        cpl_image * bkg_copy = cpl_image_duplicate(
                                       hdrl_image_get_image(
                                       jitters->limages[j]->bkg));
        cpl_image_reject_value(bkg_copy, CPL_VALUE_NOTFINITE);
 
        cpl_image * bkg_collapse = cpl_image_collapse_median_create(bkg_copy,
                                                                    0, 0, 0);
 
        double maxval = cpl_image_get_max(bkg_collapse);
        cpl_mask_threshold_image(cpl_image_get_bpm(bkg_collapse),
                                 bkg_collapse,
                                 0.7 * maxval,
                                 DBL_MAX,
                                 CPL_BINARY_0);
        cpl_image_fill_rejected(bkg_collapse, 0.0);
 
        char* filename = cpl_sprintf("bkg_vector_%d.fits", (int)j);
        cpl_vector * bkg_vector = cpl_vector_new_from_image_row(bkg_collapse, 1);
        cpl_vector_save(bkg_vector,
                       filename, CPL_TYPE_DOUBLE, NULL, CPL_IO_CREATE);
        cpl_free(filename);
        cpl_vector_delete(bkg_vector);
        }
*/
 
        cpl_image_delete(bkg_collapse);
        cpl_image_delete(bkg_copy);
    }
 
    return cpl_error_get_code();
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
cpl_error_code enlu_divide_slit_response_worker(hdrl_image * himage_2d,
                                                cpl_image * response_1d) {
 
    /* check inputs */
   
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
    cpl_ensure_code(himage_2d, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(response_1d, CPL_ERROR_NULL_INPUT);
 
    /*cpl_msg_info(cpl_func, "enlu_divide_slit_response_worker called");*/
 
    /* do it the simple but slow way, using hdrl and cpl to access the
       pixels one by one */
 
    cpl_size nx = hdrl_image_get_size_x(himage_2d);
    cpl_size ny = hdrl_image_get_size_y(himage_2d);
 
    for (cpl_size i = 1; i <= nx; i++) {
        int slit_ok = 0;
        double slit = cpl_image_get(response_1d, i, 1, &slit_ok); 
        for (cpl_size j = 1; j <= ny; j++) {
            int data_ok = 0;
            hdrl_value data = hdrl_image_get_pixel(himage_2d, i, j, &data_ok);
 
            if (data_ok==0 && slit_ok==0) {
                data.data = data.data / slit;
                data.error = data.error / slit;
            } else {
                data = (hdrl_value) {0.0, 0.0};
                data_ok = 1;
            }
            hdrl_image_set_pixel(himage_2d, i, j, data);
            if (data_ok == 1) {
                hdrl_image_reject(himage_2d, i, j);
            }
        }
    }
 
    return cpl_error_get_code();
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
cpl_matrix * enlu_linepos_2d(const hdrl_image * spectrum2d, 
                             const cpl_size slice_index,
                             const cpl_vector * guess_pos) {
 
    cpl_matrix * line_pos = NULL;
    cpl_vector * next_guess_pos = NULL;
    cpl_vector * spectrum1d = NULL;
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return NULL;
 
    cpl_ensure(spectrum2d, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(guess_pos, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(cpl_vector_get_size(guess_pos) > 0,
               CPL_ERROR_ILLEGAL_INPUT, NULL);
 
    cpl_size nlines = cpl_vector_get_size(guess_pos);
 
    const cpl_size nx = hdrl_image_get_size_x(spectrum2d);
    line_pos = cpl_matrix_new(nx, nlines);
    cpl_matrix_fill(line_pos, NAN);
 
    /* loop through slices on RH side of image. Remember that cpl_image
       indeces are 1-based, cpl_vector and cpl_matrix 0-based - fab! */
 
    next_guess_pos = cpl_vector_duplicate(guess_pos);
 
    for (cpl_size ix = slice_index; ix < nx; ix++) {
 
        /* fit the lines in this slice to get accurate peak positions */
 
        spectrum1d = cpl_vector_new_from_image_column(
                     hdrl_image_get_image_const(spectrum2d), ix+1);
    
        for (cpl_size line_id=0; line_id < nlines; line_id++) {
            double line_guess_pos = cpl_vector_get(next_guess_pos, line_id);
            double fitted_pos = enlu_linepos_1d(spectrum1d, line_guess_pos, 4);
            if (!isnan(fitted_pos)) {
 
                /* set the fit peak as the measured position of the line,
                   set next_guess_pos to the actual line positions so that
                   it can follow slow variations with ix */
 
                cpl_matrix_set(line_pos, ix, line_id, fitted_pos);
                cpl_vector_set(next_guess_pos, line_id, fitted_pos);
            }
        }
        cpl_vector_delete(spectrum1d);
    }
    cpl_vector_delete(next_guess_pos);
 
    /* now loop through slices on LH side of image */
 
    next_guess_pos = cpl_vector_duplicate(guess_pos);
 
    for (cpl_size ix = slice_index-1; ix >= 0; ix--) { 
        spectrum1d = cpl_vector_new_from_image_column(
                     hdrl_image_get_image_const(spectrum2d), ix+1);
    
        for (cpl_size line_id=0; line_id < nlines; line_id++) {
            double line_guess_pos = cpl_vector_get(next_guess_pos, line_id);
            double fitted_pos = enlu_linepos_1d(spectrum1d, line_guess_pos, 4);
            if (!isnan(fitted_pos)) {
                cpl_matrix_set(line_pos, ix, line_id, fitted_pos);
                cpl_vector_set(next_guess_pos, line_id, fitted_pos);
            }
        }
        cpl_vector_delete(spectrum1d);
    }
    cpl_vector_delete(next_guess_pos);
 
    /* Return NULL on error */
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        cpl_matrix_delete(line_pos);
        line_pos = NULL;
    }
 
    return line_pos;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
double enlu_linepos_1d(const cpl_vector * spectrum1d,
                       const double guess_pos,
                       const cpl_size half_width) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return NAN;
    cpl_ensure(spectrum1d, CPL_ERROR_NULL_INPUT, NAN);
 
    cpl_polynomial * line_fit = NULL;
    double result = NAN;
 
    cpl_size istart = (cpl_size) guess_pos - half_width;
    cpl_size istop = (cpl_size) guess_pos + half_width;
    cpl_size npos = istop - istart + 1;
 
    /* set up pixel positions of chunk */
 
    cpl_matrix * pos_chunk = cpl_matrix_new(1, npos);
    for (cpl_size i = 0; i < npos; i++) {
        cpl_matrix_set(pos_chunk, 0, i, (double) (i + istart));
    }
 
    /* get spectrum data for chunk, spec_ok goes False if any points 
       bad (=0) or are NaN, the fitting routine can't handle NaNs */
 
    cpl_vector * spectrum_chunk = cpl_vector_extract(spectrum1d,
                                                     istart, istop, 1);
    const double * spectrum_chunk_data = cpl_vector_get_data_const(
                                         spectrum_chunk);
    int spec_ok = 1;
    for (cpl_size i = 0; i < npos; i++) {
        spec_ok = spec_ok &&
                  spectrum_chunk_data[i] != 0.0 &&
                  !isnan(spectrum_chunk_data[i]);
    }
 
    /* fit a parabola to the line peak if the data are good */
 
    if (spec_ok) {
        line_fit = cpl_polynomial_new(1);
        const cpl_size maxdeg1d = 2;
        cpl_polynomial_fit(line_fit, pos_chunk, NULL, spectrum_chunk, NULL,
                           CPL_FALSE, NULL, &maxdeg1d);
 
        /* set the fit peak as the measured position of the line,
           set next_guess_pos to the actual line positions so that
           it can follow slow variations with ix */
 
        cpl_size pow = 1;
        double poly_b = cpl_polynomial_get_coeff(line_fit, &pow);
        pow = 2;
        double poly_c = cpl_polynomial_get_coeff(line_fit, &pow);
        result = -poly_b / (2.0 * poly_c);
 
        /* if the fitted line centre is outside the expected range 
           then something has probably gone wrong - errant pixel or
           something - ignore it */
 
        if (fabs(result - guess_pos) > half_width) {
            result = NAN;
        } 
    }
 
    cpl_matrix_delete(pos_chunk);
    cpl_vector_delete(spectrum_chunk);
    cpl_polynomial_delete(line_fit);
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        result = NAN;
    }
 
    return result;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
cpl_error_code enlu_trace_save(const char * pro_catg,
                               const hdrl_image * image,
                               const cpl_image * confidence,
                               const cpl_size ntraces,
                               const cpl_polynomial * traces[ntraces],
                               const cpl_size nspectra,
                               const cpl_vector * spectra[nspectra],
                               const cpl_size nlines,
                               const cpl_polynomial * lines[nlines],
                               cpl_frameset * frameset,
                               const cpl_parameterlist * parlist,
                               const char * filename,
                               const char * recipe_name) {
 
    /* checks */
   
    if (cpl_error_get_code() != CPL_ERROR_NONE) return cpl_error_get_code();
    cpl_ensure_code(pro_catg, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(image, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(frameset, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(parlist, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(filename, CPL_ERROR_NULL_INPUT);
 
    mef_extension_list   * mefs = NULL;
    cpl_propertylist     * plist = NULL;
 
    /* make output propertylist */
 
    plist = cpl_propertylist_new();
    cpl_propertylist_append_string(plist, CPL_DFS_PRO_CATG, pro_catg);
 
    /* add QC parameters  */
    /* TBD */
 
    enu_check_error_code("error constructing output propertylist");
 
    /* save the trace to a DFS-compliant MEF file */
 
    /* .. how many mefs? */
 
    cpl_size nmefs = ntraces + nspectra + nlines + 1;
    mefs = enu_mef_extension_list_new(nmefs);
    cpl_size imef = 0;
    mefs->mef[imef] = enu_mef_new_image("CONFIDENCE", confidence, NULL);
    imef++;
    for (cpl_size i = 0; i < ntraces; i++) {
        cpl_propertylist * mef_plist = cpl_propertylist_new();
        cpl_propertylist_update_string(mef_plist, "DIR", "Y");
        mefs->mef[imef] = enu_mef_new_table(
                                        "TRACE",
                                        enlu_warp_poly_save_to_table(traces[i]),
                                        mef_plist);
        imef++;
    }
    for (cpl_size i = 0; i < nspectra; i++) {
        mefs->mef[imef] = enu_mef_new_vector("SPECTRUM", spectra[i], NULL);
        imef++;
    }
    for (cpl_size i = 0; i < nlines; i++) {
        cpl_propertylist * mef_plist = cpl_propertylist_new();
        cpl_propertylist_update_string(mef_plist, "DIR", "X");
        mefs->mef[imef] = enu_mef_new_table(
                                        "LINE",
                                        enlu_warp_poly_save_to_table(lines[i]),
                                        mef_plist);
        imef++;
    }
 
    enu_dfs_save_himage(frameset,
                        parlist,
                        frameset,
                        CPL_TRUE,
                        image,
                        NULL,
                        mefs,
                        recipe_name,
                        NULL,
                        plist,
                        NULL,
                        PACKAGE "/" PACKAGE_VERSION,
                        filename);
 
 cleanup:
    enu_mef_extension_list_delete(mefs);
    cpl_propertylist_delete(plist);
 
    return cpl_error_get_code();
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
cpl_polynomial * enlu_warp_poly_load_from_table(const cpl_table * table) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return NULL;
    cpl_ensure(table, CPL_ERROR_NULL_INPUT, NULL);
 
    /* only handle polynomial degree 2, keeps things simpler */
 
    cpl_polynomial * result = cpl_polynomial_new(2);
 
    /* expect table columns 1.dim.power, 2.dim.power, coefficient */
 
    enu_check(cpl_table_has_column(table, "1.dim.power"), 
              CPL_ERROR_ILLEGAL_INPUT,
              "table does not have column '1.dim.power'");
    enu_check(cpl_table_has_column(table, "2.dim.power"), 
              CPL_ERROR_ILLEGAL_INPUT,
              "table does not have column '2.dim.power'");
    enu_check(cpl_table_has_column(table, "coefficient"), 
              CPL_ERROR_ILLEGAL_INPUT,
              "table does not have column 'coefficient'");
 
    /* loop through the rows and set the polynomial coefficients */
 
    cpl_size nrow = cpl_table_get_nrow(table);
    for (cpl_size row=0; row<nrow; row++) {
        int null1 = 0;
        int pow1 = cpl_table_get_int(table, "1.dim.power", row, &null1);
        int null2 = 0;
        int pow2 = cpl_table_get_int(table, "2.dim.power", row, &null2);
        int nullcoeff = 0;
        double coeff = cpl_table_get_double(table, "coefficient", row, 
                                            &nullcoeff);
        enu_check(!(null1 || null2 || nullcoeff), CPL_ERROR_ILLEGAL_INPUT,
                  "table contains NULL values");
 
        cpl_size pows[2] = {pow1, pow2};
        cpl_polynomial_set_coeff(result, pows, coeff);
    }
 
 cleanup:
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        cpl_polynomial_delete(result);
        result = NULL;
    }
    return result;
}
 
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
cpl_table * enlu_warp_poly_save_to_table(const cpl_polynomial * poly) {
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) return NULL;
    cpl_ensure(poly, CPL_ERROR_NULL_INPUT, NULL);
 
    cpl_table * result = NULL;
 
    /* Is the polynomial 1 or 2d? */
 
    const cpl_size dimension = cpl_polynomial_get_dimension(poly);
    enu_check(dimension==1 || dimension==2, CPL_ERROR_ILLEGAL_INPUT,
              "can only handle polynomial with dimension 1 or 2");
 
    if (dimension == 1) {
 
        const cpl_size degree = cpl_polynomial_get_degree(poly);
        result = cpl_table_new(degree + 1);
 
        /* construct table columns */
 
        cpl_table_new_column(result, "1.dim.power", CPL_TYPE_INT);
        cpl_table_new_column(result, "coefficient", CPL_TYPE_DOUBLE);
 
        cpl_size nrows = 0;
        for (cpl_size dim1_power=0; dim1_power<=degree; dim1_power++) {
            double coeff = cpl_polynomial_get_coeff(poly, &dim1_power);
            if (coeff != 0.0) {
                cpl_table_set_int(result, "1.dim.power", nrows, dim1_power);
                cpl_table_set_double(result, "coefficient", nrows, coeff);
                nrows++;
            } 
        }
        cpl_table_set_size(result, nrows);
 
    } else if (dimension == 2) {
 
        const cpl_size degree = cpl_polynomial_get_degree(poly);
        result = cpl_table_new((degree + 1) * (degree + 1));
 
        /* construct table columns */
 
        cpl_table_new_column(result, "1.dim.power", CPL_TYPE_INT);
        cpl_table_new_column(result, "2.dim.power", CPL_TYPE_INT);
        cpl_table_new_column(result, "coefficient", CPL_TYPE_DOUBLE);
 
        cpl_size nrows = 0;
        for (int dim2_power=0; dim2_power<=degree; dim2_power++) {
            for (int dim1_power=0; dim1_power<=degree; dim1_power++) {
                cpl_size pows[2] = {dim1_power, dim2_power};
                double coeff = cpl_polynomial_get_coeff(poly, pows);
                if (coeff != 0.0) {
                    cpl_table_set_int(result, "1.dim.power", nrows, dim1_power);
                    cpl_table_set_int(result, "2.dim.power", nrows, dim2_power);
                    cpl_table_set_double(result, "coefficient", nrows, coeff);
                    nrows++;
                }
            } 
        }
        cpl_table_set_size(result, nrows);
    }
 
 cleanup:
 
    if (cpl_error_get_code() != CPL_ERROR_NONE) {
        cpl_table_delete(result);
        result = NULL;
    }
    return result;
}