moo_compute_resp.c

/*
 * This file is part of the MOONS Pipeline
 * Copyright (C) 2002-2016 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                   Includes
 -----------------------------------------------------------------------------*/
#include <math.h>
#include <string.h>
#include <cpl.h>
 
#include "moo_compute_resp.h"
#include "moo_fits.h"
#include "moo_pfits.h"
#include "moo_badpix.h"
#include "moo_utils.h"
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
/*-----------------------------------------------------------------------------
                              Function codes
 -----------------------------------------------------------------------------*/
 
 
static cpl_error_code
_moo_response_fit1d(cpl_table *fit_table,
                    int degree,
                    int niter,
                    double max_frac,
                    double kappa_lo,
                    double kappa_up)
{
    double wmin =
        cpl_table_get_column_min(fit_table, MOO_COMPUTE_RESP_FIT_WAVE);
    double wmax =
        cpl_table_get_column_max(fit_table, MOO_COMPUTE_RESP_FIT_WAVE);
 
    cpl_size fit_table_size =
        cpl_table_and_selected_int(fit_table, MOO_COMPUTE_RESP_FIT_IS_USED,
                                   CPL_EQUAL_TO, 1);
    if (fit_table_size > degree) {
        int total_rejected = 0;
        moo_tcheby_polynomial *poly = NULL;
 
        cpl_array *seltab = cpl_table_where_selected(fit_table);
 
        double *fluxes =
            cpl_table_get_data_double(fit_table, MOO_COMPUTE_RESP_FIT_FLUX);
        double *wave =
            cpl_table_get_data_double(fit_table, MOO_COMPUTE_RESP_FIT_WAVE);
        int *isused =
            cpl_table_get_data_int(fit_table, MOO_COMPUTE_RESP_FIT_IS_USED);
 
        cpl_bivector *data = cpl_bivector_new(fit_table_size);
        cpl_vector *vx = cpl_bivector_get_x(data);
        cpl_vector *vy = cpl_bivector_get_y(data);
        for (int i = 0; i < fit_table_size; i++) {
            int idx = cpl_array_get_cplsize(seltab, i, NULL);
            double f = fluxes[idx];
            double w = wave[idx];
            cpl_vector_set(vx, i, w);
            cpl_vector_set(vy, i, f);
        }
 
        poly = moo_tcheby_polynomial_fit(data, degree, wmin, wmax);
        cpl_size size = fit_table_size;
 
        for (int i = 1; i <= niter; i++) {
            cpl_msg_info(__func__,
                         "iter %d / %d with %" CPL_SIZE_FORMAT
                         " data (total rejected %d)",
                         i, niter, size, total_rejected);
            cpl_vector *vdiff = cpl_vector_new(size);
 
            for (int j = 0; j < size; j++) {
                double x = cpl_vector_get(vx, j);
                double y = cpl_vector_get(vy, j);
                double t = moo_tcheby_polynomial_eval(poly, x);
                double diff = y - t;
                cpl_vector_set(vdiff, j, diff);
            }
 
            double stdev = cpl_vector_get_stdev(vdiff);
            int nb_rejected = 0;
 
            for (int j = 0; j < size; j++) {
                double diff = cpl_vector_get(vdiff, j);
                if ((diff >= kappa_lo * stdev) || (diff <= -kappa_up * stdev)) {
                    int idx = cpl_array_get_cplsize(seltab, j, NULL);
                    isused[idx] = 2;
                    nb_rejected++;
                }
            }
            cpl_vector_delete(vdiff);
 
            if (nb_rejected == size || nb_rejected == 0) {
                break;
            }
            double frac =
                (double)(nb_rejected + total_rejected) / (double)fit_table_size;
            if (frac > max_frac) {
                break;
            }
            /* OK to reject points and delete old solution */
            total_rejected += nb_rejected;
            moo_tcheby_polynomial_delete(poly);
            cpl_array_delete(seltab);
 
            cpl_table_select_all(fit_table);
            size = cpl_table_and_selected_int(fit_table,
                                              MOO_COMPUTE_RESP_FIT_IS_USED,
                                              CPL_EQUAL_TO, 2);
            seltab = cpl_table_where_selected(fit_table);
            for (int j = 0; j < size; j++) {
                int idx = cpl_array_get_cplsize(seltab, j, NULL);
                isused[idx] = 0;
            }
            cpl_array_delete(seltab);
            cpl_table_select_all(fit_table);
            size = cpl_table_and_selected_int(fit_table,
                                              MOO_COMPUTE_RESP_FIT_IS_USED,
                                              CPL_EQUAL_TO, 1);
            seltab = cpl_table_where_selected(fit_table);
 
            cpl_bivector_delete(data);
            data = cpl_bivector_new(size);
            vx = cpl_bivector_get_x(data);
            vy = cpl_bivector_get_y(data);
            for (int j = 0; j < size; j++) {
                int idx = cpl_array_get_cplsize(seltab, j, NULL);
                cpl_vector_set(vx, j, wave[idx]);
                cpl_vector_set(vy, j, fluxes[idx]);
            }
            poly = moo_tcheby_polynomial_fit(data, degree, wmin, wmax);
        }
 
        cpl_array_delete(seltab);
        cpl_table_select_all(fit_table);
 
        size =
            cpl_table_and_selected_int(fit_table, MOO_COMPUTE_RESP_FIT_IS_USED,
                                       CPL_EQUAL_TO, 2);
        seltab = cpl_table_where_selected(fit_table);
        for (int j = 0; j < size; j++) {
            int idx = cpl_array_get_cplsize(seltab, j, NULL);
            isused[idx] = 1;
        }
 
        int nrow = cpl_table_get_nrow(fit_table);
 
        for (int i = 0; i < nrow; i++) {
            double w = cpl_table_get_double(fit_table,
                                            MOO_COMPUTE_RESP_FIT_WAVE, i, NULL);
            double fit = moo_tcheby_polynomial_eval(poly, w);
            cpl_table_set_double(fit_table, MOO_COMPUTE_RESP_FIT_FIT, i, fit);
        }
 
        moo_tcheby_polynomial_delete(poly);
        cpl_bivector_delete(data);
        cpl_array_delete(seltab);
    }
    return cpl_error_get_code();
}
 
static moo_spline *
_get_atm_spline(moo_atm *atm, double airm)
{
    moo_spline *res = NULL;
    cpl_table *atm_flux = moo_atm_convert_mag_to_flux(atm, airm);
 
    int nrow = cpl_table_get_nrow(atm_flux);
    double *x = cpl_table_get_data_double(atm_flux, MOO_ATM_WAVE);
    double *y = cpl_table_get_data_double(atm_flux, MOO_ATM_FLUX);
 
    cpl_vector *vx = cpl_vector_wrap(nrow, x);
    cpl_vector *vy = cpl_vector_wrap(nrow, y);
    cpl_bivector *bv = cpl_bivector_wrap_vectors(vx, vy);
 
    res = moo_spline_create(bv);
 
    cpl_bivector_unwrap_vectors(bv);
    cpl_vector_unwrap(vx);
    cpl_vector_unwrap(vy);
    cpl_table_delete(atm_flux);
    return res;
}
 
static moo_spline *
_get_flx_spline(moo_flx *flx, const char *targname)
{
    moo_spline *flx_spline = NULL;
    cpl_table *std_spec = moo_flx_get_obj(flx, targname);
    int nrow = cpl_table_get_nrow(std_spec);
 
    double *x = cpl_table_get_data_double(std_spec, MOO_FLX_SPECTRUM_WAVE);
    double *y = cpl_table_get_data_double(std_spec, MOO_FLX_SPECTRUM_FLUX);
 
    cpl_vector *vx = cpl_vector_wrap(nrow, x);
    cpl_vector *vy = cpl_vector_wrap(nrow, y);
    cpl_bivector *bv = cpl_bivector_wrap_vectors(vx, vy);
 
    flx_spline = moo_spline_create(bv);
 
    cpl_bivector_unwrap_vectors(bv);
    cpl_vector_unwrap(vx);
    cpl_vector_unwrap(vy);
 
    cpl_table_delete(std_spec);
 
    return flx_spline;
}
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
moo_resp *
moo_compute_resp(moo_scilist *scilist,
                 moo_atm *atm,
                 moo_flx *flx,
                 moo_compute_resp_params *params)
{
    moo_resp *resp = NULL;
 
    cpl_ensure(scilist != NULL, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(atm != NULL, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(flx != NULL, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(params != NULL, CPL_ERROR_NULL_INPUT, NULL);
 
    int size = moo_scilist_get_size(scilist);
    unsigned int badpix_level = MOO_BADPIX_OUTSIDE_DATA_RANGE |
                                MOO_BADPIX_CALIB_DEFECT | MOO_BADPIX_HOT |
                                MOO_BADPIX_COSMETIC | MOO_BADPIX_LOW_QE;
 
    resp = moo_resp_create(scilist, badpix_level);
 
    cpl_msg_info(__func__, "Compute response using %d frames", size);
 
    for (int il = 0; il < size; il++) {
        const char *response_colname =
            cpl_table_get_string(resp->response_table, MOO_RESP_TABLE_RESPONSE,
                                 il);
        moo_sci *sci = moo_scilist_get(scilist, il);
        moo_target_table *target_table = moo_sci_get_target_table(sci);
 
        if (target_table->table != NULL) {
            const char *targname = moo_target_table_get_std_name(target_table);
            moo_spline *flx_spline = _get_flx_spline(flx, targname);
            double airm_start =
                moo_pfits_get_tel_airm_start(sci->primary_header);
            double airm_end = moo_pfits_get_tel_airm_end(sci->primary_header);
            double airm = (airm_start + airm_end) / 2.;
 
            moo_spline *atm_spline = _get_atm_spline(atm, airm);
            int tidx = moo_target_table_find_index(target_table, targname);
            const char *colnames[] = { MOO_TARGET_TABLE_EXPTIMERI,
                                       MOO_TARGET_TABLE_EXPTIMEYJ,
                                       MOO_TARGET_TABLE_EXPTIMEH };
 
            for (int i = 0; i < 3; i++) {
                int degree = params->degree[i];
                double frac = params->frac;
                double kappa_lo = params->kappa_lo;
                double kappa_up = params->kappa_up;
                int niter = params->niter;
                moo_sci_single *sci_single = moo_sci_get_single(sci, i);
                if (sci_single != NULL) {
                    int win_hsize = params->filter_winhsize[i];
 
                    cpl_propertylist *header =
                        moo_sci_single_get_header(sci_single);
                    double cd1_1 = moo_pfits_get_cd1_1(header);
                    double exptime =
                        cpl_table_get_double(target_table->table, colnames[i],
                                             tidx, NULL);
                    double entranceLoss = 1;
                    double div = cd1_1 * exptime * entranceLoss;
 
                    cpl_table *response = moo_resp_get_table(resp, i);
 
                    int nrow = cpl_table_get_nrow(response);
                    double *data =
                        cpl_table_get_data_double(response, response_colname);
                    double *waves =
                        cpl_table_get_data_double(response, MOO_RESP_WAVE);
 
                    cpl_vector *init_data = cpl_vector_wrap(nrow, data);
                    cpl_vector *filter_data =
                        moo_median_filter(init_data, win_hsize);
 
                    cpl_vector_unwrap(init_data);
                    for (int j = 0; j < nrow; j++) {
                        double w = waves[j];
                        double flxval = moo_spline_eval(flx_spline, w);
                        double atmval = moo_spline_eval(atm_spline, w);
                        double fval = cpl_vector_get(filter_data, j);
 
#if MOO_DEBUG_RESP
                        {
                            char *flx_colname =
                                cpl_sprintf("%s_%s", response_colname,
                                            MOO_RESP_FLX);
                            char *atm_colname =
                                cpl_sprintf("%s_%s", response_colname,
                                            MOO_RESP_ATM);
                            char *fluxfilter_colname =
                                cpl_sprintf("%s_%s", response_colname,
                                            MOO_RESP_SPECFLUXFILTER);
                            cpl_table_set_double(response, fluxfilter_colname,
                                                 j, fval);
                            cpl_table_set_double(response, atm_colname, j,
                                                 atmval);
                            cpl_table_set_double(response, flx_colname, j,
                                                 flxval);
                            cpl_free(flx_colname);
                            cpl_free(atm_colname);
                            cpl_free(fluxfilter_colname);
                        }
#endif
 
                        if (!isnan(fval)) {
                            data[j] = flxval / (fval / div / atmval);
                        }
                        else {
                            cpl_table_set_column_invalid(response,
                                                         response_colname, j,
                                                         1);
                        }
                    }
                    cpl_table *fit_table = cpl_table_new(nrow);
                    cpl_table_duplicate_column(fit_table,
                                               MOO_COMPUTE_RESP_FIT_WAVE,
                                               response, MOO_RESP_WAVE);
                    cpl_table_duplicate_column(fit_table,
                                               MOO_COMPUTE_RESP_FIT_FLUX,
                                               response, response_colname);
                    cpl_table_new_column(fit_table,
                                         MOO_COMPUTE_RESP_FIT_IS_USED,
                                         CPL_TYPE_INT);
                    cpl_table_new_column(fit_table, MOO_COMPUTE_RESP_FIT_FIT,
                                         CPL_TYPE_DOUBLE);
                    for (int k = 0; k < nrow; k++) {
                        int val = 0;
                        if (cpl_table_is_valid(fit_table,
                                               MOO_COMPUTE_RESP_FIT_FLUX, k)) {
                            val = 1;
                        }
                        cpl_table_set_int(fit_table,
                                          MOO_COMPUTE_RESP_FIT_IS_USED, k, val);
                    }
                    _moo_response_fit1d(fit_table, degree, niter, frac,
                                        kappa_lo, kappa_up);
 
#if MOO_DEBUG_RESPONSE_FIT
                    {
                        char *resname =
                            cpl_sprintf("COMPUTE_RESP_TABLE_%s_%d.fits",
                                        sci_single->extname, il);
                        cpl_table_save(fit_table, NULL, NULL, resname,
                                       CPL_IO_CREATE);
                        cpl_free(resname);
                    }
#endif
                    for (int j = 0; j < nrow; j++) {
                        double fit =
                            cpl_table_get_double(fit_table,
                                                 MOO_COMPUTE_RESP_FIT_FIT, j,
                                                 NULL);
                        cpl_table_set(response, response_colname, j, 1 / fit);
                    }
                    cpl_table_delete(fit_table);
                    cpl_vector_delete(filter_data);
                }
            }
            moo_spline_delete(flx_spline);
            moo_spline_delete(atm_spline);
        }
    }
 
    cpl_table_erase_invalid_rows(resp->response_table);
 
    return resp;
}