moo_remove_crh.c

/*
 * This file is part of the MOONS Pipeline
 * Copyright (C) 2002-2016 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
 
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
/*-----------------------------------------------------------------------------
                                   Includes
 -----------------------------------------------------------------------------*/
#include <math.h>
#include <string.h>
#include <cpl.h>
#include <hdrl.h>
#include "moo_utils.h"
#include "moo_dfs.h"
#include "moo_pfits.h"
#include "moo_qc.h"
#include "moo_params.h"
#include "moo_badpix.h"
#include "moo_single.h"
#include "moo_det.h"
#include "moo_detlist.h"
#include "moo_remove_crh.h"
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
 
/*-----------------------------------------------------------------------------
                              Function codes
 -----------------------------------------------------------------------------*/
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static cpl_image *
_moo_qual_fromlist(cpl_imagelist *list)
{
    cpl_ensure(list, CPL_ERROR_NULL_INPUT, NULL);
 
    int size = cpl_imagelist_get_size(list);
    cpl_ensure(size > 0, CPL_ERROR_ILLEGAL_INPUT, NULL);
 
    const cpl_image *qimg = cpl_imagelist_get_const(list, 0);
 
    if (size > 1) {
        int k;
 
        int nx = cpl_image_get_size_x(qimg);
        int ny = cpl_image_get_size_y(qimg);
        const int *qdata = cpl_image_get_data_int_const(qimg);
 
        for (k = 1; k < size; k++) {
            cpl_image *qimg2 = cpl_imagelist_get(list, k);
            int *qdata2 = cpl_image_get_data_int(qimg2);
            for (int i = 0; i < nx * ny; i++) {
                if (qdata[i] != qdata2[i]) {
                    cpl_msg_error("moo_remove_crh",
                                  "QUAL of first frame and %d are not equal",
                                  k);
                }
            }
        }
    }
    return cpl_image_duplicate(qimg);
}
 
static moo_single *
_moo_remove_CRH_collapse_median(hdrl_imagelist *list,
                                cpl_imagelist *quallist,
                                moo_detector_type type,
                                int num)
{
    moo_single *single = NULL;
    int j;
    cpl_ensure(list != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
 
    int size = hdrl_imagelist_get_size(list);
 
    if (size > 0) {
        hdrl_image *res = NULL;
        cpl_image *contrib = NULL;
        hdrl_imagelist_collapse_median(list, &res, &contrib);
        for (j = 0; j < size; j++) {
            hdrl_imagelist_unset(list, 0);
        }
 
        cpl_image *qual = _moo_qual_fromlist(quallist);
        single = moo_single_new(type, num);
        single->qual = qual;
        single->image = res;
 
        for (j = 0; j < size; j++) {
            cpl_imagelist_unset(quallist, 0);
        }
 
        cpl_image_delete(contrib);
    }
 
    return single;
}
 
static moo_single *
_moo_remove_CRH_collapse_mean(hdrl_imagelist *list,
                              cpl_imagelist *quallist,
                              moo_detector_type type,
                              int num)
{
    moo_single *single = NULL;
    int j;
    cpl_ensure(list != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
 
    int size = hdrl_imagelist_get_size(list);
 
    if (size > 0) {
        hdrl_image *res = NULL;
        cpl_image *contrib = NULL;
        hdrl_imagelist_collapse_mean(list, &res, &contrib);
 
        for (j = 0; j < size; j++) {
            hdrl_imagelist_unset(list, 0);
        }
 
        cpl_image *qual = _moo_qual_fromlist(quallist);
 
        single = moo_single_new(type, num);
        single->qual = qual;
        single->image = res;
 
        for (j = 0; j < size; j++) {
            cpl_imagelist_unset(quallist, 0);
        }
        cpl_image_delete(contrib);
    }
    return single;
}
 
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
static moo_single *
_moo_remove_CRH_collapse_sigclip(hdrl_imagelist *list,
                                 cpl_imagelist *quallist,
                                 moo_masklist *cosmiclist,
                                 double kappa,
                                 int niter,
                                 int *nbcrh,
                                 moo_detector_type type,
                                 int num)
{
    cpl_ensure(nbcrh != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
    cpl_ensure(list != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
    cpl_ensure(quallist != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
 
    moo_single *single = NULL;
    int j, k;
 
    int size = hdrl_imagelist_get_size(list);
    int nbcosmics = 0;
    float avg_cosmics = 0.0;
 
    if (size > 0) {
        hdrl_image *res = NULL;
        cpl_image *rejhigh = NULL;
        cpl_image *contrib = NULL;
 
        int sizex = hdrl_imagelist_get_size_x(list);
        int sizey = hdrl_imagelist_get_size_y(list);
 
        if (cosmiclist != NULL) {
            for (int i = 0; i < size; i++) {
                moo_mask *mask = moo_masklist_get(cosmiclist, i);
                moo_mask_set(mask, type, num, cpl_mask_new(sizex, sizey));
            }
        }
        moo_try_check(hdrl_imagelist_collapse_sigclip(list, kappa, kappa, niter,
                                                      &res, &contrib, NULL,
                                                      &rejhigh),
                      " ");
#if MOO_DEBUG_REMOVE_CRH
        {
            const char *extname = moo_detector_get_extname(type, num);
            char *name = cpl_sprintf("contrib_%s.fits", extname);
            cpl_image_save(contrib, name, CPL_TYPE_INT, NULL, CPL_IO_CREATE);
            cpl_free(name);
 
            name = cpl_sprintf("rejhigh_%s.fits", extname);
            cpl_image_save(rejhigh, name, CPL_TYPE_DOUBLE, NULL, CPL_IO_CREATE);
            cpl_free(name);
        }
#endif
        const double *high_data = cpl_image_get_data_double_const(rejhigh);
 
        cpl_image *qual = _moo_qual_fromlist(quallist);
        for (j = 0; j < sizex * sizey; j++) {
            if (cosmiclist != NULL) {
                for (k = 0; k < size; k++) {
                    const hdrl_image *himage =
                        hdrl_imagelist_get_const(list, k);
                    const cpl_image *image = hdrl_image_get_image_const(himage);
                    const double *img_data =
                        cpl_image_get_data_double_const(image);
                    cpl_image *qimg = cpl_imagelist_get(quallist, k);
                    cpl_mask *mask =
                        moo_masklist_get_mask(cosmiclist, k, type, num);
                    cpl_binary *mdata = cpl_mask_get_data(mask);
                    int *qdata = cpl_image_get_data_int(qimg);
 
                    double val = img_data[j];
                    double high = high_data[j];
 
                    if (val > high) {
                        nbcosmics++;
                        qdata[j] |= MOO_BADPIX_COSMIC_UNREMOVED;
                        mdata[j] = CPL_BINARY_1;
                    }
                }
            }
        }
        for (j = 0; j < size; j++) {
            hdrl_imagelist_unset(list, 0);
        }
        *nbcrh = nbcosmics;
        avg_cosmics = (float)nbcosmics / (float)size;
        cpl_msg_indent_more();
        if (cosmiclist != NULL) {
            const char *extname = moo_detector_get_extname(type, num);
            cpl_msg_info(__func__,
                         "%s: found %d cosmics in total (avg %.1f / frame) ",
                         extname, nbcosmics, avg_cosmics);
        }
        cpl_msg_indent_less();
        single = moo_single_new(type, num);
        moo_qc_set_ncrh(single->header, nbcosmics);
        moo_qc_set_ncrh_avg(single->header, avg_cosmics);
        single->qual = qual;
        single->image = res;
#if MOO_DEBUG_REMOVE_CRH
        for (j = 0; j < size; j++) {
            {
                char *name = cpl_sprintf("qual_%s_%d.fits", extname, j);
                cpl_image_save(cpl_imagelist_get(quallist, j), name,
                               CPL_TYPE_INT, NULL, CPL_IO_CREATE);
                cpl_free(name);
            }
        }
#endif
        for (j = 0; j < size; j++) {
            cpl_imagelist_unset(quallist, 0);
        }
 
 
        cpl_image_delete(rejhigh);
        cpl_image_delete(contrib);
    }
moo_try_cleanup:
    return single;
}
 
 
static moo_single *
_moo_remove_CRH_collapse(hdrl_imagelist *list,
                         cpl_imagelist *quallist,
                         moo_masklist *cosmiclist,
                         moo_crh_params *params,
                         int *nbcrh,
                         moo_detector_type type,
                         int num)
{
    moo_single *single = NULL;
 
 
    if (strcmp(params->method, MOO_CRH_METHOD_MEDIAN) == 0) {
        single = _moo_remove_CRH_collapse_median(list, quallist, type, num);
    }
    else if (strcmp(params->method, MOO_CRH_METHOD_MEAN) == 0) {
        single = _moo_remove_CRH_collapse_mean(list, quallist, type, num);
    }
    else {
        single = _moo_remove_CRH_collapse_sigclip(list, quallist, cosmiclist,
                                                  params->kappa, params->niter,
                                                  nbcrh, type, num);
    }
    return single;
}
 
 
static cpl_error_code
_moo_copy_header_kw(cpl_propertylist *res_header,
                    const cpl_propertylist *orig_header)
{
    cpl_ensure_code(res_header != NULL, CPL_ERROR_NULL_INPUT);
    cpl_ensure_code(orig_header != NULL, CPL_ERROR_NULL_INPUT);
 
    cpl_propertylist_copy_property_regexp(res_header, orig_header, "ESO DET *",
                                          0);
    cpl_propertylist_copy_property_regexp(res_header, orig_header, "HDU*", 0);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_BUNIT);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_ERRDATA);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_QUALDATA);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CRPIX1);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CRPIX2);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CRVAL1);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CRVAL2);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CTYPE1);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CTYPE2);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CD1_1);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CD1_2);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CUNIT1);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CD2_1);
    cpl_propertylist_copy_property(res_header, orig_header, MOO_PFITS_CD2_2);
    return cpl_error_get_code();
}
 
static cpl_error_code
_moo_remove_crh_single(moo_detlist *detlist,
                       moo_masklist *cosmiclist,
                       moo_det *orig_det,
                       moo_detector_type type,
                       int num,
                       unsigned int badpix_level,
                       moo_crh_params *params,
                       moo_det *res,
                       int *nbcrh_total)
{
    cpl_error_code status = CPL_ERROR_NONE;
    int nbcrh = 0;
    moo_single *single = NULL;
 
 
    moo_detlist_load_single(detlist, type, num, badpix_level);
    hdrl_imagelist *list = moo_detlist_get_image(detlist, type, num);
    cpl_imagelist *qlist = moo_detlist_get_single_qual(detlist, type, num);
 
    single = _moo_remove_CRH_collapse(list, qlist, cosmiclist, params, &nbcrh,
                                      type, num);
    if (single != NULL) {
        cpl_propertylist *header = NULL;
 
        moo_try_check(header = moo_det_get_single_header(orig_det, type, num),
                      " ");
        moo_try_check(_moo_copy_header_kw(single->header, header), " ");
        *nbcrh_total += nbcrh;
    }
moo_try_cleanup:
    moo_det_set_single(res, type, num, single);
    cpl_imagelist_unwrap(qlist);
    hdrl_imagelist_unwrap(list);
    moo_detlist_free_single(detlist, type, num);
    return status;
}
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
moo_det *
moo_remove_CRH(moo_detlist *detlist,
               moo_masklist *cosmiclist,
               moo_crh_params *params)
{
    moo_det *dres = NULL;
    cpl_ensure(detlist != NULL, CPL_ERROR_NULL_INPUT, NULL);
    cpl_ensure(detlist->size > 0, CPL_ERROR_ILLEGAL_INPUT, NULL);
    cpl_ensure(params != NULL, CPL_ERROR_NULL_INPUT, NULL);
 
    cpl_errorstate prestate = cpl_errorstate_get();
 
    if (strcmp(params->method, MOO_CRH_METHOD_MEDIAN) == 0) {
        cpl_msg_info(__func__, "Computing median of %d frames",
                     (int)detlist->size);
    }
    else if (strcmp(params->method, MOO_CRH_METHOD_MEAN) == 0) {
        cpl_msg_info(__func__, "Computing mean of %d frames",
                     (int)detlist->size);
    }
    else {
        cpl_msg_info(__func__, "Computing sigma clip of %d frames",
                     (int)detlist->size);
    }
 
    int badpix_level = MOO_BADPIX_GOOD;
 
    moo_det *orig_det = moo_detlist_get(detlist, 0);
    dres = moo_det_new();
    dres->primary_header = cpl_propertylist_duplicate(orig_det->primary_header);
    cpl_propertylist_erase_regexp(dres->primary_header, "ESO QC FRAME RAW", 0);
    moo_qc_set_ncom(dres->primary_header, detlist->size);
    cpl_errorstate prev_state = cpl_errorstate_get();
    dres->fibre_table = cpl_table_duplicate(moo_det_get_fibre_table(orig_det));
 
    if (!cpl_errorstate_is_equal(prev_state)) {
        cpl_msg_info(__func__, "DET input files do not have a fibre table.");
        cpl_errorstate_set(prev_state);
    }
 
    int nbcrh_total = 0;
    for (int num = 1; num <= 2; num++) {
        moo_try_check(_moo_remove_crh_single(detlist, cosmiclist, orig_det,
                                             MOO_TYPE_RI, num, badpix_level,
                                             params, dres, &nbcrh_total),
                      " ");
        moo_try_check(_moo_remove_crh_single(detlist, cosmiclist, orig_det,
                                             MOO_TYPE_YJ, num, badpix_level,
                                             params, dres, &nbcrh_total),
                      " ");
        moo_try_check(_moo_remove_crh_single(detlist, cosmiclist, orig_det,
                                             MOO_TYPE_H, num, badpix_level,
                                             params, dres, &nbcrh_total),
                      " ");
    }
 
    if (strcmp(params->method, MOO_CRH_METHOD_SIGCLIP) == 0) {
        moo_qc_set_ncrh_tot(dres->primary_header, nbcrh_total);
    }
 
moo_try_cleanup:
    if (!cpl_errorstate_is_equal(prestate)) {
        cpl_msg_error(__func__, "Error in remove CRH");
        cpl_errorstate_dump(prestate, CPL_FALSE, cpl_errorstate_dump_one);
        moo_det_delete(dres);
        dres = NULL;
    }
    return dres;
}