KMOS Pipeline Reference Manual  1.2.8
kmo_illumination_flat.c
00001 /* $Id: kmo_illumination_flat.c,v 1.1 2013/10/21 13:44:55 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013/10/21 13:44:55 $
00024  * $Revision: 1.1 $
00025  * $Name:  $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmclipm_priv_splines.h"
00042 
00043 #include "kmo_priv_reconstruct.h"
00044 #include "kmo_priv_functions.h"
00045 #include "kmo_priv_flat.h"
00046 #include "kmo_priv_wave_cal.h"
00047 #include "kmo_functions.h"
00048 #include "kmo_cpl_extensions.h"
00049 #include "kmo_dfs.h"
00050 #include "kmo_error.h"
00051 #include "kmo_constants.h"
00052 #include "kmo_debug.h"
00053 
00054 /*-----------------------------------------------------------------------------
00055  *                          Functions prototypes
00056  *----------------------------------------------------------------------------*/
00057 
00058 static int kmo_illumination_flat_create(cpl_plugin *);
00059 static int kmo_illumination_flat_exec(cpl_plugin *);
00060 static int kmo_illumination_flat_destroy(cpl_plugin *);
00061 static int kmo_illumination_flat(cpl_parameterlist *, cpl_frameset *);
00062 
00063 /*-----------------------------------------------------------------------------
00064  *                          Static variables
00065  *----------------------------------------------------------------------------*/
00066 
00067 static char kmo_illumination_flat_description[] =
00068 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00069 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00070 "generates the spectral calibration frame needed in this recipe. As input at\n"
00071 "least a flatfield frame is required.\n"
00072 "Contrary to kmo_illumination it doesn't use flat sky frames but rather the\n"
00073 "flatfield frames from the internal flat lamp. This recipe can be used if no\n"
00074 "acceptable flat sky frames are available.\n"
00075 "The created product, the illumination correction, can be used as input for\n"
00076 "kmo_std_star and kmo_sci_red.\n"
00077 "\n"
00078 "BASIC PARAMETERS:\n"
00079 "-----------------\n"
00080 "--imethod\n"
00081 "The interpolation method used for reconstruction.\n"
00082 "\n"
00083 "ADVANCED PARAMETERS\n"
00084 "-------------------\n"
00085 "--flux\n"
00086 "Specify if flux conservation should be applied.\n"
00087 "\n"
00088 "--neighborhoodRange\n"
00089 "Defines the range to search for neighbors during reconstruction\n"
00090 "\n"
00091 "--b_samples\n"
00092 "The number of samples in spectral direction for the reconstructed cube.\n"
00093 "Ideally this number should be greater than 2048, the detector size.\n"
00094 "\n"
00095 "--b_start\n"
00096 "--b_end\n"
00097 "Used to define manually the start and end wavelength for the reconstructed\n"
00098 "cube. By default the internally defined values are used.\n"
00099 "\n"
00100 "--cmethod\n"
00101 "Following methods of frame combination are available:\n"
00102 "   * 'ksigma' (Default)\n"
00103 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00104 "   are examined. If they deviate significantly, they will be rejected according\n"
00105 "   to the conditions:\n"
00106 "       val > mean + stdev * cpos_rej\n"
00107 "   and\n"
00108 "       val < mean - stdev * cneg_rej\n"
00109 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00110 "   parameters. In the first iteration median and percentile level are used.\n"
00111 "\n"
00112 "   * 'median'\n"
00113 "   At each pixel position the median is calculated.\n"
00114 "\n"
00115 "   * 'average'\n"
00116 "   At each pixel position the average is calculated.\n"
00117 "\n"
00118 "   * 'sum'\n"
00119 "   At each pixel position the sum is calculated.\n"
00120 "\n"
00121 "   * 'min_max'\n"
00122 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00123 "   --cmax and --cmin apply to this method.\n"
00124 "\n"
00125 "--cpos_rej\n"
00126 "--cneg_rej\n"
00127 "--citer\n"
00128 "see --cmethod='ksigma'\n"
00129 "\n"
00130 "--cmax\n"
00131 "--cmin\n"
00132 "see --cmethod='min_max'\n"
00133 "\n"
00134 "--pix_scale\n"
00135 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00136 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00137 "\n"
00138 "--suppress_extension\n"
00139 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00140 "products with the same category are produced, they will be numered consecutively\n"
00141 "starting from 0.\n"
00142 "\n"
00143 "-------------------------------------------------------------------------------\n"
00144 "  Input files:\n"
00145 "\n"
00146 "   DO                    KMOS                                                  \n"
00147 "   category              Type   Explanation                    Required #Frames\n"
00148 "   --------              -----  -----------                    -------- -------\n"
00149 "   FLAT_SKY_FLAT          F2D   Flatlamp-on exposures             Y      1-n   \n"
00150 "                                (at least 3 frames recommended)                \n"
00151 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00152 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00153 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00154 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00155 "\n"
00156 "  Output files:\n"
00157 "\n"
00158 "   DO                    KMOS\n"
00159 "   category              Type   Explanation\n"
00160 "   --------              -----  -----------\n"
00161 "   ILLUM_CORR             F2I    Illumination calibration frame   \n"
00162 "-------------------------------------------------------------------------------\n"
00163 "\n";
00164 
00165 /*-----------------------------------------------------------------------------
00166  *                              Functions code
00167  *----------------------------------------------------------------------------*/
00168 
00185 int cpl_plugin_get_info(cpl_pluginlist *list)
00186 {
00187     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00188     cpl_plugin *plugin = &recipe->interface;
00189 
00190     cpl_plugin_init(plugin,
00191                         CPL_PLUGIN_API,
00192                         KMOS_BINARY_VERSION,
00193                         CPL_PLUGIN_TYPE_RECIPE,
00194                         "kmo_illumination_flat",
00195                         "Alternative to kmo_illumination based on flatfield frames.",
00196                         kmo_illumination_flat_description,
00197                         "Alex Agudo Berbel",
00198                         "kmos-spark@mpe.mpg.de",
00199                         kmos_get_license(),
00200                         kmo_illumination_flat_create,
00201                         kmo_illumination_flat_exec,
00202                         kmo_illumination_flat_destroy);
00203 
00204     cpl_pluginlist_append(list, plugin);
00205 
00206     return 0;
00207 }
00208 
00216 static int kmo_illumination_flat_create(cpl_plugin *plugin)
00217 {
00218     cpl_recipe *recipe;
00219     cpl_parameter *p;
00220 
00221     /* Check that the plugin is part of a valid recipe */
00222     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00223         recipe = (cpl_recipe *)plugin;
00224     else
00225         return -1;
00226 
00227     /* Create the parameters list in the cpl_recipe object */
00228     recipe->parameters = cpl_parameterlist_new();
00229 
00230     /* Fill the parameters list */
00231     /* --imethod */
00232     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.imethod",
00233                                 CPL_TYPE_STRING,
00234                                 "Method to use for interpolation: "
00235                                 "[\"NN\" (nearest neighbour), "
00236                                 "\"lwNN\" (linear weighted nearest neighbor), "
00237                                 "\"swNN\" (square weighted nearest neighbor), "
00238                                 "\"MS\" (Modified Shepard's method), "
00239                                 "\"CS\" (Cubic spline)]",
00240                                 "kmos.kmo_illumination_flat",
00241                                 "CS");
00242     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00243     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00244     cpl_parameterlist_append(recipe->parameters, p);
00245 
00246     /* --neighborhoodRange */
00247     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.neighborhoodRange",
00248                                 CPL_TYPE_DOUBLE,
00249                                 "Defines the range to search for neighbors. "
00250                                 "in pixels",
00251                                 "kmos.kmo_illumination_flat",
00252                                 1.001);
00253     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00254     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00255     cpl_parameterlist_append(recipe->parameters, p);
00256 
00257     /* --flux */
00258     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.flux",
00259                                 CPL_TYPE_BOOL,
00260                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00261                                 "kmos.kmo_illumination_flat",
00262                                 FALSE);
00263     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00264     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00265     cpl_parameterlist_append(recipe->parameters, p);
00266 
00267     /* --pix_scale */
00268     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.pix_scale",
00269                                 CPL_TYPE_DOUBLE,
00270                                 "Change the pixel scale [arcsec]. "
00271                                 "Default of 0.2\" results into cubes of 14x14pix, "
00272                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00273                                 "etc.",
00274                                 "kmos.kmo_illumination_flat",
00275                                 KMOS_PIX_RESOLUTION);
00276     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00277     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00278     cpl_parameterlist_append(recipe->parameters, p);
00279 
00280     /* --suppress_extension */
00281     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.suppress_extension",
00282                                 CPL_TYPE_BOOL,
00283                                 "Suppress arbitrary filename extension. "
00284                                 "(TRUE (apply) or FALSE (don't apply)",
00285                                 "kmos.kmo_illumination_flat",
00286                                 FALSE);
00287     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00288     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00289     cpl_parameterlist_append(recipe->parameters, p);
00290 
00291     // add parameters for band-definition
00292     kmo_band_pars_create(recipe->parameters,
00293                          "kmos.kmo_illumination_flat");
00294 
00295     // add parameters for combining
00296     return kmo_combine_pars_create(recipe->parameters,
00297                                    "kmos.kmo_illumination_flat",
00298                                    DEF_REJ_METHOD,
00299                                    FALSE);
00300 }
00301 
00307 static int kmo_illumination_flat_exec(cpl_plugin *plugin)
00308 {
00309     cpl_recipe  *recipe;
00310 
00311     /* Get the recipe out of the plugin */
00312     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00313         recipe = (cpl_recipe *)plugin;
00314     else return -1;
00315 
00316     return kmo_illumination_flat(recipe->parameters, recipe->frames);
00317 }
00318 
00324 static int kmo_illumination_flat_destroy(cpl_plugin *plugin)
00325 {
00326     cpl_recipe *recipe;
00327 
00328     /* Get the recipe out of the plugin */
00329     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00330         recipe = (cpl_recipe *)plugin;
00331     else return -1 ;
00332 
00333     cpl_parameterlist_delete(recipe->parameters);
00334     return 0 ;
00335 }
00336 
00351 static int kmo_illumination_flat(cpl_parameterlist *parlist, cpl_frameset *frameset)
00352 {
00353     int              ret_val                    = 0,
00354                      nr_devices                 = 0,
00355                      ifu_nr                     = 0,
00356                      nx                         = 0,
00357                      ny                         = 0,
00358                      cmax                       = 0,
00359                      cmin                       = 0,
00360                      citer                      = 0,
00361                      *bounds                    = NULL,
00362                      cnt                        = 0,
00363                      qc_max_dev_id              = 0,
00364                      qc_max_nonunif_id          = 0,
00365                      flux                       = FALSE,
00366                      background                 = FALSE,
00367                      suppress_extension         = FALSE,
00368                      mhalf                      = 3,    //width of median filter is mhalf*2 + 1
00369                      boxsize                    = 0,
00370                      i = 0, j = 0, ix = 0, iy = 0, det_nr = 0,
00371                      firstx = 0, lastx = 0, firsty = 0, lasty = 0,
00372                      xmin = 0, xmax = 0, ymin = 0, ymax = 0;
00373     const int        *punused_ifus              = NULL;
00374     float            *pbad_pix_mask             = NULL,
00375                      *pdata                     = NULL,
00376                      *pnoise                    = NULL;
00377     double           exptime                    = 0.,
00378                      cpos_rej                   = 0.0,
00379                      cneg_rej                   = 0.0,
00380                      neighborhoodRange          = 1.001,
00381                      mean_data                  = 0.0,
00382                      qc_spat_unif               = 0.0,
00383                      qc_max_dev                 = 0.0,
00384                      qc_max_nonunif             = 0.0,
00385                      tmp_stdev                  = 0.0,
00386                      tmp_mean                   = 0.0,
00387                      rotangle                   = 0.0,
00388                      tmp_rotangle               = 0.0,
00389                      rotangle_found             = 0.0,
00390                      pix_scale                  = 0.0;
00391     char             *keyword                   = NULL,
00392                      *fn_lut                    = NULL,
00393                      *suffix                    = NULL,
00394                      *fn_suffix                 = NULL,
00395                      *extname                   = NULL,
00396                      *filter                    = NULL,
00397                      content[256];
00398     const char       *method                    = NULL,
00399                      *cmethod                   = NULL,
00400                      *filter_id_l               = NULL,
00401                      *filter_id                 = NULL,
00402                      *tmp_str                   = NULL;
00403     cpl_array        *calTimestamp              = NULL,
00404                      **unused_ifus_before       = NULL,
00405                      **unused_ifus_after        = NULL;
00406     cpl_frame        *frame                     = NULL,
00407                      *xcalFrame                 = NULL,
00408                      *ycalFrame                 = NULL,
00409                      *lcalFrame                 = NULL;
00410     cpl_image        *img_in                    = NULL,
00411                      *img_dark                  = NULL,
00412                      *img_flat                  = NULL,
00413                      *combined_data             = NULL,
00414                      *xcal                      = NULL,
00415                      *ycal                      = NULL,
00416                      *lcal                      = NULL,
00417                      *bad_pix_mask              = NULL,
00418                      *data_ifu                  = NULL,
00419                      *noise_ifu                 = NULL,
00420                      **stored_data_images       = NULL,
00421                      **stored_noise_images      = NULL,
00422                      *tmp_img                   = NULL;
00423     cpl_imagelist    *cube_data                 = NULL,
00424                      *detector_in               = NULL,
00425                      **stored_data_cubes        = NULL;
00426     cpl_propertylist *main_header               = NULL,
00427                      *tmp_header                = NULL,
00428                      *sub_header                = NULL,
00429                      **stored_sub_headers       = NULL;
00430     cpl_table        *band_table                = NULL;
00431     cpl_vector       *identified_slices         = NULL,
00432                      *calAngles                 = NULL;
00433     main_fits_desc   desc_sky,
00434                      desc_xcal,
00435                      desc_ycal,
00436                      desc_lcal;
00437     gridDefinition   gd;
00438     enum kmo_frame_type fr_type;
00439 
00440     KMO_TRY
00441     {
00442         kmo_init_fits_desc(&desc_sky);
00443         kmo_init_fits_desc(&desc_xcal);
00444         kmo_init_fits_desc(&desc_ycal);
00445         kmo_init_fits_desc(&desc_lcal);
00446 
00447         /* --- check input --- */
00448         KMO_TRY_ASSURE((parlist != NULL) &&
00449                        (frameset != NULL),
00450                        CPL_ERROR_NULL_INPUT,
00451                        "Not all input data is provided!");
00452 
00453         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) >= 1,
00454                        CPL_ERROR_ILLEGAL_INPUT,
00455                        "One or more FLAT_SKY_FLAT frames are required!");
00456 
00457         if (cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) < 3) {
00458             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00459                                       "3 FLAT_SKY_FLAT frames!");
00460         }
00461 
00462         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00463                        CPL_ERROR_ILLEGAL_INPUT,
00464                        "Exactly one XCAL frame is required!");
00465 
00466         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00467                        CPL_ERROR_ILLEGAL_INPUT,
00468                        "Exactly one YCAL frame is required!");
00469 
00470         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00471                        CPL_ERROR_ILLEGAL_INPUT,
00472                        "Exactly one LCAL frame is required!");
00473 
00474         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00475                        CPL_ERROR_ILLEGAL_INPUT,
00476                        "Exactly one WAVE_BAND frame is required!");
00477 
00478         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination_flat") == 1,
00479                        CPL_ERROR_ILLEGAL_INPUT,
00480                        "Cannot identify RAW and CALIB frames!");
00481 
00482         /* --- get parameters --- */
00483         cpl_msg_info("", "--- Parameter setup for kmo_illumination_flat ---");
00484 
00485         KMO_TRY_EXIT_IF_NULL(
00486             method = kmo_dfs_get_parameter_string(parlist, "kmos.kmo_illumination_flat.imethod"));
00487 
00488         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00489                        (strcmp(method, "lwNN") == 0) ||
00490                        (strcmp(method, "swNN") == 0) ||
00491                        (strcmp(method, "MS") == 0) ||
00492                        (strcmp(method, "CS") == 0),
00493                        CPL_ERROR_ILLEGAL_INPUT,
00494                        "method must be either \"NN\", \"lwNN\", "
00495                        "\"swNN\", \"MS\" or \"CS\"!");
00496 
00497         KMO_TRY_EXIT_IF_ERROR(
00498             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.imethod"));
00499 
00500         neighborhoodRange = kmo_dfs_get_parameter_double(parlist, "kmos.kmo_illumination_flat.neighborhoodRange");
00501         KMO_TRY_CHECK_ERROR_STATE();
00502 
00503         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00504                        CPL_ERROR_ILLEGAL_INPUT,
00505                        "neighborhoodRange must be greater than 0.0");
00506 
00507         KMO_TRY_EXIT_IF_ERROR(
00508             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.neighborhoodRange"));
00509 
00510         flux = kmo_dfs_get_parameter_bool(parlist, "kmos.kmo_illumination_flat.flux");
00511         KMO_TRY_ASSURE((flux == 0) || (flux == 1),
00512                        CPL_ERROR_ILLEGAL_INPUT,
00513                        "flux must be either FALSE or TRUE!");
00514         KMO_TRY_EXIT_IF_ERROR(
00515             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.flux"));
00516 
00517         pix_scale = kmo_dfs_get_parameter_double(parlist, "kmos.kmo_illumination_flat.pix_scale");
00518         KMO_TRY_CHECK_ERROR_STATE();
00519         KMO_TRY_EXIT_IF_ERROR(
00520            kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.pix_scale"));
00521         KMO_TRY_ASSURE((pix_scale >= 0.01) && (pix_scale <= 0.4),
00522                        CPL_ERROR_ILLEGAL_INPUT,
00523                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00524                        "with 7x7 to 280x280 pixels)!");
00525 
00526         suppress_extension = kmo_dfs_get_parameter_bool(parlist, "kmos.kmo_illumination_flat.suppress_extension");
00527         KMO_TRY_CHECK_ERROR_STATE();
00528         KMO_TRY_EXIT_IF_ERROR(
00529             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.suppress_extension"));
00530 
00531         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00532                        CPL_ERROR_ILLEGAL_INPUT,
00533                        "suppress_extension must be TRUE or FALSE!");
00534 
00535         kmo_band_pars_load(parlist, "kmos.kmo_illumination_flat");
00536 
00537         KMO_TRY_EXIT_IF_ERROR(
00538             kmo_combine_pars_load(parlist, "kmos.kmo_illumination_flat",
00539                                   &cmethod, &cpos_rej, &cneg_rej,
00540                                   &citer, &cmin, &cmax, FALSE));
00541         cpl_msg_info("", "-------------------------------------------");
00542 
00543         // check if filter_id, grating_id and rotator offset match for all
00544         // detectors
00545         KMO_TRY_EXIT_IF_ERROR(
00546             kmo_check_frameset_setup(frameset, FLAT_SKY_FLAT, TRUE, FALSE, TRUE));
00547         KMO_TRY_EXIT_IF_ERROR(
00548             kmo_check_frame_setup(frameset, FLAT_SKY_FLAT, XCAL, TRUE, FALSE, TRUE));
00549         KMO_TRY_EXIT_IF_ERROR(
00550             kmo_check_frame_setup(frameset, XCAL, YCAL, TRUE, FALSE, TRUE));
00551         KMO_TRY_EXIT_IF_ERROR(
00552             kmo_check_frame_setup(frameset, XCAL, LCAL, TRUE, FALSE, TRUE));
00553 
00554         KMO_TRY_EXIT_IF_NULL(
00555             frame = kmo_dfs_get_frame(frameset, XCAL));
00556         KMO_TRY_EXIT_IF_NULL(
00557             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00558 
00559         KMO_TRY_EXIT_IF_ERROR(
00560             kmo_check_frame_setup_md5_xycal(frameset));
00561         KMO_TRY_EXIT_IF_ERROR(
00562             kmo_check_frame_setup_md5(frameset));
00563 
00564         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00565         cpl_msg_info("", "(grating 1, 2 & 3)");
00566 
00567         // check which IFUs are active for all frames
00568         KMO_TRY_EXIT_IF_NULL(
00569             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00570 
00571         KMO_TRY_EXIT_IF_NULL(
00572             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00573 
00574         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00575 
00576         // load desc for XCAL and check
00577         KMO_TRY_EXIT_IF_NULL(
00578             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00579         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00580         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00581                                       "be in KMOS-format!");
00582         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00583                        (desc_xcal.ex_badpix == FALSE) &&
00584                        (desc_xcal.fits_type == f2d_fits) &&
00585                        (desc_xcal.frame_type == detector_frame),
00586                        CPL_ERROR_ILLEGAL_INPUT,
00587                        "XCAL isn't in the correct format!!!");
00588         nx = desc_xcal.naxis1;
00589         ny = desc_xcal.naxis2;
00590         nr_devices = desc_xcal.nr_ext;
00591 
00592         // load desc for YCAL and check
00593         KMO_TRY_EXIT_IF_NULL(
00594             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00595         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00596         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00597                                       "be in KMOS-format!");
00598         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00599                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00600                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00601                        (desc_ycal.frame_type == desc_xcal.frame_type),
00602                        CPL_ERROR_ILLEGAL_INPUT,
00603                        "YCAL isn't in the correct format!!!");
00604         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00605                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00606                        CPL_ERROR_ILLEGAL_INPUT,
00607                        "XCAL and YCAL frame haven't same dimensions! "
00608                        "(x,y): (%d,%d) vs (%d,%d)",
00609                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00610 
00611         // load desc for LCAL and check
00612         KMO_TRY_EXIT_IF_NULL(
00613             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00614         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00615         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00616                                       "be in KMOS-format!");
00617         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00618                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00619                        (desc_lcal.frame_type == desc_xcal.frame_type),
00620                        CPL_ERROR_ILLEGAL_INPUT,
00621                        "LCAL isn't in the correct format!!!");
00622         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00623                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00624                        CPL_ERROR_ILLEGAL_INPUT,
00625                        "XCAL and LCAL frame haven't same dimensions! "
00626                        "(x,y): (%d,%d) vs (%d,%d)",
00627                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00628         KMO_TRY_EXIT_IF_NULL(
00629             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00630 
00631         // load desc for FLAT_SKY_FLAT and check
00632         nr_devices = KMOS_NR_DETECTORS;
00633         KMO_TRY_EXIT_IF_NULL(
00634             frame = kmo_dfs_get_frame(frameset, FLAT_SKY_FLAT));
00635 
00636         KMO_TRY_EXIT_IF_NULL(
00637             main_header = kmclipm_propertylist_load(
00638                                          cpl_frame_get_filename(frame), 0));
00639         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00640         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00641         kmclipm_strip_angle(&rotangle);
00642         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00643         KMO_TRY_CHECK_ERROR_STATE("EXPTIME keyword in main header "
00644                                   "missing!");
00645         cpl_propertylist_delete(main_header); main_header = NULL;
00646 
00647         cnt = 1;
00648         while (frame != NULL) {
00649             KMO_TRY_EXIT_IF_NULL(
00650                 main_header = kmclipm_propertylist_load(
00651                                              cpl_frame_get_filename(frame), 0));
00652 
00653             desc_sky = kmo_identify_fits_header(
00654                         cpl_frame_get_filename(frame));
00655             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY_FLAT frame doesn't seem to "
00656                                           "be in KMOS-format!");
00657             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00658                            (desc_sky.ex_badpix == FALSE) &&
00659                            (desc_sky.fits_type == raw_fits) &&
00660                            (desc_sky.frame_type == detector_frame),
00661                            CPL_ERROR_ILLEGAL_INPUT,
00662                            "FLAT_SKY_FLAT isn't in the correct format!!!");
00663             kmo_free_fits_desc(&desc_sky);
00664             kmo_init_fits_desc(&desc_sky);
00665 
00666             KMO_TRY_ASSURE(
00667                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00668                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE),
00669                 CPL_ERROR_ILLEGAL_INPUT,
00670                 "Arc lamps must be switched off!");
00671 
00672             KMO_TRY_ASSURE(cpl_propertylist_get_double(main_header, EXPTIME) == exptime,
00673                            CPL_ERROR_ILLEGAL_INPUT,
00674                            "EXPTIME isn't the same for all frames: (is %g and %g).",
00675                            cpl_propertylist_get_double(main_header, EXPTIME), exptime);
00676 
00677             // assert that filters have correct IDs and that all detectors of
00678             // all input frames have the same filter set
00679             for (i = 1; i <= KMOS_NR_DETECTORS; i++) {
00680                 // ESO INS FILTi ID
00681                 KMO_TRY_EXIT_IF_NULL(
00682                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00683                 KMO_TRY_EXIT_IF_NULL(
00684                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00685 
00686                 KMO_TRY_EXIT_IF_NULL(
00687                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00688                 cpl_free(keyword); keyword = NULL;
00689 
00690                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00691                                (strcmp(filter_id, "YJ") == 0) ||
00692                                (strcmp(filter_id, "H") == 0) ||
00693                                (strcmp(filter_id, "K") == 0) ||
00694                                (strcmp(filter_id, "HK") == 0),
00695                                CPL_ERROR_ILLEGAL_INPUT,
00696                                "Filter ID in primary header must be either 'IZ', "
00697                                "'YJ', 'H', 'K' or " "'HK' !");
00698 
00699                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00700                                CPL_ERROR_ILLEGAL_INPUT,
00701                                "Filter IDs must be the same for FLAT_SKY_FLAT frame"
00702                                " and lcal frame!"
00703                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00704                                i, cpl_frame_get_filename(frame),
00705                                filter_id, filter_id_l);
00706 
00707                 // ESO INS GRATi ID
00708                 KMO_TRY_EXIT_IF_NULL(
00709                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00710                 KMO_TRY_EXIT_IF_NULL(
00711                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00712 
00713                 KMO_TRY_EXIT_IF_NULL(
00714                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00715                 cpl_free(keyword); keyword = NULL;
00716 
00717                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00718                                (strcmp(filter_id, "YJ") == 0) ||
00719                                (strcmp(filter_id, "H") == 0) ||
00720                                (strcmp(filter_id, "K") == 0) ||
00721                                (strcmp(filter_id, "HK") == 0),
00722                                CPL_ERROR_ILLEGAL_INPUT,
00723                                "Grating ID in primary header must be either "
00724                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00725 
00726                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00727                                CPL_ERROR_ILLEGAL_INPUT,
00728                                "Grating IDs must be the same for FLAT_SKY_FLAT frame"
00729                                " and lcal frame!"
00730                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00731                                i, cpl_frame_get_filename(frame),
00732                                filter_id, filter_id_l);
00733 
00734                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00735                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00736                 kmclipm_strip_angle(&tmp_rotangle);
00737                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00738                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00739                         CPL_ERROR_ILLEGAL_INPUT,
00740                         "OCS ROT NAANGLE of sky flat frames differ too much: %f %f",
00741                         rotangle, tmp_rotangle);
00742             }
00743             cpl_propertylist_delete(main_header); main_header = NULL;
00744 
00745             // get next FLAT_SKY_FLAT frame
00746             frame = kmo_dfs_get_frame(frameset, NULL);
00747             KMO_TRY_CHECK_ERROR_STATE();
00748             cnt++;
00749         }
00750 
00751         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00752 
00753         if (cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) == 1) {
00754             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00755                             "since there is only one input frame! (The output "
00756                             "file won't have any noise extensions)");
00757 
00758             cmethod = "average";
00759         }
00760 
00761         KMO_TRY_EXIT_IF_NULL(
00762             frame = kmo_dfs_get_frame(frameset, FLAT_SKY_FLAT));
00763         KMO_TRY_EXIT_IF_NULL(
00764             main_header = kmo_dfs_load_primary_header(frameset, FLAT_SKY_FLAT));
00765         KMO_TRY_EXIT_IF_NULL(
00766             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
00767         KMO_TRY_EXIT_IF_NULL(
00768             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
00769         cpl_free(keyword); keyword = NULL;
00770 
00771         // setup grid definition, wavelength start and end points will be set
00772         // in the detector loop
00773         KMO_TRY_EXIT_IF_ERROR(
00774             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale, 0.));
00775 
00776         // create filename for LUT
00777         KMO_TRY_EXIT_IF_NULL(
00778             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
00779 
00780         // extract bounds
00781         KMO_TRY_EXIT_IF_NULL(
00782             tmp_header = kmo_dfs_load_primary_header(frameset, XCAL));
00783         KMO_TRY_EXIT_IF_NULL(
00784             bounds = kmclipm_extract_bounds(tmp_header));
00785         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00786 
00787         // get timestamps of xcal, ycal & lcal
00788         KMO_TRY_EXIT_IF_NULL(
00789             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
00790 
00791         // create arrays to hold reconstructed data and noise cubes and
00792         // their headers
00793         KMO_TRY_EXIT_IF_NULL(
00794             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00795                                                             sizeof(cpl_imagelist*)));
00796         KMO_TRY_EXIT_IF_NULL(
00797             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00798                                                          sizeof(cpl_image*)));
00799         KMO_TRY_EXIT_IF_NULL(
00800             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00801                                                           sizeof(cpl_image*)));
00802         KMO_TRY_EXIT_IF_NULL(
00803             stored_sub_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00804                                                                 sizeof(cpl_propertylist*)));
00805         KMO_TRY_EXIT_IF_NULL(
00806             calAngles = cpl_vector_new(3));
00807 
00808         //
00809         // loop through all detectors
00810         //
00811         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
00812             cpl_msg_info("","Processing detector No. %d", det_nr);
00813 
00814             KMO_TRY_EXIT_IF_NULL(
00815                 detector_in = cpl_imagelist_new());
00816 
00817             // load data of det_nr of all FLAT_SKY_FLAT frames into an imagelist
00818             KMO_TRY_EXIT_IF_NULL(
00819                 img_in = kmo_dfs_load_image(frameset, FLAT_SKY_FLAT, det_nr, FALSE, TRUE, NULL));
00820 
00821             cnt = 0;
00822             while (img_in != NULL) {
00823                 cpl_imagelist_set(detector_in, img_in, cnt);
00824                 KMO_TRY_CHECK_ERROR_STATE();
00825 
00826                 /* load same extension of next FLAT_SKY_FLAT frame*/
00827                 img_in = kmo_dfs_load_image(frameset, NULL, det_nr, FALSE, TRUE, NULL);
00828                 KMO_TRY_CHECK_ERROR_STATE();
00829 
00830                 cnt++;
00831             }
00832 
00833             //
00834             // process imagelist
00835             //
00836 
00837             // combine imagelist (data only) and create noise (stdev of data)
00838             cpl_msg_info("","Combining frames...");
00839             KMO_TRY_EXIT_IF_ERROR(
00840                 kmclipm_combine_frames(detector_in,
00841                                        NULL,
00842                                        NULL,
00843                                        cmethod,
00844                                        cpos_rej,
00845                                        cneg_rej,
00846                                        citer,
00847                                        cmax,
00848                                        cmin,
00849                                        &combined_data,
00850                                        NULL,
00851                                        -1.0));
00852 
00853             if (img_dark == NULL) {
00854                 KMO_TRY_EXIT_IF_NULL(
00855                     img_dark = cpl_image_duplicate(combined_data));
00856                 KMO_TRY_EXIT_IF_ERROR(
00857                     cpl_image_multiply_scalar(img_dark, 0));
00858             }
00859 
00860             if (img_flat == NULL) {
00861                 KMO_TRY_EXIT_IF_NULL(
00862                     img_flat = cpl_image_duplicate(combined_data));
00863                 KMO_TRY_EXIT_IF_ERROR(
00864                     cpl_image_multiply_scalar(img_flat, 0));
00865                 cpl_image_add_scalar(img_flat, 1);
00866             }
00867 
00868             if (kmclipm_omit_warning_one_slice > 10) {
00869 // AA: commmented this out: Too unclear for the user, no benefit to know about this number
00870 //                cpl_msg_warning(cpl_func, "Previous warning (number of "
00871 //                                          "identified slices) occured %d times.",
00872 //                                kmclipm_omit_warning_one_slice);
00873                 kmclipm_omit_warning_one_slice = FALSE;
00874             }
00875 
00876             cpl_imagelist_delete(detector_in); detector_in = NULL;
00877 
00878             // load calibration files
00879             KMO_TRY_EXIT_IF_NULL(
00880                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
00881                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00882 
00883             KMO_TRY_EXIT_IF_ERROR(
00884                 cpl_vector_set(calAngles, 0, rotangle_found));
00885             KMO_TRY_EXIT_IF_NULL(
00886                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
00887                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00888             KMO_TRY_EXIT_IF_ERROR(
00889                 cpl_vector_set(calAngles, 1, rotangle_found));
00890             KMO_TRY_EXIT_IF_NULL(
00891                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
00892                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00893             KMO_TRY_EXIT_IF_ERROR(
00894                 cpl_vector_set(calAngles, 2, rotangle_found));
00895 
00896             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
00897             KMO_TRY_EXIT_IF_NULL(
00898                 bad_pix_mask = cpl_image_duplicate(xcal));
00899 
00900             KMO_TRY_EXIT_IF_NULL(
00901                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
00902             for (ix = 0; ix < nx; ix++) {
00903                 for (iy = 0; iy < ny; iy++) {
00904                     if (isnan(pbad_pix_mask[ix+nx*iy])) {
00905                         pbad_pix_mask[ix+nx*iy] = 0.;
00906                     } else {
00907                         pbad_pix_mask[ix+nx*iy] = 1.;
00908                     }
00909                 }
00910             }
00911             KMO_TRY_CHECK_ERROR_STATE();
00912 
00913             //
00914             // reconstruct
00915             //
00916             print_warning_once_reconstruct = FALSE;
00917 
00918             char *tmp_band_method = getenv("KMO_BAND_METHOD");
00919             int band_method = 0;
00920             if (tmp_band_method != NULL) {
00921                 band_method = atoi(tmp_band_method);
00922             }
00923 
00924             // ESO INS FILTi ID
00925             KMO_TRY_EXIT_IF_NULL(
00926                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
00927                                       IFU_FILTID_POSTFIX));
00928             KMO_TRY_EXIT_IF_NULL(
00929                 filter_id = cpl_propertylist_get_string(main_header, keyword));
00930             cpl_free(keyword); keyword = NULL;
00931 
00932             KMO_TRY_EXIT_IF_NULL(
00933                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
00934             KMO_TRY_EXIT_IF_ERROR(
00935                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
00936                                              band_table));
00937             cpl_table_delete(band_table); band_table = NULL;
00938 
00939             cpl_msg_info("","Reconstructing cubes...");
00940             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
00941                 // update sub-header
00942                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + i + 1;
00943 
00944                 // load raw image and sub-header
00945                 KMO_TRY_EXIT_IF_NULL(
00946                     sub_header = kmo_dfs_load_sub_header(frameset, FLAT_SKY_FLAT,
00947                                                          det_nr, FALSE));
00948 
00949                 KMO_TRY_EXIT_IF_NULL(
00950                     punused_ifus = cpl_array_get_data_int_const(
00951                                                   unused_ifus_after[det_nr-1]));
00952 
00953                 // check if IFU is valid according to main header keywords &
00954                 // calibration files
00955                 KMO_TRY_EXIT_IF_NULL(
00956                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
00957                                           IFU_VALID_POSTFIX));
00958                 KMO_TRY_CHECK_ERROR_STATE();
00959 
00960                 // just to see if keyword exists
00961                 cpl_propertylist_get_string(main_header, keyword);
00962                 cpl_free(keyword); keyword = NULL;
00963 
00964                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
00965                     (bounds[2*(ifu_nr-1)] != -1) &&
00966                     (bounds[2*(ifu_nr-1)+1] != -1) &&
00967                     (punused_ifus[i] == 0))
00968                 {
00969                     // IFU is valid
00970                     cpl_error_reset();
00971 
00972                     // calculate WCS
00973                     KMO_TRY_EXIT_IF_ERROR(
00974                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
00975 
00976                     // reconstruct data
00977                     KMO_TRY_EXIT_IF_ERROR(
00978                         kmo_reconstruct_sci_image(ifu_nr,
00979                                                 bounds[2*(ifu_nr-1)],
00980                                                 bounds[2*(ifu_nr-1)+1],
00981                                                 combined_data,
00982                                                 NULL,
00983                                                 img_dark,
00984                                                 NULL,
00985                                                 img_flat,
00986                                                 NULL,
00987                                                 xcal,
00988                                                 ycal,
00989                                                 lcal,
00990                                                 &gd,
00991                                                 calTimestamp,
00992                                                 calAngles,
00993                                                 fn_lut,
00994                                                 &cube_data,
00995                                                 NULL,
00996                                                 flux,
00997                                                 background,
00998                                                 NULL,
00999                                                 NULL,
01000                                                 NULL));
01001                     KMO_TRY_CHECK_ERROR_STATE();
01002                 } else {
01003                     // IFU is invalid
01004                     cpl_error_reset();
01005                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01006 
01007                 // save output
01008                 KMO_TRY_EXIT_IF_NULL(
01009                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01010 
01011                 KMO_TRY_EXIT_IF_ERROR(
01012                     kmclipm_update_property_string(sub_header, EXTNAME,
01013                                                    extname,
01014                                                    "FITS extension name"));
01015 
01016                 cpl_free(extname); extname = NULL;
01017 
01018                 // store cube and sub header into array for later
01019                 stored_data_cubes[ifu_nr - 1] = cube_data;
01020                 stored_sub_headers[ifu_nr - 1] = sub_header;
01021 
01022                 cpl_image_delete(data_ifu); data_ifu = NULL;
01023                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01024                 cube_data = NULL;
01025             } // for i IFUs
01026 
01027             // free memory
01028             cpl_image_delete(combined_data); combined_data = NULL;
01029             cpl_image_delete(xcal); xcal = NULL;
01030             cpl_image_delete(ycal); ycal = NULL;
01031             cpl_image_delete(lcal); lcal = NULL;
01032             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01033         } // for nr_devices
01034 
01035         cpl_image_delete(img_dark); img_dark = NULL;
01036         cpl_image_delete(img_flat); img_flat = NULL;
01037 
01038         //
01039         // collapse cubes using rejection and apply median filtering
01040         //
01041         KMO_TRY_EXIT_IF_NULL(
01042             identified_slices = cpl_vector_new(gd.l.dim));
01043         KMO_TRY_EXIT_IF_ERROR(
01044             cpl_vector_fill(identified_slices, 1));
01045 
01046         cpl_msg_info("","Collapsing cubes...");
01047         for (j = 0; j < nr_devices; j++) {
01048             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01049                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01050                 KMO_TRY_EXIT_IF_NULL(
01051                     punused_ifus = cpl_array_get_data_int_const(
01052                                                   unused_ifus_after[j]));
01053                 if (punused_ifus[i] == 0) {
01054                     if (stored_data_cubes[ifu_nr] != NULL) {
01055                         KMO_TRY_EXIT_IF_ERROR(
01056                             kmclipm_make_image(stored_data_cubes[ifu_nr],
01057                                                NULL,
01058                                                &stored_data_images[ifu_nr],
01059                                                &stored_noise_images[ifu_nr],
01060                                                identified_slices,
01061                                                cmethod, cpos_rej, cneg_rej,
01062                                                citer, cmax, cmin));
01063                     }
01064 
01065                     //
01066                     // apply median smoothing
01067                     //
01068 
01069                     // taking care of edges (IFUs 1-16 top/bottom, IFUs 17-24 left/right)
01070                     if (ifu_nr+1 <= 2*KMOS_IFUS_PER_DETECTOR) {
01071                         firstx = 0;
01072                         lastx = 13;
01073                         firsty = 1;
01074                         lasty = 12;
01075                     } else {
01076                         firstx = 1;
01077                         lastx= 12;
01078                         firsty = 0;
01079                         lasty = 13;
01080                     }
01081 
01082                     KMO_TRY_EXIT_IF_NULL(
01083                         tmp_img = cpl_image_duplicate(stored_data_images[ifu_nr]));
01084                     KMO_TRY_EXIT_IF_NULL(
01085                         pdata = cpl_image_get_data_float(tmp_img));
01086                     KMO_TRY_EXIT_IF_NULL(
01087                         pnoise = cpl_image_get_data_float(stored_noise_images[ifu_nr]));
01088                     nx = cpl_image_get_size_x(tmp_img);
01089                     ny = cpl_image_get_size_y(tmp_img);
01090                     KMO_TRY_CHECK_ERROR_STATE();
01091 
01092                     // median filtering
01093                     for (ix = 0; ix < nx; ix++) {
01094                         for (iy = 0; iy < ny; iy++) {
01095                             if (ix-mhalf > firstx) { xmin = ix-mhalf; } else { xmin = firstx; }
01096                             if (ix+mhalf < lastx)  { xmax = ix+mhalf; } else { xmax = lastx; }
01097                             if (iy-mhalf > firsty) { ymin = iy-mhalf; } else { ymin = firsty; }
01098                             if (iy+mhalf < lasty)  { ymax = iy+mhalf; } else { ymax = lasty; }
01099 
01100                             pdata[ix+nx*iy] = cpl_image_get_median_window(
01101                                                                 stored_data_images[ifu_nr],
01102                                                                 xmin+1, ymin+1, xmax+1, ymax+1);
01103                             KMO_TRY_CHECK_ERROR_STATE();
01104 
01105                             if (stored_noise_images[ifu_nr] != NULL) {
01106                                 boxsize = (xmax-xmin+1)*(ymax-ymin+1);
01107                                 pnoise[ix+nx*iy] /= boxsize; //sqrt(boxsize*boxsize)
01108                             }
01109 
01110                         }
01111                     }
01112 
01113                     // replace images
01114                     cpl_image_delete(stored_data_images[ifu_nr]);
01115                     stored_data_images[ifu_nr] = tmp_img;
01116                 } else {
01117                     // IFU is invalid
01118                 }
01119             } // end for (i) ifu_nr
01120         } // end for (j) det_nr
01121         cpl_vector_delete(identified_slices); identified_slices = NULL;
01122 
01123         // normalise all IFUs of a detector as a group.
01124         // Calculate mean of each IFU, add up and divide by number of successful
01125         // averaged IFUs.
01126         // Then divide all valid IFUs with mean value
01127         for (j = 0; j < nr_devices; j++) {
01128             cnt = 0;
01129             mean_data = 0;
01130             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01131                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01132                 if (stored_data_images[ifu_nr] != NULL) {
01133                     KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[ifu_nr]) <
01134                                    cpl_image_get_size_x(stored_data_images[ifu_nr])*
01135                                    cpl_image_get_size_y(stored_data_images[ifu_nr]),
01136                                    CPL_ERROR_ILLEGAL_INPUT,
01137                                    "The collapsed image contains only invalid values!");
01138                     mean_data += cpl_image_get_mean(stored_data_images[ifu_nr]);
01139                     KMO_TRY_CHECK_ERROR_STATE();
01140                     cnt++;
01141                 }
01142 
01143             } // end for (i) ifu_nr
01144             mean_data /= cnt;
01145             if (mean_data != 0.0) {
01146                 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01147                     ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01148                     if (stored_data_images[ifu_nr] != NULL) {
01149                         KMO_TRY_EXIT_IF_ERROR(
01150                             cpl_image_divide_scalar(stored_data_images[ifu_nr], mean_data));
01151                     }
01152                     if (stored_noise_images[ifu_nr] != NULL) {
01153                         KMO_TRY_EXIT_IF_ERROR(
01154                             cpl_image_divide_scalar(stored_noise_images[ifu_nr], mean_data));
01155                     }
01156                 } // end for (i) ifu_nr
01157             } else {
01158                 cpl_msg_warning(cpl_func, "Data couldn't be normalised (mean = 0.0)!");
01159             }
01160         } // end for (j) det_nr
01161 
01162         //
01163         // invert data and noise
01164         //
01165         double old_val  = 0.,
01166                new_val  = 0.;
01167         for (j = 0; j < nr_devices; j++) {
01168             cnt = 0;
01169             mean_data = 0;
01170             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01171                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01172                 if (stored_data_images[ifu_nr] != NULL) {
01173                     // invert data
01174                     KMO_TRY_EXIT_IF_NULL(
01175                         pdata = cpl_image_get_data_float(stored_data_images[ifu_nr]));
01176                     if (stored_noise_images[ifu_nr] != NULL) {
01177                         KMO_TRY_EXIT_IF_NULL(
01178                             pnoise = cpl_image_get_data_float(stored_noise_images[ifu_nr]));
01179                     }
01180                     for (ix = 0; ix < nx; ix++) {
01181                         for (iy = 0; iy < ny; iy++) {
01182                             old_val = pdata[ix+nx*iy];
01183                             pdata[ix+nx*iy] = 1. / pdata[ix+nx*iy];
01184                             if (stored_noise_images[ifu_nr] != NULL) {
01185                                 new_val = pdata[ix+nx*iy];
01186                                 pnoise[ix+nx*iy] = sqrt(pow(new_val, 2) *
01187                                                         pow(pnoise[ix+nx*iy], 2) /
01188                                                         pow(old_val, 2));
01189                             }
01190                         }
01191                     }
01192                 }
01193             } // end for (i) ifu_nr
01194         } // end for (j) det_nr
01195 
01196         // calculate qc parameters on normalised data
01197         qc_spat_unif = 0.0;
01198         cnt = 0;
01199         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01200             if (stored_data_images[i] != NULL) {
01201                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01202                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01203 
01204                 qc_spat_unif += pow(tmp_mean-1, 2);
01205                 if (fabs(tmp_mean) > qc_max_dev) {
01206                     qc_max_dev = tmp_mean-1;
01207                     qc_max_dev_id = i+1;
01208                 }
01209                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01210                     qc_max_nonunif = tmp_stdev;
01211                     qc_max_nonunif_id = i+1;
01212                 }
01213                 KMO_TRY_CHECK_ERROR_STATE();
01214                 cnt++;
01215             }
01216         }
01217         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01218 
01219         //
01220         // save data
01221         //
01222 
01223         // update which IFUs are not used
01224         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01225 
01226         KMO_TRY_EXIT_IF_ERROR(
01227             kmo_set_unused_ifus(unused_ifus_after, main_header,
01228                                 "kmo_illumination_flat"));
01229 
01230         cpl_msg_info("","Saving data...");
01231 
01232         KMO_TRY_EXIT_IF_ERROR(
01233             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01234                                            "[adu] uniformity of illumination correction"));
01235         KMO_TRY_EXIT_IF_ERROR(
01236             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01237                                            "[adu] max. deviation from unity"));
01238         KMO_TRY_EXIT_IF_ERROR(
01239             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01240                                         "[] IFU ID with max. dev. from unity"));
01241         KMO_TRY_EXIT_IF_ERROR(
01242             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01243                                            "[adu] max. stdev of illumination corr."));
01244         KMO_TRY_EXIT_IF_ERROR(
01245             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01246                                         "[] IFU ID with max. stdev in illum. corr."));
01247 
01248         if (!suppress_extension) {
01249             KMO_TRY_EXIT_IF_NULL(
01250                 fn_suffix = cpl_sprintf("%s", suffix));
01251         } else {
01252             KMO_TRY_EXIT_IF_NULL(
01253                 fn_suffix = cpl_sprintf("%s", ""));
01254         }
01255         KMO_TRY_EXIT_IF_ERROR(
01256             kmo_dfs_save_main_header(frameset, ILLUM_CORR_FLAT, fn_suffix, frame,
01257                                      main_header, parlist, cpl_func));
01258 
01259         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01260             KMO_TRY_EXIT_IF_ERROR(
01261                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR_FLAT, fn_suffix,
01262                                    stored_sub_headers[i], 0./0.));
01263 
01264             KMO_TRY_EXIT_IF_NULL(
01265                 tmp_str = cpl_propertylist_get_string(stored_sub_headers[i], EXTNAME));
01266             KMO_TRY_EXIT_IF_ERROR(
01267                 kmo_extname_extractor(tmp_str, &fr_type, &ifu_nr, content));
01268             KMO_TRY_EXIT_IF_NULL(
01269                 extname = kmo_extname_creator(ifu_frame, ifu_nr,
01270                                               EXT_NOISE));
01271             KMO_TRY_EXIT_IF_ERROR(
01272                 kmclipm_update_property_string(stored_sub_headers[i], EXTNAME,
01273                                                extname, "FITS extension name"));
01274             cpl_free(extname); extname = NULL;
01275 
01276             KMO_TRY_EXIT_IF_ERROR(
01277                 kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR_FLAT,
01278                                    fn_suffix, stored_sub_headers[i], 0./0.));
01279         }
01280     }
01281     KMO_CATCH
01282     {
01283         KMO_CATCH_MSG();
01284         ret_val = -1;
01285     }
01286     kmo_free_fits_desc(&desc_sky);
01287     kmo_free_fits_desc(&desc_xcal);
01288     kmo_free_fits_desc(&desc_ycal);
01289     kmo_free_fits_desc(&desc_lcal);
01290     cpl_image_delete(combined_data); combined_data = NULL;
01291     cpl_image_delete(xcal); xcal = NULL;
01292     cpl_image_delete(ycal); ycal = NULL;
01293     cpl_image_delete(lcal); lcal = NULL;
01294     cpl_image_delete(img_dark); img_dark = NULL;
01295     cpl_image_delete(img_flat); img_flat = NULL;
01296     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01297     cpl_free(bounds); bounds = NULL;
01298     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01299     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01300     cpl_free(fn_lut); fn_lut = NULL;
01301     cpl_free(suffix); suffix = NULL;
01302     cpl_free(fn_suffix); fn_suffix = NULL;
01303     cpl_free(filter); filter = NULL;
01304     if (calAngles != NULL) {
01305         cpl_vector_delete(calAngles); calAngles = NULL;
01306     }
01307     cpl_propertylist_delete(main_header); main_header = NULL;
01308     for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01309         if (stored_data_cubes != NULL) {
01310             cpl_imagelist_delete(stored_data_cubes[i]);
01311             stored_data_cubes[i] = NULL;
01312         }
01313         if (stored_data_images != NULL) {
01314             cpl_image_delete(stored_data_images[i]);
01315             stored_data_images[i] = NULL;
01316         }
01317         if (stored_noise_images != NULL) {
01318             cpl_image_delete(stored_noise_images[i]);
01319             stored_noise_images[i] = NULL;
01320         }
01321         if (stored_sub_headers != NULL) {
01322             cpl_propertylist_delete(stored_sub_headers[i]);
01323             stored_sub_headers[i] = NULL;
01324         }
01325     }
01326     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01327     cpl_free(stored_data_images); stored_data_images = NULL;
01328     cpl_free(stored_noise_images); stored_noise_images = NULL;
01329     cpl_free(stored_sub_headers); stored_sub_headers = NULL;
01330 
01331     return ret_val;
01332 }
01333