KMOS Pipeline Reference Manual  1.2.8
kmo_illumination.c
00001 /* $Id: kmo_illumination.c,v 1.65 2013-10-21 13:44:54 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013-10-21 13:44:54 $
00024  * $Revision: 1.65 $
00025  * $Name: not supported by cvs2svn $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmclipm_priv_splines.h"
00042 
00043 #include "kmo_priv_reconstruct.h"
00044 #include "kmo_priv_functions.h"
00045 #include "kmo_priv_flat.h"
00046 #include "kmo_priv_wave_cal.h"
00047 #include "kmo_functions.h"
00048 #include "kmo_cpl_extensions.h"
00049 #include "kmo_dfs.h"
00050 #include "kmo_error.h"
00051 #include "kmo_constants.h"
00052 #include "kmo_debug.h"
00053 
00054 /*-----------------------------------------------------------------------------
00055  *                          Functions prototypes
00056  *----------------------------------------------------------------------------*/
00057 
00058 static int kmo_illumination_create(cpl_plugin *);
00059 static int kmo_illumination_exec(cpl_plugin *);
00060 static int kmo_illumination_destroy(cpl_plugin *);
00061 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00062 
00063 /*-----------------------------------------------------------------------------
00064  *                          Static variables
00065  *----------------------------------------------------------------------------*/
00066 
00067 static char kmo_illumination_description[] =
00068 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00069 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00070 "generates the spectral calibration frame needed in this recipe. As input at\n"
00071 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00072 "bration frames are required.\n"
00073 "The created product, the illumination correction, can be used as input for\n"
00074 "kmo_std_star and kmo_sci_red.\n"
00075 "\n"
00076 "BASIC PARAMETERS:\n"
00077 "-----------------\n"
00078 "--imethod\n"
00079 "The interpolation method used for reconstruction.\n"
00080 "\n"
00081 "--range\n"
00082 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00083 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00084 "\n"
00085 "ADVANCED PARAMETERS\n"
00086 "-------------------\n"
00087 "--flux\n"
00088 "Specify if flux conservation should be applied.\n"
00089 "\n"
00090 "--add-all\n"
00091 "By default the first FLAT_SKY frame is omitted, since in the\n"
00092 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00093 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00094 "should be considered, set this parameter to TRUE.\n"
00095 "\n"
00096 "--neighborhoodRange\n"
00097 "Defines the range to search for neighbors during reconstruction\n"
00098 "\n"
00099 "--b_samples\n"
00100 "The number of samples in spectral direction for the reconstructed cube.\n"
00101 "Ideally this number should be greater than 2048, the detector size.\n"
00102 "\n"
00103 "--b_start\n"
00104 "--b_end\n"
00105 "Used to define manually the start and end wavelength for the reconstructed\n"
00106 "cube. By default the internally defined values are used.\n"
00107 "\n"
00108 "--cmethod\n"
00109 "Following methods of frame combination are available:\n"
00110 "   * 'ksigma' (Default)\n"
00111 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00112 "   are examined. If they deviate significantly, they will be rejected according\n"
00113 "   to the conditions:\n"
00114 "       val > mean + stdev * cpos_rej\n"
00115 "   and\n"
00116 "       val < mean - stdev * cneg_rej\n"
00117 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00118 "   parameters. In the first iteration median and percentile level are used.\n"
00119 "\n"
00120 "   * 'median'\n"
00121 "   At each pixel position the median is calculated.\n"
00122 "\n"
00123 "   * 'average'\n"
00124 "   At each pixel position the average is calculated.\n"
00125 "\n"
00126 "   * 'sum'\n"
00127 "   At each pixel position the sum is calculated.\n"
00128 "\n"
00129 "   * 'min_max'\n"
00130 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00131 "   --cmax and --cmin apply to this method.\n"
00132 "\n"
00133 "--cpos_rej\n"
00134 "--cneg_rej\n"
00135 "--citer\n"
00136 "see --cmethod='ksigma'\n"
00137 "\n"
00138 "--cmax\n"
00139 "--cmin\n"
00140 "see --cmethod='min_max'\n"
00141 "\n"
00142 "--pix_scale\n"
00143 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00144 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00145 "\n"
00146 "--suppress_extension\n"
00147 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00148 "products with the same category are produced, they will be numered consecutively\n"
00149 "starting from 0.\n"
00150 "\n"
00151 "-------------------------------------------------------------------------------\n"
00152 "  Input files:\n"
00153 "\n"
00154 "   DO                    KMOS                                                  \n"
00155 "   category              Type   Explanation                    Required #Frames\n"
00156 "   --------              -----  -----------                    -------- -------\n"
00157 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00158 "                                (at least 3 frames recommended)                \n"
00159 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00160 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00161 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00162 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00163 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00164 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00165 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00166 "\n"
00167 "  Output files:\n"
00168 "\n"
00169 "   DO                    KMOS\n"
00170 "   category              Type   Explanation\n"
00171 "   --------              -----  -----------\n"
00172 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00173 "   If FLAT_EDGE is provided: \n"
00174 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00175 "                                slitlets of all IFUs of all detectors\n"
00176 "-------------------------------------------------------------------------------\n"
00177 "\n";
00178 
00179 /*-----------------------------------------------------------------------------
00180  *                              Functions code
00181  *----------------------------------------------------------------------------*/
00182 
00199 int cpl_plugin_get_info(cpl_pluginlist *list)
00200 {
00201     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00202     cpl_plugin *plugin = &recipe->interface;
00203 
00204     cpl_plugin_init(plugin,
00205                         CPL_PLUGIN_API,
00206                         KMOS_BINARY_VERSION,
00207                         CPL_PLUGIN_TYPE_RECIPE,
00208                         "kmo_illumination",
00209                         "Create a calibration frame to correct spatial "
00210                         "non-uniformity of flatfield.",
00211                         kmo_illumination_description,
00212                         "Alex Agudo Berbel",
00213                         "kmos-spark@mpe.mpg.de",
00214                         kmos_get_license(),
00215                         kmo_illumination_create,
00216                         kmo_illumination_exec,
00217                         kmo_illumination_destroy);
00218 
00219     cpl_pluginlist_append(list, plugin);
00220 
00221     return 0;
00222 }
00223 
00231 static int kmo_illumination_create(cpl_plugin *plugin)
00232 {
00233     cpl_recipe *recipe;
00234     cpl_parameter *p;
00235 
00236     /* Check that the plugin is part of a valid recipe */
00237     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00238         recipe = (cpl_recipe *)plugin;
00239     else
00240         return -1;
00241 
00242     /* Create the parameters list in the cpl_recipe object */
00243     recipe->parameters = cpl_parameterlist_new();
00244 
00245     /* Fill the parameters list */
00246     /* --imethod */
00247     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00248                                 CPL_TYPE_STRING,
00249                                 "Method to use for interpolation: "
00250                                 "[\"NN\" (nearest neighbour), "
00251                                 "\"lwNN\" (linear weighted nearest neighbor), "
00252                                 "\"swNN\" (square weighted nearest neighbor), "
00253                                 "\"MS\" (Modified Shepard's method), "
00254                                 "\"CS\" (Cubic spline)]",
00255                                 "kmos.kmo_illumination",
00256                                 "CS");
00257     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00258     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00259     cpl_parameterlist_append(recipe->parameters, p);
00260 
00261     /* --neighborhoodRange */
00262     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00263                                 CPL_TYPE_DOUBLE,
00264                                 "Defines the range to search for neighbors. "
00265                                 "in pixels",
00266                                 "kmos.kmo_illumination",
00267                                 1.001);
00268     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00269     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00270     cpl_parameterlist_append(recipe->parameters, p);
00271 
00272     /* --range */
00273     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00274                                 CPL_TYPE_STRING,
00275                                 "The spectral ranges to combine when collapsing "
00276                                 "the reconstructed cubes. e.g. "
00277                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00278                                 "kmos.kmo_illumination",
00279                                 "");
00280     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00281     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00282     cpl_parameterlist_append(recipe->parameters, p);
00283 
00284     /* --flux */
00285     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00286                                 CPL_TYPE_BOOL,
00287                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00288                                 "kmos.kmo_illumination",
00289                                 FALSE);
00290     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00291     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00292     cpl_parameterlist_append(recipe->parameters, p);
00293 
00294     /* --add-all */
00295     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00296                                 CPL_TYPE_BOOL,
00297                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00298                                 "TRUE: don't perform any checks, add them all",
00299                                 "kmos.kmo_illumination",
00300                                 FALSE);
00301     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00302     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00303     cpl_parameterlist_append(recipe->parameters, p);
00304 
00305     /* --pix_scale */
00306     p = cpl_parameter_new_value("kmos.kmo_illumination.pix_scale",
00307                                 CPL_TYPE_DOUBLE,
00308                                 "Change the pixel scale [arcsec]. "
00309                                 "Default of 0.2\" results into cubes of 14x14pix, "
00310                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00311                                 "etc.",
00312                                 "kmos.kmo_illumination",
00313                                 KMOS_PIX_RESOLUTION);
00314     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00315     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00316     cpl_parameterlist_append(recipe->parameters, p);
00317 
00318     /* --suppress_extension */
00319     p = cpl_parameter_new_value("kmos.kmo_illumination.suppress_extension",
00320                                 CPL_TYPE_BOOL,
00321                                 "Suppress arbitrary filename extension. "
00322                                 "(TRUE (apply) or FALSE (don't apply)",
00323                                 "kmos.kmo_illumination",
00324                                 FALSE);
00325     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00326     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00327     cpl_parameterlist_append(recipe->parameters, p);
00328 
00329     // add parameters for band-definition
00330     kmo_band_pars_create(recipe->parameters,
00331                          "kmos.kmo_illumination");
00332 
00333     // add parameters for combining
00334     return kmo_combine_pars_create(recipe->parameters,
00335                                    "kmos.kmo_illumination",
00336                                    DEF_REJ_METHOD,
00337                                    FALSE);
00338 }
00339 
00345 static int kmo_illumination_exec(cpl_plugin *plugin)
00346 {
00347     cpl_recipe  *recipe;
00348 
00349     /* Get the recipe out of the plugin */
00350     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00351         recipe = (cpl_recipe *)plugin;
00352     else return -1;
00353 
00354     return kmo_illumination(recipe->parameters, recipe->frames);
00355 }
00356 
00362 static int kmo_illumination_destroy(cpl_plugin *plugin)
00363 {
00364     cpl_recipe *recipe;
00365 
00366     /* Get the recipe out of the plugin */
00367     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00368         recipe = (cpl_recipe *)plugin;
00369     else return -1 ;
00370 
00371     cpl_parameterlist_delete(recipe->parameters);
00372     return 0 ;
00373 }
00374 
00389 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00390 {
00391     int              ret_val                    = 0,
00392                      nr_devices                 = 0,
00393                      ifu_nr                     = 0,
00394                      nx                         = 0,
00395                      ny                         = 0,
00396                      process_noise              = FALSE,
00397                      cmax                       = 0,
00398                      cmin                       = 0,
00399                      citer                      = 0,
00400                      *bounds                    = NULL,
00401                      cnt                        = 0,
00402                      qc_max_dev_id              = 0,
00403                      qc_max_nonunif_id          = 0,
00404                      flux                       = FALSE,
00405                      background                 = FALSE,
00406                      add_all_sky                = FALSE,
00407                      same_exptime               = TRUE,
00408                      suppress_extension         = FALSE,
00409                      has_flat_edge              = FALSE,
00410                      i = 0, j = 0, x = 0, y = 0, ix = 0, iy = 0, det_nr = 0, edgeNr = 0;
00411     const int        *punused_ifus              = NULL;
00412     float            *pbad_pix_mask             = NULL;
00413     double           exptime                    = 0.,
00414                      exptime1                   = 0.,
00415                      exptime2                   = 0.,
00416                      cpos_rej                   = 0.0,
00417                      cneg_rej                   = 0.0,
00418                      neighborhoodRange          = 1.001,
00419                      mean_data                  = 0.0,
00420                      ifu_crpix                  = 0.0,
00421                      ifu_crval                  = 0.0,
00422                      ifu_cdelt                  = 0.0,
00423                      qc_spat_unif               = 0.0,
00424                      qc_max_dev                 = 0.0,
00425                      qc_max_nonunif             = 0.0,
00426                      tmp_stdev                  = 0.0,
00427                      tmp_mean                   = 0.0,
00428                      rotangle                   = 0.0,
00429                      tmp_rotangle               = 0.0,
00430                      rotangle_found             = 0.0,
00431                      pix_scale                  = 0.0;
00432     char             *keyword                   = NULL,
00433                      *fn_lut                    = NULL,
00434                      *suffix                    = NULL,
00435                      *fn_suffix                 = NULL,
00436                      *extname                   = NULL,
00437                      *filter                    = NULL;
00438     const char       *method                    = NULL,
00439                      *cmethod                   = NULL,
00440                      *filter_id_l               = NULL,
00441                      *filter_id                 = NULL,
00442                      *ranges_txt                = NULL;
00443     cpl_array        *calTimestamp              = NULL,
00444                      **unused_ifus_before       = NULL,
00445                      **unused_ifus_after        = NULL;
00446     cpl_frame        *frame                     = NULL,
00447                      *xcalFrame                 = NULL,
00448                      *ycalFrame                 = NULL,
00449                      *lcalFrame                 = NULL;
00450     cpl_frameset     *frameset_sky              = NULL;
00451     cpl_image        *img_in                    = NULL,
00452                      *img_dark                  = NULL,
00453                      *img_dark_noise            = NULL,
00454                      *img_flat                  = NULL,
00455                      *img_flat_noise            = NULL,
00456                      *combined_data             = NULL,
00457                      *combined_noise            = NULL,
00458                      *xcal                      = NULL,
00459                      *ycal                      = NULL,
00460                      *lcal                      = NULL,
00461                      *bad_pix_mask              = NULL,
00462                      *data_ifu                  = NULL,
00463                      *noise_ifu                 = NULL,
00464                      **stored_data_images       = NULL,
00465                      **stored_noise_images      = NULL;
00466     cpl_imagelist    *cube_data                 = NULL,
00467                      *cube_noise                = NULL,
00468                      *detector_in               = NULL,
00469                      **stored_data_cubes        = NULL,
00470                      **stored_noise_cubes       = NULL;
00471     cpl_matrix       **edgepars                 = NULL;
00472     cpl_propertylist *main_header               = NULL,
00473                      *tmp_header                = NULL,
00474                      *sub_header                = NULL,
00475                      **stored_sub_data_headers  = NULL,
00476                      **stored_sub_noise_headers = NULL;
00477     cpl_table        *band_table                = NULL,
00478                      ***edge_table_sky          = NULL,
00479                      **edge_table_flat          = NULL;
00480     cpl_vector       *ranges                    = NULL,
00481                      *identified_slices         = NULL,
00482                      *calAngles                 = NULL,
00483                      **slitlet_ids              = NULL,
00484                      *shift_vec                 = NULL,
00485                      *edge_vec                  = NULL;
00486     main_fits_desc   desc_sky,
00487                      desc_dark,
00488                      desc_flat,
00489                      desc_xcal,
00490                      desc_ycal,
00491                      desc_lcal;
00492     gridDefinition   gd;
00493 
00494     KMO_TRY
00495     {
00496         kmo_init_fits_desc(&desc_sky);
00497         kmo_init_fits_desc(&desc_dark);
00498         kmo_init_fits_desc(&desc_flat);
00499         kmo_init_fits_desc(&desc_xcal);
00500         kmo_init_fits_desc(&desc_ycal);
00501         kmo_init_fits_desc(&desc_lcal);
00502 
00503         /* --- check input --- */
00504         KMO_TRY_ASSURE((parlist != NULL) &&
00505                        (frameset != NULL),
00506                        CPL_ERROR_NULL_INPUT,
00507                        "Not all input data is provided!");
00508 
00509         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00510                        CPL_ERROR_ILLEGAL_INPUT,
00511                        "One or more FLAT_SKY frames are required!");
00512 
00513         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00514             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00515                                       "3 FLAT_SKY frames!");
00516         }
00517 
00518         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00519                        CPL_ERROR_ILLEGAL_INPUT,
00520                        "Exactly one MASTER_DARK frame is required!");
00521 
00522         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00523                        CPL_ERROR_ILLEGAL_INPUT,
00524                        "Exactly one MASTER_FLAT frame is required!");
00525 
00526         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00527                        CPL_ERROR_ILLEGAL_INPUT,
00528                        "Exactly one XCAL frame is required!");
00529 
00530         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00531                        CPL_ERROR_ILLEGAL_INPUT,
00532                        "Exactly one YCAL frame is required!");
00533 
00534         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00535                        CPL_ERROR_ILLEGAL_INPUT,
00536                        "Exactly one LCAL frame is required!");
00537 
00538         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00539                        CPL_ERROR_ILLEGAL_INPUT,
00540                        "Exactly one WAVE_BAND frame is required!");
00541 
00542         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00543                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00544                        CPL_ERROR_ILLEGAL_INPUT,
00545                        "Exactly one FLAT_EDGE frame is required!");
00546 
00547         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00548 
00549         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00550                        CPL_ERROR_ILLEGAL_INPUT,
00551                        "Cannot identify RAW and CALIB frames!");
00552 
00553         /* --- get parameters --- */
00554         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00555 
00556         KMO_TRY_EXIT_IF_NULL(
00557             method = kmo_dfs_get_parameter_string(parlist,
00558                                               "kmos.kmo_illumination.imethod"));
00559 
00560         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00561                        (strcmp(method, "lwNN") == 0) ||
00562                        (strcmp(method, "swNN") == 0) ||
00563                        (strcmp(method, "MS") == 0) ||
00564                        (strcmp(method, "CS") == 0),
00565                        CPL_ERROR_ILLEGAL_INPUT,
00566                        "method must be either \"NN\", \"lwNN\", "
00567                        "\"swNN\", \"MS\" or \"CS\"!");
00568 
00569         KMO_TRY_EXIT_IF_ERROR(
00570             kmo_dfs_print_parameter_help(parlist,
00571                                         "kmos.kmo_illumination.imethod"));
00572 
00573         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00574                 "kmos.kmo_illumination.neighborhoodRange");
00575         KMO_TRY_CHECK_ERROR_STATE();
00576 
00577         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00578                 CPL_ERROR_ILLEGAL_INPUT,
00579                 "neighborhoodRange must be greater than 0.0");
00580 
00581         KMO_TRY_EXIT_IF_ERROR(
00582             kmo_dfs_print_parameter_help(parlist,
00583                                     "kmos.kmo_illumination.neighborhoodRange"));
00584 
00585         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00586                                                   "kmos.kmo_illumination.range");
00587         KMO_TRY_CHECK_ERROR_STATE();
00588 
00589         KMO_TRY_EXIT_IF_ERROR(
00590             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00591 
00592         ranges = kmo_identify_ranges(ranges_txt);
00593         KMO_TRY_CHECK_ERROR_STATE();
00594 
00595         flux = kmo_dfs_get_parameter_bool(parlist,
00596                                           "kmos.kmo_illumination.flux");
00597 
00598         KMO_TRY_ASSURE((flux == 0) ||
00599                        (flux == 1),
00600                        CPL_ERROR_ILLEGAL_INPUT,
00601                        "flux must be either FALSE or TRUE!");
00602 
00603         KMO_TRY_EXIT_IF_ERROR(
00604             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00605 
00606         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00607                                                  "kmos.kmo_illumination.add-all");
00608 
00609         KMO_TRY_ASSURE((add_all_sky == 0) ||
00610                        (add_all_sky == 1),
00611                        CPL_ERROR_ILLEGAL_INPUT,
00612                        "add_all must be either FALSE or TRUE!");
00613 
00614         KMO_TRY_EXIT_IF_ERROR(
00615             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00616 
00617         pix_scale = kmo_dfs_get_parameter_double(parlist,
00618                                         "kmos.kmo_illumination.pix_scale");
00619         KMO_TRY_CHECK_ERROR_STATE();
00620         KMO_TRY_EXIT_IF_ERROR(
00621            kmo_dfs_print_parameter_help(parlist,
00622                                        "kmos.kmo_illumination.pix_scale"));
00623         KMO_TRY_ASSURE((pix_scale >= 0.01) &&
00624                        (pix_scale <= 0.4),
00625                        CPL_ERROR_ILLEGAL_INPUT,
00626                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00627                        "with 7x7 to 280x280 pixels)!");
00628 
00629         suppress_extension = kmo_dfs_get_parameter_bool(parlist,
00630                                           "kmos.kmo_illumination.suppress_extension");
00631         KMO_TRY_CHECK_ERROR_STATE();
00632         KMO_TRY_EXIT_IF_ERROR(
00633             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.suppress_extension"));
00634 
00635         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00636                        CPL_ERROR_ILLEGAL_INPUT,
00637                        "suppress_extension must be TRUE or FALSE!");
00638 
00639         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00640 
00641         KMO_TRY_EXIT_IF_ERROR(
00642             kmo_combine_pars_load(parlist,
00643                                   "kmos.kmo_illumination",
00644                                   &cmethod,
00645                                   &cpos_rej,
00646                                   &cneg_rej,
00647                                   &citer,
00648                                   &cmin,
00649                                   &cmax,
00650                                   FALSE));
00651         cpl_msg_info("", "-------------------------------------------");
00652 
00653         // check if filter_id, grating_id and rotator offset match for all
00654         // detectors
00655         KMO_TRY_EXIT_IF_ERROR(
00656             kmo_check_frameset_setup(frameset, FLAT_SKY,
00657                                        TRUE, FALSE, TRUE));
00658         KMO_TRY_EXIT_IF_ERROR(
00659             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00660                                        TRUE, FALSE, TRUE));
00661         KMO_TRY_EXIT_IF_ERROR(
00662             kmo_check_frame_setup(frameset, XCAL, YCAL,
00663                                        TRUE, FALSE, TRUE));
00664         KMO_TRY_EXIT_IF_ERROR(
00665             kmo_check_frame_setup(frameset, XCAL, LCAL,
00666                                        TRUE, FALSE, TRUE));
00667         KMO_TRY_EXIT_IF_ERROR(
00668             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00669                                        TRUE, FALSE, TRUE));
00670 
00671         KMO_TRY_EXIT_IF_NULL(
00672             frame = kmo_dfs_get_frame(frameset, XCAL));
00673         KMO_TRY_EXIT_IF_NULL(
00674             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00675 
00676         KMO_TRY_EXIT_IF_ERROR(
00677             kmo_check_frame_setup_md5_xycal(frameset));
00678         KMO_TRY_EXIT_IF_ERROR(
00679             kmo_check_frame_setup_md5(frameset));
00680 
00681         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00682         cpl_msg_info("", "(grating 1, 2 & 3)");
00683 
00684         // check which IFUs are active for all frames
00685         KMO_TRY_EXIT_IF_NULL(
00686             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00687 
00688         KMO_TRY_EXIT_IF_NULL(
00689             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00690 
00691         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00692 
00693         // load desc for MASTER_DARK and check
00694         KMO_TRY_EXIT_IF_NULL(
00695             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00696         desc_dark = kmo_identify_fits_header(
00697                     cpl_frame_get_filename(frame));
00698         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00699                                       "be in KMOS-format!");
00700         KMO_TRY_ASSURE((desc_dark.nr_ext == 2*KMOS_NR_DETECTORS) &&
00701                        (desc_dark.ex_badpix == FALSE) &&
00702                        (desc_dark.fits_type == f2d_fits) &&
00703                        (desc_dark.frame_type == detector_frame),
00704                        CPL_ERROR_ILLEGAL_INPUT,
00705                        "MASTER_DARK isn't in the correct format!!!");
00706         nx = desc_dark.naxis1;
00707         ny = desc_dark.naxis2;
00708 
00709         // load desc for MASTER_FLAT and check
00710         KMO_TRY_EXIT_IF_NULL(
00711             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00712         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00713         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00714                                       "be in KMOS-format!");
00715         KMO_TRY_ASSURE((desc_flat.nr_ext % (2*KMOS_NR_DETECTORS) == 0) &&
00716                        (desc_flat.ex_badpix == FALSE) &&
00717                        (desc_flat.fits_type == f2d_fits) &&
00718                        (desc_flat.frame_type == detector_frame),
00719                        CPL_ERROR_ILLEGAL_INPUT,
00720                        "MASTER_FLAT isn't in the correct format!!!");
00721 
00722         // load desc for XCAL and check
00723         KMO_TRY_EXIT_IF_NULL(
00724             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00725         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00726         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00727                                       "be in KMOS-format!");
00728         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00729                        (desc_xcal.ex_badpix == FALSE) &&
00730                        (desc_xcal.fits_type == f2d_fits) &&
00731                        (desc_xcal.frame_type == detector_frame),
00732                        CPL_ERROR_ILLEGAL_INPUT,
00733                        "XCAL isn't in the correct format!!!");
00734         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00735                        (desc_xcal.naxis2 == ny),
00736                        CPL_ERROR_ILLEGAL_INPUT,
00737                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00738                        "(x,y): (%d,%d) vs (%d,%d)",
00739                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00740 
00741         nr_devices = desc_xcal.nr_ext;
00742 
00743         // load desc for YCAL and check
00744         KMO_TRY_EXIT_IF_NULL(
00745             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00746         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00747         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00748                                       "be in KMOS-format!");
00749         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00750                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00751                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00752                        (desc_ycal.frame_type == desc_xcal.frame_type),
00753                        CPL_ERROR_ILLEGAL_INPUT,
00754                        "YCAL isn't in the correct format!!!");
00755         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00756                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00757                        CPL_ERROR_ILLEGAL_INPUT,
00758                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00759                        "(x,y): (%d,%d) vs (%d,%d)",
00760                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00761 
00762         // load desc for LCAL and check
00763         KMO_TRY_EXIT_IF_NULL(
00764             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00765         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00766         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00767                                       "be in KMOS-format!");
00768         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00769                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00770                        (desc_lcal.frame_type == desc_xcal.frame_type),
00771                        CPL_ERROR_ILLEGAL_INPUT,
00772                        "LCAL isn't in the correct format!!!");
00773         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00774                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00775                        CPL_ERROR_ILLEGAL_INPUT,
00776                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00777                        "(x,y): (%d,%d) vs (%d,%d)",
00778                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00779         KMO_TRY_EXIT_IF_NULL(
00780             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00781 
00782         // load desc for FLAT_SKY and check
00783         nr_devices = KMOS_NR_DETECTORS;
00784         KMO_TRY_EXIT_IF_NULL(
00785             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00786 
00787         KMO_TRY_EXIT_IF_NULL(
00788             main_header = kmclipm_propertylist_load(
00789                                          cpl_frame_get_filename(frame), 0));
00790         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00791         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00792         kmclipm_strip_angle(&rotangle);
00793         cpl_propertylist_delete(main_header); main_header = NULL;
00794 
00795         cnt = 1;
00796         while (frame != NULL) {
00797             KMO_TRY_EXIT_IF_NULL(
00798                 main_header = kmclipm_propertylist_load(
00799                                              cpl_frame_get_filename(frame), 0));
00800 
00801             desc_sky = kmo_identify_fits_header(
00802                         cpl_frame_get_filename(frame));
00803             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00804                                           "be in KMOS-format!");
00805             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00806                            (desc_sky.ex_badpix == FALSE) &&
00807                            (desc_sky.fits_type == raw_fits) &&
00808                            (desc_sky.frame_type == detector_frame),
00809                            CPL_ERROR_ILLEGAL_INPUT,
00810                            "FLAT_SKY isn't in the correct format!!!");
00811             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00812                            (desc_sky.naxis2 == ny) &&
00813                            (desc_sky.nr_ext == nr_devices),
00814                            CPL_ERROR_ILLEGAL_INPUT,
00815                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00816                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00817                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00818             kmo_free_fits_desc(&desc_sky);
00819             kmo_init_fits_desc(&desc_sky);
00820 
00821             KMO_TRY_ASSURE(
00822                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00823                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00824                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00825                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00826                 CPL_ERROR_ILLEGAL_INPUT,
00827                 "All lamps must be switched off for the FLAT_SKY frames!");
00828 
00829             // assert that filters have correct IDs and that all detectors of
00830             // all input frames have the same filter set
00831             for (i = 1; i <= KMOS_NR_DETECTORS; i++) {
00832                 // ESO INS FILTi ID
00833                 KMO_TRY_EXIT_IF_NULL(
00834                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00835                 KMO_TRY_EXIT_IF_NULL(
00836                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00837 
00838                 KMO_TRY_EXIT_IF_NULL(
00839                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00840                 cpl_free(keyword); keyword = NULL;
00841 
00842                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00843                                (strcmp(filter_id, "YJ") == 0) ||
00844                                (strcmp(filter_id, "H") == 0) ||
00845                                (strcmp(filter_id, "K") == 0) ||
00846                                (strcmp(filter_id, "HK") == 0),
00847                                CPL_ERROR_ILLEGAL_INPUT,
00848                                "Filter ID in primary header must be either 'IZ', "
00849                                "'YJ', 'H', 'K' or " "'HK' !");
00850 
00851                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00852                                CPL_ERROR_ILLEGAL_INPUT,
00853                                "Filter IDs must be the same for FLAT_SKY frame"
00854                                " and lcal frame!"
00855                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00856                                i, cpl_frame_get_filename(frame),
00857                                filter_id, filter_id_l);
00858 
00859                 // ESO INS GRATi ID
00860                 KMO_TRY_EXIT_IF_NULL(
00861                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00862                 KMO_TRY_EXIT_IF_NULL(
00863                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00864 
00865                 KMO_TRY_EXIT_IF_NULL(
00866                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00867                 cpl_free(keyword); keyword = NULL;
00868 
00869                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00870                                (strcmp(filter_id, "YJ") == 0) ||
00871                                (strcmp(filter_id, "H") == 0) ||
00872                                (strcmp(filter_id, "K") == 0) ||
00873                                (strcmp(filter_id, "HK") == 0),
00874                                CPL_ERROR_ILLEGAL_INPUT,
00875                                "Grating ID in primary header must be either "
00876                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00877 
00878                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00879                                CPL_ERROR_ILLEGAL_INPUT,
00880                                "Grating IDs must be the same for FLAT_SKY frame"
00881                                " and lcal frame!"
00882                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00883                                i, cpl_frame_get_filename(frame),
00884                                filter_id, filter_id_l);
00885 
00886                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00887                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00888                 kmclipm_strip_angle(&tmp_rotangle);
00889                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00890                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00891                         CPL_ERROR_ILLEGAL_INPUT,
00892                         "OCS ROT NAANGLE of sky flat frames differ too much: %f %f",
00893                         rotangle, tmp_rotangle);
00894             }
00895             cpl_propertylist_delete(main_header); main_header = NULL;
00896 
00897             // get next FLAT_SKY frame
00898             frame = kmo_dfs_get_frame(frameset, NULL);
00899             KMO_TRY_CHECK_ERROR_STATE();
00900             cnt++;
00901         }
00902 
00903         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00904 
00905         //
00906         // noise will be propagated when:
00907         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00908         // 2 FLAT_SKY frames are provided.
00909         // Otherwise noise will be ignored.
00910         //
00911         if (desc_dark.ex_noise &&
00912             desc_flat.ex_noise &&
00913             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00914             process_noise = TRUE;
00915         }
00916 
00917         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00918             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00919                             "since there is only one input frame! (The output "
00920                             "file won't have any noise extensions)");
00921 
00922             cmethod = "average";
00923         }
00924 
00925         //
00926         // Check whether 1st FLAT_SKY should be omitted
00927         //
00928         KMO_TRY_EXIT_IF_NULL(
00929             frameset_sky = cpl_frameset_new());
00930 
00931         if (add_all_sky) {
00932             // just add all FLAT_SKY frames without check
00933             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00934             while (frame != NULL) {
00935                 KMO_TRY_EXIT_IF_ERROR(
00936                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00937                 frame = kmo_dfs_get_frame(frameset, NULL);
00938             }
00939             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00940         } else {
00941             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00942             KMO_TRY_EXIT_IF_NULL(
00943                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00944 
00945             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00946                 // just one FLAT_SKY, always add
00947                 KMO_TRY_EXIT_IF_ERROR(
00948                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00949                 KMO_TRY_CHECK_ERROR_STATE();
00950             } else {
00951                 // several FLAT_SKY frames, check exptime
00952 
00953                 // get exptime 1
00954                 KMO_TRY_EXIT_IF_NULL(
00955                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00956                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00957                 KMO_TRY_CHECK_ERROR_STATE();
00958                 cpl_propertylist_delete(main_header); main_header = NULL;
00959 
00960                 // get exptime 2
00961                 frame = kmo_dfs_get_frame(frameset, NULL);
00962                 KMO_TRY_EXIT_IF_NULL(
00963                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00964                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00965                 KMO_TRY_CHECK_ERROR_STATE();
00966                 cpl_propertylist_delete(main_header); main_header = NULL;
00967 
00968                 // loop remaining frames
00969                 same_exptime = TRUE;
00970                 frame = kmo_dfs_get_frame(frameset, NULL);
00971                 while (same_exptime && (frame != NULL)) {
00972                     KMO_TRY_EXIT_IF_NULL(
00973                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00974                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00975                     KMO_TRY_CHECK_ERROR_STATE();
00976                     cpl_propertylist_delete(main_header); main_header = NULL;
00977                     if (fabs(exptime-exptime2) > 0.01) {
00978                         // not same
00979                         same_exptime = FALSE;
00980                     }
00981                     frame = kmo_dfs_get_frame(frameset, NULL);
00982                 }
00983 
00984                 if (same_exptime) {
00985                     // frame [2,n] have same exptime, add them
00986                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00987                     KMO_TRY_EXIT_IF_NULL(
00988                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00989                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00990                     KMO_TRY_CHECK_ERROR_STATE();
00991                     cpl_propertylist_delete(main_header); main_header = NULL;
00992                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00993                     frame = kmo_dfs_get_frame(frameset, NULL);
00994                     while (frame != NULL) {
00995                         KMO_TRY_EXIT_IF_ERROR(
00996                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00997                         frame = kmo_dfs_get_frame(frameset, NULL);
00998                     }
00999                     if (fabs(exptime1-exptime2) < 0.01) {
01000                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
01001                                             "It has anyway been omitted since we assume it is an acquisition frame. "
01002                                             "If you want to add it anyway call this recipe with the --add-all parameter");
01003                     }
01004                 } else {
01005                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
01006                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
01007                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
01008                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
01009                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
01010                     while (frame != NULL) {
01011                         KMO_TRY_EXIT_IF_NULL(
01012                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
01013                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
01014                         KMO_TRY_CHECK_ERROR_STATE();
01015                         cpl_propertylist_delete(main_header); main_header = NULL;
01016                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
01017                         frame = kmo_dfs_get_frame(frameset, NULL);
01018                     }
01019                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
01020                     KMO_TRY_CHECK_ERROR_STATE();
01021                 }
01022             }
01023         }
01024 
01025         KMO_TRY_EXIT_IF_NULL(
01026             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
01027         KMO_TRY_EXIT_IF_NULL(
01028             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
01029         KMO_TRY_EXIT_IF_NULL(
01030             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
01031         KMO_TRY_EXIT_IF_NULL(
01032             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
01033         cpl_free(keyword); keyword = NULL;
01034 
01035         //
01036         // set default band-specific ranges for collapsing
01037         //
01038         if (ranges == NULL) {
01039             if (strcmp(filter, "IZ") == 0) {
01040                 ranges_txt = "0.81,1.05";
01041             } else if (strcmp(filter, "YJ") == 0) {
01042                 ranges_txt = "1.025,1.3";
01043             } else if (strcmp(filter, "H") == 0) {
01044                 ranges_txt = "1.5,1.7";
01045             } else if (strcmp(filter, "K") == 0) {
01046                 ranges_txt = "2.1,2.35";
01047             } else if (strcmp(filter, "HK") == 0) {
01048                 ranges_txt = "1.5,1.7;2.1,2.35";
01049 //                ranges_txt = "1.5,1.7";
01050             } else {
01051                 KMO_TRY_ASSURE(1 == 0,
01052                                CPL_ERROR_ILLEGAL_INPUT,
01053                                "We really shouldn't get here...");
01054             }
01055             cpl_msg_info("", "Spectral range to collapse has been set to [%s] um for this band.", ranges_txt);
01056             ranges = kmo_identify_ranges(ranges_txt);
01057             KMO_TRY_CHECK_ERROR_STATE();
01058         }
01059 
01060         // setup grid definition, wavelength start and end points will be set
01061         // in the detector loop
01062         KMO_TRY_EXIT_IF_ERROR(
01063             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale, 0.));
01064 
01065         // create filename for LUT
01066         KMO_TRY_EXIT_IF_NULL(
01067             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01068 
01069         // extract bounds
01070         KMO_TRY_EXIT_IF_NULL(
01071             tmp_header = kmo_dfs_load_primary_header(frameset, XCAL));
01072         KMO_TRY_EXIT_IF_NULL(
01073             bounds = kmclipm_extract_bounds(tmp_header));
01074         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01075 
01076         // get timestamps of xcal, ycal & lcal
01077         KMO_TRY_EXIT_IF_NULL(
01078             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01079 
01080         // create arrays to hold reconstructed data and noise cubes and
01081         // their headers
01082         KMO_TRY_EXIT_IF_NULL(
01083             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01084                                                             sizeof(cpl_imagelist*)));
01085         KMO_TRY_EXIT_IF_NULL(
01086             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01087                                                              sizeof(cpl_imagelist*)));
01088         KMO_TRY_EXIT_IF_NULL(
01089             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01090                                                          sizeof(cpl_image*)));
01091         KMO_TRY_EXIT_IF_NULL(
01092             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01093                                                           sizeof(cpl_image*)));
01094         KMO_TRY_EXIT_IF_NULL(
01095             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01096                                                                      sizeof(cpl_propertylist*)));
01097         KMO_TRY_EXIT_IF_NULL(
01098             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01099                                                                       sizeof(cpl_propertylist*)));
01100         KMO_TRY_EXIT_IF_NULL(
01101             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01102                                                       sizeof(cpl_table**)));
01103         KMO_TRY_EXIT_IF_NULL(
01104             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01105                                                       sizeof(cpl_table*)));
01106         KMO_TRY_EXIT_IF_NULL(
01107             calAngles = cpl_vector_new(3));
01108 
01109         //
01110         // loop through all detectors
01111         //
01112         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01113             cpl_msg_info("","Processing detector No. %d", det_nr);
01114 
01115             KMO_TRY_EXIT_IF_NULL(
01116                 detector_in = cpl_imagelist_new());
01117 
01118             // load data of det_nr of all FLAT_SKY frames into an imagelist
01119             KMO_TRY_EXIT_IF_NULL(
01120                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01121 
01122             cnt = 0;
01123             while (img_in != NULL) {
01124                 cpl_imagelist_set(detector_in, img_in, cnt);
01125                 KMO_TRY_CHECK_ERROR_STATE();
01126 
01127                 /* load same extension of next FLAT_SKY frame*/
01128                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01129                 KMO_TRY_CHECK_ERROR_STATE();
01130 
01131                 cnt++;
01132             }
01133 
01134             //
01135             // process imagelist
01136             //
01137 
01138             // combine imagelist (data only) and create noise (stdev of data)
01139             cpl_msg_info("","Combining frames...");
01140             if (process_noise) {
01141                 KMO_TRY_EXIT_IF_ERROR(
01142                     kmclipm_combine_frames(detector_in,
01143                                            NULL,
01144                                            NULL,
01145                                            cmethod,
01146                                            cpos_rej,
01147                                            cneg_rej,
01148                                            citer,
01149                                            cmax,
01150                                            cmin,
01151                                            &combined_data,
01152                                            &combined_noise,
01153                                            -1.0));
01154             } else {
01155                 KMO_TRY_EXIT_IF_ERROR(
01156                     kmclipm_combine_frames(detector_in,
01157                                            NULL,
01158                                            NULL,
01159                                            cmethod,
01160                                            cpos_rej,
01161                                            cneg_rej,
01162                                            citer,
01163                                            cmax,
01164                                            cmin,
01165                                            &combined_data,
01166                                            NULL,
01167                                            -1.0));
01168             }
01169 
01170             if (kmclipm_omit_warning_one_slice > 10) {
01171 // AA: commmented this out: Too unclear for the user, no benefit to know about this number
01172 //                cpl_msg_warning(cpl_func, "Previous warning (number of "
01173 //                                          "identified slices) occured %d times.",
01174 //                                kmclipm_omit_warning_one_slice);
01175                 kmclipm_omit_warning_one_slice = FALSE;
01176             }
01177 
01178             cpl_imagelist_delete(detector_in); detector_in = NULL;
01179 
01180             // load calibration files
01181             KMO_TRY_EXIT_IF_NULL(
01182                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01183                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01184 
01185             KMO_TRY_EXIT_IF_ERROR(
01186                 cpl_vector_set(calAngles, 0, rotangle_found));
01187             KMO_TRY_EXIT_IF_NULL(
01188                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01189                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01190             KMO_TRY_EXIT_IF_ERROR(
01191                 cpl_vector_set(calAngles, 1, rotangle_found));
01192             KMO_TRY_EXIT_IF_NULL(
01193                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01194                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01195             KMO_TRY_EXIT_IF_ERROR(
01196                 cpl_vector_set(calAngles, 2, rotangle_found));
01197 
01198             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01199             KMO_TRY_EXIT_IF_NULL(
01200                 bad_pix_mask = cpl_image_duplicate(xcal));
01201 
01202             KMO_TRY_EXIT_IF_NULL(
01203                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01204             for (x = 0; x < nx; x++) {
01205                 for (y = 0; y < ny; y++) {
01206                     if (isnan(pbad_pix_mask[x+nx*y])) {
01207                         pbad_pix_mask[x+nx*y] = 0.;
01208                     } else {
01209                         pbad_pix_mask[x+nx*y] = 1.;
01210                     }
01211                 }
01212             }
01213             KMO_TRY_CHECK_ERROR_STATE();
01214 
01215             //
01216             // calculate SKYFLAT_EDGE
01217             //
01218             if (has_flat_edge) {
01219                 // get edge-edgepars from FLAT_SKY
01220                 KMO_TRY_EXIT_IF_ERROR(
01221                     kmo_calc_edgepars(combined_data,
01222                                       unused_ifus_after[det_nr-1],
01223                                       bad_pix_mask,
01224                                       det_nr,
01225                                       &slitlet_ids,
01226                                       &edgepars));
01227                 KMO_TRY_CHECK_ERROR_STATE();
01228 
01229                 // copy edgepars to table for saving later on
01230                 KMO_TRY_EXIT_IF_NULL(
01231                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01232 
01233                 if (edgepars != NULL) {
01234                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01235                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01236                     }
01237                     cpl_free(edgepars); edgepars = NULL;
01238                 }
01239                 if (slitlet_ids != NULL) {
01240                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01241                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01242                     }
01243                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01244                 }
01245                 KMO_TRY_CHECK_ERROR_STATE();
01246 
01247                 //
01248                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01249                 //
01250 
01251                 // load flat_edge from MASTER_FLAT
01252                 KMO_TRY_EXIT_IF_NULL(
01253                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01254                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01255                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01256 
01257                     KMO_TRY_EXIT_IF_NULL(
01258                         punused_ifus = cpl_array_get_data_int_const(unused_ifus_after[det_nr-1]));
01259                     if (punused_ifus[j] == 0) {
01260                         KMO_TRY_EXIT_IF_NULL(
01261                             edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01262                                                                         ifu_nr, rotangle, 0, &tmp_rotangle));
01263                     }
01264                 }
01265 
01266                 //
01267                 // calculate shift value
01268                 //
01269 
01270                 KMO_TRY_EXIT_IF_NULL(
01271                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01272                 KMO_TRY_EXIT_IF_NULL(
01273                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01274 
01275                 // get shift values for each IFU by comparing all edge parameters,
01276                 // rejecting and applying median
01277                 int row = 1024; // middle of frame
01278                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01279                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01280                     for (edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01281                         if (edge_table_flat[j] != NULL) {
01282                             double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01283                             double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01284                             cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01285                         }
01286                     }
01287 
01288                     // reject deviating edge-differences
01289                     kmclipm_vector *kv = NULL;
01290                     KMO_TRY_EXIT_IF_NULL(
01291                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01292                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01293 
01294                     // set shift value for each IFU
01295                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01296                     kmclipm_vector_delete(kv); kv = NULL;
01297                 }
01298                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01299                 KMO_TRY_CHECK_ERROR_STATE();
01300 
01301                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01302                     cpl_table_delete(edge_table_flat[j]);
01303                     edge_table_flat[j] = NULL;
01304                 }
01305 
01306                 // take median of all IFU-shift-values
01307                 double shift_val = -cpl_vector_get_median(shift_vec);
01308                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01309 
01310                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01311 
01312                 int     xdim                = cpl_image_get_size_x(combined_data),
01313                         ydim                = cpl_image_get_size_y(combined_data);
01314                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01315                         *array_out          = NULL;
01316                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01317                         *pcombined_noise    = NULL;
01318     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01319                 if (process_noise) {
01320                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01321                 }
01322 
01323                 for (iy = 0; iy < ydim; iy++) {
01324                     // cubic spline
01325                     for (ix = 0; ix < xdim; ix++) {
01326                         array_in[ix] = pcombined_data[ix+iy*xdim];
01327                     }
01328                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01329                                                     xdim, shift_val, 1.0,
01330                                                     NATURAL);
01331                     for (ix = 0; ix < xdim; ix++) {
01332                       pcombined_data[ix+iy*xdim] = array_out[ix];
01333                     }
01334                     cpl_free(array_out);
01335 
01336     //                // linear
01337     //                for (ix = 1; ix < xdim; ix++) {
01338     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01339     //                                     pcombined_data[(ix-1)+iy*xdim];
01340     //                }
01341     //                for (ix = 1; ix < xdim; ix++) {
01342     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01343     //                }
01344 
01345                     if (process_noise) {
01346                         // cubic spline
01347                         for (ix = 0; ix < xdim; ix++) {
01348                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01349                         }
01350                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01351                                                         xdim, shift_val, 1.0,
01352                                                         NATURAL);
01353                         for (ix = 0; ix < xdim; ix++) {
01354                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01355                         }
01356                         cpl_free(array_out);
01357 
01358     //                    // linear
01359     //                    for (ix = 1; ix < xdim; ix++) {
01360     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01361     //                                         pcombined_noise[(ix-1)+iy*xdim];
01362     //                    }
01363     //                    for (ix = 1; ix < xdim; ix++) {
01364     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01365     //                    }
01366                     }
01367                 }
01368                 cpl_free(array_in); array_in = NULL;
01369             }
01370             //
01371             // reconstruct
01372             //
01373             // load MASTER_DARK and MASTER_FLAT
01374             KMO_TRY_EXIT_IF_NULL(
01375                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01376                                               det_nr, FALSE, FALSE, NULL));
01377 
01378             if (process_noise) {
01379                 KMO_TRY_EXIT_IF_NULL(
01380                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01381                                                         det_nr, TRUE, FALSE, NULL));
01382             }
01383 
01384             KMO_TRY_EXIT_IF_NULL(
01385                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01386                                                   rotangle, FALSE, NULL,
01387                                                   &rotangle_found, -1, 0, 0));
01388 
01389             if (process_noise) {
01390                 KMO_TRY_EXIT_IF_NULL(
01391                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01392                                                             rotangle, FALSE, NULL,
01393                                                             &rotangle_found, -1, 0, 0));
01394             }
01395 
01396             char *tmp_band_method = getenv("KMO_BAND_METHOD");
01397             int band_method = 0;
01398             if (tmp_band_method != NULL) {
01399                 band_method = atoi(tmp_band_method);
01400             }
01401 
01402             // ESO INS FILTi ID
01403             KMO_TRY_EXIT_IF_NULL(
01404                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01405                                       IFU_FILTID_POSTFIX));
01406             KMO_TRY_EXIT_IF_NULL(
01407                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01408             cpl_free(keyword); keyword = NULL;
01409 
01410             KMO_TRY_EXIT_IF_NULL(
01411                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01412             KMO_TRY_EXIT_IF_ERROR(
01413                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
01414                                              band_table));
01415             cpl_table_delete(band_table); band_table = NULL;
01416 
01417             cpl_msg_info("","Reconstructing cubes...");
01418             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01419                 // update sub-header
01420                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01421 
01422                 // load raw image and sub-header
01423                 KMO_TRY_EXIT_IF_NULL(
01424                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01425                                                          det_nr, FALSE));
01426 
01427                 KMO_TRY_EXIT_IF_NULL(
01428                     punused_ifus = cpl_array_get_data_int_const(
01429                                                   unused_ifus_after[det_nr-1]));
01430 
01431                 // check if IFU is valid according to main header keywords &
01432                 // calibration files
01433                 KMO_TRY_EXIT_IF_NULL(
01434                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01435                                           IFU_VALID_POSTFIX));
01436                 KMO_TRY_CHECK_ERROR_STATE();
01437                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01438                 cpl_free(keyword); keyword = NULL;
01439 
01440                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01441                     (bounds[2*(ifu_nr-1)] != -1) &&
01442                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01443                     (punused_ifus[j] == 0))
01444                 {
01445                     // IFU is valid
01446                     cpl_error_reset();
01447 
01448                     // calculate WCS
01449                     KMO_TRY_EXIT_IF_ERROR(
01450                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01451 
01452                     // reconstruct data
01453                     KMO_TRY_EXIT_IF_ERROR(
01454                         kmo_reconstruct_sci_image(ifu_nr,
01455                                                 bounds[2*(ifu_nr-1)],
01456                                                 bounds[2*(ifu_nr-1)+1],
01457                                                 combined_data,
01458                                                 combined_noise,
01459                                                 img_dark,
01460                                                 img_dark_noise,
01461                                                 img_flat,
01462                                                 img_flat_noise,
01463                                                 xcal,
01464                                                 ycal,
01465                                                 lcal,
01466                                                 &gd,
01467                                                 calTimestamp,
01468                                                 calAngles,
01469                                                 fn_lut,
01470                                                 &cube_data,
01471                                                 &cube_noise,
01472                                                 flux,
01473                                                 background,
01474                                                 NULL,
01475                                                 NULL,
01476                                                 NULL));
01477                     KMO_TRY_CHECK_ERROR_STATE();
01478                 } else {
01479                     // IFU is invalid
01480                     cpl_error_reset();
01481                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01482 
01483                 // save output
01484                 KMO_TRY_EXIT_IF_NULL(
01485                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01486 
01487                 KMO_TRY_EXIT_IF_ERROR(
01488                     kmclipm_update_property_string(sub_header, EXTNAME,
01489                                                    extname,
01490                                                    "FITS extension name"));
01491 
01492                 cpl_free(extname); extname = NULL;
01493 
01494                 // store cube and sub header into array for later
01495                 stored_data_cubes[ifu_nr - 1] = cube_data;
01496                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01497 
01498                 if (process_noise) {
01499                     KMO_TRY_EXIT_IF_NULL(
01500                         sub_header = cpl_propertylist_duplicate(
01501                                            stored_sub_data_headers[ifu_nr - 1]));
01502                     KMO_TRY_EXIT_IF_NULL(
01503                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01504                                                       EXT_NOISE));
01505 
01506                     KMO_TRY_EXIT_IF_ERROR(
01507                         kmclipm_update_property_string(sub_header,
01508                                                 EXTNAME,
01509                                                 extname,
01510                                                 "FITS extension name"));
01511 
01512                     cpl_free(extname); extname = NULL;
01513 
01514                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01515                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01516                 }
01517                 cpl_image_delete(data_ifu); data_ifu = NULL;
01518                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01519                 cube_data = NULL;
01520                 cube_noise = NULL;
01521             } // for j IFUs
01522 
01523             // free memory
01524             cpl_image_delete(combined_data); combined_data = NULL;
01525             cpl_image_delete(combined_noise); combined_noise = NULL;
01526             cpl_image_delete(xcal); xcal = NULL;
01527             cpl_image_delete(ycal); ycal = NULL;
01528             cpl_image_delete(lcal); lcal = NULL;
01529             cpl_image_delete(img_dark); img_dark = NULL;
01530             cpl_image_delete(img_flat); img_flat = NULL;
01531             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01532             if (process_noise) {
01533                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01534                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01535             }
01536         } // for nr_devices
01537 
01538         cpl_free(edge_table_flat); edge_table_flat = NULL;
01539 
01540         // collapse cubes using rejection
01541         cpl_msg_info("","Collapsing cubes...");
01542         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01543             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01544                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01545 
01546                 KMO_TRY_EXIT_IF_NULL(
01547                     punused_ifus = cpl_array_get_data_int_const(
01548                                                   unused_ifus_after[det_nr-1]));
01549                 if (punused_ifus[j] == 0) {
01550                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01551                         // IFU is valid
01552                         ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01553                                                                 CRPIX3);
01554                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01555                                        "CRPIX3 keyword in FITS-header is missing!");
01556 
01557                         ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01558                                                                 CRVAL3);
01559                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01560                                        "CRVAL3 keyword in FITS-header is missing!");
01561 
01562                         ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01563                                                                 CDELT3);
01564                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01565                                        "CDELT3 keyword in FITS-header is missing!");
01566 
01567                         KMO_TRY_EXIT_IF_NULL(
01568                             identified_slices = kmo_identify_slices(ranges,
01569                                                                     ifu_crpix,
01570                                                                     ifu_crval,
01571                                                                     ifu_cdelt,
01572                                                                     gd.l.dim));
01573                     }/* else {
01574                         KMO_TRY_EXIT_IF_NULL(
01575                             identified_slices = cpl_vector_new(gd.l.dim));
01576                         cpl_vector_fill(identified_slices, 1.);
01577                     }*/
01578 
01579                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01580                         KMO_TRY_EXIT_IF_ERROR(
01581                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01582                                                stored_noise_cubes[ifu_nr-1],
01583                                                &stored_data_images[ifu_nr-1],
01584                                                &stored_noise_images[ifu_nr-1],
01585                                                identified_slices,
01586                                                cmethod, cpos_rej, cneg_rej,
01587                                                citer, cmax, cmin));
01588                     }
01589                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01590                 } else {
01591                     // IFU is invalid
01592                 }
01593             }
01594         }
01595 
01596         // normalise all IFUs of a detector as a group.
01597         // Calculate mean of each IFU, add up and divide by number of successful
01598         // averaged IFUs.
01599         // Then divide all valid IFUs with mean value
01600         int jj;
01601         for (jj = 0; jj < nr_devices; jj++) {
01602             cnt = 0;
01603             mean_data = 0;
01604             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01605                 ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01606                 if (stored_data_images[ifu_nr] != NULL) {
01607                     KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[ifu_nr]) <
01608                                    cpl_image_get_size_x(stored_data_images[ifu_nr])*
01609                                    cpl_image_get_size_y(stored_data_images[ifu_nr]),
01610                                    CPL_ERROR_ILLEGAL_INPUT,
01611                                    "The collapsed, dark-subtracted image contains "
01612                                    "only invalid values! Probably the provided "
01613                                    "FLAT_SKY frames are exactly the same as the "
01614                                    "frames used for MASTER_DARK calculation.");
01615 
01616                     mean_data += cpl_image_get_mean(stored_data_images[ifu_nr]);
01617                     KMO_TRY_CHECK_ERROR_STATE();
01618                     cnt++;
01619                 }
01620 
01621             }
01622             mean_data /= cnt;
01623 
01624             if (mean_data != 0.0) {
01625                 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01626                     ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01627                     if (stored_data_images[ifu_nr] != NULL) {
01628                         KMO_TRY_EXIT_IF_ERROR(
01629                             cpl_image_divide_scalar(stored_data_images[ifu_nr],
01630                                                     mean_data));
01631                     }
01632                 }
01633             } else {
01634                 cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01635                                           "(mean = 0.0)!");
01636             }
01637 
01638             if (process_noise) {
01639                 if (mean_data != 0.0) {
01640                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01641                         ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01642                         if (stored_noise_images[ifu_nr] != NULL) {
01643                             KMO_TRY_EXIT_IF_ERROR(
01644                                 cpl_image_divide_scalar(stored_noise_images[ifu_nr],
01645                                                         mean_data));
01646                         }
01647                     }
01648                 } else {
01649                     cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01650                                               "(mean = 0.0)!");
01651                 }
01652             }
01653         } // end for(jj)
01654 
01655         // calculate qc parameters on normalised data
01656         qc_spat_unif = 0.0;
01657         cnt = 0;
01658         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01659             if (stored_data_images[i] != NULL) {
01660                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01661                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01662 
01663                 qc_spat_unif += pow(tmp_mean-1, 2);
01664                 if (fabs(tmp_mean) > qc_max_dev) {
01665                     qc_max_dev = tmp_mean-1;
01666                     qc_max_dev_id = i+1;
01667                 }
01668                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01669                     qc_max_nonunif = tmp_stdev;
01670                     qc_max_nonunif_id = i+1;
01671                 }
01672                 KMO_TRY_CHECK_ERROR_STATE();
01673                 cnt++;
01674             }
01675         }
01676         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01677 
01678         //
01679         // save data
01680         //
01681 
01682         // update which IFUs are not used
01683         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01684 
01685         KMO_TRY_EXIT_IF_ERROR(
01686             kmo_set_unused_ifus(unused_ifus_after, main_header,
01687                                 "kmo_illumination"));
01688 
01689         cpl_msg_info("","Saving data...");
01690 
01691         KMO_TRY_EXIT_IF_ERROR(
01692             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01693                                            "[adu] uniformity of illumination correction"));
01694         KMO_TRY_EXIT_IF_ERROR(
01695             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01696                                            "[adu] max. deviation from unity"));
01697         KMO_TRY_EXIT_IF_ERROR(
01698             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01699                                         "[] IFU ID with max. dev. from unity"));
01700         KMO_TRY_EXIT_IF_ERROR(
01701             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01702                                            "[adu] max. stdev of illumination corr."));
01703         KMO_TRY_EXIT_IF_ERROR(
01704             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01705                                         "[] IFU ID with max. stdev in illum. corr."));
01706 
01707         if (!suppress_extension) {
01708             KMO_TRY_EXIT_IF_NULL(
01709                 fn_suffix = cpl_sprintf("%s", suffix));
01710         } else {
01711             KMO_TRY_EXIT_IF_NULL(
01712                 fn_suffix = cpl_sprintf("%s", ""));
01713         }
01714         KMO_TRY_EXIT_IF_ERROR(
01715             kmo_dfs_save_main_header(frameset, ILLUM_CORR, fn_suffix, frame,
01716                                      main_header, parlist, cpl_func));
01717 
01718         if (has_flat_edge) {
01719             KMO_TRY_EXIT_IF_ERROR(
01720                 kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, fn_suffix, frame,
01721                                          main_header, parlist, cpl_func));
01722         }
01723 
01724         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01725             KMO_TRY_EXIT_IF_ERROR(
01726                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, fn_suffix,
01727                                    stored_sub_data_headers[i], 0./0.));
01728 
01729             if (process_noise) {
01730                 KMO_TRY_EXIT_IF_ERROR(
01731                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01732                                        fn_suffix, stored_sub_noise_headers[i], 0./0.));
01733             }
01734         }
01735 
01736         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01737             for (ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01738                 KMO_TRY_EXIT_IF_ERROR(
01739                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01740                                                 CAL_IFU_NR,
01741                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01742                                                 "IFU Number {1..24}"));
01743                 KMO_TRY_EXIT_IF_ERROR(
01744                     kmclipm_update_property_double(
01745                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01746                                                 CAL_ROTANGLE,
01747                                                 rotangle_found,
01748                                                 "[deg] Rotator relative to nasmyth"));
01749                 if (has_flat_edge) {
01750                     // save edge-parameters as product
01751                     KMO_TRY_EXIT_IF_ERROR(
01752                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, fn_suffix,
01753                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01754                 }
01755             }
01756         }
01757     }
01758     KMO_CATCH
01759     {
01760         KMO_CATCH_MSG();
01761         ret_val = -1;
01762     }
01763     kmo_free_fits_desc(&desc_sky);
01764     kmo_free_fits_desc(&desc_dark);
01765     kmo_free_fits_desc(&desc_flat);
01766     kmo_free_fits_desc(&desc_xcal);
01767     kmo_free_fits_desc(&desc_ycal);
01768     kmo_free_fits_desc(&desc_lcal);
01769     cpl_image_delete(combined_data); combined_data = NULL;
01770     cpl_image_delete(combined_noise); combined_noise = NULL;
01771     cpl_image_delete(xcal); xcal = NULL;
01772     cpl_image_delete(ycal); ycal = NULL;
01773     cpl_image_delete(lcal); lcal = NULL;
01774     cpl_image_delete(img_dark); img_dark = NULL;
01775     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01776     cpl_image_delete(img_flat); img_flat = NULL;
01777     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01778     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01779     cpl_free(bounds); bounds = NULL;
01780     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01781     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01782     cpl_free(fn_lut); fn_lut = NULL;
01783     cpl_free(suffix); suffix = NULL;
01784     cpl_free(fn_suffix); fn_suffix = NULL;
01785     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01786     cpl_vector_delete(ranges); ranges = NULL;
01787     cpl_free(filter); filter = NULL;
01788     if (calAngles != NULL) {
01789         cpl_vector_delete(calAngles); calAngles = NULL;
01790     }
01791     cpl_propertylist_delete(main_header); main_header = NULL;
01792     for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01793         if (stored_data_cubes != NULL) {
01794             cpl_imagelist_delete(stored_data_cubes[i]);
01795             stored_data_cubes[i] = NULL;
01796         }
01797         if (stored_noise_cubes != NULL) {
01798             cpl_imagelist_delete(stored_noise_cubes[i]);
01799             stored_noise_cubes[i] = NULL;
01800         }
01801         if (stored_data_images != NULL) {
01802             cpl_image_delete(stored_data_images[i]);
01803             stored_data_images[i] = NULL;
01804         }
01805         if (stored_noise_images != NULL) {
01806             cpl_image_delete(stored_noise_images[i]);
01807             stored_noise_images[i] = NULL;
01808         }
01809         if (stored_sub_data_headers != NULL) {
01810             cpl_propertylist_delete(stored_sub_data_headers[i]);
01811             stored_sub_data_headers[i] = NULL;
01812         }
01813         if (stored_sub_noise_headers != NULL) {
01814             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01815             stored_sub_noise_headers[i] = NULL;
01816         }
01817     }
01818     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01819     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01820     cpl_free(stored_data_images); stored_data_images = NULL;
01821     cpl_free(stored_noise_images); stored_noise_images = NULL;
01822     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01823     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01824     if (edge_table_sky != NULL) {
01825         for (i = 0; i < KMOS_NR_DETECTORS; i++) {
01826             if (edge_table_sky[i] != NULL) {
01827                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01828                     cpl_table_delete(edge_table_sky[i][j]);
01829                     edge_table_sky[i][j] = NULL;
01830                 }
01831                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01832             }
01833         }
01834         cpl_free(edge_table_sky); edge_table_sky = NULL;
01835     }
01836     if (edge_table_flat != NULL) {
01837         for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01838             cpl_table_delete(edge_table_flat[j]);
01839             edge_table_flat[j] = NULL;
01840         }
01841         cpl_free(edge_table_flat); edge_table_flat = NULL;
01842     }
01843 
01844     return ret_val;
01845 }
01846