KMOS Pipeline Reference Manual  1.2.4
kmo_illumination.c
00001 /* $Id: kmo_illumination.c,v 1.58 2013/07/25 09:45:32 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013/07/25 09:45:32 $
00024  * $Revision: 1.58 $
00025  * $Name: kmosp_v1_2_4__20130807 $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmo_priv_reconstruct.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_priv_flat.h"
00044 #include "kmo_priv_wave_cal.h"
00045 #include "kmclipm_priv_splines.h"
00046 #include "kmo_functions.h"
00047 #include "kmo_cpl_extensions.h"
00048 #include "kmo_dfs.h"
00049 #include "kmo_error.h"
00050 #include "kmo_constants.h"
00051 #include "kmo_debug.h"
00052 
00053 /*-----------------------------------------------------------------------------
00054  *                          Functions prototypes
00055  *----------------------------------------------------------------------------*/
00056 
00057 static int kmo_illumination_create(cpl_plugin *);
00058 static int kmo_illumination_exec(cpl_plugin *);
00059 static int kmo_illumination_destroy(cpl_plugin *);
00060 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00061 
00062 /*-----------------------------------------------------------------------------
00063  *                          Static variables
00064  *----------------------------------------------------------------------------*/
00065 
00066 static char kmo_illumination_description[] =
00067 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00068 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00069 "generates the spectral calibration frame needed in this recipe. As input at\n"
00070 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00071 "bration frames are required.\n"
00072 "\n"
00073 "BASIC PARAMETERS:\n"
00074 "-----------------\n"
00075 "--imethod\n"
00076 "The interpolation method used for reconstruction.\n"
00077 "\n"
00078 "--range\n"
00079 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00080 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00081 "\n"
00082 "ADVANCED PARAMETERS\n"
00083 "-------------------\n"
00084 "--flux\n"
00085 "Specify if flux conservation should be applied.\n"
00086 "\n"
00087 "--add-all\n"
00088 "By default the first FLAT_SKY frame is omitted, since in the\n"
00089 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00090 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00091 "should be considered, set this parameter to TRUE.\n"
00092 "\n"
00093 "--neighborhoodRange\n"
00094 "Defines the range to search for neighbors during reconstruction\n"
00095 "\n"
00096 "--b_samples\n"
00097 "The number of samples in spectral direction for the reconstructed cube.\n"
00098 "Ideally this number should be greater than 2048, the detector size.\n"
00099 "\n"
00100 "--b_start\n"
00101 "--b_end\n"
00102 "Used to define manually the start and end wavelength for the reconstructed\n"
00103 "cube. By default the internally defined values are used.\n"
00104 "\n"
00105 "--cmethod\n"
00106 "Following methods of frame combination are available:\n"
00107 "   * 'ksigma' (Default)\n"
00108 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00109 "   are examined. If they deviate significantly, they will be rejected according\n"
00110 "   to the conditions:\n"
00111 "       val > mean + stdev * cpos_rej\n"
00112 "   and\n"
00113 "       val < mean - stdev * cneg_rej\n"
00114 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00115 "   parameters. In the first iteration median and percentile level are used.\n"
00116 "\n"
00117 "   * 'median'\n"
00118 "   At each pixel position the median is calculated.\n"
00119 "\n"
00120 "   * 'average'\n"
00121 "   At each pixel position the average is calculated.\n"
00122 "\n"
00123 "   * 'sum'\n"
00124 "   At each pixel position the sum is calculated.\n"
00125 "\n"
00126 "   * 'min_max'\n"
00127 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00128 "   --cmax and --cmin apply to this method.\n"
00129 "\n"
00130 "--cpos_rej\n"
00131 "--cneg_rej\n"
00132 "--citer\n"
00133 "see --cmethod='ksigma'\n"
00134 "\n"
00135 "--cmax\n"
00136 "--cmin\n"
00137 "see --cmethod='min_max'\n"
00138 "\n"
00139 "--pix_scale\n"
00140 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00141 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00142 "\n"
00143 "--suppress_extension\n"
00144 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00145 "products with the same category are produced, they will be numered consecutively\n"
00146 "starting from 0.\n"
00147 "\n"
00148 "-------------------------------------------------------------------------------\n"
00149 "  Input files:\n"
00150 "\n"
00151 "   DO                    KMOS                                                  \n"
00152 "   category              Type   Explanation                    Required #Frames\n"
00153 "   --------              -----  -----------                    -------- -------\n"
00154 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00155 "                                (at least 3 frames recommended)                \n"
00156 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00157 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00158 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00159 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00160 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00161 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00162 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00163 "\n"
00164 "  Output files:\n"
00165 "\n"
00166 "   DO                    KMOS\n"
00167 "   category              Type   Explanation\n"
00168 "   --------              -----  -----------\n"
00169 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00170 "   If FLAT_EDGE is provided: \n"
00171 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00172 "                                slitlets of all IFUs of all detectors\n"
00173 "-------------------------------------------------------------------------------\n"
00174 "\n";
00175 
00176 /*-----------------------------------------------------------------------------
00177  *                              Functions code
00178  *----------------------------------------------------------------------------*/
00179 
00196 int cpl_plugin_get_info(cpl_pluginlist *list)
00197 {
00198     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00199     cpl_plugin *plugin = &recipe->interface;
00200 
00201     cpl_plugin_init(plugin,
00202                         CPL_PLUGIN_API,
00203                         KMOS_BINARY_VERSION,
00204                         CPL_PLUGIN_TYPE_RECIPE,
00205                         "kmo_illumination",
00206                         "Create a calibration frame to correct spatial "
00207                         "non-uniformity of flatfield.",
00208                         kmo_illumination_description,
00209                         "Alex Agudo Berbel",
00210                         "kmos-spark@mpe.mpg.de",
00211                         kmos_get_license(),
00212                         kmo_illumination_create,
00213                         kmo_illumination_exec,
00214                         kmo_illumination_destroy);
00215 
00216     cpl_pluginlist_append(list, plugin);
00217 
00218     return 0;
00219 }
00220 
00228 static int kmo_illumination_create(cpl_plugin *plugin)
00229 {
00230     cpl_recipe *recipe;
00231     cpl_parameter *p;
00232 
00233     /* Check that the plugin is part of a valid recipe */
00234     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00235         recipe = (cpl_recipe *)plugin;
00236     else
00237         return -1;
00238 
00239     /* Create the parameters list in the cpl_recipe object */
00240     recipe->parameters = cpl_parameterlist_new();
00241 
00242     /* Fill the parameters list */
00243     /* --imethod */
00244     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00245                                 CPL_TYPE_STRING,
00246                                 "Method to use for interpolation: "
00247                                 "[\"NN\" (nearest neighbour), "
00248                                 "\"lwNN\" (linear weighted nearest neighbor),"
00249                                 "\"swNN\" (square weighted nearest neighbor)"
00250                                 "\"MS\" (Modified Shepard's method)"
00251                                 "\"CS\" (Cubic spline)]",
00252                                 "kmos.kmo_illumination",
00253                                 "CS");
00254     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00255     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00256     cpl_parameterlist_append(recipe->parameters, p);
00257 
00258     /* --neighborhoodRange */
00259     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00260                                 CPL_TYPE_DOUBLE,
00261                                 "Defines the range to search for neighbors."
00262                                 "in pixels",
00263                                 "kmos.kmo_illumination",
00264                                 1.001);
00265     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00266     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00267     cpl_parameterlist_append(recipe->parameters, p);
00268 
00269     /* --range */
00270     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00271                                 CPL_TYPE_STRING,
00272                                 "The spectral ranges to combine when collapsing"
00273                                 "the reconstructed cubes. e.g."
00274                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00275                                 "kmos.kmo_illumination",
00276                                 "");
00277     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00278     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00279     cpl_parameterlist_append(recipe->parameters, p);
00280 
00281     /* --flux */
00282     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00283                                 CPL_TYPE_BOOL,
00284                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00285                                 "kmos.kmo_illumination",
00286                                 FALSE);
00287     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00288     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00289     cpl_parameterlist_append(recipe->parameters, p);
00290 
00291     /* --add-all */
00292     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00293                                 CPL_TYPE_BOOL,
00294                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00295                                 "TRUE: don't perform any checks, add them all",
00296                                 "kmos.kmo_illumination",
00297                                 FALSE);
00298     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00299     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00300     cpl_parameterlist_append(recipe->parameters, p);
00301 
00302     /* --pix_scale */
00303     p = cpl_parameter_new_value("kmos.kmo_illumination.pix_scale",
00304                                 CPL_TYPE_DOUBLE,
00305                                 "Change the pixel scale [arcsec]. "
00306                                 "Default of 0.2\" results into cubes of 14x14pix, "
00307                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00308                                 "etc.",
00309                                 "kmos.kmo_illumination",
00310                                 KMOS_PIX_RESOLUTION);
00311     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00312     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00313     cpl_parameterlist_append(recipe->parameters, p);
00314 
00315     /* --suppress_extension */
00316     p = cpl_parameter_new_value("kmos.kmo_illumination.suppress_extension",
00317                                 CPL_TYPE_BOOL,
00318                                 "Suppress arbitrary filename extension."
00319                                 "(TRUE (apply) or FALSE (don't apply)",
00320                                 "kmos.kmo_illumination",
00321                                 FALSE);
00322     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00323     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00324     cpl_parameterlist_append(recipe->parameters, p);
00325 
00326     // add parameters for band-definition
00327     kmo_band_pars_create(recipe->parameters,
00328                          "kmos.kmo_illumination");
00329 
00330     // add parameters for combining
00331     return kmo_combine_pars_create(recipe->parameters,
00332                                    "kmos.kmo_illumination",
00333                                    DEF_REJ_METHOD,
00334                                    FALSE);
00335 }
00336 
00342 static int kmo_illumination_exec(cpl_plugin *plugin)
00343 {
00344     cpl_recipe  *recipe;
00345 
00346     /* Get the recipe out of the plugin */
00347     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00348         recipe = (cpl_recipe *)plugin;
00349     else return -1;
00350 
00351     return kmo_illumination(recipe->parameters, recipe->frames);
00352 }
00353 
00359 static int kmo_illumination_destroy(cpl_plugin *plugin)
00360 {
00361     cpl_recipe *recipe;
00362 
00363     /* Get the recipe out of the plugin */
00364     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00365         recipe = (cpl_recipe *)plugin;
00366     else return -1 ;
00367 
00368     cpl_parameterlist_delete(recipe->parameters);
00369     return 0 ;
00370 }
00371 
00386 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00387 {
00388     int              ret_val                    = 0,
00389                      nr_devices                 = 0,
00390                      ifu_nr                     = 0,
00391                      nx                         = 0,
00392                      ny                         = 0,
00393                      process_noise              = FALSE,
00394                      cmax                       = 0,
00395                      cmin                       = 0,
00396                      citer                      = 0,
00397                      *bounds                    = NULL,
00398                      cnt                        = 0,
00399                      qc_max_dev_id              = 0,
00400                      qc_max_nonunif_id          = 0,
00401                      flux                       = FALSE,
00402                      background                 = FALSE,
00403                      add_all_sky                = FALSE,
00404                      same_exptime               = TRUE,
00405                      suppress_extension         = FALSE,
00406                      has_flat_edge              = FALSE,
00407                      i = 0, j = 0, x = 0, y = 0, ix = 0, iy = 0, det_nr = 0, edgeNr = 0;
00408     const int        *punused_ifus              = NULL;
00409     float            *pbad_pix_mask             = NULL;
00410     double           exptime                    = 0.,
00411                      exptime1                   = 0.,
00412                      exptime2                   = 0.,
00413                      cpos_rej                   = 0.0,
00414                      cneg_rej                   = 0.0,
00415                      neighborhoodRange          = 1.001,
00416                      mean_data                  = 0.0,
00417                      ifu_crpix                  = 0.0,
00418                      ifu_crval                  = 0.0,
00419                      ifu_cdelt                  = 0.0,
00420                      qc_spat_unif               = 0.0,
00421                      qc_max_dev                 = 0.0,
00422                      qc_max_nonunif             = 0.0,
00423                      tmp_stdev                  = 0.0,
00424                      tmp_mean                   = 0.0,
00425                      rotangle                   = 0.0,
00426                      tmp_rotangle               = 0.0,
00427                      rotangle_found             = 0.0,
00428                      pix_scale                  = 0.0;
00429     char             *keyword                   = NULL,
00430                      *fn_lut                    = NULL,
00431                      *suffix                    = NULL,
00432                      *fn_suffix                 = NULL,
00433                      *extname                   = NULL,
00434                      *filter                    = NULL;
00435     const char       *method                    = NULL,
00436                      *cmethod                   = NULL,
00437                      *filter_id_l               = NULL,
00438                      *filter_id                 = NULL,
00439                      *ranges_txt                = NULL;
00440     cpl_array        *calTimestamp              = NULL,
00441                      **unused_ifus_before       = NULL,
00442                      **unused_ifus_after        = NULL;
00443     cpl_frame        *frame                     = NULL,
00444                      *xcalFrame                 = NULL,
00445                      *ycalFrame                 = NULL,
00446                      *lcalFrame                 = NULL;
00447     cpl_frameset     *frameset_sky              = NULL;
00448     cpl_image        *img_in                    = NULL,
00449                      *img_dark                  = NULL,
00450                      *img_dark_noise            = NULL,
00451                      *img_flat                  = NULL,
00452                      *img_flat_noise            = NULL,
00453                      *combined_data             = NULL,
00454                      *combined_noise            = NULL,
00455                      *xcal                      = NULL,
00456                      *ycal                      = NULL,
00457                      *lcal                      = NULL,
00458                      *bad_pix_mask              = NULL,
00459                      *data_ifu                  = NULL,
00460                      *noise_ifu                 = NULL,
00461                      **stored_data_images       = NULL,
00462                      **stored_noise_images      = NULL;
00463     cpl_imagelist    *cube_data                 = NULL,
00464                      *cube_noise                = NULL,
00465                      *detector_in               = NULL,
00466                      **stored_data_cubes        = NULL,
00467                      **stored_noise_cubes       = NULL;
00468     cpl_matrix       **edgepars                 = NULL;
00469     cpl_propertylist *main_header               = NULL,
00470                      *tmp_header                = NULL,
00471                      *sub_header                = NULL,
00472                      **stored_sub_data_headers  = NULL,
00473                      **stored_sub_noise_headers = NULL;
00474     cpl_table        *band_table                = NULL,
00475                      ***edge_table_sky          = NULL,
00476                      **edge_table_flat          = NULL;
00477     cpl_vector       *ranges                    = NULL,
00478                      *identified_slices         = NULL,
00479                      *calAngles                 = NULL,
00480                      **slitlet_ids              = NULL,
00481                      *shift_vec                 = NULL,
00482                      *edge_vec                  = NULL;
00483     main_fits_desc   desc_sky,
00484                      desc_dark,
00485                      desc_flat,
00486                      desc_xcal,
00487                      desc_ycal,
00488                      desc_lcal;
00489     gridDefinition   gd;
00490 
00491     KMO_TRY
00492     {
00493         kmo_init_fits_desc(&desc_sky);
00494         kmo_init_fits_desc(&desc_dark);
00495         kmo_init_fits_desc(&desc_flat);
00496         kmo_init_fits_desc(&desc_xcal);
00497         kmo_init_fits_desc(&desc_ycal);
00498         kmo_init_fits_desc(&desc_lcal);
00499 
00500         /* --- check input --- */
00501         KMO_TRY_ASSURE((parlist != NULL) &&
00502                        (frameset != NULL),
00503                        CPL_ERROR_NULL_INPUT,
00504                        "Not all input data is provided!");
00505 
00506         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00507                        CPL_ERROR_ILLEGAL_INPUT,
00508                        "One or more FLAT_SKY frames are required!");
00509 
00510         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00511             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00512                                       "3 FLAT_SKY frames!");
00513         }
00514 
00515         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00516                        CPL_ERROR_ILLEGAL_INPUT,
00517                        "Exactly one MASTER_DARK frame is required!");
00518 
00519         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00520                        CPL_ERROR_ILLEGAL_INPUT,
00521                        "Exactly one MASTER_FLAT frame is required!");
00522 
00523         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00524                        CPL_ERROR_ILLEGAL_INPUT,
00525                        "Exactly one XCAL frame is required!");
00526 
00527         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00528                        CPL_ERROR_ILLEGAL_INPUT,
00529                        "Exactly one YCAL frame is required!");
00530 
00531         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00532                        CPL_ERROR_ILLEGAL_INPUT,
00533                        "Exactly one LCAL frame is required!");
00534 
00535         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00536                        CPL_ERROR_ILLEGAL_INPUT,
00537                        "Exactly one WAVE_BAND frame is required!");
00538 
00539         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00540                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00541                        CPL_ERROR_ILLEGAL_INPUT,
00542                        "Exactly one FLAT_EDGE frame is required!");
00543 
00544         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00545 
00546         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00547                        CPL_ERROR_ILLEGAL_INPUT,
00548                        "Cannot identify RAW and CALIB frames!");
00549 
00550         /* --- get parameters --- */
00551         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00552 
00553         KMO_TRY_EXIT_IF_NULL(
00554             method = kmo_dfs_get_parameter_string(parlist,
00555                                               "kmos.kmo_illumination.imethod"));
00556 
00557         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00558                        (strcmp(method, "lwNN") == 0) ||
00559                        (strcmp(method, "swNN") == 0) ||
00560                        (strcmp(method, "MS") == 0) ||
00561                        (strcmp(method, "CS") == 0),
00562                        CPL_ERROR_ILLEGAL_INPUT,
00563                        "method must be either \"NN\", \"lwNN\", "
00564                        "\"swNN\", \"MS\" or \"CS\"!");
00565 
00566         KMO_TRY_EXIT_IF_ERROR(
00567             kmo_dfs_print_parameter_help(parlist,
00568                                         "kmos.kmo_illumination.imethod"));
00569 
00570         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00571                 "kmos.kmo_illumination.neighborhoodRange");
00572         KMO_TRY_CHECK_ERROR_STATE();
00573 
00574         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00575                 CPL_ERROR_ILLEGAL_INPUT,
00576                 "neighborhoodRange must be greater than 0.0");
00577 
00578         KMO_TRY_EXIT_IF_ERROR(
00579             kmo_dfs_print_parameter_help(parlist,
00580                                     "kmos.kmo_illumination.neighborhoodRange"));
00581 
00582         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00583                                                   "kmos.kmo_illumination.range");
00584         KMO_TRY_CHECK_ERROR_STATE();
00585 
00586         KMO_TRY_EXIT_IF_ERROR(
00587             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00588 
00589         ranges = kmo_identify_ranges(ranges_txt);
00590         KMO_TRY_CHECK_ERROR_STATE();
00591 
00592         flux = kmo_dfs_get_parameter_bool(parlist,
00593                                           "kmos.kmo_illumination.flux");
00594 
00595         KMO_TRY_ASSURE((flux == 0) ||
00596                        (flux == 1),
00597                        CPL_ERROR_ILLEGAL_INPUT,
00598                        "flux must be either FALSE or TRUE!");
00599 
00600         KMO_TRY_EXIT_IF_ERROR(
00601             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00602 
00603         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00604                                                  "kmos.kmo_illumination.add-all");
00605 
00606         KMO_TRY_ASSURE((add_all_sky == 0) ||
00607                        (add_all_sky == 1),
00608                        CPL_ERROR_ILLEGAL_INPUT,
00609                        "add_all must be either FALSE or TRUE!");
00610 
00611         KMO_TRY_EXIT_IF_ERROR(
00612             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00613 
00614         pix_scale = kmo_dfs_get_parameter_double(parlist,
00615                                         "kmos.kmo_illumination.pix_scale");
00616         KMO_TRY_CHECK_ERROR_STATE();
00617         KMO_TRY_EXIT_IF_ERROR(
00618            kmo_dfs_print_parameter_help(parlist,
00619                                        "kmos.kmo_illumination.pix_scale"));
00620         KMO_TRY_ASSURE((pix_scale >= 0.01) &&
00621                        (pix_scale <= 0.4),
00622                        CPL_ERROR_ILLEGAL_INPUT,
00623                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00624                        "with 7x7 to 280x280 pixels)!");
00625 
00626         suppress_extension = kmo_dfs_get_parameter_bool(parlist,
00627                                           "kmos.kmo_illumination.suppress_extension");
00628         KMO_TRY_CHECK_ERROR_STATE();
00629         KMO_TRY_EXIT_IF_ERROR(
00630             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.suppress_extension"));
00631 
00632         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00633                        CPL_ERROR_ILLEGAL_INPUT,
00634                        "suppress_extension must be TRUE or FALSE!");
00635 
00636         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00637 
00638         KMO_TRY_EXIT_IF_ERROR(
00639             kmo_combine_pars_load(parlist,
00640                                   "kmos.kmo_illumination",
00641                                   &cmethod,
00642                                   &cpos_rej,
00643                                   &cneg_rej,
00644                                   &citer,
00645                                   &cmin,
00646                                   &cmax,
00647                                   FALSE));
00648         cpl_msg_info("", "-------------------------------------------");
00649 
00650         // check if filter_id, grating_id and rotator offset match for all
00651         // detectors
00652         KMO_TRY_EXIT_IF_ERROR(
00653             kmo_check_frameset_setup(frameset, FLAT_SKY,
00654                                        TRUE, FALSE, TRUE));
00655         KMO_TRY_EXIT_IF_ERROR(
00656             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00657                                        TRUE, FALSE, TRUE));
00658         KMO_TRY_EXIT_IF_ERROR(
00659             kmo_check_frame_setup(frameset, XCAL, YCAL,
00660                                        TRUE, FALSE, TRUE));
00661         KMO_TRY_EXIT_IF_ERROR(
00662             kmo_check_frame_setup(frameset, XCAL, LCAL,
00663                                        TRUE, FALSE, TRUE));
00664         KMO_TRY_EXIT_IF_ERROR(
00665             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00666                                        TRUE, FALSE, TRUE));
00667 
00668         KMO_TRY_EXIT_IF_NULL(
00669             frame = kmo_dfs_get_frame(frameset, XCAL));
00670         KMO_TRY_EXIT_IF_NULL(
00671             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00672 
00673         KMO_TRY_EXIT_IF_ERROR(
00674             kmo_check_frame_setup_md5_xycal(frameset));
00675         KMO_TRY_EXIT_IF_ERROR(
00676             kmo_check_frame_setup_md5(frameset));
00677 
00678         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00679         cpl_msg_info("", "(grating 1, 2 & 3)");
00680 
00681         // check which IFUs are active for all frames
00682         KMO_TRY_EXIT_IF_NULL(
00683             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00684 
00685         KMO_TRY_EXIT_IF_NULL(
00686             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00687 
00688         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00689 
00690         // load desc for MASTER_DARK and check
00691         KMO_TRY_EXIT_IF_NULL(
00692             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00693         desc_dark = kmo_identify_fits_header(
00694                     cpl_frame_get_filename(frame));
00695         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00696                                       "be in KMOS-format!");
00697         KMO_TRY_ASSURE((desc_dark.nr_ext == 2*KMOS_NR_DETECTORS) &&
00698                        (desc_dark.ex_badpix == FALSE) &&
00699                        (desc_dark.fits_type == f2d_fits) &&
00700                        (desc_dark.frame_type == detector_frame),
00701                        CPL_ERROR_ILLEGAL_INPUT,
00702                        "MASTER_DARK isn't in the correct format!!!");
00703         nx = desc_dark.naxis1;
00704         ny = desc_dark.naxis2;
00705 
00706         // load desc for MASTER_FLAT and check
00707         KMO_TRY_EXIT_IF_NULL(
00708             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00709         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00710         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00711                                       "be in KMOS-format!");
00712         KMO_TRY_ASSURE((desc_flat.nr_ext % (2*KMOS_NR_DETECTORS) == 0) &&
00713                        (desc_flat.ex_badpix == FALSE) &&
00714                        (desc_flat.fits_type == f2d_fits) &&
00715                        (desc_flat.frame_type == detector_frame),
00716                        CPL_ERROR_ILLEGAL_INPUT,
00717                        "MASTER_FLAT isn't in the correct format!!!");
00718 
00719         // load desc for XCAL and check
00720         KMO_TRY_EXIT_IF_NULL(
00721             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00722         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00723         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00724                                       "be in KMOS-format!");
00725         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00726                        (desc_xcal.ex_badpix == FALSE) &&
00727                        (desc_xcal.fits_type == f2d_fits) &&
00728                        (desc_xcal.frame_type == detector_frame),
00729                        CPL_ERROR_ILLEGAL_INPUT,
00730                        "XCAL isn't in the correct format!!!");
00731         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00732                        (desc_xcal.naxis2 == ny),
00733                        CPL_ERROR_ILLEGAL_INPUT,
00734                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00735                        "(x,y): (%d,%d) vs (%d,%d)",
00736                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00737 
00738         nr_devices = desc_xcal.nr_ext;
00739 
00740         // load desc for YCAL and check
00741         KMO_TRY_EXIT_IF_NULL(
00742             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00743         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00744         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00745                                       "be in KMOS-format!");
00746         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00747                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00748                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00749                        (desc_ycal.frame_type == desc_xcal.frame_type),
00750                        CPL_ERROR_ILLEGAL_INPUT,
00751                        "YCAL isn't in the correct format!!!");
00752         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00753                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00754                        CPL_ERROR_ILLEGAL_INPUT,
00755                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00756                        "(x,y): (%d,%d) vs (%d,%d)",
00757                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00758 
00759         // load desc for LCAL and check
00760         KMO_TRY_EXIT_IF_NULL(
00761             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00762         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00763         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00764                                       "be in KMOS-format!");
00765         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00766                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00767                        (desc_lcal.frame_type == desc_xcal.frame_type),
00768                        CPL_ERROR_ILLEGAL_INPUT,
00769                        "LCAL isn't in the correct format!!!");
00770         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00771                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00772                        CPL_ERROR_ILLEGAL_INPUT,
00773                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00774                        "(x,y): (%d,%d) vs (%d,%d)",
00775                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00776         KMO_TRY_EXIT_IF_NULL(
00777             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00778 
00779         // load desc for FLAT_SKY and check
00780         nr_devices = KMOS_NR_DETECTORS;
00781         KMO_TRY_EXIT_IF_NULL(
00782             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00783 
00784         KMO_TRY_EXIT_IF_NULL(
00785             main_header = kmclipm_propertylist_load(
00786                                          cpl_frame_get_filename(frame), 0));
00787         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00788         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00789         kmclipm_strip_angle(&rotangle);
00790         cpl_propertylist_delete(main_header); main_header = NULL;
00791 
00792         cnt = 1;
00793         while (frame != NULL) {
00794             KMO_TRY_EXIT_IF_NULL(
00795                 main_header = kmclipm_propertylist_load(
00796                                              cpl_frame_get_filename(frame), 0));
00797 
00798             desc_sky = kmo_identify_fits_header(
00799                         cpl_frame_get_filename(frame));
00800             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00801                                           "be in KMOS-format!");
00802             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00803                            (desc_sky.ex_badpix == FALSE) &&
00804                            (desc_sky.fits_type == raw_fits) &&
00805                            (desc_sky.frame_type == detector_frame),
00806                            CPL_ERROR_ILLEGAL_INPUT,
00807                            "FLAT_SKY isn't in the correct format!!!");
00808             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00809                            (desc_sky.naxis2 == ny) &&
00810                            (desc_sky.nr_ext == nr_devices),
00811                            CPL_ERROR_ILLEGAL_INPUT,
00812                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00813                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00814                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00815             kmo_free_fits_desc(&desc_sky);
00816             kmo_init_fits_desc(&desc_sky);
00817 
00818             KMO_TRY_ASSURE(
00819                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00820                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00821                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00822                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00823                 CPL_ERROR_ILLEGAL_INPUT,
00824                 "All lamps must be switched off for the FLAT_SKY frames!");
00825 
00826             // assert that filters have correct IDs and that all detectors of
00827             // all input frames have the same filter set
00828             for (i = 1; i <= KMOS_NR_DETECTORS; i++) {
00829                 // ESO INS FILTi ID
00830                 KMO_TRY_EXIT_IF_NULL(
00831                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00832                 KMO_TRY_EXIT_IF_NULL(
00833                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00834 
00835                 KMO_TRY_EXIT_IF_NULL(
00836                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00837                 cpl_free(keyword); keyword = NULL;
00838 
00839                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00840                                (strcmp(filter_id, "YJ") == 0) ||
00841                                (strcmp(filter_id, "H") == 0) ||
00842                                (strcmp(filter_id, "K") == 0) ||
00843                                (strcmp(filter_id, "HK") == 0),
00844                                CPL_ERROR_ILLEGAL_INPUT,
00845                                "Filter ID in primary header must be either 'IZ', "
00846                                "'YJ', 'H', 'K' or " "'HK' !");
00847 
00848                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00849                                CPL_ERROR_ILLEGAL_INPUT,
00850                                "Filter IDs must be the same for FLAT_SKY frame"
00851                                " and lcal frame!"
00852                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00853                                i, cpl_frame_get_filename(frame),
00854                                filter_id, filter_id_l);
00855 
00856                 // ESO INS GRATi ID
00857                 KMO_TRY_EXIT_IF_NULL(
00858                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00859                 KMO_TRY_EXIT_IF_NULL(
00860                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00861 
00862                 KMO_TRY_EXIT_IF_NULL(
00863                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00864                 cpl_free(keyword); keyword = NULL;
00865 
00866                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00867                                (strcmp(filter_id, "YJ") == 0) ||
00868                                (strcmp(filter_id, "H") == 0) ||
00869                                (strcmp(filter_id, "K") == 0) ||
00870                                (strcmp(filter_id, "HK") == 0),
00871                                CPL_ERROR_ILLEGAL_INPUT,
00872                                "Grating ID in primary header must be either "
00873                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00874 
00875                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00876                                CPL_ERROR_ILLEGAL_INPUT,
00877                                "Grating IDs must be the same for FLAT_SKY frame"
00878                                " and lcal frame!"
00879                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00880                                i, cpl_frame_get_filename(frame),
00881                                filter_id, filter_id_l);
00882 
00883                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00884                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00885                 kmclipm_strip_angle(&tmp_rotangle);
00886                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00887                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00888                         CPL_ERROR_ILLEGAL_INPUT,
00889                         "OCS ROT NAANGLE of sky flat frames differ to much: %f %f",
00890                         rotangle, tmp_rotangle);
00891             }
00892             cpl_propertylist_delete(main_header); main_header = NULL;
00893 
00894             // get next FLAT_SKY frame
00895             frame = kmo_dfs_get_frame(frameset, NULL);
00896             KMO_TRY_CHECK_ERROR_STATE();
00897             cnt++;
00898         }
00899 
00900         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00901 
00902         //
00903         // noise will be propagated when:
00904         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00905         // 2 FLAT_SKY frames are provided.
00906         // Otherwise noise will be ignored.
00907         //
00908         if (desc_dark.ex_noise &&
00909             desc_flat.ex_noise &&
00910             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00911             process_noise = TRUE;
00912         }
00913 
00914         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00915             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00916                             "since there is only one input frame! (The output "
00917                             "file won't have any noise extensions)");
00918 
00919             cmethod = "average";
00920         }
00921 
00922         //
00923         // Check whether 1st FLAT_SKY should be omitted
00924         //
00925         KMO_TRY_EXIT_IF_NULL(
00926             frameset_sky = cpl_frameset_new());
00927 
00928         if (add_all_sky) {
00929             // just add all FLAT_SKY frames without check
00930             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00931             while (frame != NULL) {
00932                 KMO_TRY_EXIT_IF_ERROR(
00933                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00934                 frame = kmo_dfs_get_frame(frameset, NULL);
00935             }
00936             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00937         } else {
00938             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00939             KMO_TRY_EXIT_IF_NULL(
00940                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00941 
00942             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00943                 // just one FLAT_SKY, always add
00944                 KMO_TRY_EXIT_IF_ERROR(
00945                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00946                 KMO_TRY_CHECK_ERROR_STATE();
00947             } else {
00948                 // several FLAT_SKY frames, check exptime
00949 
00950                 // get exptime 1
00951                 KMO_TRY_EXIT_IF_NULL(
00952                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00953                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00954                 KMO_TRY_CHECK_ERROR_STATE();
00955                 cpl_propertylist_delete(main_header); main_header = NULL;
00956 
00957                 // get exptime 2
00958                 frame = kmo_dfs_get_frame(frameset, NULL);
00959                 KMO_TRY_EXIT_IF_NULL(
00960                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00961                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00962                 KMO_TRY_CHECK_ERROR_STATE();
00963                 cpl_propertylist_delete(main_header); main_header = NULL;
00964 
00965                 // loop remaining frames
00966                 same_exptime = TRUE;
00967                 frame = kmo_dfs_get_frame(frameset, NULL);
00968                 while (same_exptime && (frame != NULL)) {
00969                     KMO_TRY_EXIT_IF_NULL(
00970                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00971                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00972                     KMO_TRY_CHECK_ERROR_STATE();
00973                     cpl_propertylist_delete(main_header); main_header = NULL;
00974                     if (fabs(exptime-exptime2) > 0.01) {
00975                         // not same
00976                         same_exptime = FALSE;
00977                     }
00978                     frame = kmo_dfs_get_frame(frameset, NULL);
00979                 }
00980 
00981                 if (same_exptime) {
00982                     // frame [2,n] have same exptime, add them
00983                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00984                     KMO_TRY_EXIT_IF_NULL(
00985                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00986                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00987                     KMO_TRY_CHECK_ERROR_STATE();
00988                     cpl_propertylist_delete(main_header); main_header = NULL;
00989                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00990                     frame = kmo_dfs_get_frame(frameset, NULL);
00991                     while (frame != NULL) {
00992                         KMO_TRY_EXIT_IF_ERROR(
00993                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00994                         frame = kmo_dfs_get_frame(frameset, NULL);
00995                     }
00996                     if (fabs(exptime1-exptime2) < 0.01) {
00997                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
00998                                             "It has anyway been omitted since we assume it is an acquisition frame. "
00999                                             "If you want to add it anyway call this recipe with the --add-all parameter");
01000                     }
01001                 } else {
01002                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
01003                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
01004                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
01005                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
01006                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
01007                     while (frame != NULL) {
01008                         KMO_TRY_EXIT_IF_NULL(
01009                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
01010                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
01011                         KMO_TRY_CHECK_ERROR_STATE();
01012                         cpl_propertylist_delete(main_header); main_header = NULL;
01013                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
01014                         frame = kmo_dfs_get_frame(frameset, NULL);
01015                     }
01016                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
01017                     KMO_TRY_CHECK_ERROR_STATE();
01018                 }
01019             }
01020         }
01021 
01022         KMO_TRY_EXIT_IF_NULL(
01023             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
01024         KMO_TRY_EXIT_IF_NULL(
01025             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
01026         KMO_TRY_EXIT_IF_NULL(
01027             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
01028         KMO_TRY_EXIT_IF_NULL(
01029             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
01030         cpl_free(keyword); keyword = NULL;
01031 
01032         //
01033         // set default band-specific ranges for collapsing
01034         //
01035         if (ranges == NULL) {
01036             if (strcmp(filter, "IZ") == 0) {
01037                 ranges_txt = "0.81,1.05";
01038             } else if (strcmp(filter, "YJ") == 0) {
01039                 ranges_txt = "1.025,1.3";
01040             } else if (strcmp(filter, "H") == 0) {
01041                 ranges_txt = "1.5,1.7";
01042             } else if (strcmp(filter, "K") == 0) {
01043                 ranges_txt = "2.1,2.35";
01044             } else if (strcmp(filter, "HK") == 0) {
01045                 ranges_txt = "1.5,1.7;2.1,2.35";
01046 //                ranges_txt = "1.5,1.7";
01047             } else {
01048                 KMO_TRY_ASSURE(1 == 0,
01049                                CPL_ERROR_ILLEGAL_INPUT,
01050                                "We really shouldn't get here...");
01051             }
01052             cpl_msg_info("", "Spectral range to collapse has been set to %s um for this band.", ranges_txt);
01053             ranges = kmo_identify_ranges(ranges_txt);
01054             KMO_TRY_CHECK_ERROR_STATE();
01055         }
01056 
01057         // setup grid definition, wavelength start and end points will be set
01058         // in the detector loop
01059         KMO_TRY_EXIT_IF_ERROR(
01060             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale));
01061 
01062         // create filename for LUT
01063         KMO_TRY_EXIT_IF_NULL(
01064             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01065 
01066         // extract bounds
01067         KMO_TRY_EXIT_IF_NULL(
01068             tmp_header = kmo_dfs_load_primary_header(frameset, XCAL));
01069         KMO_TRY_EXIT_IF_NULL(
01070             bounds = kmclipm_extract_bounds(tmp_header));
01071         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01072 
01073         // get timestamps of xcal, ycal & lcal
01074         KMO_TRY_EXIT_IF_NULL(
01075             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01076 
01077         // create arrays to hold reconstructed data and noise cubes and
01078         // their headers
01079         KMO_TRY_EXIT_IF_NULL(
01080             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01081                                                             sizeof(cpl_imagelist*)));
01082         KMO_TRY_EXIT_IF_NULL(
01083             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01084                                                              sizeof(cpl_imagelist*)));
01085         KMO_TRY_EXIT_IF_NULL(
01086             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01087                                                          sizeof(cpl_image*)));
01088         KMO_TRY_EXIT_IF_NULL(
01089             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01090                                                           sizeof(cpl_image*)));
01091         KMO_TRY_EXIT_IF_NULL(
01092             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01093                                                                      sizeof(cpl_propertylist*)));
01094         KMO_TRY_EXIT_IF_NULL(
01095             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01096                                                                       sizeof(cpl_propertylist*)));
01097         KMO_TRY_EXIT_IF_NULL(
01098             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01099                                                       sizeof(cpl_table**)));
01100         KMO_TRY_EXIT_IF_NULL(
01101             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01102                                                       sizeof(cpl_table*)));
01103         KMO_TRY_EXIT_IF_NULL(
01104             calAngles = cpl_vector_new(3));
01105 
01106         //
01107         // loop through all detectors
01108         //
01109         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01110             cpl_msg_info("","Processing detector No. %d", det_nr);
01111 
01112             KMO_TRY_EXIT_IF_NULL(
01113                 detector_in = cpl_imagelist_new());
01114 
01115             // load data of det_nr of all FLAT_SKY frames into an imagelist
01116             KMO_TRY_EXIT_IF_NULL(
01117                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01118 
01119             cnt = 0;
01120             while (img_in != NULL) {
01121                 cpl_imagelist_set(detector_in, img_in, cnt);
01122                 KMO_TRY_CHECK_ERROR_STATE();
01123 
01124                 /* load same extension of next FLAT_SKY frame*/
01125                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01126                 KMO_TRY_CHECK_ERROR_STATE();
01127 
01128                 cnt++;
01129             }
01130 
01131             //
01132             // process imagelist
01133             //
01134 
01135             // combine imagelist (data only) and create noise (stdev of data)
01136             cpl_msg_info("","Combining frames...");
01137             if (process_noise) {
01138                 KMO_TRY_EXIT_IF_ERROR(
01139                     kmclipm_combine_frames(detector_in,
01140                                            NULL,
01141                                            NULL,
01142                                            cmethod,
01143                                            cpos_rej,
01144                                            cneg_rej,
01145                                            citer,
01146                                            cmax,
01147                                            cmin,
01148                                            &combined_data,
01149                                            &combined_noise,
01150                                            -1.0));
01151             } else {
01152                 KMO_TRY_EXIT_IF_ERROR(
01153                     kmclipm_combine_frames(detector_in,
01154                                            NULL,
01155                                            NULL,
01156                                            cmethod,
01157                                            cpos_rej,
01158                                            cneg_rej,
01159                                            citer,
01160                                            cmax,
01161                                            cmin,
01162                                            &combined_data,
01163                                            NULL,
01164                                            -1.0));
01165             }
01166 
01167             if (kmclipm_omit_warning_one_slice > 10) {
01168 // AA: commmented this out: Too unclear for the user, no benefit to know about this number
01169 //                cpl_msg_warning(cpl_func, "Previous warning (number of "
01170 //                                          "identified slices) occured %d times.",
01171 //                                kmclipm_omit_warning_one_slice);
01172                 kmclipm_omit_warning_one_slice = FALSE;
01173             }
01174 
01175             cpl_imagelist_delete(detector_in); detector_in = NULL;
01176 
01177             // load calibration files
01178             KMO_TRY_EXIT_IF_NULL(
01179                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01180                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01181 
01182             KMO_TRY_EXIT_IF_ERROR(
01183                 cpl_vector_set(calAngles, 0, rotangle_found));
01184             KMO_TRY_EXIT_IF_NULL(
01185                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01186                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01187             KMO_TRY_EXIT_IF_ERROR(
01188                 cpl_vector_set(calAngles, 1, rotangle_found));
01189             KMO_TRY_EXIT_IF_NULL(
01190                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01191                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01192             KMO_TRY_EXIT_IF_ERROR(
01193                 cpl_vector_set(calAngles, 2, rotangle_found));
01194 
01195             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01196             KMO_TRY_EXIT_IF_NULL(
01197                 bad_pix_mask = cpl_image_duplicate(xcal));
01198 
01199             KMO_TRY_EXIT_IF_NULL(
01200                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01201             for (x = 0; x < nx; x++) {
01202                 for (y = 0; y < ny; y++) {
01203                     if (isnan(pbad_pix_mask[x+nx*y])) {
01204                         pbad_pix_mask[x+nx*y] = 0.;
01205                     } else {
01206                         pbad_pix_mask[x+nx*y] = 1.;
01207                     }
01208                 }
01209             }
01210             KMO_TRY_CHECK_ERROR_STATE();
01211 
01212             //
01213             // calculate SKYFLAT_EDGE
01214             //
01215             if (has_flat_edge) {
01216                 // get edge-edgepars from FLAT_SKY
01217                 KMO_TRY_EXIT_IF_ERROR(
01218                     kmo_calc_edgepars(combined_data,
01219                                       unused_ifus_after[det_nr-1],
01220                                       bad_pix_mask,
01221                                       det_nr,
01222                                       &slitlet_ids,
01223                                       &edgepars));
01224                 KMO_TRY_CHECK_ERROR_STATE();
01225 
01226                 // copy edgepars to table for saving later on
01227                 KMO_TRY_EXIT_IF_NULL(
01228                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01229 
01230                 if (edgepars != NULL) {
01231                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01232                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01233                     }
01234                     cpl_free(edgepars); edgepars = NULL;
01235                 }
01236                 if (slitlet_ids != NULL) {
01237                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01238                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01239                     }
01240                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01241                 }
01242                 KMO_TRY_CHECK_ERROR_STATE();
01243 
01244                 //
01245                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01246                 //
01247 
01248                 // load flat_edge from MASTER_FLAT
01249                 KMO_TRY_EXIT_IF_NULL(
01250                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01251                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01252                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01253 
01254                     KMO_TRY_EXIT_IF_NULL(
01255                         punused_ifus = cpl_array_get_data_int_const(unused_ifus_after[det_nr-1]));
01256                     if (punused_ifus[j] == 0) {
01257                         KMO_TRY_EXIT_IF_NULL(
01258                             edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01259                                                                         ifu_nr, rotangle, 0, &tmp_rotangle));
01260                     }
01261                 }
01262 
01263                 //
01264                 // calculate shift value
01265                 //
01266 
01267                 KMO_TRY_EXIT_IF_NULL(
01268                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01269                 KMO_TRY_EXIT_IF_NULL(
01270                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01271 
01272                 // get shift values for each IFU by comparing all edge parameters,
01273                 // rejecting and applying median
01274                 int row = 1024; // middle of frame
01275                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01276                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01277                     for (edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01278                         if (edge_table_flat[j] != NULL) {
01279                             double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01280                             double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01281                             cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01282                         }
01283                     }
01284 
01285                     // reject deviating edge-differences
01286                     kmclipm_vector *kv = NULL;
01287                     KMO_TRY_EXIT_IF_NULL(
01288                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01289                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01290 
01291                     // set shift value for each IFU
01292                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01293                     kmclipm_vector_delete(kv); kv = NULL;
01294                 }
01295                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01296                 KMO_TRY_CHECK_ERROR_STATE();
01297 
01298                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01299                     cpl_table_delete(edge_table_flat[j]);
01300                     edge_table_flat[j] = NULL;
01301                 }
01302 
01303                 // take median of all IFU-shift-values
01304                 double shift_val = -cpl_vector_get_median(shift_vec);
01305                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01306 
01307                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01308 
01309                 int     xdim                = cpl_image_get_size_x(combined_data),
01310                         ydim                = cpl_image_get_size_y(combined_data);
01311                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01312                         *array_out          = NULL;
01313                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01314                         *pcombined_noise    = NULL;
01315     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01316                 if (process_noise) {
01317                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01318                 }
01319 
01320                 for (iy = 0; iy < ydim; iy++) {
01321                     // cubic spline
01322                     for (ix = 0; ix < xdim; ix++) {
01323                         array_in[ix] = pcombined_data[ix+iy*xdim];
01324                     }
01325                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01326                                                     xdim, shift_val, 1.0,
01327                                                     NATURAL);
01328                     for (ix = 0; ix < xdim; ix++) {
01329                       pcombined_data[ix+iy*xdim] = array_out[ix];
01330                     }
01331                     cpl_free(array_out);
01332 
01333     //                // linear
01334     //                for (ix = 1; ix < xdim; ix++) {
01335     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01336     //                                     pcombined_data[(ix-1)+iy*xdim];
01337     //                }
01338     //                for (ix = 1; ix < xdim; ix++) {
01339     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01340     //                }
01341 
01342                     if (process_noise) {
01343                         // cubic spline
01344                         for (ix = 0; ix < xdim; ix++) {
01345                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01346                         }
01347                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01348                                                         xdim, shift_val, 1.0,
01349                                                         NATURAL);
01350                         for (ix = 0; ix < xdim; ix++) {
01351                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01352                         }
01353                         cpl_free(array_out);
01354 
01355     //                    // linear
01356     //                    for (ix = 1; ix < xdim; ix++) {
01357     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01358     //                                         pcombined_noise[(ix-1)+iy*xdim];
01359     //                    }
01360     //                    for (ix = 1; ix < xdim; ix++) {
01361     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01362     //                    }
01363                     }
01364                 }
01365                 cpl_free(array_in); array_in = NULL;
01366             }
01367             //
01368             // reconstruct
01369             //
01370             // load MASTER_DARK and MASTER_FLAT
01371             KMO_TRY_EXIT_IF_NULL(
01372                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01373                                               det_nr, FALSE, FALSE, NULL));
01374 
01375             if (process_noise) {
01376                 KMO_TRY_EXIT_IF_NULL(
01377                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01378                                                         det_nr, TRUE, FALSE, NULL));
01379             }
01380 
01381             KMO_TRY_EXIT_IF_NULL(
01382                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01383                                                   rotangle, FALSE, NULL,
01384                                                   &rotangle_found, -1, 0, 0));
01385 
01386             if (process_noise) {
01387                 KMO_TRY_EXIT_IF_NULL(
01388                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01389                                                             rotangle, FALSE, NULL,
01390                                                             &rotangle_found, -1, 0, 0));
01391             }
01392 
01393             char *tmp_band_method = getenv("KMO_BAND_METHOD");
01394             int band_method = 0;
01395             if (tmp_band_method != NULL) {
01396                 band_method = atoi(tmp_band_method);
01397             }
01398 
01399             // ESO INS FILTi ID
01400             KMO_TRY_EXIT_IF_NULL(
01401                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01402                                       IFU_FILTID_POSTFIX));
01403             KMO_TRY_EXIT_IF_NULL(
01404                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01405             cpl_free(keyword); keyword = NULL;
01406 
01407             KMO_TRY_EXIT_IF_NULL(
01408                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01409             KMO_TRY_EXIT_IF_ERROR(
01410                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
01411                                              band_table));
01412             cpl_table_delete(band_table); band_table = NULL;
01413 
01414             cpl_msg_info("","Reconstructing cubes...");
01415             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01416                 // update sub-header
01417                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01418 
01419                 // load raw image and sub-header
01420                 KMO_TRY_EXIT_IF_NULL(
01421                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01422                                                          det_nr, FALSE));
01423 
01424                 KMO_TRY_EXIT_IF_NULL(
01425                     punused_ifus = cpl_array_get_data_int_const(
01426                                                   unused_ifus_after[det_nr-1]));
01427 
01428                 // check if IFU is valid according to main header keywords &
01429                 // calibration files
01430                 KMO_TRY_EXIT_IF_NULL(
01431                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01432                                           IFU_VALID_POSTFIX));
01433                 KMO_TRY_CHECK_ERROR_STATE();
01434                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01435                 cpl_free(keyword); keyword = NULL;
01436 
01437                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01438                     (bounds[2*(ifu_nr-1)] != -1) &&
01439                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01440                     (punused_ifus[j] == 0))
01441                 {
01442                     // IFU is valid
01443                     cpl_error_reset();
01444 
01445                     // calculate WCS
01446                     KMO_TRY_EXIT_IF_ERROR(
01447                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01448 
01449                     // reconstruct data
01450                     KMO_TRY_EXIT_IF_ERROR(
01451                         kmo_reconstruct_sci_image(ifu_nr,
01452                                                 bounds[2*(ifu_nr-1)],
01453                                                 bounds[2*(ifu_nr-1)+1],
01454                                                 combined_data,
01455                                                 combined_noise,
01456                                                 img_dark,
01457                                                 img_dark_noise,
01458                                                 img_flat,
01459                                                 img_flat_noise,
01460                                                 xcal,
01461                                                 ycal,
01462                                                 lcal,
01463                                                 &gd,
01464                                                 calTimestamp,
01465                                                 calAngles,
01466                                                 fn_lut,
01467                                                 &cube_data,
01468                                                 &cube_noise,
01469                                                 flux,
01470                                                 background));
01471                     KMO_TRY_CHECK_ERROR_STATE();
01472                 } else {
01473                     // IFU is invalid
01474                     cpl_error_reset();
01475                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01476 
01477                 // save output
01478                 KMO_TRY_EXIT_IF_NULL(
01479                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01480 
01481                 KMO_TRY_EXIT_IF_ERROR(
01482                     kmclipm_update_property_string(sub_header, EXTNAME,
01483                                                    extname,
01484                                                    "FITS extension name"));
01485 
01486                 cpl_free(extname); extname = NULL;
01487 
01488                 // store cube and sub header into array for later
01489                 stored_data_cubes[ifu_nr - 1] = cube_data;
01490                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01491 
01492                 if (process_noise) {
01493                     KMO_TRY_EXIT_IF_NULL(
01494                         sub_header = cpl_propertylist_duplicate(
01495                                            stored_sub_data_headers[ifu_nr - 1]));
01496                     KMO_TRY_EXIT_IF_NULL(
01497                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01498                                                       EXT_NOISE));
01499 
01500                     KMO_TRY_EXIT_IF_ERROR(
01501                         kmclipm_update_property_string(sub_header,
01502                                                 EXTNAME,
01503                                                 extname,
01504                                                 "FITS extension name"));
01505 
01506                     cpl_free(extname); extname = NULL;
01507 
01508                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01509                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01510                 }
01511                 cpl_image_delete(data_ifu); data_ifu = NULL;
01512                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01513                 cube_data = NULL;
01514                 cube_noise = NULL;
01515             } // for j IFUs
01516 
01517             // free memory
01518             cpl_image_delete(combined_data); combined_data = NULL;
01519             cpl_image_delete(combined_noise); combined_noise = NULL;
01520             cpl_image_delete(xcal); xcal = NULL;
01521             cpl_image_delete(ycal); ycal = NULL;
01522             cpl_image_delete(lcal); lcal = NULL;
01523             cpl_image_delete(img_dark); img_dark = NULL;
01524             cpl_image_delete(img_flat); img_flat = NULL;
01525             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01526             if (process_noise) {
01527                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01528                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01529             }
01530         } // for nr_devices
01531 
01532         cpl_free(edge_table_flat); edge_table_flat = NULL;
01533 
01534         // collapse cubes using rejection
01535         cpl_msg_info("","Collapsing cubes...");
01536         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01537             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01538                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01539 
01540                 KMO_TRY_EXIT_IF_NULL(
01541                     punused_ifus = cpl_array_get_data_int_const(
01542                                                   unused_ifus_after[det_nr-1]));
01543                 if (punused_ifus[j] == 0) {
01544                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01545                     // IFU is valid
01546                     ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01547                                                             CRPIX3);
01548                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01549                                    "CRPIX3 keyword in FITS-header is missing!");
01550 
01551                     ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01552                                                             CRVAL3);
01553                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01554                                    "CRVAL3 keyword in FITS-header is missing!");
01555 
01556                     ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01557                                                             CDELT3);
01558                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01559                                    "CDELT3 keyword in FITS-header is missing!");
01560 
01561                     KMO_TRY_EXIT_IF_NULL(
01562                         identified_slices = kmo_identify_slices(ranges,
01563                                                                 ifu_crpix,
01564                                                                 ifu_crval,
01565                                                                 ifu_cdelt,
01566                                                                 gd.l.dim));
01567                     }/* else {
01568                         KMO_TRY_EXIT_IF_NULL(
01569                             identified_slices = cpl_vector_new(gd.l.dim));
01570                         cpl_vector_fill(identified_slices, 1.);
01571                     }*/
01572 
01573                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01574                         KMO_TRY_EXIT_IF_ERROR(
01575                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01576                                                stored_noise_cubes[ifu_nr-1],
01577                                                &stored_data_images[ifu_nr-1],
01578                                                &stored_noise_images[ifu_nr-1],
01579                                                identified_slices,
01580                                                cmethod, cpos_rej, cneg_rej,
01581                                                citer, cmax, cmin));
01582                     }
01583                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01584                 } else {
01585                     // IFU is invalid
01586                 }
01587             }
01588         }
01589 
01590         // normalise all IFUs as a group.
01591         // Calculate mean of each IFU, add up and divide by number of successful
01592         // averaged IFUs.
01593         // Then divide all valid IFUs with mean value
01594         cnt = 0;
01595         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01596             if (stored_data_images[i] != NULL) {
01597                 KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[i]) <
01598                                cpl_image_get_size_x(stored_data_images[i])*
01599                                cpl_image_get_size_y(stored_data_images[i]),
01600                                CPL_ERROR_ILLEGAL_INPUT,
01601                                "The collapsed, dark-subtracted image contains "
01602                                "only invalid values! Probably the provided "
01603                                "FLAT_SKY frames are exactly the same as the "
01604                                "frames used for MASTER_DARK calculation.");
01605 
01606                 mean_data += cpl_image_get_mean(stored_data_images[i]);
01607                 KMO_TRY_CHECK_ERROR_STATE();
01608                 cnt++;
01609             }
01610 
01611         }
01612         mean_data /= cnt;
01613 
01614         if (mean_data != 0.0) {
01615             for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01616                 if (stored_data_images[i] != NULL) {
01617                     KMO_TRY_EXIT_IF_ERROR(
01618                         cpl_image_divide_scalar(stored_data_images[i],
01619                                                 mean_data));
01620                 }
01621             }
01622         } else {
01623             cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01624                                       "(mean = 0.0)!");
01625         }
01626 
01627         if (process_noise) {
01628             if (mean_data != 0.0) {
01629                 for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01630                     if (stored_noise_images[i] != NULL) {
01631                         KMO_TRY_EXIT_IF_ERROR(
01632                             cpl_image_divide_scalar(stored_noise_images[i],
01633                                                     mean_data));
01634                     }
01635                 }
01636             } else {
01637                 cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01638                                           "(mean = 0.0)!");
01639             }
01640         }
01641 
01642         // calculate qc parameters on normalised data
01643         qc_spat_unif = 0.0;
01644         cnt = 0;
01645         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01646             if (stored_data_images[i] != NULL) {
01647                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01648                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01649 
01650                 qc_spat_unif += pow(tmp_mean-1, 2);
01651                 if (fabs(tmp_mean) > qc_max_dev) {
01652                     qc_max_dev = tmp_mean-1;
01653                     qc_max_dev_id = i+1;
01654                 }
01655                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01656                     qc_max_nonunif = tmp_stdev;
01657                     qc_max_nonunif_id = i+1;
01658                 }
01659                 KMO_TRY_CHECK_ERROR_STATE();
01660                 cnt++;
01661             }
01662         }
01663         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01664 
01665         //
01666         // save data
01667         //
01668 
01669         // update which IFUs are not used
01670         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01671 
01672         KMO_TRY_EXIT_IF_ERROR(
01673             kmo_set_unused_ifus(unused_ifus_after, main_header,
01674                                 "kmo_illumination"));
01675 
01676         cpl_msg_info("","Saving data...");
01677 
01678         KMO_TRY_EXIT_IF_ERROR(
01679             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01680                                            "[adu] uniformity of illumination correction"));
01681         KMO_TRY_EXIT_IF_ERROR(
01682             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01683                                            "[adu] max. deviation from unity"));
01684         KMO_TRY_EXIT_IF_ERROR(
01685             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01686                                         "[] IFU ID with max. dev. from unity"));
01687         KMO_TRY_EXIT_IF_ERROR(
01688             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01689                                            "[adu] max. stdev of illumination corr."));
01690         KMO_TRY_EXIT_IF_ERROR(
01691             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01692                                         "[] IFU ID with max. stdev in illum. corr."));
01693 
01694         if (!suppress_extension) {
01695             KMO_TRY_EXIT_IF_NULL(
01696                 fn_suffix = cpl_sprintf("%s", suffix));
01697         } else {
01698             KMO_TRY_EXIT_IF_NULL(
01699                 fn_suffix = cpl_sprintf("%s", ""));
01700         }
01701         KMO_TRY_EXIT_IF_ERROR(
01702             kmo_dfs_save_main_header(frameset, ILLUM_CORR, fn_suffix, frame,
01703                                      main_header, parlist, cpl_func));
01704 
01705         KMO_TRY_EXIT_IF_ERROR(
01706             kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, fn_suffix, frame,
01707                                      main_header, parlist, cpl_func));
01708 
01709         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01710             KMO_TRY_EXIT_IF_ERROR(
01711                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, fn_suffix,
01712                                    stored_sub_data_headers[i], 0./0.));
01713 
01714             if (process_noise) {
01715                 KMO_TRY_EXIT_IF_ERROR(
01716                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01717                                        fn_suffix, stored_sub_noise_headers[i], 0./0.));
01718             }
01719         }
01720 
01721         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01722             for (ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01723                 KMO_TRY_EXIT_IF_ERROR(
01724                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01725                                                 CAL_IFU_NR,
01726                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01727                                                 "IFU Number {1..24}"));
01728                 KMO_TRY_EXIT_IF_ERROR(
01729                     kmclipm_update_property_double(
01730                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01731                                                 CAL_ROTANGLE,
01732                                                 rotangle_found,
01733                                                 "[deg] Rotator relative to nasmyth"));
01734                 if (has_flat_edge) {
01735                     // save edge-parameters as product
01736                     KMO_TRY_EXIT_IF_ERROR(
01737                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, fn_suffix,
01738                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01739                 }
01740             }
01741         }
01742     }
01743     KMO_CATCH
01744     {
01745         KMO_CATCH_MSG();
01746         ret_val = -1;
01747     }
01748     kmo_free_fits_desc(&desc_sky);
01749     kmo_free_fits_desc(&desc_dark);
01750     kmo_free_fits_desc(&desc_flat);
01751     kmo_free_fits_desc(&desc_xcal);
01752     kmo_free_fits_desc(&desc_ycal);
01753     kmo_free_fits_desc(&desc_lcal);
01754     cpl_image_delete(combined_data); combined_data = NULL;
01755     cpl_image_delete(combined_noise); combined_noise = NULL;
01756     cpl_image_delete(xcal); xcal = NULL;
01757     cpl_image_delete(ycal); ycal = NULL;
01758     cpl_image_delete(lcal); lcal = NULL;
01759     cpl_image_delete(img_dark); img_dark = NULL;
01760     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01761     cpl_image_delete(img_flat); img_flat = NULL;
01762     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01763     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01764     cpl_free(bounds); bounds = NULL;
01765     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01766     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01767     cpl_free(fn_lut); fn_lut = NULL;
01768     cpl_free(suffix); suffix = NULL;
01769     cpl_free(fn_suffix); fn_suffix = NULL;
01770     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01771     cpl_vector_delete(ranges); ranges = NULL;
01772     cpl_free(filter); filter = NULL;
01773     if (calAngles != NULL) {
01774         cpl_vector_delete(calAngles); calAngles = NULL;
01775     }
01776     cpl_propertylist_delete(main_header); main_header = NULL;
01777     for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01778         if (stored_data_cubes != NULL) {
01779             cpl_imagelist_delete(stored_data_cubes[i]);
01780             stored_data_cubes[i] = NULL;
01781         }
01782         if (stored_noise_cubes != NULL) {
01783             cpl_imagelist_delete(stored_noise_cubes[i]);
01784             stored_noise_cubes[i] = NULL;
01785         }
01786         if (stored_data_images != NULL) {
01787             cpl_image_delete(stored_data_images[i]);
01788             stored_data_images[i] = NULL;
01789         }
01790         if (stored_noise_images != NULL) {
01791             cpl_image_delete(stored_noise_images[i]);
01792             stored_noise_images[i] = NULL;
01793         }
01794         if (stored_sub_data_headers != NULL) {
01795             cpl_propertylist_delete(stored_sub_data_headers[i]);
01796             stored_sub_data_headers[i] = NULL;
01797         }
01798         if (stored_sub_noise_headers != NULL) {
01799             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01800             stored_sub_noise_headers[i] = NULL;
01801         }
01802     }
01803     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01804     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01805     cpl_free(stored_data_images); stored_data_images = NULL;
01806     cpl_free(stored_noise_images); stored_noise_images = NULL;
01807     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01808     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01809     if (edge_table_sky != NULL) {
01810         for (i = 0; i < KMOS_NR_DETECTORS; i++) {
01811             if (edge_table_sky[i] != NULL) {
01812                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01813                     cpl_table_delete(edge_table_sky[i][j]);
01814                     edge_table_sky[i][j] = NULL;
01815                 }
01816                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01817             }
01818         }
01819         cpl_free(edge_table_sky); edge_table_sky = NULL;
01820     }
01821     if (edge_table_flat != NULL) {
01822         for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01823             cpl_table_delete(edge_table_flat[j]);
01824             edge_table_flat[j] = NULL;
01825         }
01826         cpl_free(edge_table_flat); edge_table_flat = NULL;
01827     }
01828 
01829     return ret_val;
01830 }
01831