HAWKI Pipeline Reference Manual  1.8.12
hawki_cal_flat.c
1 /* $Id: hawki_cal_flat.c,v 1.25 2012/01/11 15:02:18 cgarcia Exp $
2  *
3  * This file is part of the HAWKI Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: cgarcia $
23  * $Date: 2012/01/11 15:02:18 $
24  * $Revision: 1.25 $
25  * $Name: hawki-1_8_12 $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include <math.h>
37 #include <string.h>
38 #include <cpl.h>
39 
40 #include "irplib_utils.h"
41 
42 #include "hawki_utils.h"
43 #include "hawki_image_stats.h"
44 #include "hawki_load.h"
45 #include "hawki_save.h"
46 #include "hawki_pfits.h"
47 #include "hawki_dfs.h"
48 
49 /*-----------------------------------------------------------------------------
50  Functions prototypes
51  -----------------------------------------------------------------------------*/
52 
53 static int hawki_cal_flat_create(cpl_plugin *) ;
54 static int hawki_cal_flat_exec(cpl_plugin *) ;
55 static int hawki_cal_flat_destroy(cpl_plugin *) ;
56 static int hawki_cal_flat(cpl_parameterlist *, cpl_frameset *) ;
57 
58 static int hawki_cal_flat_retrieve_input_param
59 (cpl_parameterlist * parlist);
60 static cpl_imagelist ** hawki_cal_flat_reduce(
61  cpl_frameset * flatframes,
62  const char * dark_file,
63  cpl_table ** raw_flat_stats,
64  cpl_table ** raw_flat_odd_column_stats,
65  cpl_table ** raw_flat_even_column_stats,
66  cpl_table ** raw_flat_odd_row_stats,
67  cpl_table ** raw_flat_even_row_stats,
68  cpl_vector ** selected);
69 static int hawki_cal_flat_clean_outliers(cpl_image *, cpl_imagelist *,
70  cpl_imagelist *, cpl_vector *, cpl_image **) ;
71 static int hawki_cal_flat_save
72 (cpl_imagelist ** flat,
73  cpl_table ** raw_flat_stats,
74  cpl_table ** raw_flat_odd_column_stats,
75  cpl_table ** raw_flat_even_column_stats,
76  cpl_table ** raw_flat_odd_row_stats,
77  cpl_table ** raw_flat_even_row_stats,
78  cpl_vector ** raw_selected,
79  int set_nb,
80  const cpl_frame * bpmdark,
81  cpl_frameset * flatframes,
82  cpl_frameset * calibframes,
83  cpl_parameterlist * parlist,
84  cpl_frameset * set_tot);
85 static int hawki_cal_flat_compare(const cpl_frame *, const cpl_frame *) ;
86 static cpl_imagelist * hawki_cal_flat_merge_bpms
87 (const cpl_frame * bpm_orig,
88  cpl_imagelist * bpm_to_add);
89 static int hawki_cal_flat_select
90 (cpl_vector * meds,
91  cpl_vector * rms,
92  int auto_flag,
93  int auto_max_bins,
94  double min_level,
95  double max_level,
96  double max_rms,
97  int min_nframes,
98  cpl_vector * selection);
99 static cpl_vector * hawki_cal_flat_extract_vector(cpl_vector *,
100  cpl_vector *) ;
101 
102 /*-----------------------------------------------------------------------------
103  Static variables
104  -----------------------------------------------------------------------------*/
105 
106 static struct {
107  /* Inputs */
108  int llx ;
109  int lly ;
110  int urx ;
111  int ury ;
112  int normalise ;
113  int second_pass ;
114  double sigma_badres ;
115  double sigma_bpm ;
116  double lowval_bpm ;
117  double highval_bpm ;
118  int select_auto ;
119  int select_auto_max_bins;
120  double select_min_level[HAWKI_NB_DETECTORS];
121  double select_max_level[HAWKI_NB_DETECTORS];
122  double select_max_rms[HAWKI_NB_DETECTORS];
123  int select_min_nframes ;
124  int extra_stats;
125 } hawki_cal_flat_config ;
126 
127 static struct {
128  /* Outputs */
129  int nb_badpix[HAWKI_NB_DETECTORS];
130  double norm[HAWKI_NB_DETECTORS];
131  double med_stdev[HAWKI_NB_DETECTORS];
132  double med_avg[HAWKI_NB_DETECTORS];
133  double med_med[HAWKI_NB_DETECTORS];
134  double med_min[HAWKI_NB_DETECTORS];
135  double med_max[HAWKI_NB_DETECTORS];
136 } hawki_cal_flat_outputs;
137 
138 static char hawki_cal_flat_description[] =
139 "hawki_cal_flat -- HAWKĪI imaging flat-field creation from twillight images.\n"
140 "The input of the recipe files listed in the Set Of Frames (sof-file)\n"
141 "must be tagged as:\n"
142 "raw-file.fits "HAWKI_CAL_FLAT_RAW" or\n"
143 "Optional inputs are:\n"
144 "bpmdark-file.fits "HAWKI_CALPRO_BPM_HOT"\n"
145 "dark-file.fits "HAWKI_CALPRO_DARK"\n"
146 "dark_err-file.fits "HAWKI_CALPRO_DARK_ERR"\n"
147 "The recipe creates as an output:\n"
148 "hawki_cal_flat_setxx.fits ("HAWKI_CALPRO_FLAT"): Master flat for filter xx\n"
149 "hawki_cal_flat_err_setxx.fits ("HAWKI_CALPRO_FLAT_ERRMAP"): Master flat residuals\n"
150 "hawki_cal_flat_bpmflat_setxx.fits ("HAWKI_CALPRO_BPM_COLD"): BPM from the flat\n"
151 "hawki_cal_flat_stats_setxx.fits ("HAWKI_CALPRO_FLAT_STATS"): Stats of the individual flats\n"
152 "Optionally it also creates:\n"
153 "hawki_cal_flat_bpm_setxx.fits ("HAWKI_CALPRO_BPM"): Bad pixel mask combining bpm from dark and flat\n"
154 "hawki_cal_flat_stats_ec_setxx.fits ("HAWKI_CALPRO_FLAT_STATS_EVEN_COL"): Stats of the individual flats for even columns\n"
155 "hawki_cal_flat_stats_oc_setxx.fits ("HAWKI_CALPRO_FLAT_STATS_ODD_COL"): Stats of the individual flats for odd columns\n"
156 "hawki_cal_flat_stats_er_setxx.fits ("HAWKI_CALPRO_FLAT_STATS_EVEN_ROW"): Stats of the individual flats for even rows\n"
157 "hawki_cal_flat_stats_or_setxx.fits ("HAWKI_CALPRO_FLAT_STATS_ODD_ROW"): Stats of the individual flats for odd rows\n"
158 "Return code:\n"
159 "esorex exits with an error code of 0 if the recipe completes successfully\n"
160 "or 1 otherwise";
161 
162 
163 
164 
165 
166 /*-----------------------------------------------------------------------------
167  Functions code
168  -----------------------------------------------------------------------------*/
169 
170 /*----------------------------------------------------------------------------*/
178 /*----------------------------------------------------------------------------*/
179 int cpl_plugin_get_info(cpl_pluginlist * list)
180 {
181  cpl_recipe * recipe = cpl_calloc(1, sizeof(*recipe)) ;
182  cpl_plugin * plugin = &recipe->interface ;
183 
184  cpl_plugin_init(plugin,
185  CPL_PLUGIN_API,
186  HAWKI_BINARY_VERSION,
187  CPL_PLUGIN_TYPE_RECIPE,
188  "hawki_cal_flat",
189  "Twillight flat recipe",
190  hawki_cal_flat_description,
191  "Cesar Enrique Garcia Dabo",
192  PACKAGE_BUGREPORT,
194  hawki_cal_flat_create,
195  hawki_cal_flat_exec,
196  hawki_cal_flat_destroy) ;
197 
198  cpl_pluginlist_append(list, plugin) ;
199 
200  return 0;
201 }
202 
203 /*----------------------------------------------------------------------------*/
212 /*----------------------------------------------------------------------------*/
213 static int hawki_cal_flat_create(cpl_plugin * plugin)
214 {
215  cpl_recipe * recipe ;
216  cpl_parameter * p ;
217 
218  /* Get the recipe out of the plugin */
219  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
220  recipe = (cpl_recipe *)plugin ;
221  else return -1 ;
222 
223  /* Create the parameters list in the cpl_recipe object */
224  recipe->parameters = cpl_parameterlist_new() ;
225 
226  /* Fill the parameters list */
227  /* --zone */
228  p = cpl_parameter_new_value("hawki.hawki_cal_flat.zone",
229  CPL_TYPE_STRING,
230  "Stats zone",
231  "hawki.hawki_cal_flat",
232  "1,1,2048,2048") ;
233  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "zone") ;
234  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
235  cpl_parameterlist_append(recipe->parameters, p) ;
236 
237  /* --normalise */
238  p = cpl_parameter_new_value("hawki.hawki_cal_flat.normalise",
239  CPL_TYPE_BOOL, "Flag to apply the normalisation",
240  "hawki.hawki_cal_flat", FALSE) ;
241  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "normalise") ;
242  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
243  cpl_parameterlist_append(recipe->parameters, p) ;
244 
245  /* --second_pass */
246  p = cpl_parameter_new_value("hawki.hawki_cal_flat.second_pass",
247  CPL_TYPE_BOOL, "Flag to apply a second pass computation",
248  "hawki.hawki_cal_flat", TRUE) ;
249  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "second_pass") ;
250  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
251  cpl_parameterlist_append(recipe->parameters, p) ;
252 
253  /* --sigma_badres */
254  p = cpl_parameter_new_value("hawki.hawki_cal_flat.sigma_badres",
255  CPL_TYPE_DOUBLE, "sigma for detection of bad flat results",
256  "hawki.hawki_cal_flat", 1.0) ;
257  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "sigma_badres") ;
258  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
259  cpl_parameterlist_append(recipe->parameters, p) ;
260 
261  /* --sigma_bpm */
262  p = cpl_parameter_new_value("hawki.hawki_cal_flat.sigma_bpm",
263  CPL_TYPE_DOUBLE, "sigma for detection of bad pixels",
264  "hawki.hawki_cal_flat", 10.0) ;
265  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "sigma_bpm") ;
266  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
267  cpl_parameterlist_append(recipe->parameters, p) ;
268 
269  /* --lowval_bpm */
270  p = cpl_parameter_new_value("hawki.hawki_cal_flat.lowval_bpm",
271  CPL_TYPE_DOUBLE, "values of the flat below this will be included "
272  "in the bpm. In units of final flat (normalised if normalise is on)",
273  "hawki.hawki_cal_flat", .1) ;
274  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "lowval_bpm") ;
275  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
276  cpl_parameterlist_append(recipe->parameters, p) ;
277 
278  /* --highval_bpm */
279  p = cpl_parameter_new_value("hawki.hawki_cal_flat.highval_bpm",
280  CPL_TYPE_DOUBLE, "values of the flat above this will be included "
281  "in the bpm. In units of final flat (normalized if normalise is on)",
282  "hawki.hawki_cal_flat", 10.0) ;
283  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "highval_bpm") ;
284  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
285  cpl_parameterlist_append(recipe->parameters, p) ;
286 
287  /* --select_auto */
288  p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_auto",
289  CPL_TYPE_BOOL, "Flag to automatically select the good input frames",
290  "hawki.hawki_cal_flat", TRUE);
291  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_auto") ;
292  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
293  cpl_parameterlist_append(recipe->parameters, p) ;
294 
295  /* --select_auto_max_bins */
296  p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_auto_max_bins",
297  CPL_TYPE_INT, "Maximum number of frames requested",
298  "hawki.hawki_cal_flat", 10) ;
299  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_auto_max_bins");
300  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
301  cpl_parameterlist_append(recipe->parameters, p) ;
302 
303  /* --select_min_level */
304  p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_min_level",
305  CPL_TYPE_STRING, "Minimum ADU level for frames selection",
306  "hawki.hawki_cal_flat", "-1.0") ;
307  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_min_level") ;
308  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
309  cpl_parameterlist_append(recipe->parameters, p) ;
310 
311  /* --select_max_level */
312  p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_max_level",
313  CPL_TYPE_STRING, "Maximum ADU level for frames selection",
314  "hawki.hawki_cal_flat", "25000");
315  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_max_level");
316  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
317  cpl_parameterlist_append(recipe->parameters, p);
318 
319  /* --select_max_rms */
320  p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_max_rms",
321  CPL_TYPE_STRING, "Maximum RMS for frames selection",
322  "hawki.hawki_cal_flat", "4000");
323  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_max_rms");
324  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
325  cpl_parameterlist_append(recipe->parameters, p);
326 
327  /* --select_min_nframes */
328  p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_min_nframes",
329  CPL_TYPE_INT, "Minimum number of frames requested",
330  "hawki.hawki_cal_flat", 3);
331  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_min_nframes") ;
332  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
333  cpl_parameterlist_append(recipe->parameters, p) ;
334 
335  /* --extra_stats */
336  p = cpl_parameter_new_value("hawki.hawki_cal_flat.extra_stats",
337  CPL_TYPE_BOOL, "Request for even/odd column/rows statistics",
338  "hawki.hawki_cal_flat", FALSE) ;
339  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "extra_stats") ;
340  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
341  cpl_parameterlist_append(recipe->parameters, p) ;
342 
343  /* Return */
344  return 0;
345 }
346 
347 /*----------------------------------------------------------------------------*/
353 /*----------------------------------------------------------------------------*/
354 static int hawki_cal_flat_exec(cpl_plugin * plugin)
355 {
356  cpl_recipe * recipe ;
357 
358  /* Get the recipe out of the plugin */
359  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
360  recipe = (cpl_recipe *)plugin ;
361  else return -1 ;
362 
363  /* Issue a banner */
365 
366  return hawki_cal_flat(recipe->parameters, recipe->frames) ;
367 }
368 
369 /*----------------------------------------------------------------------------*/
375 /*----------------------------------------------------------------------------*/
376 static int hawki_cal_flat_destroy(cpl_plugin * plugin)
377 {
378  cpl_recipe * recipe ;
379 
380  /* Get the recipe out of the plugin */
381  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
382  recipe = (cpl_recipe *)plugin ;
383  else return -1 ;
384 
385  cpl_parameterlist_delete(recipe->parameters) ;
386  return 0 ;
387 }
388 
389 /*----------------------------------------------------------------------------*/
396 /*----------------------------------------------------------------------------*/
397 static int hawki_cal_flat(
398  cpl_parameterlist * parlist,
399  cpl_frameset * framelist)
400 {
401  cpl_size * labels ;
402  cpl_size nlabels ;
403  cpl_frameset * flatframes ;
404  int nflats;
405  const char * dark ;
406  const char * dark_err;
407  const cpl_frame * bpmdark ;
408  cpl_imagelist ** twflat ;
409  cpl_table ** raw_flat_stats;
410  cpl_table ** raw_flat_odd_column_stats = NULL;
411  cpl_table ** raw_flat_even_column_stats = NULL;
412  cpl_table ** raw_flat_odd_row_stats = NULL;
413  cpl_table ** raw_flat_even_row_stats = NULL;
414  cpl_vector ** raw_selected;
415  cpl_size i;
416  int j ;
417  cpl_errorstate error_prevstate = cpl_errorstate_get();
418 
419  /* Retrieve input parameters */
420  if(hawki_cal_flat_retrieve_input_param(parlist))
421  {
422  cpl_msg_error(__func__, "Wrong parameters");
423  return -1;
424  }
425 
426  /* Identify the RAW and CALIB frames in the input frameset */
427  if (hawki_dfs_set_groups(framelist)) {
428  cpl_msg_error(__func__, "Cannot identify RAW and CALIB frames") ;
429  return -1;
430  }
431 
432  /* Retrieve raw frames */
433  if ((flatframes = hawki_extract_frameset(framelist,
434  HAWKI_CAL_FLAT_RAW)) == NULL) {
435  cpl_msg_error(__func__, "Cannot find flat frames in the input list (%s)",
436  HAWKI_CAL_FLAT_RAW);
437  return -1 ;
438  }
439 
440  /* Retrieve calibration frames */
441  bpmdark = cpl_frameset_find_const(framelist, HAWKI_CALPRO_BPM_HOT);
442  dark = hawki_extract_first_filename(framelist, HAWKI_CALPRO_DARK);
443  dark_err = hawki_extract_first_filename(framelist, HAWKI_CALPRO_DARK_ERR);
444 
445  /* Labelise all input flat frames */
446  labels = cpl_frameset_labelise(flatframes, hawki_cal_flat_compare,
447  &nlabels);
448  if (labels == NULL) {
449  cpl_msg_error(__func__, "Cannot labelise input frames") ;
450  cpl_frameset_delete(flatframes);
451  return -1;
452  }
453 
454  /* Extract sets and reduce each of them */
455  for (i=0 ; i<nlabels ; i++)
456  {
457  cpl_frameset * this_filter_flats;
458 
459  /* Reduce data set nb i */
460  cpl_msg_info(__func__, "Reduce data set no %d out of %d", i+1, nlabels);
461  cpl_msg_indent_more() ;
462  this_filter_flats = cpl_frameset_extract(flatframes, labels, i) ;
463  nflats = cpl_frameset_get_size(this_filter_flats);
464 
465  /* Allocate and initialize statistics */
466  raw_flat_stats = cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
467  raw_selected = cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_vector*));
468  for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
469  {
470  raw_selected[j] = cpl_vector_new(nflats);
471  raw_flat_stats[j] = cpl_table_new(nflats);
472  }
473  /* Initialize the statistics table */
474  hawki_image_stats_initialize(raw_flat_stats);
475  if(hawki_cal_flat_config.extra_stats)
476  {
477  raw_flat_odd_column_stats =
478  cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
479  raw_flat_even_column_stats =
480  cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
481  raw_flat_odd_row_stats =
482  cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
483  raw_flat_even_row_stats =
484  cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
485  for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
486  {
487  raw_flat_odd_column_stats[j] = cpl_table_new(nflats);
488  raw_flat_even_column_stats[j] = cpl_table_new(nflats);
489  raw_flat_odd_row_stats[j] = cpl_table_new(nflats);
490  raw_flat_even_row_stats[j] = cpl_table_new(nflats);
491  }
492  /* Initialize the statistics table */
493  hawki_image_stats_initialize(raw_flat_odd_column_stats);
494  hawki_image_stats_initialize(raw_flat_even_column_stats);
495  hawki_image_stats_initialize(raw_flat_odd_row_stats);
496  hawki_image_stats_initialize(raw_flat_even_row_stats);
497  }
498 
499  /* Reduce */
500  if ((twflat = hawki_cal_flat_reduce
501  (this_filter_flats,
502  dark,
503  raw_flat_stats,
504  raw_flat_odd_column_stats,
505  raw_flat_even_column_stats,
506  raw_flat_odd_row_stats,
507  raw_flat_even_row_stats,
508  raw_selected)) == NULL)
509  {
510  for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
511  {
512  cpl_table_delete(raw_flat_stats[j]);
513  cpl_vector_delete(raw_selected[j]);
514  }
515  cpl_free(raw_flat_stats);
516  cpl_free(raw_selected);
517  if(hawki_cal_flat_config.extra_stats)
518  {
519  for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
520  {
521  cpl_table_delete(raw_flat_odd_column_stats[j]);
522  cpl_table_delete(raw_flat_even_column_stats[j]);
523  cpl_table_delete(raw_flat_odd_row_stats[j]);
524  cpl_table_delete(raw_flat_even_row_stats[j]);
525  }
526  cpl_free(raw_flat_odd_column_stats);
527  cpl_free(raw_flat_even_column_stats);
528  cpl_free(raw_flat_odd_row_stats);
529  cpl_free(raw_flat_even_row_stats);
530  }
531  cpl_frameset_delete(this_filter_flats);
532  cpl_frameset_delete(flatframes);
533  cpl_free(labels);
534  cpl_msg_error(__func__, "Cannot reduce set nb %d", i+1) ;
535  return 1;
536 
537  } else {
538  /* Save the products */
539  cpl_frameset * calib_frames;
540 
541  cpl_msg_info(__func__, "Save the products") ;
542  calib_frames = cpl_frameset_new();
543  if(bpmdark)
544  cpl_frameset_insert(calib_frames, cpl_frame_duplicate(bpmdark));
545  if(dark)
546  cpl_frameset_insert(calib_frames,
547  cpl_frame_duplicate(cpl_frameset_find_const(framelist,
548  HAWKI_CALPRO_DARK)));
549  if(dark_err)
550  cpl_frameset_insert(calib_frames,
551  cpl_frame_duplicate(cpl_frameset_find_const(framelist,
552  HAWKI_CALPRO_DARK_ERR)));
553  hawki_cal_flat_save
554  (twflat, raw_flat_stats,
555  raw_flat_odd_column_stats,
556  raw_flat_even_column_stats,
557  raw_flat_odd_row_stats,
558  raw_flat_even_row_stats,
559  raw_selected,
560  i+1, bpmdark, this_filter_flats, calib_frames,
561  parlist, framelist);
562  cpl_imagelist_delete(twflat[0]);
563  cpl_imagelist_delete(twflat[1]);
564  cpl_imagelist_delete(twflat[2]);
565  if (hawki_cal_flat_config.second_pass)
566  cpl_imagelist_delete(twflat[3]);
567  cpl_free(twflat);
568  for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
569  {
570  cpl_table_delete(raw_flat_stats[j]);
571  cpl_vector_delete(raw_selected[j]);
572  }
573  cpl_free(raw_flat_stats);
574  cpl_free(raw_selected);
575  if(hawki_cal_flat_config.extra_stats)
576  {
577  for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
578  {
579  cpl_table_delete(raw_flat_odd_column_stats[j]);
580  cpl_table_delete(raw_flat_even_column_stats[j]);
581  cpl_table_delete(raw_flat_odd_row_stats[j]);
582  cpl_table_delete(raw_flat_even_row_stats[j]);
583  }
584  cpl_free(raw_flat_odd_column_stats);
585  cpl_free(raw_flat_even_column_stats);
586  cpl_free(raw_flat_odd_row_stats);
587  cpl_free(raw_flat_even_row_stats);
588  }
589  cpl_frameset_delete(calib_frames);
590  }
591  cpl_msg_indent_less();
592  cpl_frameset_delete(this_filter_flats);
593  }
594 
595  if(!cpl_errorstate_is_equal(error_prevstate))
596  cpl_msg_warning(__func__,"Probably some data could not be saved. "
597  "Check permisions or disk space");
598 
599 
600  /* Free and return */
601  cpl_frameset_delete(flatframes);
602  cpl_free(labels);
603 
604  /* Return */
605  if (cpl_error_get_code()) return -1 ;
606  else return 0;
607 }
608 
609 /*----------------------------------------------------------------------------*/
620 /*----------------------------------------------------------------------------*/
621 static cpl_imagelist ** hawki_cal_flat_reduce(
622  cpl_frameset * flatframes,
623  const char * dark_file,
624  cpl_table ** raw_flat_stats,
625  cpl_table ** raw_flat_odd_column_stats,
626  cpl_table ** raw_flat_even_column_stats,
627  cpl_table ** raw_flat_odd_row_stats,
628  cpl_table ** raw_flat_even_row_stats,
629  cpl_vector ** selected)
630 {
631  int nima ;
632  cpl_image * ima_cur ;
633  cpl_image * big_ima ;
634  cpl_image * big_badres ;
635  cpl_vector * medians[HAWKI_NB_DETECTORS];
636  cpl_vector * stdevs[HAWKI_NB_DETECTORS];
637  cpl_vector * sub_medians ;
638  cpl_imagelist * in_quad ;
639  cpl_imagelist ** results ;
640  cpl_imagelist * res_quad[4] ;
641  cpl_image * err_quad[4] ;
642  cpl_image * badres_mask[4] ;
643  cpl_image * flat_image ;
644  cpl_image * dark ;
645  cpl_propertylist * plist;
646  double gradient ;
647  double flat_dit;
648  cpl_image * bpmflat;
649  int j, k ;
650  int idet;
651 
652  /* Test entries */
653  if (flatframes == NULL) return NULL ;
654 
655  /* Initialise */
656  nima = cpl_frameset_get_size(flatframes) ;
657  if (nima < 3) {
658  cpl_msg_error(__func__, "Not enough frames (%d)", nima) ;
659  return NULL ;
660  }
661  for (idet=0 ; idet<HAWKI_NB_DETECTORS ; idet++)
662  hawki_cal_flat_outputs.norm[idet] = 1.0 ;
663 
664  /* Compute statistics */
665  cpl_msg_info(__func__, "Compute statistics") ;
666  cpl_msg_indent_more() ;
667  /* Loop on the HAWKI_NB_DETECTORS chips */
668  for (idet=0 ; idet<HAWKI_NB_DETECTORS ; idet++)
669  {
670 
671  /* Compute some stats on input images */
672  cpl_msg_info(__func__, "Chip number %d", idet+1) ;
673  cpl_msg_info(__func__, "image min max med rms") ;
674  cpl_msg_info(__func__, "--------------------------------------------") ;
675  medians[idet] = cpl_vector_new(nima);
676  stdevs[idet] = cpl_vector_new(nima);
677  for (j=0 ; j<nima ; j++)
678  {
679  /* Load the image */
680  ima_cur = hawki_load_image(flatframes, j, idet+1, CPL_TYPE_FLOAT) ;
681 
682  /* Compute the stats */
684  (raw_flat_stats,
685  ima_cur,
686  hawki_cal_flat_config.llx,
687  hawki_cal_flat_config.lly,
688  hawki_cal_flat_config.urx,
689  hawki_cal_flat_config.ury,
690  idet,
691  j) !=0 )
692  {
693  cpl_msg_error(__func__, "Cannot compute stats on image %d",j+1);
694  cpl_msg_indent_less() ;
695  cpl_image_delete(ima_cur);
696  for (k=0 ; k<=idet ; k++) cpl_vector_delete(medians[k]) ;
697  for (k=0 ; k<=idet ; k++) cpl_vector_delete(stdevs[k]) ;
698  return NULL ;
699  }
700 
701  if(hawki_cal_flat_config.extra_stats)
702  {
703  if(hawki_image_stats_odd_even_column_row_fill_from_image
704  (raw_flat_odd_column_stats,
705  raw_flat_even_column_stats,
706  raw_flat_odd_row_stats,
707  raw_flat_even_row_stats,
708  ima_cur,
709  idet,
710  j) !=0 )
711  {
712  cpl_msg_error(__func__, "Cannot compute extra stats");
713  cpl_msg_indent_less() ;
714  cpl_image_delete(ima_cur);
715  for (k=0 ; k<=idet ; k++) cpl_vector_delete(medians[k]) ;
716  for (k=0 ; k<=idet ; k++) cpl_vector_delete(stdevs[k]) ;
717  return NULL ;
718  }
719  }
720  cpl_vector_set(medians[idet], j, cpl_table_get_double
721  (raw_flat_stats[idet],HAWKI_COL_STAT_MED,j,NULL )) ;
722  cpl_vector_set(stdevs[idet], j, cpl_table_get_double
723  (raw_flat_stats[idet],HAWKI_COL_STAT_RMS,j,NULL )) ;
724  cpl_msg_info(__func__, "%02d %10.2f %10.2f %10.2f %10.2f",
725  j+1,
726  cpl_table_get_double(raw_flat_stats[idet],
727  HAWKI_COL_STAT_MIN,j,NULL ),
728  cpl_table_get_double(raw_flat_stats[idet],
729  HAWKI_COL_STAT_MAX,j,NULL ),
730  cpl_table_get_double(raw_flat_stats[idet],
731  HAWKI_COL_STAT_MED,j,NULL ),
732  cpl_table_get_double(raw_flat_stats[idet],
733  HAWKI_COL_STAT_RMS,j,NULL ));
734  if (cpl_table_get_double
735  (raw_flat_stats[idet],HAWKI_COL_STAT_MED,j,NULL ) < 1e-6)
736  {
737  cpl_msg_error(__func__, "image %d has negative flux: aborting",
738  j+1) ;
739  cpl_msg_indent_less() ;
740  for (k=0 ; k<=idet ; k++) cpl_vector_delete(medians[k]) ;
741  for (k=0 ; k<=idet ; k++) cpl_vector_delete(stdevs[k]) ;
742  return NULL ;
743  }
744  cpl_image_delete(ima_cur);
745  }
746  cpl_msg_info(__func__, "--------------------------------------------") ;
747 
748  /* Compute min max stdev and mean of the medians */
749  hawki_cal_flat_outputs.med_min[idet] =
750  cpl_vector_get_min(medians[idet]);
751  hawki_cal_flat_outputs.med_max[idet] =
752  cpl_vector_get_max(medians[idet]);
753  hawki_cal_flat_outputs.med_avg[idet] =
754  cpl_vector_get_mean(medians[idet]);
755  hawki_cal_flat_outputs.med_med[idet] =
756  cpl_vector_get_median_const(medians[idet]);
757  hawki_cal_flat_outputs.med_stdev[idet] =
758  cpl_vector_get_stdev(medians[idet]);
759 
760  /* See if flux gradient is large enough for a correct fit */
761  gradient=fabs(hawki_cal_flat_outputs.med_max[idet]/
762  hawki_cal_flat_outputs.med_min[idet]) ;
763  if (gradient < 4.0) {
764  /* cpl_msg_warning(__func__, "Low flux gradient: %g", gradient) ;*/
765  }
766  }
767  cpl_msg_indent_less() ;
768 
769  /* Allocate for results */
770  results = cpl_malloc(4 * sizeof(cpl_imagelist*)) ;
771  results[0] = cpl_imagelist_new() ;
772  results[1] = cpl_imagelist_new() ;
773  results[2] = cpl_imagelist_new() ;
774  if (hawki_cal_flat_config.second_pass) results[3] = cpl_imagelist_new() ;
775  else results[3] = NULL ;
776 
777  cpl_msg_info(__func__, "Compute the flat") ;
778  cpl_msg_indent_more() ;
779  /* Loop on the HAWKI_NB_DETECTORS chips */
780  for (idet=0 ; idet<HAWKI_NB_DETECTORS ; idet++) {
781  cpl_msg_info(__func__, "Chip number %d", idet+1) ;
782  cpl_msg_indent_more() ;
783 
784  /* Frames selection */
785  cpl_msg_info(__func__, "Apply the frames selection");
786  if ((hawki_cal_flat_select(medians[idet],
787  stdevs[idet],
788  hawki_cal_flat_config.select_auto,
789  hawki_cal_flat_config.select_auto_max_bins,
790  hawki_cal_flat_config.select_min_level[idet],
791  hawki_cal_flat_config.select_max_level[idet],
792  hawki_cal_flat_config.select_max_rms[idet],
793  hawki_cal_flat_config.select_min_nframes,
794  selected[idet])) == -1)
795  {
796  cpl_msg_error(__func__, "Cannot apply the frames selection") ;
797  cpl_imagelist_delete(results[0]) ;
798  cpl_imagelist_delete(results[1]) ;
799  cpl_imagelist_delete(results[2]) ;
800  if (hawki_cal_flat_config.second_pass)
801  cpl_imagelist_delete(results[3]) ;
802  cpl_free(results) ;
803  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
804  cpl_vector_delete(medians[k]) ;
805  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
806  cpl_vector_delete(stdevs[k]) ;
807  return NULL ;
808  }
809 
810  /* Apply the medians subselection */
811  sub_medians = hawki_cal_flat_extract_vector
812  (medians[idet], selected[idet]) ;
813 
814  /* Loop on the 4 quadrants */
815  for (j=0 ; j<4 ; j++) {
816  /* Load input image chips */
817  if ((in_quad = hawki_load_quadrants(flatframes, idet+1, j+1,
818  CPL_TYPE_FLOAT))==NULL) {
819  cpl_msg_error(__func__, "Cannot load the raw quadrants") ;
820  cpl_imagelist_delete(results[0]) ;
821  cpl_imagelist_delete(results[1]) ;
822  cpl_imagelist_delete(results[2]) ;
823  if (hawki_cal_flat_config.second_pass)
824  cpl_imagelist_delete(results[3]) ;
825  cpl_free(results) ;
826  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
827  cpl_vector_delete(medians[k]) ;
828  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
829  cpl_vector_delete(stdevs[k]) ;
830  cpl_vector_delete(sub_medians) ;
831  return NULL ;
832  }
833 
834  /* Apply subselection of the frames */
835  cpl_imagelist_erase(in_quad, selected[idet]);
836 
837  /* Apply dark correction to all planes if requested */
838  if (dark_file) {
839  if (j==0) cpl_msg_info(__func__, "Subtracting dark") ;
840  /* Load dark */
841  if ((dark = hawki_load_quadrant_from_file(dark_file,
842  idet+1, j+1, CPL_TYPE_FLOAT)) == NULL) {
843  cpl_msg_error(__func__, "Cannot load the dark quadrants") ;
844  cpl_imagelist_delete(in_quad) ;
845  cpl_imagelist_delete(results[0]) ;
846  cpl_imagelist_delete(results[1]) ;
847  cpl_imagelist_delete(results[2]) ;
848  if (hawki_cal_flat_config.second_pass)
849  cpl_imagelist_delete(results[3]) ;
850  cpl_free(results) ;
851  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
852  cpl_vector_delete(medians[k]) ;
853  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
854  cpl_vector_delete(stdevs[k]) ;
855  cpl_vector_delete(sub_medians) ;
856  return NULL ;
857  }
858 
859  /* Multiply by the dit */
860  if ((plist=cpl_propertylist_load
861  (cpl_frame_get_filename
862  (cpl_frameset_get_first_const(flatframes)), 0)) == NULL)
863  {
864  cpl_msg_error(__func__, "Cannot get header from frame");
865  cpl_imagelist_delete(in_quad) ;
866  cpl_imagelist_delete(results[0]) ;
867  cpl_imagelist_delete(results[1]) ;
868  cpl_imagelist_delete(results[2]) ;
869  if (hawki_cal_flat_config.second_pass)
870  cpl_imagelist_delete(results[3]) ;
871  cpl_free(results) ;
872  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
873  cpl_vector_delete(medians[k]) ;
874  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
875  cpl_vector_delete(stdevs[k]) ;
876  cpl_vector_delete(sub_medians) ;
877  cpl_image_delete(dark);
878  return NULL ;
879  }
880  flat_dit = hawki_pfits_get_dit(plist);
881  cpl_image_multiply_scalar(dark, flat_dit);
882  cpl_propertylist_delete(plist);
883 
884  /* Dark correction */
885  cpl_imagelist_subtract_image(in_quad, dark) ;
886  cpl_image_delete(dark) ;
887  if (cpl_error_get_code() != CPL_ERROR_NONE) {
888  cpl_msg_warning(__func__,"Cannot subtract the dark frame");
889  cpl_error_reset() ;
890  }
891  }
892 
893  /* Fit slopes */
894  err_quad[j] = cpl_image_duplicate(cpl_imagelist_get(in_quad, 0)) ;
895  res_quad[j] = cpl_fit_imagelist_polynomial(sub_medians, in_quad,
896  0, 1, CPL_FALSE, CPL_TYPE_FLOAT, err_quad[j]) ;
897  if (res_quad[j] == NULL) {
898  cpl_msg_error(__func__, "Cannot create twilight flat-field") ;
899  cpl_imagelist_delete(results[0]) ;
900  cpl_imagelist_delete(results[1]) ;
901  cpl_imagelist_delete(results[2]) ;
902  if (hawki_cal_flat_config.second_pass)
903  cpl_imagelist_delete(results[3]) ;
904  cpl_free(results) ;
905  cpl_imagelist_delete(in_quad) ;
906  for (k=0 ; k<j ; k++) cpl_imagelist_delete(res_quad[k]) ;
907  for (k=0 ; k<=j ; k++) cpl_image_delete(err_quad[k]) ;
908  for (k=0 ; k<j ; k++)
909  if (badres_mask[k]) cpl_image_delete(badres_mask[k]) ;
910  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
911  cpl_vector_delete(medians[k]) ;
912  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
913  cpl_vector_delete(stdevs[k]) ;
914  cpl_vector_delete(sub_medians) ;
915  return NULL ;
916  }
917 
918  /* Handle the pixels with a high error */
919  badres_mask[j] = NULL ;
920  if (hawki_cal_flat_config.second_pass) {
921  if (j==0) cpl_msg_info(__func__,
922  "Second pass to clean the outliers") ;
923  if (hawki_cal_flat_clean_outliers(err_quad[j], res_quad[j],
924  in_quad, sub_medians, &(badres_mask[j])) == -1) {
925  cpl_msg_error(__func__, "Cannot clean the outliers") ;
926  cpl_imagelist_delete(results[0]) ;
927  cpl_imagelist_delete(results[1]) ;
928  cpl_imagelist_delete(results[2]) ;
929  cpl_imagelist_delete(results[3]) ;
930  cpl_free(results) ;
931  cpl_imagelist_delete(in_quad) ;
932  for (k=0 ; k<=j ; k++) cpl_imagelist_delete(res_quad[k]) ;
933  for (k=0 ; k<=j ; k++) cpl_image_delete(err_quad[k]) ;
934  for (k=0 ; k<=j ; k++) cpl_image_delete(badres_mask[k]) ;
935  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
936  cpl_vector_delete(medians[k]) ;
937  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
938  cpl_vector_delete(stdevs[k]) ;
939  cpl_vector_delete(sub_medians) ;
940  cpl_msg_indent_less() ;
941  return NULL ;
942  }
943  }
944  cpl_imagelist_delete(in_quad) ;
945  }
946  cpl_vector_delete(sub_medians) ;
947 
948  /* Rebuild the quadrants and put in results */
949  /* Rebuild RESULTS */
950  big_ima = hawki_rebuild_quadrants(
951  cpl_imagelist_get(res_quad[0],1),
952  cpl_imagelist_get(res_quad[1],1),
953  cpl_imagelist_get(res_quad[2],1),
954  cpl_imagelist_get(res_quad[3],1)) ;
955  for (j=0 ; j<4 ; j++) cpl_imagelist_delete(res_quad[j]) ;
956  cpl_imagelist_set(results[0], big_ima, idet) ;
957  if (big_ima == NULL) {
958  cpl_msg_error(__func__, "Cannot rebuild the image") ;
959  cpl_imagelist_delete(results[0]) ;
960  cpl_imagelist_delete(results[1]) ;
961  cpl_imagelist_delete(results[2]) ;
962  if (hawki_cal_flat_config.second_pass)
963  cpl_imagelist_delete(results[3]) ;
964  cpl_free(results) ;
965  for (j=0 ; j<4 ; j++) cpl_image_delete(err_quad[j]) ;
966  for (j=0 ; j<4 ; j++)
967  if (badres_mask[j]) cpl_image_delete(badres_mask[j]) ;
968  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
969  cpl_vector_delete(medians[k]) ;
970  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
971  cpl_vector_delete(stdevs[k]) ;
972  return NULL ;
973  }
974 
975  /* Rebuild ERROR */
976  big_ima = hawki_rebuild_quadrants(err_quad[0], err_quad[1],
977  err_quad[2], err_quad[3]) ;
978  for (j=0 ; j<4 ; j++) cpl_image_delete(err_quad[j]) ;
979  if (big_ima == NULL) {
980  cpl_msg_error(__func__, "Cannot rebuild the image") ;
981  cpl_imagelist_delete(results[0]) ;
982  cpl_imagelist_delete(results[1]) ;
983  cpl_imagelist_delete(results[2]) ;
984  if (hawki_cal_flat_config.second_pass)
985  cpl_imagelist_delete(results[3]) ;
986  cpl_free(results) ;
987  for (j=0 ; j<4 ; j++)
988  if (badres_mask[j]) cpl_image_delete(badres_mask[j]) ;
989  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
990  cpl_vector_delete(medians[k]) ;
991  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
992  cpl_vector_delete(stdevs[k]) ;
993  return NULL ;
994  }
995  cpl_imagelist_set(results[1], big_ima, idet) ;
996 
997  /* Rebuild BADRES_MASK */
998  big_badres = hawki_rebuild_quadrants(badres_mask[0], badres_mask[1],
999  badres_mask[2], badres_mask[3]) ;
1000  if (hawki_cal_flat_config.second_pass) {
1001  for (j=0 ; j<4 ; j++) cpl_image_delete(badres_mask[j]) ;
1002  cpl_imagelist_set(results[3], big_badres, idet) ;
1003  }
1004 
1005  if (hawki_cal_flat_config.normalise) {
1006  /* Normalize gain */
1007  cpl_msg_info(__func__, "Normalise the flat") ;
1008  flat_image = cpl_imagelist_get(results[0], idet) ;
1009  hawki_cal_flat_outputs.norm[idet] =
1010  cpl_image_get_median(flat_image) ;
1011  cpl_image_divide_scalar
1012  (flat_image, hawki_cal_flat_outputs.norm[idet]);
1013  if (cpl_error_get_code()) {
1014  cpl_msg_error(__func__, "Cannot normalise") ;
1015  cpl_imagelist_delete(results[0]) ;
1016  cpl_imagelist_delete(results[1]) ;
1017  cpl_imagelist_delete(results[2]) ;
1018  if (hawki_cal_flat_config.second_pass)
1019  cpl_imagelist_delete(results[3]) ;
1020  cpl_free(results) ;
1021  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
1022  cpl_vector_delete(medians[k]) ;
1023  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++)
1024  cpl_vector_delete(stdevs[k]) ;
1025  return NULL ;
1026  }
1027  }
1028 
1029  /* BPM from the flat */
1030  cpl_msg_info(__func__, "Compute the BPM from the flat") ;
1031  bpmflat=hawki_compute_flatbpm(cpl_imagelist_get(results[0],idet),
1032  hawki_cal_flat_config.sigma_bpm,
1033  hawki_cal_flat_config.lowval_bpm,
1034  hawki_cal_flat_config.highval_bpm);
1035 
1036  cpl_imagelist_set(results[2], bpmflat, idet) ;
1037  hawki_cal_flat_outputs.nb_badpix[idet]=
1038  (int)cpl_image_get_flux(bpmflat);
1039 
1040  cpl_msg_indent_less() ;
1041  }
1042  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) cpl_vector_delete(medians[k]) ;
1043  for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) cpl_vector_delete(stdevs[k]) ;
1044  cpl_msg_indent_less() ;
1045 
1046  return results ;
1047 }
1048 
1049 /*----------------------------------------------------------------------------*/
1059 /*----------------------------------------------------------------------------*/
1060 static int hawki_cal_flat_clean_outliers(
1061  cpl_image * error,
1062  cpl_imagelist * fit_res,
1063  cpl_imagelist * raw,
1064  cpl_vector * medians,
1065  cpl_image ** recomp_mask)
1066 {
1067  cpl_mask * recompute ;
1068  cpl_binary * precompute ;
1069  double med, stdev, threshold1, threshold2 ;
1070  int nx, ny, pos, nbad, nima, out, ind, pix ;
1071  cpl_image * cur_ima ;
1072  float * pimaf ;
1073  double val, fit_val, a, b, max ;
1074  cpl_vector * z_pix ;
1075  double * pz_pix ;
1076  cpl_image * onepix ;
1077  cpl_vector * med_purged ;
1078  cpl_imagelist * raw_purged ;
1079  cpl_imagelist * fit_one ;
1080  cpl_image * err_one ;
1081  int i, j, k ;
1082 
1083  /* Check entries */
1084  if (error == NULL) return -1 ;
1085  if (fit_res == NULL) return -1 ;
1086  if (raw == NULL) return -1 ;
1087 
1088  /* Initialise */
1089  if (recomp_mask) *recomp_mask = NULL ;
1090 
1091  /* Identify the places to recompute */
1092  med = cpl_image_get_median_dev(error, &stdev) ;
1093  threshold1 = med - hawki_cal_flat_config.sigma_badres * stdev ;
1094  threshold2 = med + hawki_cal_flat_config.sigma_badres * stdev ;
1095  recompute = cpl_mask_threshold_image_create(error,threshold1,threshold2) ;
1096  cpl_mask_not(recompute) ;
1097 
1098  if ((nbad=cpl_mask_count(recompute)) == 0) {
1099  if (recomp_mask)
1100  *recomp_mask = cpl_image_new_from_mask(recompute) ;
1101  cpl_mask_delete(recompute) ;
1102  return 0 ;
1103  }
1104  nx = cpl_image_get_size_x(error) ;
1105  ny = cpl_image_get_size_y(error) ;
1106  nima = cpl_imagelist_get_size(raw) ;
1107 
1108  /* Get access to the mask */
1109  precompute = cpl_mask_get_data(recompute) ;
1110  for (j=0 ; j<ny ; j++) {
1111  for (i=0 ; i<nx ; i++) {
1112  pos = i + j*nx ;
1113  /* The pixel has to be recomputed */
1114  if (precompute[pos] == CPL_BINARY_1) {
1115  /* Get the pix_val-fit in a vector */
1116  z_pix = cpl_vector_new(nima) ;
1117  for (k=0 ; k<nima ; k++) {
1118  cur_ima = cpl_imagelist_get(fit_res, 0) ;
1119  pimaf = cpl_image_get_data_float(cur_ima) ;
1120  a = pimaf[pos] ;
1121  cur_ima = cpl_imagelist_get(fit_res, 1) ;
1122  pimaf = cpl_image_get_data_float(cur_ima) ;
1123  b = pimaf[pos] ;
1124  med = cpl_vector_get(medians, k) ;
1125  fit_val = a + b * med ;
1126  cur_ima = cpl_imagelist_get(raw, k) ;
1127  pimaf = cpl_image_get_data_float(cur_ima) ;
1128  cpl_vector_set(z_pix, k, (double)(pimaf[pos]-fit_val)) ;
1129  }
1130 
1131  /* Identify the outlier */
1132  out = -1 ;
1133  max = -1.0 ;
1134  pz_pix = cpl_vector_get_data(z_pix) ;
1135  for (k=0 ; k<nima ; k++) {
1136  if (fabs(pz_pix[k]) > max) {
1137  max = fabs(pz_pix[k]) ;
1138  out = k ;
1139  }
1140  }
1141  cpl_vector_delete(z_pix) ;
1142 
1143  /* Create 1-pixel purged image list and the purged medians */
1144  med_purged = cpl_vector_new(nima-1) ;
1145  raw_purged = cpl_imagelist_new() ;
1146  ind = 0 ;
1147  for (k=0 ; k<nima ; k++) {
1148  if (k != out) {
1149  /* Fill raw_purged */
1150  cur_ima = cpl_imagelist_get(raw, k) ;
1151  onepix=cpl_image_extract(cur_ima, i+1, j+1, i+1, j+1) ;
1152  cpl_imagelist_set(raw_purged, onepix, ind) ;
1153  /* Fill med_purged */
1154  cpl_vector_set(med_purged, ind,
1155  cpl_vector_get(medians, k)) ;
1156  ind ++;
1157  }
1158  }
1159 
1160  /* Perform the fit */
1161  err_one = cpl_image_duplicate(cpl_imagelist_get(raw_purged,0));
1162  fit_one = cpl_fit_imagelist_polynomial(med_purged, raw_purged,
1163  0, 1, CPL_FALSE, CPL_TYPE_FLOAT, err_one) ;
1164  if (fit_one == NULL) {
1165  cpl_msg_error(__func__, "Cannot fit in second pass") ;
1166  cpl_mask_delete(recompute) ;
1167  cpl_vector_delete(med_purged) ;
1168  cpl_imagelist_delete(raw_purged) ;
1169  cpl_image_delete(err_one) ;
1170  return -1 ;
1171  }
1172  cpl_vector_delete(med_purged) ;
1173  cpl_imagelist_delete(raw_purged) ;
1174 
1175  /* Write the result in the input */
1176  val = cpl_image_get(err_one, 1, 1, &pix) ;
1177  cpl_image_set(error, i+1, j+1, val) ;
1178  cpl_image_delete(err_one) ;
1179 
1180  cur_ima = cpl_imagelist_get(fit_one, 0) ;
1181  val = cpl_image_get(cur_ima, 1, 1, &pix) ;
1182  cur_ima = cpl_imagelist_get(fit_res, 0) ;
1183  cpl_image_set(cur_ima, i+1, j+1, val) ;
1184 
1185  cur_ima = cpl_imagelist_get(fit_one, 1) ;
1186  val = cpl_image_get(cur_ima, 1, 1, &pix) ;
1187  cur_ima = cpl_imagelist_get(fit_res, 1) ;
1188  cpl_image_set(cur_ima, i+1, j+1, val) ;
1189  cpl_imagelist_delete(fit_one) ;
1190  }
1191  }
1192  }
1193  if (recomp_mask)
1194  *recomp_mask = cpl_image_new_from_mask(recompute) ;
1195  cpl_mask_delete(recompute) ;
1196 
1197  /* Return */
1198  if (cpl_error_get_code()) return -1 ;
1199  else return 0 ;
1200 }
1201 
1202 /*----------------------------------------------------------------------------*/
1214 /*----------------------------------------------------------------------------*/
1215 static int hawki_cal_flat_save
1216 (cpl_imagelist ** flat,
1217  cpl_table ** raw_flat_stats,
1218  cpl_table ** raw_flat_odd_column_stats,
1219  cpl_table ** raw_flat_even_column_stats,
1220  cpl_table ** raw_flat_odd_row_stats,
1221  cpl_table ** raw_flat_even_row_stats,
1222  cpl_vector ** raw_selected,
1223  int set_nb,
1224  const cpl_frame * bpmdark,
1225  cpl_frameset * flat_frames,
1226  cpl_frameset * calib_frames,
1227  cpl_parameterlist * parlist,
1228  cpl_frameset * set_tot)
1229 {
1230  cpl_propertylist ** qclists ;
1231  cpl_imagelist * bpm ;
1232  const cpl_frame * ref_frame ;
1233  cpl_frameset * used_frames;
1234  char * filename ;
1235  cpl_propertylist * inputlist ;
1236  int ext_nb ;
1237  const char * recipe_name = "hawki_cal_flat" ;
1238  int i ;
1239  int iflat;
1240  int nflat;
1241  int nused;
1242  char key_name[72];
1243 
1244  /* Get the reference frame */
1245  ref_frame = irplib_frameset_get_first_from_group(flat_frames, CPL_FRAME_GROUP_RAW) ;
1246 
1247  /* Create the QC lists */
1248  qclists = cpl_malloc(HAWKI_NB_DETECTORS * sizeof(cpl_propertylist*)) ;
1249  for (i=0 ; i<HAWKI_NB_DETECTORS ; i++) {
1250  qclists[i] = cpl_propertylist_new() ;
1251 
1252  /* Add the raw flat selection keywords */
1253  nflat = cpl_vector_get_size(raw_selected[i]);
1254  nused = 0;
1255  for(iflat = 0; iflat < nflat; ++iflat)
1256  {
1257  snprintf(key_name, 72, "ESO QC RAW%02d USED", iflat + 1);
1258  cpl_propertylist_append_bool
1259  (qclists[i], key_name,
1260  (cpl_vector_get(raw_selected[i], iflat) + 1) / 2);
1261  cpl_table_set_int
1262  (raw_flat_stats[i],HAWKI_COL_STAT_USED, iflat,
1263  cpl_vector_get(raw_selected[i], iflat));
1264  if(hawki_cal_flat_config.extra_stats)
1265  {
1266  cpl_table_set_int
1267  (raw_flat_odd_column_stats[i],HAWKI_COL_STAT_USED, iflat,
1268  cpl_vector_get(raw_selected[i], iflat));
1269  cpl_table_set_int
1270  (raw_flat_even_column_stats[i],HAWKI_COL_STAT_USED, iflat,
1271  cpl_vector_get(raw_selected[i], iflat));
1272  cpl_table_set_int
1273  (raw_flat_odd_row_stats[i],HAWKI_COL_STAT_USED, iflat,
1274  cpl_vector_get(raw_selected[i], iflat));
1275  cpl_table_set_int
1276  (raw_flat_even_row_stats[i],HAWKI_COL_STAT_USED, iflat,
1277  cpl_vector_get(raw_selected[i], iflat));
1278  }
1279  if(cpl_vector_get(raw_selected[i], iflat) == 1)
1280  nused++;
1281  }
1282 
1283  /* Add the master flat statistics keywords */
1284  cpl_propertylist_append_int(qclists[i], "ESO QC FLAT NBADPIX",
1285  hawki_cal_flat_outputs.nb_badpix[i]);
1286  cpl_propertylist_append_double(qclists[i], "ESO QC FLAT NORM",
1287  hawki_cal_flat_outputs.norm[i]) ;
1288  cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDMEAN",
1289  hawki_cal_flat_outputs.med_avg[i]) ;
1290  cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDMED",
1291  hawki_cal_flat_outputs.med_med[i]) ;
1292  cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDSTDEV",
1293  hawki_cal_flat_outputs.med_stdev[i]) ;
1294  cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDMIN",
1295  hawki_cal_flat_outputs.med_min[i]) ;
1296  cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDMAX",
1297  hawki_cal_flat_outputs.med_max[i]) ;
1298  cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDRANG",
1299  hawki_cal_flat_outputs.med_max[i] -
1300  hawki_cal_flat_outputs.med_min[i]);
1301  cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDNRANG",
1302  (hawki_cal_flat_outputs.med_max[i] -
1303  hawki_cal_flat_outputs.med_min[i]) /
1304  hawki_cal_flat_outputs.med_med[i]);
1305  cpl_propertylist_append_int(qclists[i], "ESO QC DATANCOM",
1306  nused) ;
1307 
1308  /* Propagate some keywords from input raw frame extensions */
1309  ext_nb = hawki_get_ext_from_detector(
1310  cpl_frame_get_filename(ref_frame), i+1);
1311  inputlist = cpl_propertylist_load_regexp(
1312  cpl_frame_get_filename(ref_frame), ext_nb,
1313  HAWKI_HEADER_EXT_FORWARD, 0) ;
1314  cpl_propertylist_append(qclists[i], inputlist) ;
1315 
1316  /* Cleaning */
1317  cpl_propertylist_delete(inputlist) ;
1318  }
1319  /* Statistics of the raw images in the QC */
1320  hawki_image_stats_stats(raw_flat_stats, qclists);
1321 
1322  /* Get the used frames */
1323  used_frames = cpl_frameset_duplicate(flat_frames);
1324  for(i = 0; i< cpl_frameset_get_size(calib_frames); ++i)
1325  cpl_frameset_insert(used_frames,
1326  cpl_frame_duplicate(cpl_frameset_get_frame(calib_frames, i)));
1327 
1328  /* Write the flat image */
1329  filename = cpl_sprintf("hawki_cal_flat_set%02d.fits", set_nb) ;
1330  hawki_imagelist_save(set_tot,
1331  parlist,
1332  used_frames,
1333  flat[0],
1334  recipe_name,
1335  HAWKI_CALPRO_FLAT,
1336  HAWKI_PROTYPE_FLAT,
1337  NULL,
1338  (const cpl_propertylist**)qclists,
1339  filename) ;
1340  cpl_free(filename) ;
1341 
1342  /* Write the error map */
1343  filename = cpl_sprintf("hawki_cal_flat_err_set%02d.fits", set_nb) ;
1344  hawki_imagelist_save(set_tot,
1345  parlist,
1346  used_frames,
1347  flat[1],
1348  recipe_name,
1349  HAWKI_CALPRO_FLAT_ERRMAP,
1350  HAWKI_PROTYPE_ERRMAP,
1351  NULL,
1352  (const cpl_propertylist**)qclists,
1353  filename) ;
1354  cpl_free(filename) ;
1355 
1356  /* Write the Cold pixels map */
1357  filename = cpl_sprintf("hawki_cal_flat_bpmflat_set%02d.fits", set_nb) ;
1358  hawki_imagelist_save(set_tot,
1359  parlist,
1360  used_frames,
1361  flat[2],
1362  recipe_name,
1363  HAWKI_CALPRO_BPM_COLD,
1364  HAWKI_PROTYPE_BPM,
1365  NULL,
1366  (const cpl_propertylist**)qclists,
1367  filename) ;
1368  cpl_free(filename) ;
1369 
1370  if (flat[3] != NULL) {
1371  /* Write the recomputed map */
1372  filename=cpl_sprintf("hawki_cal_flat_recomputed_set%02d.fits", set_nb) ;
1373  hawki_imagelist_save(set_tot,
1374  parlist,
1375  used_frames,
1376  flat[3],
1377  recipe_name,
1378  HAWKI_CALPRO_FLAT_RECOMPUTED,
1379  HAWKI_PROTYPE_FLAT,
1380  NULL,
1381  (const cpl_propertylist**)qclists,
1382  filename) ;
1383  cpl_free(filename) ;
1384  }
1385 
1386  /* If the HOT pixel map is passed, merge with the COLD one */
1387  if (bpmdark != NULL) {
1388  if ((bpm = hawki_cal_flat_merge_bpms(bpmdark, flat[2])) == NULL) {
1389  cpl_msg_error(__func__, "Cannot merge bad pixel maps") ;
1390  } else {
1391  filename=cpl_sprintf("hawki_cal_flat_bpm_set%02d.fits", set_nb) ;
1392  /* Get the used frames for statistics */
1393  hawki_imagelist_save(set_tot,
1394  parlist,
1395  used_frames,
1396  bpm,
1397  recipe_name,
1398  HAWKI_CALPRO_BPM,
1399  HAWKI_PROTYPE_BPM,
1400  NULL,
1401  (const cpl_propertylist**)qclists,
1402  filename) ;
1403  cpl_free(filename) ;
1404  cpl_imagelist_delete(bpm) ;
1405  }
1406  }
1407  cpl_frameset_delete(used_frames);
1408 
1409  /* Get the used frames for statistics */
1410  used_frames = cpl_frameset_duplicate(flat_frames);
1411 
1412  /* Write the table with the statistics */
1413  filename = cpl_sprintf("hawki_cal_flat_stats_set%02d.fits", set_nb) ;
1414  hawki_tables_save(set_tot,
1415  parlist,
1416  used_frames,
1417  (const cpl_table **)raw_flat_stats,
1418  recipe_name,
1419  HAWKI_CALPRO_FLAT_STATS,
1420  HAWKI_PROTYPE_FLAT_STATS,
1421  NULL,
1422  (const cpl_propertylist **)qclists,
1423  filename) ;
1424  cpl_free(filename) ;
1425 
1426  if(hawki_cal_flat_config.extra_stats)
1427  {
1428  filename = cpl_sprintf("hawki_cal_flat_stats_ec_set%02d.fits", set_nb);
1429  hawki_tables_save(set_tot,
1430  parlist,
1431  used_frames,
1432  (const cpl_table **)raw_flat_even_column_stats,
1433  recipe_name,
1434  HAWKI_CALPRO_FLAT_STATS_EVEN_COL,
1435  HAWKI_PROTYPE_FLAT_STATS_EVEN_COL,
1436  NULL,
1437  (const cpl_propertylist **)qclists,
1438  filename) ;
1439  cpl_free(filename) ;
1440  filename = cpl_sprintf("hawki_cal_flat_stats_oc_set%02d.fits", set_nb);
1441  hawki_tables_save(set_tot,
1442  parlist,
1443  used_frames,
1444  (const cpl_table **)raw_flat_odd_column_stats,
1445  recipe_name,
1446  HAWKI_CALPRO_FLAT_STATS_ODD_COL,
1447  HAWKI_PROTYPE_FLAT_STATS_ODD_COL,
1448  NULL,
1449  (const cpl_propertylist **)qclists,
1450  filename) ;
1451  cpl_free(filename) ;
1452  filename = cpl_sprintf("hawki_cal_flat_stats_er_set%02d.fits", set_nb);
1453  hawki_tables_save(set_tot,
1454  parlist,
1455  used_frames,
1456  (const cpl_table **)raw_flat_even_row_stats,
1457  recipe_name,
1458  HAWKI_CALPRO_FLAT_STATS_EVEN_ROW,
1459  HAWKI_PROTYPE_FLAT_STATS_EVEN_ROW,
1460  NULL,
1461  (const cpl_propertylist **)qclists,
1462  filename) ;
1463  cpl_free(filename) ;
1464  filename = cpl_sprintf("hawki_cal_flat_stats_or_set%02d.fits", set_nb);
1465  hawki_tables_save(set_tot,
1466  parlist,
1467  used_frames,
1468  (const cpl_table **)raw_flat_odd_row_stats,
1469  recipe_name,
1470  HAWKI_CALPRO_FLAT_STATS_ODD_ROW,
1471  HAWKI_PROTYPE_FLAT_STATS_ODD_ROW,
1472  NULL,
1473  (const cpl_propertylist **)qclists,
1474  filename) ;
1475  cpl_free(filename) ;
1476  }
1477  cpl_frameset_delete(used_frames);
1478 
1479  /* Free and return */
1480  for (i=0 ; i<HAWKI_NB_DETECTORS ; i++) {
1481  cpl_propertylist_delete(qclists[i]) ;
1482  }
1483  cpl_free(qclists) ;
1484  if (cpl_error_get_code()) return -1 ;
1485  return 0;
1486 }
1487 
1488 /*----------------------------------------------------------------------------*/
1495 /*----------------------------------------------------------------------------*/
1496 static int hawki_cal_flat_compare(
1497  const cpl_frame * frame1,
1498  const cpl_frame * frame2)
1499 {
1500  int comparison ;
1501  cpl_propertylist * plist1 ;
1502  cpl_propertylist * plist2 ;
1503  const char * sval1,
1504  * sval2 ;
1505  double dval1, dval2 ;
1506 
1507 
1508  /* Test entries */
1509  if (frame1==NULL || frame2==NULL) return -1 ;
1510 
1511  /* Get property lists */
1512  if ((plist1=cpl_propertylist_load(cpl_frame_get_filename(frame1),
1513  0)) == NULL) {
1514  cpl_msg_error(__func__, "getting header from reference frame");
1515  return -1 ;
1516  }
1517  if ((plist2=cpl_propertylist_load(cpl_frame_get_filename(frame2),
1518  0)) == NULL) {
1519  cpl_msg_error(__func__, "getting header from reference frame");
1520  cpl_propertylist_delete(plist1) ;
1521  return -1 ;
1522  }
1523 
1524  /* Test status */
1525  if (cpl_error_get_code()) {
1526  cpl_propertylist_delete(plist1) ;
1527  cpl_propertylist_delete(plist2) ;
1528  return -1 ;
1529  }
1530 
1531  comparison = 1 ;
1532 
1533  /* Compare filters */
1534  sval1 = hawki_pfits_get_filter(plist1) ;
1535  sval2 = hawki_pfits_get_filter(plist2) ;
1536  if (cpl_error_get_code()) {
1537  cpl_msg_error(__func__, "cannot get the filter");
1538  cpl_propertylist_delete(plist1) ;
1539  cpl_propertylist_delete(plist2) ;
1540  return -1 ;
1541  }
1542  if (strcmp(sval1, sval2)) comparison = 0 ;
1543 
1544  /* Compare DITs */
1545  dval1 = hawki_pfits_get_dit(plist1) ;
1546  dval2 = hawki_pfits_get_dit(plist2) ;
1547  if (cpl_error_get_code()) {
1548  cpl_msg_error(__func__, "cannot get the DIT");
1549  cpl_propertylist_delete(plist1) ;
1550  cpl_propertylist_delete(plist2) ;
1551  return -1 ;
1552  }
1553  if (fabs(dval1-dval2) > 1e-4) comparison = 0 ;
1554 
1555  cpl_propertylist_delete(plist1) ;
1556  cpl_propertylist_delete(plist2) ;
1557  return comparison ;
1558 }
1559 
1560 /*----------------------------------------------------------------------------*/
1567 /*----------------------------------------------------------------------------*/
1568 static cpl_imagelist * hawki_cal_flat_merge_bpms
1569 (const cpl_frame * bpm_orig,
1570  cpl_imagelist * bpm_to_merge)
1571 {
1572  cpl_imagelist * merged ;
1573  cpl_imagelist * bpm_orig_im;
1574  cpl_image * tmp ;
1575  int i ;
1576 
1577  /* Test entries */
1578  if (bpm_orig==NULL || bpm_to_merge ==NULL) return NULL ;
1579 
1580  /* Create merged */
1581  merged = cpl_imagelist_new() ;
1582 
1583  /* Load the bpm_file */
1584  bpm_orig_im = hawki_load_frame(bpm_orig, CPL_TYPE_INT);
1585  if(bpm_orig_im == NULL)
1586  {
1587  cpl_msg_error(__func__, "Cannot read existing bpm %s",
1588  cpl_frame_get_filename(bpm_orig));
1589  cpl_imagelist_delete(merged);
1590  return NULL;
1591  }
1592 
1593  /* Loop on the chips */
1594  for (i=0 ; i<HAWKI_NB_DETECTORS ; i++) {
1595 
1596  /* Merge */
1597  tmp = cpl_image_add_create(cpl_imagelist_get(bpm_orig_im, i),
1598  cpl_imagelist_get(bpm_to_merge, i));
1599  cpl_image_multiply(cpl_imagelist_get(bpm_orig_im, i),
1600  cpl_imagelist_get(bpm_to_merge, i));
1601  cpl_image_subtract(tmp, cpl_imagelist_get(bpm_orig_im, i)) ;
1602 
1603  /* Store */
1604  cpl_imagelist_set(merged, tmp, i) ;
1605  }
1606 
1607  /* Clean-up and return */
1608  cpl_imagelist_delete(bpm_orig_im);
1609  return merged ;
1610 }
1611 
1612 /*----------------------------------------------------------------------------*/
1623 /*----------------------------------------------------------------------------*/
1624 static int hawki_cal_flat_select
1625 (cpl_vector * meds,
1626  cpl_vector * rms,
1627  int auto_flag,
1628  int auto_max_bins,
1629  double min_level,
1630  double max_level,
1631  double max_rms,
1632  int min_nframes,
1633  cpl_vector * selection)
1634 {
1635  double * pselection ;
1636  double * pmeds ;
1637  double * prms ;
1638  int nvals ;
1639  int nsel ;
1640  double first_val, last_val, bin_val, dist, dist_min;
1641  double bin_low_val;
1642  double bin_up_val;
1643  int nbins, ind_closest ;
1644  int ibin;
1645  int ival;
1646 
1647  /* Test entries */
1648  if (meds == NULL) return -1;
1649  if (rms == NULL) return -1;
1650  if (selection == NULL) return -1;
1651  if(cpl_vector_get_size(meds) != cpl_vector_get_size(selection))
1652  {
1653  cpl_msg_error(__func__,
1654  "The meds and selection vectors have not the same size");
1655  return -1;
1656  }
1657 
1658  /* Initialise. All the frames are selected by default */
1659  nvals = cpl_vector_get_size(meds);
1660  pmeds = cpl_vector_get_data(meds);
1661  prms = cpl_vector_get_data(rms);
1662  cpl_vector_fill(selection, 1.0);
1663 
1664  /* Pointer to selection */
1665  pselection = cpl_vector_get_data(selection) ;
1666 
1667  /* First select based on minimum level, if applies */
1668  if (min_level >= 0.0)
1669  {
1670  for (ival=0 ; ival<nvals ; ival++)
1671  {
1672  if (pmeds[ival] < min_level)
1673  pselection[ival] = -1.0 ;
1674  }
1675  }
1676 
1677  /* Second select based on maximum level, if applies */
1678  if (max_level >= 0.0)
1679  {
1680  for (ival=0 ; ival<nvals ; ival++)
1681  {
1682  if (pmeds[ival] > max_level)
1683  pselection[ival] = -1.0 ;
1684  }
1685  }
1686 
1687  /* Third select based on rms maximum level, if applies */
1688  if (max_rms >= 0.0)
1689  {
1690  for (ival=0 ; ival<nvals ; ival++)
1691  {
1692  if (prms[ival] > max_rms)
1693  pselection[ival] = -1.0 ;
1694  }
1695  }
1696 
1697  /* Apply the histogram selection */
1698  if (auto_flag)
1699  {
1700  /* Automatic */
1701  cpl_vector * auto_selection;
1702  auto_selection = cpl_vector_new(nvals);
1703  cpl_vector_fill(auto_selection, -1.0);
1704  cpl_msg_info(__func__, "Automatic dynamic range selection");
1705  first_val = hawki_vector_get_min_select(meds, selection);
1706  last_val = hawki_vector_get_max_select(meds, selection);
1707 
1708  nbins = nvals ;
1709  if (auto_max_bins > 0 && auto_max_bins < nvals) nbins = auto_max_bins;
1710  for (ibin=0 ; ibin<nbins ; ibin++)
1711  {
1712  if(ibin == 0)
1713  bin_val = first_val + (last_val-first_val)*(ibin+1)/nbins ;
1714  else if(ibin == nbins - 1)
1715  bin_val = first_val + (last_val-first_val)*(ibin)/nbins ;
1716  else
1717  bin_val = first_val + (last_val-first_val)*(ibin+0.5)/nbins ;
1718  bin_low_val = first_val + (last_val-first_val)*(ibin)/nbins ;
1719  bin_up_val = first_val + (last_val-first_val)*(ibin+1)/nbins ;
1720 
1721  /* Select the closest */
1722  dist_min = fabs(pmeds[0] - bin_val) ;
1723  ind_closest = -1;
1724  for (ival=0 ; ival<nvals ; ival++)
1725  {
1726  dist = fabs(pmeds[ival] - bin_val) ;
1727  if (dist < dist_min && pselection[ival] != -1)
1728  {
1729  dist_min = dist;
1730  ind_closest = ival;
1731  }
1732  }
1733  if(ind_closest != -1)
1734  cpl_vector_set(auto_selection, ind_closest, 1.0);
1735  }
1736  /* Use the auto_selection */
1737  cpl_vector_copy(selection, auto_selection);
1738  cpl_vector_delete(auto_selection);
1739  }
1740 
1741  /* Print and count the selected frames */
1742  nsel = 0;
1743  cpl_msg_indent_more();
1744  for (ival=0 ; ival<nvals ; ival++)
1745  {
1746  if(pselection[ival] != -1)
1747  {
1748  cpl_msg_info(__func__, "Selected frame %d", ival+1) ;
1749  nsel++;
1750  }
1751  }
1752  cpl_msg_indent_less();
1753 
1754  /* Check the number of selected against min_nframes */
1755  if (nsel == 0) {
1756  cpl_msg_error(__func__, "No frame selected. Check selection criteria");
1757  return -1;
1758  }
1759  if (min_nframes >= 0) {
1760  if (nsel < min_nframes) {
1761  cpl_msg_error(__func__, "Not enough selected frames (%d < %d)",
1762  nsel, min_nframes) ;
1763  return -1;
1764  }
1765  }
1766 
1767  return 0;
1768 }
1769 
1770 /*----------------------------------------------------------------------------*/
1777 /*----------------------------------------------------------------------------*/
1778 static cpl_vector * hawki_cal_flat_extract_vector(
1779  cpl_vector * in,
1780  cpl_vector * selection)
1781 {
1782  int nvals ;
1783  cpl_vector * selected ;
1784  double * pin ;
1785  double * pselection ;
1786  double * pselected ;
1787  int nselected ;
1788  int i ;
1789 
1790  /* Test entries */
1791  if (in == NULL) return NULL ;
1792  if (selection == NULL) return NULL ;
1793 
1794  /* Initialise */
1795  nvals = cpl_vector_get_size(in) ;
1796  nselected = 0 ;
1797  pin = cpl_vector_get_data(in) ;
1798  pselection = cpl_vector_get_data(selection) ;
1799 
1800  /* Test entries */
1801  if (nvals != cpl_vector_get_size(selection)) return NULL ;
1802 
1803  /* Count the selected values */
1804  for (i=0 ; i<nvals ; i++) {
1805  if (pselection[i] > 0.0) nselected++ ;
1806  }
1807  if (nselected == 0) return NULL ;
1808 
1809  /* Create the selected vector */
1810  selected = cpl_vector_new(nselected) ;
1811  pselected = cpl_vector_get_data(selected) ;
1812 
1813  nselected = 0 ;
1814  for (i=0 ; i<nvals ; i++) {
1815  if (pselection[i] > 0.0) {
1816  pselected[nselected] = pin[i] ;
1817  nselected++ ;
1818  }
1819  }
1820  return selected ;
1821 }
1822 
1823 static int hawki_cal_flat_retrieve_input_param
1824 (cpl_parameterlist * parlist)
1825 {
1826  const char * sval;
1827  cpl_parameter * par;
1828  int idet;
1829 
1830  /* Initialise */
1831  par = NULL ;
1832  for (idet=0 ; idet<HAWKI_NB_DETECTORS ; idet++) {
1833  hawki_cal_flat_outputs.nb_badpix[idet] = -1 ;
1834  hawki_cal_flat_outputs.med_stdev[idet] = -1.0 ;
1835  hawki_cal_flat_outputs.med_avg[idet] = -1.0 ;
1836  hawki_cal_flat_outputs.med_med[idet] = -1.0 ;
1837  hawki_cal_flat_outputs.med_min[idet] = -1.0 ;
1838  hawki_cal_flat_outputs.med_max[idet] = -1.0 ;
1839  }
1840 
1841  /* Retrieve input parameters */
1842  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.zone") ;
1843  sval = cpl_parameter_get_string(par) ;
1844  if (sscanf(sval, "%d,%d,%d,%d",
1845  &hawki_cal_flat_config.llx,
1846  &hawki_cal_flat_config.lly,
1847  &hawki_cal_flat_config.urx,
1848  &hawki_cal_flat_config.ury)!=4)
1849  {
1850  return -1 ;
1851  }
1852  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.sigma_badres") ;
1853  hawki_cal_flat_config.sigma_badres = cpl_parameter_get_double(par) ;
1854  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.sigma_bpm") ;
1855  hawki_cal_flat_config.sigma_bpm = cpl_parameter_get_double(par) ;
1856  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.lowval_bpm") ;
1857  hawki_cal_flat_config.lowval_bpm = cpl_parameter_get_double(par) ;
1858  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.highval_bpm") ;
1859  hawki_cal_flat_config.highval_bpm = cpl_parameter_get_double(par) ;
1860  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.normalise") ;
1861  hawki_cal_flat_config.normalise = cpl_parameter_get_bool(par) ;
1862  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.second_pass") ;
1863  hawki_cal_flat_config.second_pass = cpl_parameter_get_bool(par) ;
1864  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.select_auto") ;
1865  hawki_cal_flat_config.select_auto = cpl_parameter_get_bool(par) ;
1866  par = cpl_parameterlist_find(parlist,
1867  "hawki.hawki_cal_flat.select_auto_max_bins") ;
1868  hawki_cal_flat_config.select_auto_max_bins = cpl_parameter_get_int(par);
1869  par = cpl_parameterlist_find(parlist,
1870  "hawki.hawki_cal_flat.select_min_level") ;
1871  sval = cpl_parameter_get_string(par);
1872  if (sscanf(sval, "%lf,%lf,%lf,%lf",
1873  &hawki_cal_flat_config.select_min_level[0],
1874  &hawki_cal_flat_config.select_min_level[1],
1875  &hawki_cal_flat_config.select_min_level[2],
1876  &hawki_cal_flat_config.select_min_level[3])!=4)
1877  {
1878  if (sscanf(sval, "%f", &hawki_cal_flat_config.select_min_level[0])!=1)
1879  {
1880  return -1;
1881  }
1882  else
1883  {
1884  hawki_cal_flat_config.select_min_level[1] =
1885  hawki_cal_flat_config.select_min_level[0];
1886  hawki_cal_flat_config.select_min_level[2] =
1887  hawki_cal_flat_config.select_min_level[0];
1888  hawki_cal_flat_config.select_min_level[3] =
1889  hawki_cal_flat_config.select_min_level[0];
1890  }
1891  }
1892  par = cpl_parameterlist_find(parlist,
1893  "hawki.hawki_cal_flat.select_max_level") ;
1894  sval = cpl_parameter_get_string(par);
1895  if (sscanf(sval, "%lf,%lf,%lf,%lf",
1896  &hawki_cal_flat_config.select_max_level[0],
1897  &hawki_cal_flat_config.select_max_level[1],
1898  &hawki_cal_flat_config.select_max_level[2],
1899  &hawki_cal_flat_config.select_max_level[3])!=4)
1900  {
1901  if (sscanf(sval, "%f", &hawki_cal_flat_config.select_max_level[0])!=1)
1902  {
1903  return -1;
1904  }
1905  else
1906  {
1907  hawki_cal_flat_config.select_max_level[1] =
1908  hawki_cal_flat_config.select_max_level[0];
1909  hawki_cal_flat_config.select_max_level[2] =
1910  hawki_cal_flat_config.select_max_level[0];
1911  hawki_cal_flat_config.select_max_level[3] =
1912  hawki_cal_flat_config.select_max_level[0];
1913  }
1914  }
1915  par = cpl_parameterlist_find(parlist,
1916  "hawki.hawki_cal_flat.select_max_rms") ;
1917  sval = cpl_parameter_get_string(par);
1918  if (sscanf(sval, "%lf,%lf,%lf,%lf",
1919  hawki_cal_flat_config.select_max_rms,
1920  hawki_cal_flat_config.select_max_rms+1,
1921  hawki_cal_flat_config.select_max_rms+2,
1922  hawki_cal_flat_config.select_max_rms+3)!=4)
1923  {
1924  if (sscanf(sval, "%f", &hawki_cal_flat_config.select_max_rms[0])!=1)
1925  {
1926  return -1;
1927  }
1928  else
1929  {
1930  hawki_cal_flat_config.select_max_rms[1] =
1931  hawki_cal_flat_config.select_max_rms[0];
1932  hawki_cal_flat_config.select_max_rms[2] =
1933  hawki_cal_flat_config.select_max_rms[0];
1934  hawki_cal_flat_config.select_max_rms[3] =
1935  hawki_cal_flat_config.select_max_rms[0];
1936  }
1937  }
1938  par = cpl_parameterlist_find(parlist,
1939  "hawki.hawki_cal_flat.select_min_nframes") ;
1940  hawki_cal_flat_config.select_min_nframes = cpl_parameter_get_int(par) ;
1941  par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.extra_stats") ;
1942  hawki_cal_flat_config.extra_stats = cpl_parameter_get_bool(par) ;
1943 
1944  if(hawki_cal_flat_config.select_auto_max_bins != -1 &&
1945  !hawki_cal_flat_config.select_auto)
1946  {
1947  cpl_msg_error(__func__,"Max bins is only allowed with select_auto");
1948  return -1;
1949  }
1950 
1951  return 0;
1952 }