GRAVI Pipeline Reference Manual  0.7.11
gravi_all_flat.c
1 /* $Id: gravi_all_flat.c,v 1.29 2009/02/10 09:16:12 llundin Exp $
2  *
3  * This file is part of the GRAVI Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: llundin $
23  * $Date: 2009/02/10 09:16:12 $
24  * $Revision: 1.29 $
25  * $Name: $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include <cpl.h>
37 #include <stdio.h>
38 #include <string.h>
39 
40 #include "gravi_calib.h"
41 #include "gravi_utils.h"
42 #include "gravi_pfits.h"
43 #include "gravi_dfs.h"
44 #include <time.h>
45 #include "gravi_data.h"
46 
47 
48 /*-----------------------------------------------------------------------------
49  Private function prototypes
50  -----------------------------------------------------------------------------*/
51 
52 static int gravi_all_flat_create(cpl_plugin *);
53 static int gravi_all_flat_exec(cpl_plugin *);
54 static int gravi_all_flat_destroy(cpl_plugin *);
55 static int gravi_all_flat(cpl_frameset *, const cpl_parameterlist *);
56 
57 /*-----------------------------------------------------------------------------
58  Static variables
59  -----------------------------------------------------------------------------*/
60 
61 static char gravi_all_flat_description[] =
62 "It produces the master flat containing : \n"
63 " - the profile of each spectrum of the SC beam combiner,\n"
64 " - the mean spectrum for SC and FT.\n"
65 "This recipe also produces a bad pixel map, where each pixels where \n"
66 "Dark_rms > rms(Dark_rms_i)*bad-dark-threshold are flag as 1.\n"
67 "\n"
68 "Description DO category\n"
69 "Required input :\n"
70 " Raw flat file (4 files) " GRAVI_FLAT "\n"
71 " Raw dark file " GRAVI_DARK "\n"
72 "or optionally\n"
73 " Master dark " DARK "\n"
74 " Bad pixel map " BAD "\n"
75 "Output :\n"
76 " Profile map " FLAT "\n"
77 " Bad pixel map " BAD "\n"
78 "\n";
79 
80 /*-----------------------------------------------------------------------------
81  Function code
82  -----------------------------------------------------------------------------*/
83 
84 /*----------------------------------------------------------------------------*/
94 /*----------------------------------------------------------------------------*/
95 int cpl_plugin_get_info(cpl_pluginlist * list)
96 {
97  cpl_recipe * recipe = cpl_calloc(1, sizeof *recipe );
98  cpl_plugin * plugin = &recipe->interface;
99 
100  if (cpl_plugin_init(plugin,
101  CPL_PLUGIN_API,
102  GRAVI_BINARY_VERSION,
103  CPL_PLUGIN_TYPE_RECIPE,
104  "gravi_all_flat",
105  "This recipe is used to reduce the flat data.",
106  gravi_all_flat_description,
107  "Firstname Lastname",
108  PACKAGE_BUGREPORT,
109  gravi_get_license(),
110  gravi_all_flat_create,
111  gravi_all_flat_exec,
112  gravi_all_flat_destroy)) {
113  cpl_msg_error(cpl_func, "Plugin initialization failed");
114  (void)cpl_error_set_where(cpl_func);
115  return 1;
116  }
117 
118  if (cpl_pluginlist_append(list, plugin)) {
119  cpl_msg_error(cpl_func, "Error adding plugin to list");
120  (void)cpl_error_set_where(cpl_func);
121  return 1;
122  }
123 
124  return 0;
125 }
126 
127 /*----------------------------------------------------------------------------*/
135 /*----------------------------------------------------------------------------*/
136 static int gravi_all_flat_create(cpl_plugin * plugin)
137 {
138  cpl_recipe * recipe;
139  cpl_parameter * p;
140 
141  /* Do not create the recipe if an error code is already set */
142  if (cpl_error_get_code() != CPL_ERROR_NONE) {
143  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
144  cpl_func, __LINE__, cpl_error_get_where());
145  return (int)cpl_error_get_code();
146  }
147 
148  if (plugin == NULL) {
149  cpl_msg_error(cpl_func, "Null plugin");
150  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
151  }
152 
153  /* Verify plugin type */
154  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
155  cpl_msg_error(cpl_func, "Plugin is not a recipe");
156  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
157  }
158 
159  /* Get the recipe */
160  recipe = (cpl_recipe *)plugin;
161 
162  /* Create the parameters list in the cpl_recipe object */
163  recipe->parameters = cpl_parameterlist_new();
164  if (recipe->parameters == NULL) {
165  cpl_msg_error(cpl_func, "Parameter list allocation failed");
166  cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT);
167  }
168 
169  /* Fill the parameters list */
170  /* -- */
171  p = cpl_parameter_new_value("gravi.gravi_all_flat."
172  "profile_width", CPL_TYPE_INT, "Width on which the "
173  "spectral element is fitted (High Resolution mode)",
174  "gravi.gravi_all_flat", 6);
175  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "profile-width");
176  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
177  cpl_parameterlist_append(recipe->parameters, p);
178 
179  p = cpl_parameter_new_value("gravi."
180  "flat_param.Bad_dark_threshold", CPL_TYPE_INT, "Rms noise "
181  "factor for bad pixel detection", "gravi.gravi_all_flat", 10);
182  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "bad-dark-threshold");
183  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
184  cpl_parameterlist_append(recipe->parameters, p);
185 
186 
187  return 0;
188 }
189 
190 /*----------------------------------------------------------------------------*/
196 /*----------------------------------------------------------------------------*/
197 static int gravi_all_flat_exec(cpl_plugin * plugin)
198 {
199 
200  cpl_recipe * recipe;
201  int recipe_status;
202  cpl_errorstate initial_errorstate = cpl_errorstate_get();
203 
204  /* Return immediately if an error code is already set */
205  if (cpl_error_get_code() != CPL_ERROR_NONE) {
206  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
207  cpl_func, __LINE__, cpl_error_get_where());
208  return (int)cpl_error_get_code();
209  }
210 
211  if (plugin == NULL) {
212  cpl_msg_error(cpl_func, "Null plugin");
213  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
214  }
215 
216  /* Verify plugin type */
217  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
218  cpl_msg_error(cpl_func, "Plugin is not a recipe");
219  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
220  }
221 
222  /* Get the recipe */
223  recipe = (cpl_recipe *)plugin;
224 
225  /* Verify parameter and frame lists */
226  if (recipe->parameters == NULL) {
227  cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
228  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
229  }
230  if (recipe->frames == NULL) {
231  cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
232  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
233  }
234 
235  /* Invoke the recipe */
236  recipe_status = gravi_all_flat(recipe->frames, recipe->parameters);
237 
238  /* Ensure DFS-compliance of the products */
239  if (cpl_dfs_update_product_header(recipe->frames)) {
240  if (!recipe_status) recipe_status = (int)cpl_error_get_code();
241  }
242 
243  if (!cpl_errorstate_is_equal(initial_errorstate)) {
244  /* Dump the error history since recipe execution start.
245  At this point the recipe cannot recover from the error */
246  cpl_errorstate_dump(initial_errorstate, CPL_FALSE, NULL);
247  }
248 
249  return recipe_status;
250 }
251 
252 /*----------------------------------------------------------------------------*/
258 /*----------------------------------------------------------------------------*/
259 static int gravi_all_flat_destroy(cpl_plugin * plugin)
260 {
261  cpl_recipe * recipe;
262 
263  if (plugin == NULL) {
264  cpl_msg_error(cpl_func, "Null plugin");
265  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
266  }
267 
268  /* Verify plugin type */
269  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
270  cpl_msg_error(cpl_func, "Plugin is not a recipe");
271  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
272  }
273 
274  /* Get the recipe */
275  recipe = (cpl_recipe *)plugin;
276 
277  cpl_parameterlist_delete(recipe->parameters);
278 
279  return 0;
280 }
281 
282 
283 /*----------------------------------------------------------------------------*/
290 /*----------------------------------------------------------------------------*/
291 static int gravi_all_flat(cpl_frameset * frameset,
292  const cpl_parameterlist * parlist)
293 {
294  cpl_frameset * flat_frameset, * dark_frameset, * darkcalib_frameset, * badpix_frameset;
295  cpl_propertylist * applist, * primary_hdr, * bad_primary_hdr;
296  cpl_frame * frame, *_frame;
297  const char * filename, * filename_, * temp;
298  char * output;
299  gravi_data * profile_map, * dark_file, * dark_map, * bad_map;
300  gravi_data ** flat_data;
301  double gain_ft, gain_sc;
302  int nb_frame, i;
303 
304  /* Identify the RAW and CALIB frames in the input frameset */
305  cpl_ensure_code(gravi_dfs_set_groups(frameset) == CPL_ERROR_NONE,
306  cpl_error_get_code()) ;
307 
308  /* - Extract the sets of frame for the flat */
309  flat_frameset = gravi_frameset_extract_flat(frameset);
310  darkcalib_frameset = gravi_frameset_extract_dark_file(frameset);
311  dark_frameset = gravi_frameset_extract_dark(frameset);
312  badpix_frameset = gravi_frameset_extract_badpix(frameset);
313  if (cpl_frameset_is_empty(flat_frameset) ||
314  ((cpl_frameset_is_empty(dark_frameset)) &&
315  cpl_frameset_is_empty(darkcalib_frameset))) {
316  /* To use this recipe the frameset must contain at least
317  * one flat frame. */
318  cpl_frameset_delete(flat_frameset);
319  cpl_frameset_delete(dark_frameset);
320  cpl_frameset_delete(darkcalib_frameset);
321  cpl_frameset_delete(badpix_frameset);
322  return (int)cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
323  "No dark or flat frame on the inputs frameset") ;
324  }
325 
326 // //test
327 // gravi_data * output_data = gravi_data_new(0);
328 // cpl_table * oi_target = cpl_table_new (2);
329 // cpl_table_new_column (oi_target, "TARGET_ID", CPL_TYPE_INT);
330 // cpl_table_set_column_savetype(oi_target, "TARGET_ID", CPL_TYPE_SHORT);
331 // cpl_table_set_int (oi_target, "TARGET_ID", 0, 1);
332 // cpl_table_set_int (oi_target, "TARGET_ID", 1, 2);
333 // cpl_table_set_column_savetype(oi_target, "TARGET_ID", CPL_TYPE_SHORT);
334 // cpl_table_new_column (oi_target, "TARGET", CPL_TYPE_STRING);
335 // cpl_table_set_string(oi_target, "TARGET", 0, "AB");
336 // cpl_table_set_column_format(oi_target, "TARGET", "%s8");
337 // cpl_table_new_column (oi_target, "RAEP0", CPL_TYPE_DOUBLE);
338 // cpl_table_new_column (oi_target, "DECEP0", CPL_TYPE_DOUBLE);
339 // cpl_table_set_double (oi_target, "RAEP0", 0, 12);
340 // cpl_table_set_double (oi_target, "RAEP0", 1, 13);
341 // cpl_table_set_double (oi_target, "DECEP0", 0, 14);
342 // cpl_table_set_double (oi_target, "DECEP0", 1, 15);
343 //
344 // gravi_data_add(output_data, cpl_propertylist_new(), oi_target);
345 // gravi_data_save_data(output_data, "test2.fits", CPL_IO_CREATE );
346 // cpl_table_save(oi_target, NULL, cpl_propertylist_new(), "test.fits", CPL_IO_CREATE);
347 // cpl_table_save(gravi_data_get_table_x(output_data, 0), NULL, cpl_propertylist_new(), "test3.fits", CPL_IO_CREATE);
348 
349 
350 
351  /* Get the number of the flat frame contained in the frameset */
352 
353  nb_frame = cpl_frameset_get_size(flat_frameset);
354  /* Check that the number of flat frames is upper than 4 */
355  if (nb_frame < 4){
356  cpl_frameset_delete(flat_frameset);
357  cpl_frameset_delete(dark_frameset);
358  cpl_frameset_delete(darkcalib_frameset);
359  cpl_frameset_delete(badpix_frameset);
360  return (int)cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
361  "The number of flat frames must be more than 4");
362  }
363 
364  /* Extract the dark map */
365  if (!cpl_frameset_is_empty(dark_frameset)){
366  _frame = cpl_frameset_get_position(dark_frameset, 0);
367  filename_ = cpl_frame_get_filename(_frame);
368  temp = strrchr(filename_, '/');
369  filename = temp ? temp + 1 : filename_;
370  cpl_msg_info (cpl_func, "This file %s is a dark file", filename);
371 
372  dark_file = gravi_data_load(filename_);
373  dark_map = gravi_compute_dark(dark_file);
374 
375  gravi_data_delete(dark_file);
376  }
377  else if (!cpl_frameset_is_empty(darkcalib_frameset)){
378  _frame = cpl_frameset_get_position(darkcalib_frameset, 0);
379  cpl_frameset_insert (dark_frameset, cpl_frame_duplicate (_frame));
380  filename_ = cpl_frame_get_filename(_frame);
381  temp = strrchr(filename_, '/');
382  filename = temp ? temp + 1 : filename_;
383 
384  cpl_msg_info (cpl_func, "This file %s is a dark file already "
385  "computed", filename);
386  dark_map = gravi_data_load(filename_);
387  }
388  else {
389  cpl_frameset_delete(flat_frameset);
390  cpl_frameset_delete(dark_frameset);
391  cpl_frameset_delete(darkcalib_frameset);
392  cpl_frameset_delete(badpix_frameset);
393 
394  gravi_data_delete(dark_map);
395  return (int)cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
396  "No dark frames in the frameset");
397  }
398 
399  /* Compute the bad pixel map */
400  if (!cpl_frameset_is_empty(badpix_frameset)){
401  _frame = cpl_frameset_get_position(badpix_frameset, 0);
402  filename_ = cpl_frame_get_filename(_frame);
403  temp = strrchr(filename_, '/');
404  filename = temp ? temp + 1 : filename_;
405  cpl_msg_info (cpl_func, "This file %s is a bad pixel map", filename);
406 
407  bad_map = gravi_data_load(filename_);
408  }
409 
410  else {
411 
412  cpl_msg_info (cpl_func, "This file %s is a bad pixel map", filename);
413  bad_map = gravi_compute_badpix(dark_map, parlist);
414 
415  if (cpl_error_get_code()) {
416  gravi_data_delete(dark_map);
417  gravi_data_delete(bad_map);
418 
419  cpl_frameset_delete(flat_frameset);
420  cpl_frameset_delete(dark_frameset);
421  cpl_frameset_delete(darkcalib_frameset);
422  cpl_frameset_delete(badpix_frameset);
423  return (int)cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_OUTPUT,
424  "Error while computing the bad pixel map");
425 
426  }
427  /* Save the bad pixel map */
428  bad_primary_hdr = gravi_data_get_propertylist (bad_map,
429  GRAVI_PRIMARY_HDR_NAME_EXT);
430  applist = gravi_propertylist_get_qc (bad_primary_hdr);
431 
432  cpl_propertylist_append_string(applist, CPL_DFS_PRO_CATG, BAD);
433 
434  frame = cpl_frameset_get_position (dark_frameset, 0);
435  temp = strrchr(filename, '/');
436  filename = temp?temp+1:filename;
437 
438  // output = cpl_sprintf("badpix.%s", filename);
439  output = gravi_data_product_name (filename, "badpix");
440 
441  if (gravi_data_save (bad_map, frameset, output, parlist,
442  dark_frameset, frame, "gravi_all_flat", applist)
443  != CPL_ERROR_NONE){
444 
445  gravi_data_delete(bad_map);
446 
447  gravi_data_delete(dark_map);
448  cpl_frameset_delete(flat_frameset);
449  cpl_frameset_delete(dark_frameset);
450  cpl_frameset_delete(darkcalib_frameset);
451  cpl_propertylist_delete(applist);
452  cpl_frameset_delete(badpix_frameset);
453  return (int) cpl_error_set_message(cpl_func,
454  CPL_ERROR_ILLEGAL_OUTPUT, "Could not save the bad pixel map");
455  }
456  cpl_propertylist_delete (applist);
457  cpl_free (output);
458  }
459 
460  /* Extract the gravi datas from the flat frame set and the dark map from
461  * the dark frame set */
462 
463  flat_data = cpl_malloc(nb_frame * sizeof(gravi_data *));
464 
465  for(i = 0; i < nb_frame; i++){
466  frame = cpl_frameset_get_position(flat_frameset, i);
467  filename = cpl_frame_get_filename(frame);
468 
469  flat_data[i] = gravi_data_load(filename);
470  }
471 
472  /* Compute the flat calib data of a set of flat frames */
473 
474  /* Compute the profile map */
475 
476  profile_map = gravi_compute_profile(flat_data, dark_map, bad_map,
477  nb_frame, parlist);
478 
479  if (profile_map == NULL) {
480 
481  gravi_data_delete(dark_map);
482  gravi_data_delete(profile_map);
483  for(i = 0; i < 4; i++){
484  gravi_data_delete(flat_data[i]);
485  }
486  cpl_free(flat_data);
487  cpl_frameset_delete(flat_frameset);
488  cpl_frameset_delete(dark_frameset);
489  cpl_frameset_delete(darkcalib_frameset);
490  cpl_frameset_delete(badpix_frameset);
491  return (int)cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_OUTPUT,
492  "Error while computing the profile map");
493  }
494 
495  /* Compute the gain */
496  clock_t start, end;
497  start = clock();
498  gain_ft = gravi_compute_gain(flat_data, nb_frame, dark_map, bad_map,
499  GRAVI_IMAGING_DATA_FT_EXT);
500  cpl_msg_info (cpl_func, "QC_MEANGAIN_FT = %e", gain_ft);
501 
502  gain_sc = gravi_compute_gain(flat_data, nb_frame, dark_map, bad_map,
503  GRAVI_IMAGING_DATA_SC_EXT);
504  cpl_msg_info (cpl_func, "QC_MEANGAIN_SC = %e", gain_sc);
505  end = clock();
506  printf ("Execution time gravi_compute_gain : %f\n", (end - start) / (double)CLOCKS_PER_SEC);
507 
508  gravi_data_delete(bad_map);
509 
510  for(i = 0; i < nb_frame; i++){
511  gravi_data_delete(flat_data[i]);
512  }
513  cpl_free(flat_data);
514  if (cpl_error_get_code()) {
515  gravi_data_delete(dark_map);
516  gravi_data_delete(profile_map);
517  cpl_frameset_delete(flat_frameset);
518  cpl_frameset_delete(dark_frameset);
519  cpl_frameset_delete(darkcalib_frameset);
520  cpl_frameset_delete(badpix_frameset);
521  return (int)cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_OUTPUT,
522  "Error while computing the gain map");
523  }
524 
525  primary_hdr = gravi_data_get_propertylist(profile_map,
526  GRAVI_PRIMARY_HDR_NAME_EXT);
527 
528  /* Add QC parameters */
529  cpl_propertylist_append_double (primary_hdr, QC_MEANGAIN_SC, gain_sc);
530  cpl_propertylist_set_comment( primary_hdr, QC_MEANGAIN_SC, "[ADU/e-] Mean gain of SC detector" );
531  cpl_propertylist_append_double (primary_hdr, QC_MEANGAIN_FT, gain_ft);
532  cpl_propertylist_set_comment( primary_hdr, QC_MEANGAIN_FT, "[ADU/e-] Mean gain of FT detector" );
533 
534  /* Create product frame, add DataFlow keywords, save the file, log the
535  * saved file in the input frameset*/
536  applist = gravi_propertylist_get_qc(primary_hdr);
537  cpl_propertylist_append_string(applist, CPL_DFS_PRO_CATG, FLAT);
538 
539  frame = cpl_frameset_get_position (flat_frameset, 0);
540 
541  temp = strrchr(filename, '/');
542 
543  // output = cpl_sprintf("master_profile.%s", filename);
544  output = gravi_data_product_name (filename, "flat");
545  if (gravi_data_save (profile_map, frameset, output, parlist,
546  frameset, frame, "gravi_all_flat", applist)
547  != CPL_ERROR_NONE){
548 
549  gravi_data_delete(profile_map);
550  gravi_data_delete(dark_map);
551  cpl_frameset_delete(flat_frameset);
552  cpl_frameset_delete(dark_frameset);
553  cpl_frameset_delete(darkcalib_frameset);
554  cpl_frameset_delete(badpix_frameset);
555  cpl_propertylist_delete(applist);
556  return (int) cpl_error_set_message(cpl_func,
557  CPL_ERROR_ILLEGAL_OUTPUT, "Could not save the profile_map");
558  }
559  cpl_propertylist_delete(applist);
560  cpl_free (output);
561 
562  /* Deallocation of all variables */
563 
564 
565  gravi_data_delete(profile_map);
566  gravi_data_delete(dark_map);
567  cpl_frameset_delete(flat_frameset);
568  cpl_frameset_delete(badpix_frameset);
569  cpl_frameset_delete(darkcalib_frameset);
570  cpl_frameset_delete(dark_frameset);
571 
572  return (int)cpl_error_get_code();
573 }
574 
575