HAWKI Pipeline Reference Manual  1.8.12
hawki_step_basic_calib.c
1 /* $Id: hawki_step_basic_calib.c,v 1.13 2011/01/31 11:03:23 cgarcia Exp $
2  *
3  * This file is part of the HAWKI Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: cgarcia $
23  * $Date: 2011/01/31 11:03:23 $
24  * $Revision: 1.13 $
25  * $Name: hawki-1_8_12 $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include <string.h>
37 #include <math.h>
38 #include <cpl.h>
39 
40 #include "irplib_utils.h"
41 #include "hawki_utils.h"
42 #include "hawki_calib.h"
43 #include "hawki_load.h"
44 #include "hawki_save.h"
45 #include "hawki_pfits.h"
46 #include "hawki_dfs.h"
47 
48 /*-----------------------------------------------------------------------------
49  Define
50  -----------------------------------------------------------------------------*/
51 
52 #define NEGLIG_OFF_DIFF 0.1
53 #define SQR(x) ((x)*(x))
54 
55 /*-----------------------------------------------------------------------------
56  Functions prototypes
57  -----------------------------------------------------------------------------*/
58 
59 static int hawki_step_basic_calib_create(cpl_plugin *) ;
60 static int hawki_step_basic_calib_exec(cpl_plugin *) ;
61 static int hawki_step_basic_calib_destroy(cpl_plugin *) ;
62 static int hawki_step_basic_calib(cpl_parameterlist *, cpl_frameset *) ;
63 
64 static int hawki_step_basic_calib_applycal_qc_save
65 (cpl_frameset * raw_target,
66  const cpl_frame * flat,
67  const cpl_frame * dark,
68  const cpl_frame * bpm,
69  const char * filename_postfix,
70  const char * procat,
71  const char * protype,
72  cpl_parameterlist * recipe_parlist,
73  cpl_frameset * recipe_framelist);
74 static void hawki_step_basic_calib_qc(void);
75 static int hawki_step_basic_calib_save
76 (cpl_imagelist * reduced,
77  const char * filename_postfix,
78  const char * procat,
79  const char * protype,
80  int iserie,
81  cpl_frameset * used_frameset,
82  cpl_parameterlist * recipe_parlist,
83  cpl_frameset * recipe_framelist);
84 
85 /*-----------------------------------------------------------------------------
86  Static variables
87  -----------------------------------------------------------------------------*/
88 
89 static char hawki_step_basic_calib_description[] =
90 "hawki_step_basic_calib -- hawki basic reduction utility (flat, dark).\n"
91 "The files listed in the Set Of Frames (sof-file) must be tagged:\n"
92 "raw-file.fits "HAWKI_IMG_JITTER_RAW" or\n"
93 "raw-file.fits "HAWKI_IMG_JITTER_SKY_RAW" \n"
94 "flat-file.fits "HAWKI_CALPRO_FLAT" \n"
95 "dark-file.fits "HAWKI_CALPRO_DARK" \n"
96 "bpm-file.fits "HAWKI_CALPRO_BPM" \n";
97 
98 /*-----------------------------------------------------------------------------
99  Functions code
100  -----------------------------------------------------------------------------*/
101 
102 /*----------------------------------------------------------------------------*/
110 /*----------------------------------------------------------------------------*/
111 int cpl_plugin_get_info(cpl_pluginlist * list)
112 {
113  cpl_recipe * recipe = cpl_calloc(1, sizeof(*recipe)) ;
114  cpl_plugin * plugin = &recipe->interface ;
115 
116  cpl_plugin_init(plugin,
117  CPL_PLUGIN_API,
118  HAWKI_BINARY_VERSION,
119  CPL_PLUGIN_TYPE_RECIPE,
120  "hawki_step_basic_calib",
121  "Basic reduction recipe",
122  hawki_step_basic_calib_description,
123  "Cesar Enrique Garcia Dabo",
124  "cgarcia@eso.org",
126  hawki_step_basic_calib_create,
127  hawki_step_basic_calib_exec,
128  hawki_step_basic_calib_destroy) ;
129 
130  cpl_pluginlist_append(list, plugin) ;
131 
132  return 0;
133 }
134 
135 /*----------------------------------------------------------------------------*/
144 /*----------------------------------------------------------------------------*/
145 static int hawki_step_basic_calib_create(cpl_plugin * plugin)
146 {
147  cpl_recipe * recipe ;
148  /* cpl_parameter * p ; */
149 
150  /* Get the recipe out of the plugin */
151  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
152  recipe = (cpl_recipe *)plugin ;
153  else return -1 ;
154 
155  /* Create the parameters list in the cpl_recipe object */
156  recipe->parameters = cpl_parameterlist_new() ;
157 
158  /* Fill the parameters list */
159  /* None.. */
160 
161  /* Return */
162  return 0;
163 }
164 
165 /*----------------------------------------------------------------------------*/
171 /*----------------------------------------------------------------------------*/
172 static int hawki_step_basic_calib_exec(cpl_plugin * plugin)
173 {
174  cpl_recipe * recipe ;
175 
176  /* Get the recipe out of the plugin */
177  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
178  recipe = (cpl_recipe *)plugin ;
179  else return -1 ;
180 
181  /* Issue a banner */
183 
184  return hawki_step_basic_calib(recipe->parameters, recipe->frames) ;
185 }
186 
187 /*----------------------------------------------------------------------------*/
193 /*----------------------------------------------------------------------------*/
194 static int hawki_step_basic_calib_destroy(cpl_plugin * plugin)
195 {
196  cpl_recipe * recipe ;
197 
198  /* Get the recipe out of the plugin */
199  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
200  recipe = (cpl_recipe *)plugin ;
201  else return -1 ;
202 
203  cpl_parameterlist_delete(recipe->parameters) ;
204  return 0 ;
205 }
206 
207 /*----------------------------------------------------------------------------*/
214 /*----------------------------------------------------------------------------*/
215 static int hawki_step_basic_calib(
216  cpl_parameterlist * parlist,
217  cpl_frameset * framelist)
218 {
219  const cpl_frame * flat;
220  const cpl_frame * dark;
221  const cpl_frame * bpm;
222  cpl_frameset * objframes ;
223  cpl_frameset * skyframes ;
224 
225  /* Identify the RAW and CALIB frames in the input frameset */
226  if (hawki_dfs_set_groups(framelist)) {
227  cpl_msg_error(__func__, "Cannot identify RAW and CALIB frames") ;
228  return -1 ;
229  }
230 
231  /* Retrieve calibration data */
232  cpl_msg_info(__func__, "Identifying calibration data");
233  flat = cpl_frameset_find_const(framelist, HAWKI_CALPRO_FLAT);
234  dark = cpl_frameset_find_const(framelist, HAWKI_CALPRO_DARK);
235  bpm = cpl_frameset_find_const(framelist, HAWKI_CALPRO_BPM);
236  if (flat == NULL && dark == NULL && bpm == NULL)
237  {
238  cpl_msg_error(__func__, "No calibration data provided (%s and/or %s and/or %s)",
239  HAWKI_CALPRO_FLAT, HAWKI_CALPRO_DARK, HAWKI_CALPRO_BPM);
240  return -1 ;
241  }
242 
243  /* Retrieve raw frames */
244  cpl_msg_info(__func__, "Identifying objects and sky data");
245  objframes = hawki_extract_frameset(framelist, HAWKI_IMG_JITTER_RAW) ;
246  /* Retrieve sky frames */
247  skyframes = hawki_extract_frameset(framelist, HAWKI_IMG_JITTER_SKY_RAW) ;
248  if (objframes == NULL && skyframes == NULL)
249  {
250  cpl_msg_error(__func__, "Cannot find objs (%s) or sky frames (%s) in the input list",
251  HAWKI_IMG_JITTER_RAW, HAWKI_IMG_JITTER_SKY_RAW);
252  return -1 ;
253  }
254 
255  /* Applying the calibrations, QC and saving the results */
256  if (objframes != NULL)
257  {
258  const char * procat = HAWKI_CALPRO_BASICCALIBRATED;
259  const char * protype = HAWKI_PROTYPE_BASICCALIBRATED;
260  cpl_msg_info(__func__, "Apply the basic reduction to object frames");
261  hawki_step_basic_calib_applycal_qc_save
262  (objframes, flat, dark, bpm, "obj", procat, protype,
263  parlist, framelist);
264  cpl_frameset_delete(objframes);
265  }
266  if (skyframes != NULL)
267  {
268  const char * procat = HAWKI_CALPRO_SKY_BASICCALIBRATED;
269  const char * protype = HAWKI_PROTYPE_SKY_BASICCALIBRATED;
270  cpl_msg_info(__func__, "Apply the basic reduction to sky frames");
271  hawki_step_basic_calib_applycal_qc_save
272  (skyframes, flat, dark, bpm, "sky", procat, protype,
273  parlist, framelist);
274  cpl_frameset_delete(skyframes);
275  }
276 
277  /* return */
278  if (cpl_error_get_code()) return -1 ;
279  else return 0 ;
280 }
281 
282 /*----------------------------------------------------------------------------*/
292 /*----------------------------------------------------------------------------*/
293 static int hawki_step_basic_calib_applycal_qc_save
294 (cpl_frameset * raw_target,
295  const cpl_frame * flat,
296  const cpl_frame * dark,
297  const cpl_frame * bpm,
298  const char * filename_postfix,
299  const char * procat,
300  const char * protype,
301  cpl_parameterlist * recipe_parlist,
302  cpl_frameset * recipe_framelist)
303 {
304  cpl_imagelist * flat_images;
305  cpl_imagelist * dark_images;
306  cpl_imagelist * bpm_images;
307  cpl_frameset * calib_frameset;
308  cpl_propertylist * plist ;
309  cpl_errorstate error_prevstate = cpl_errorstate_get();
310 
311  double science_dit;
312  int iframe;
313  int ntarget;
314 
315  /* Initializing the pointers */
316  flat_images = NULL;
317  dark_images = NULL;
318  bpm_images = NULL;
319 
320  /* Indentation */
321  cpl_msg_indent_more();
322 
323  /* Initializating the calibration frameset */
324  calib_frameset = cpl_frameset_new();
325 
326  /* Loading the calibration files */
327  cpl_msg_info(__func__, "Loading the calibration data") ;
328  if(flat != NULL)
329  {
330  flat_images = hawki_load_frame(flat, CPL_TYPE_FLOAT);
331  if(flat_images == NULL)
332  {
333  cpl_msg_error(__func__, "Error reading flat") ;
334  cpl_frameset_delete(calib_frameset);
335  return -1;
336  }
337  cpl_frameset_insert(calib_frameset, cpl_frame_duplicate(flat));
338  }
339  if(dark != NULL)
340  {
341  dark_images = hawki_load_frame(dark, CPL_TYPE_FLOAT);
342  if(dark_images == NULL)
343  {
344  cpl_msg_error(__func__, "Error reading dark") ;
345  cpl_imagelist_delete(flat_images);
346  cpl_frameset_delete(calib_frameset);
347  return -1;
348  }
349  cpl_frameset_insert(calib_frameset, cpl_frame_duplicate(dark));
350  }
351  if(bpm != NULL)
352  {
353  bpm_images = hawki_load_frame(bpm, CPL_TYPE_INT);
354  if(bpm_images == NULL)
355  {
356  cpl_msg_error(__func__, "Error reading bpm") ;
357  cpl_imagelist_delete(flat_images);
358  cpl_imagelist_delete(dark_images);
359  cpl_frameset_delete(calib_frameset);
360  return -1;
361  }
362  cpl_frameset_insert(calib_frameset, cpl_frame_duplicate(bpm));
363  }
364 
365  /* Multiply the dark image by the science exposure time */
366  if(dark != NULL)
367  {
368  if ((plist=cpl_propertylist_load
369  (cpl_frame_get_filename
370  (cpl_frameset_get_first_const(raw_target)), 0)) == NULL)
371  {
372  cpl_msg_error(__func__, "Cannot get header from frame");
373  cpl_imagelist_delete(flat_images);
374  cpl_imagelist_delete(dark_images);
375  cpl_frameset_delete(calib_frameset);
376  return -1 ;
377  }
378  science_dit = hawki_pfits_get_dit(plist);
379  cpl_imagelist_multiply_scalar(dark_images, science_dit);
380  cpl_propertylist_delete(plist);
381  }
382 
383  /* Loop on the number of frames */
384  ntarget = cpl_frameset_get_size(raw_target);
385  cpl_msg_info(__func__, "Looping the science frames: %d frames", ntarget);
386  for( iframe = 0 ; iframe < ntarget ; ++iframe)
387  {
388  /* Local storage variables */
389  cpl_frame * target_frame;
390  cpl_imagelist * target_images;
391  cpl_frameset * used_frameset;
392  target_images = NULL;
393 
394  /* Loading the target */
395  cpl_msg_indent_more();
396  cpl_msg_info(__func__, "Loading frame %d", iframe+1) ;
397  target_frame = cpl_frameset_get_frame(raw_target, iframe);
398  if(target_frame != NULL)
399  target_images = hawki_load_frame(target_frame, CPL_TYPE_FLOAT);
400  if(target_images == NULL)
401  {
402  cpl_msg_error(__func__, "Error reading frame") ;
403  cpl_imagelist_delete(flat_images);
404  cpl_imagelist_delete(dark_images);
405  cpl_imagelist_delete(bpm_images);
406  return -1;
407  }
408  /* Creating the used frameset */
409  used_frameset = cpl_frameset_duplicate(calib_frameset);
410  cpl_frameset_insert(used_frameset, cpl_frame_duplicate(target_frame));
411 
412  /* TODO: Creating the variance array */
413  /* cpl_create_variance_image(); */
414 
415  /* Applying the calibrations */
416  cpl_msg_info(__func__, "Calibrating frame") ;
418  (target_images, flat_images, dark_images, bpm_images) == -1)
419  {
420  cpl_msg_error(__func__, "Cannot calibrate frame") ;
421  cpl_imagelist_delete(flat_images);
422  cpl_imagelist_delete(dark_images);
423  cpl_imagelist_delete(bpm_images);
424  cpl_imagelist_delete(target_images);
425  cpl_frameset_delete(used_frameset);
426  cpl_frameset_delete(calib_frameset);
427  cpl_msg_indent_less() ;
428  cpl_msg_indent_less() ;
429  return -1 ;
430  }
431 
432  /* Compute quality control */
433  hawki_step_basic_calib_qc();
434 
435  /* Save the products */
436  cpl_msg_info(__func__, "Save the products") ;
437  if (hawki_step_basic_calib_save
438  (target_images, filename_postfix, procat, protype, iframe,
439  used_frameset, recipe_parlist,
440  recipe_framelist) == -1)
441  {
442  cpl_msg_error(__func__, "Cannot save the products") ;
443  cpl_imagelist_delete(flat_images);
444  cpl_imagelist_delete(dark_images);
445  cpl_imagelist_delete(bpm_images);
446  cpl_imagelist_delete(target_images);
447  cpl_frameset_delete(used_frameset);
448  cpl_frameset_delete(calib_frameset);
449  cpl_msg_indent_less() ;
450  cpl_msg_indent_less() ;
451  return -1 ;
452  }
453 
454  /* Delete the target and the used frameset */
455  cpl_imagelist_delete(target_images);
456  cpl_frameset_delete(used_frameset);
457  cpl_msg_indent_less();
458  }
459 
460  cpl_msg_indent_less();
461  /* Delete the calibration files */
462  cpl_imagelist_delete(flat_images);
463  cpl_imagelist_delete(dark_images);
464  cpl_imagelist_delete(bpm_images);
465  cpl_frameset_delete(calib_frameset);
466 
467  if(!cpl_errorstate_is_equal(error_prevstate))
468  {
469  cpl_msg_warning(__func__,"Probably some data could not be saved. "
470  "Check permisions or disk space");
471  cpl_errorstate_set(CPL_ERROR_NONE);
472  return 1;
473  }
474  return 0;
475 }
476 
477 /*----------------------------------------------------------------------------*/
484 /*----------------------------------------------------------------------------*/
485 static void hawki_step_basic_calib_qc(void)
486 {
487  /* Compute QC parameters from the combined image */
488  //cpl_msg_info(__func__, "Compute QC parameters from the reduced images") ;
489 }
490 
491 /*----------------------------------------------------------------------------*/
501 /*----------------------------------------------------------------------------*/
502 static int hawki_step_basic_calib_save
503 (cpl_imagelist * reduced,
504  const char * filename_postfix,
505  const char * procat,
506  const char * protype,
507  int iserie,
508  cpl_frameset * used_frameset,
509  cpl_parameterlist * recipe_parlist,
510  cpl_frameset * recipe_framelist)
511 {
512  const cpl_frame * raw_reference;
513  cpl_propertylist * proplist;
514  cpl_propertylist ** extproplists;
515  char filename[256];
516  cpl_propertylist * inputlist ;
517  int ext_nb ;
518  const char * recipe_name = "hawki_step_basic_calib";
519  int i;
520 
521  /* Get the reference frame (the raw frame) */
522  raw_reference = irplib_frameset_get_first_from_group
523  (used_frameset, CPL_FRAME_GROUP_RAW);
524 
525  /* Create the prop lists */
526  cpl_msg_indent_more();
527  proplist = cpl_propertylist_new();
528  inputlist = cpl_propertylist_load_regexp(
529  cpl_frame_get_filename(raw_reference), 0,
530  HAWKI_HEADER_EXT_FORWARD, 0);
531  cpl_propertylist_append(proplist, inputlist);
532  cpl_propertylist_delete(inputlist);
533  extproplists = cpl_malloc(HAWKI_NB_DETECTORS * sizeof(cpl_propertylist*));
534  for (i=0 ; i<HAWKI_NB_DETECTORS ; i++)
535  {
536  /* Get the extension number */
538  (cpl_frame_get_filename(raw_reference), i+1);
539 
540  /* Allocate this pro perty list */
541  extproplists[i] = cpl_propertylist_new();
542 
543  /* Propagate some keywords from input raw frame extensions */
544  inputlist = cpl_propertylist_load_regexp(
545  cpl_frame_get_filename(raw_reference), ext_nb,
546  HAWKI_HEADER_EXT_FORWARD, 0);
547  cpl_propertylist_append(extproplists[i], inputlist);
548  cpl_propertylist_delete(inputlist);
549  inputlist = cpl_propertylist_load_regexp(
550  cpl_frame_get_filename(raw_reference), ext_nb,
551  HAWKI_HEADER_WCS, 0);
552  cpl_propertylist_append(extproplists[i], inputlist);
553  cpl_propertylist_delete(inputlist);
554  }
555 
556  /* Write the image */
557  snprintf(filename, 256, "hawki_step_basic_calib_%s%03d.fits",
558  filename_postfix, iserie+1);
559  hawki_imagelist_save(recipe_framelist,
560  recipe_parlist,
561  used_frameset,
562  reduced,
563  recipe_name,
564  procat,
565  protype,
566  (const cpl_propertylist*)proplist,
567  (const cpl_propertylist**)extproplists,
568  filename);
569 
570  /* Free and return */
571  cpl_msg_indent_less();
572  cpl_propertylist_delete(proplist) ;
573  for (i=0 ; i<HAWKI_NB_DETECTORS ; i++)
574  {
575  cpl_propertylist_delete(extproplists[i]) ;
576  }
577  cpl_free(extproplists) ;
578  return 0;
579 }