GRAVI Pipeline Reference Manual  1.2.3
gravity_postprocess.c
1 /* $Id: gravity_postprocess.c,v 1.29 2011/12/3 09:16:12 nazouaoui Exp $
2  *
3  * This file is part of the GRAVI Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: nazouaoui $
23  * $Date: 2011/12/3 09:16:12 $
24  * $Revision: 1.29 $
25  * $Name: $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include <cpl.h>
37 #include <stdio.h>
38 #include <string.h>
39 #include <time.h>
40 
41 #include "gravi_data.h"
42 #include "gravi_pfits.h"
43 #include "gravi_dfs.h"
44 
45 #include "gravi_utils.h"
46 
47 #include "gravi_calib.h"
48 #include "gravi_p2vmred.h"
49 #include "gravi_eop.h"
50 #include "gravi_metrology.h"
51 
52 #include "gravi_vis.h"
53 #include "gravi_tf.h"
54 
55 #include "gravi_preproc.h"
56 
57 /*-----------------------------------------------------------------------------
58  Private function prototypes
59  -----------------------------------------------------------------------------*/
60 
61 static int gravity_postprocess_create(cpl_plugin *);
62 static int gravity_postprocess_exec(cpl_plugin *);
63 static int gravity_postprocess_destroy(cpl_plugin *);
64 static int gravity_postprocess(cpl_frameset *, const cpl_parameterlist *);
65 
66 /*-----------------------------------------------------------------------------
67  Static variables
68  -----------------------------------------------------------------------------*/
69 
70 static char gravity_postprocess_short[] = "Post-process the products, to fine-tune their content.";
71 static char gravity_postprocess_description[] =
72  "This recipe allows to manipulate the product of the GRAVITY pipeline, mostly the VIS. It permits to merge several files together into a single VIS file with all observations; to average the observations of one or several VIS file to increse the SNR; to remove some data (FT, SC); and to resample the SC observation with spectral binning.\n"
73  "\n"
74  "The list of input files can be P2VMRED, VIS, VIS_CALIBRATED (or even RAW for some parameters). However they should all be compatible in term of setup and observed objets !! Note that the recipe performs only litle checks of the input file content and structure. Thus the user shall ensure the input files are conformable (same polarisation and spectral mode for instante)\n"
75  GRAVI_RECIPE_FLOW"\n"
76  "* Load the files\n"
77  "* Execute request from user\n"
78  "* Write product\n"
79  GRAVI_RECIPE_INPUT"\n"
80  "Input files : see above\n"
81  GRAVI_RECIPE_OUTPUT"\n"
82  "POSTPROCESSED : Output file\n"
83  "";
84 
85 /*-----------------------------------------------------------------------------
86  Function code
87  -----------------------------------------------------------------------------*/
88 
89 /*----------------------------------------------------------------------------*/
99 /*----------------------------------------------------------------------------*/
100 int cpl_plugin_get_info(cpl_pluginlist * list)
101 {
102  cpl_recipe * recipe = cpl_calloc(1, sizeof *recipe );
103  cpl_plugin * plugin = &recipe->interface;
104 
105  if (cpl_plugin_init(plugin,
106  CPL_PLUGIN_API,
107  GRAVI_BINARY_VERSION,
108  CPL_PLUGIN_TYPE_RECIPE,
109  "gravity_postprocess",
110  gravity_postprocess_short,
111  gravity_postprocess_description,
112  "Nabih Azouaoui, Vincent Lapeyrere, JB. Le Bouquin",
113  PACKAGE_BUGREPORT,
115  gravity_postprocess_create,
116  gravity_postprocess_exec,
117  gravity_postprocess_destroy)) {
118  cpl_msg_error(cpl_func, "Plugin initialization failed");
119  (void)cpl_error_set_where(cpl_func);
120  return 1;
121  }
122 
123  if (cpl_pluginlist_append(list, plugin)) {
124  cpl_msg_error(cpl_func, "Error adding plugin to list");
125  (void)cpl_error_set_where(cpl_func);
126  return 1;
127  }
128 
129  return 0;
130 }
131 
132 /*----------------------------------------------------------------------------*/
140 /*----------------------------------------------------------------------------*/
141 static int gravity_postprocess_create(cpl_plugin * plugin)
142 {
143  cpl_recipe * recipe;
144  cpl_parameter * p;
145 
146  /* Do not create the recipe if an error code is already set */
147  if (cpl_error_get_code() != CPL_ERROR_NONE) {
148  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
149  cpl_func, __LINE__, cpl_error_get_where());
150  return (int)cpl_error_get_code();
151  }
152 
153  if (plugin == NULL) {
154  cpl_msg_error(cpl_func, "Null plugin");
155  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
156  }
157 
158  /* Verify plugin type */
159  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
160  cpl_msg_error(cpl_func, "Plugin is not a recipe");
161  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
162  }
163 
164  /* Get the recipe */
165  recipe = (cpl_recipe *)plugin;
166 
167  /* Create the parameters list in the cpl_recipe object */
168  recipe->parameters = cpl_parameterlist_new();
169  if (recipe->parameters == NULL) {
170  cpl_msg_error(cpl_func, "Parameter list allocation failed");
171  cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT);
172  }
173 
174  /* Fill the parameters list */
175 
176  /* Averaging */
177  gravi_parameter_add_average_vis (recipe->parameters);
178  gravi_parameter_add_force_uncertainties (recipe->parameters);
179 
180  /* Force */
181  p = cpl_parameter_new_value ("gravity.postprocess.force-merge", CPL_TYPE_BOOL,
182  "Force merging even if inconsistent data",
183  "gravity.postprocess", FALSE);
184  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "force-merge");
185  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
186  cpl_parameterlist_append (recipe->parameters, p);
187 
188  /* Remove FT */
189  p = cpl_parameter_new_value ("gravity.postprocess.remove-ft", CPL_TYPE_BOOL,
190  "Remove FT extensions",
191  "gravity.postprocess", FALSE);
192  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "remove-ft");
193  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
194  cpl_parameterlist_append (recipe->parameters, p);
195 
196  /* Remove SC */
197  p = cpl_parameter_new_value ("gravity.postprocess.remove-sc", CPL_TYPE_BOOL,
198  "Remove SC extensions",
199  "gravity.postprocess", FALSE);
200  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "remove-sc");
201  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
202  cpl_parameterlist_append (recipe->parameters, p);
203 
204  /* Remove OPDC */
205  p = cpl_parameter_new_value ("gravity.postprocess.remove-opdc", CPL_TYPE_BOOL,
206  "Remove OPDC extensions",
207  "gravity.postprocess", FALSE);
208  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "remove-opdc");
209  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
210  cpl_parameterlist_append (recipe->parameters, p);
211 
212  /* Remove MET */
213  p = cpl_parameter_new_value ("gravity.postprocess.remove-met", CPL_TYPE_BOOL,
214  "Remove METROLOGY related extensions",
215  "gravity.postprocess", FALSE);
216  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "remove-met");
217  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
218  cpl_parameterlist_append (recipe->parameters, p);
219 
220  /* Resamp SC */
221  p = cpl_parameter_new_value ("gravity.postprocess.nbin-lambda-sc", CPL_TYPE_INT,
222  "Bin SC extensions in spectral dimension",
223  "gravity.postprocess", 0);
224  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "nbin-lambda-sc");
225  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
226  cpl_parameterlist_append (recipe->parameters, p);
227 
228  return 0;
229 }
230 
231 /*----------------------------------------------------------------------------*/
237 /*----------------------------------------------------------------------------*/
238 static int gravity_postprocess_exec(cpl_plugin * plugin)
239 {
240 
241  cpl_recipe * recipe;
242  int recipe_status;
243  cpl_errorstate initial_errorstate = cpl_errorstate_get();
244 
245 
246  /* Return immediately if an error code is already set */
247  if (cpl_error_get_code() != CPL_ERROR_NONE) {
248  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
249  cpl_func, __LINE__, cpl_error_get_where());
250  return (int)cpl_error_get_code();
251  }
252 
253  if (plugin == NULL) {
254  cpl_msg_error(cpl_func, "Null plugin");
255  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
256  }
257 
258  /* Verify plugin type */
259  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
260  cpl_msg_error(cpl_func, "Plugin is not a recipe");
261  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
262  }
263 
264  /* Get the recipe */
265  recipe = (cpl_recipe *)plugin;
266 
267  /* Verify parameter and frame lists */
268  if (recipe->parameters == NULL) {
269  cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
270  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
271  }
272  if (recipe->frames == NULL) {
273  cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
274  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
275  }
276 
277  /* Invoke the recipe */
278  recipe_status = gravity_postprocess(recipe->frames, recipe->parameters);
279 
280  /* Ensure DFS-compliance of the products */
281  if (cpl_dfs_update_product_header(recipe->frames)) {
282  if (!recipe_status){
283  recipe_status = (int)cpl_error_get_code();
284  }
285  }
286 
287  if (!cpl_errorstate_is_equal(initial_errorstate)) {
288  /* Dump the error history since recipe execution start.
289  At this point the recipe cannot recover from the error */
290  cpl_errorstate_dump(initial_errorstate, CPL_FALSE, NULL);
291  }
292 
293  return recipe_status;
294 }
295 
296 /*----------------------------------------------------------------------------*/
302 /*----------------------------------------------------------------------------*/
303 static int gravity_postprocess_destroy(cpl_plugin * plugin)
304 {
305  cpl_recipe * recipe;
306 
307  if (plugin == NULL) {
308  cpl_msg_error(cpl_func, "Null plugin");
309  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
310  }
311 
312  /* Verify plugin type */
313  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
314  cpl_msg_error(cpl_func, "Plugin is not a recipe");
315  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
316  }
317 
318  /* Get the recipe */
319  recipe = (cpl_recipe *)plugin;
320 
321  cpl_parameterlist_delete(recipe->parameters);
322 
323  return 0;
324 }
325 
326 
327 /*----------------------------------------------------------------------------*/
335 /*----------------------------------------------------------------------------*/
336 static int gravity_postprocess(cpl_frameset * frameset,
337  const cpl_parameterlist * parlist)
338 {
339  cpl_frameset *used_frameset=NULL;
340  cpl_frame * frame=NULL, *frame_merged=NULL;
341 
342  gravi_data * data_merged=NULL, * data=NULL;
343 
344  /* Message */
345  gravity_print_banner ();
346  cpl_msg_set_time_on();
347  cpl_msg_set_component_on();
348  gravi_msg_function_start(1);
349 
350  cpl_ensure_code(gravi_dfs_set_groups(frameset) == CPL_ERROR_NONE,
351  cpl_error_get_code());
352 
353  cpl_size nframe = cpl_frameset_get_size (frameset);
354  int force = gravi_param_get_bool (parlist, "gravity.postprocess.force-merge");
355 
356  /* To use this recipe the frameset must not be
357  * empty and have at least two frames */
358  if ( nframe<1 ) {
359  cpl_error_set_message (cpl_func, CPL_ERROR_ILLEGAL_INPUT,
360  "Not enough frames in the frameset");
361  goto cleanup;
362  }
363 
364  /* Check the frame have all the same TAG
365  * (cannot merge different type of frame) */
366 
367  /* Insert calibration frame into the used frameset */
368  used_frameset = cpl_frameset_new();
369 
370 
371  /* Loop on other frames to append them*/
372  for (cpl_size f = 0; f < nframe ; f++ ) {
373 
374  /* Load the frame */
375  frame = cpl_frameset_get_position (frameset, f);
376  data = gravi_data_load_frame (frame, used_frameset);
377 
378  /* Remove some data */
379  if (gravi_param_get_bool (parlist, "gravity.postprocess.remove-ft")) {
380  cpl_msg_info (cpl_func, "Erase FT");
381  gravi_data_erase_type (data, "_FT");
382  CPLCHECK_CLEAN ("Cannot delete FT");
383  }
384 
385  if (gravi_param_get_bool (parlist, "gravity.postprocess.remove-sc")) {
386  cpl_msg_info (cpl_func, "Erase SC");
387  gravi_data_erase_type (data, "_SC");
388  CPLCHECK_CLEAN ("Cannot delete SC");
389  }
390 
391  if (gravi_param_get_bool (parlist, "gravity.postprocess.remove-opdc")) {
392  cpl_msg_info (cpl_func, "Erase OPDC");
393  gravi_data_erase_type (data, "OPDC");
394  CPLCHECK_CLEAN ("Cannot delete OPDC");
395  }
396 
397  if (gravi_param_get_bool (parlist, "gravity.postprocess.remove-met")) {
398  cpl_msg_info (cpl_func, "Erase MET");
399  gravi_data_erase_type (data, "METROLOGY");
400  gravi_data_erase_type (data, "VIS_MET");
401  CPLCHECK_CLEAN ("Cannot delete MET");
402  }
403 
404  /* Force uncertainties */
405  gravi_force_uncertainties (data, parlist);
406  CPLCHECK_CLEAN ("Cannot force uncertainties");
407 
408 
409  if (f == 0) {
410  /* Use the first frame for merging */
411  frame_merged = frame;
412  data_merged = data; data = NULL;
413  }
414  else {
415  /* Merge for first frame */
416  gravi_data_append (data_merged, data, force);
417  FREE (gravi_data_delete, data);
418  }
419 
420  CPLCHECK_CLEAN ("Cannot append frames");
421  }
422 
423 
424  /* Co-add them if required (FIXME: and if VIS) */
425  if (gravi_param_get_bool (parlist, "gravity.postprocess.average-vis")) {
426 
427  gravi_msg_warning ("FIXME", "Average the different observations = EXPERIMENTAL");
428  gravi_average_vis (data_merged);
429 
430  CPLCHECK_CLEAN ("Cannot average VIS");
431  }
432 
433  /* Resample them if required (FIXME: and if VIS) */
434  cpl_size resamp_sc = gravi_param_get_int (parlist, "gravity.postprocess.nbin-lambda-sc");
435  if ( resamp_sc > 1) {
436 
437  gravi_msg_warning ("FIXME", "Resamp the SC data = EXPERIMENTAL");
438  gravi_vis_resamp (data_merged, resamp_sc);
439 
440  CPLCHECK_CLEAN ("Cannot resamp SC");
441  }
442 
443 
444  /* Recompute the TIME column from the MJD column
445  * in all OIFITS tables to follow standard */
446  gravi_vis_mjd_to_time (data_merged);
447 
448  /* Save the output data file based on the first frame of the frameset */
449  gravi_data_save_new (data_merged, frameset, NULL, NULL, parlist,
450  used_frameset, frame_merged, "gravity_postprocess",
451  NULL, "POSTPROCESSED");
452 
453  CPLCHECK_CLEAN ("Cannot save the POSTPROCESSED product");
454 
455  /* Terminate the function */
456  goto cleanup;
457 
458 cleanup:
459  /* Deallocation of all variables */
460  cpl_msg_info(cpl_func,"Memory cleanup");
461 
462  FREE (gravi_data_delete,data);
463  FREE (gravi_data_delete,data_merged);
464  FREE (cpl_frameset_delete,used_frameset);
465 
466  gravi_msg_function_exit(1);
467  return (int)cpl_error_get_code();
468 }
cpl_error_code gravi_vis_resamp(gravi_data *oi_data, cpl_size nsamp)
Re-bin the SC table by nsamp consecutive spectral bins.
Definition: gravi_vis.c:3273
gravi_data * gravi_data_load_frame(cpl_frame *frame, cpl_frameset *used_frameset)
Load a FITS file and create a gravi_data.
Definition: gravi_data.c:573
cpl_error_code gravi_data_erase_type(gravi_data *self, const char *type)
Erase all extension related to an instrument (SC, FT, MET...)
Definition: gravi_data.c:1867
cpl_error_code gravi_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.
Definition: gravi_dfs.c:78
cpl_error_code gravi_vis_mjd_to_time(gravi_data *vis_data)
Recompute the TIME column of all OIFITS extension from the MJD column, following the OIFITS standard ...
Definition: gravi_vis.c:2403
const char * gravi_get_license(void)
Get the pipeline copyright and license.
Definition: gravi_utils.c:104
cpl_error_code gravi_data_save_new(gravi_data *self, cpl_frameset *allframes, const char *filename, const char *suffix, const cpl_parameterlist *parlist, cpl_frameset *usedframes, cpl_frame *frame, const char *recipe, cpl_propertylist *applist, const char *proCatg)
Save a gravi data in a CPL-complian FITS file.
Definition: gravi_data.c:896
cpl_error_code gravi_average_vis(gravi_data *oi_data)
Coadd the observations together.
Definition: gravi_vis.c:2750
void gravi_data_delete(gravi_data *self)
Delete a gravi data.
Definition: gravi_data.c:137
cpl_error_code gravi_force_uncertainties(gravi_data *oi_data, const cpl_parameterlist *parlist)
Force uncertainties.
Definition: gravi_vis.c:2660
cpl_error_code gravi_data_append(gravi_data *first, const gravi_data *second, int force)
Append a gravi_data into another existing one.
Definition: gravi_data.c:290