GRAVI Pipeline Reference Manual  0.9.10
gravity_postprocess.c
1 /* $Id: gravity_postprocess.c,v 1.29 2011/12/3 09:16:12 nazouaoui Exp $
2  *
3  * This file is part of the GRAVI Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: nazouaoui $
23  * $Date: 2011/12/3 09:16:12 $
24  * $Revision: 1.29 $
25  * $Name: $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include <cpl.h>
37 #include <stdio.h>
38 #include <string.h>
39 #include <time.h>
40 
41 #include "gravi_data.h"
42 #include "gravi_pfits.h"
43 #include "gravi_dfs.h"
44 
45 #include "gravi_utils.h"
46 
47 #include "gravi_calib.h"
48 #include "gravi_p2vmred.h"
49 #include "gravi_eop.h"
50 #include "gravi_metrology.h"
51 
52 #include "gravi_vis.h"
53 #include "gravi_tf.h"
54 
55 #include "gravi_preproc.h"
56 
57 /*-----------------------------------------------------------------------------
58  Private function prototypes
59  -----------------------------------------------------------------------------*/
60 
61 static int gravity_postprocess_create(cpl_plugin *);
62 static int gravity_postprocess_exec(cpl_plugin *);
63 static int gravity_postprocess_destroy(cpl_plugin *);
64 static int gravity_postprocess(cpl_frameset *, const cpl_parameterlist *);
65 
66 /*-----------------------------------------------------------------------------
67  Static variables
68  -----------------------------------------------------------------------------*/
69 
70 static char gravity_postprocess_short[] = "Post-process the products, to fine-tune their content.";
71 static char gravity_postprocess_description[] =
72  "The recipe allows to manipulate the product of the GRAVITY pipeline, mostly the VIS. It permits to merge several files together into a single VIS file with all observations; to average the observations of one or several VIS file to increse the SNR; to remove some data (FT, SC); and to resample the SC observation with spectral bining.\n"
73  "\n"
74  "The list of input file can be P2VMRED, VIS, VIS_CALIBRATED (or even RAW for some parameters). However they should all be compatible in term of setup and observed objet !! Note that the recipe performs only little checks of the input file content and structure. Thus the user shall ensure the input files are conformable (same polarisation and spectral mode for instante)";
75 
76 /*-----------------------------------------------------------------------------
77  Function code
78  -----------------------------------------------------------------------------*/
79 
80 /*----------------------------------------------------------------------------*/
90 /*----------------------------------------------------------------------------*/
91 int cpl_plugin_get_info(cpl_pluginlist * list)
92 {
93  cpl_recipe * recipe = cpl_calloc(1, sizeof *recipe );
94  cpl_plugin * plugin = &recipe->interface;
95 
96  if (cpl_plugin_init(plugin,
97  CPL_PLUGIN_API,
98  GRAVI_BINARY_VERSION,
99  CPL_PLUGIN_TYPE_RECIPE,
100  "gravity_postprocess",
101  gravity_postprocess_short,
102  gravity_postprocess_description,
103  "Nabih Azouaoui, Vincent Lapeyrere, JB. Le Bouquin",
104  PACKAGE_BUGREPORT,
105  gravi_get_license(),
106  gravity_postprocess_create,
107  gravity_postprocess_exec,
108  gravity_postprocess_destroy)) {
109  cpl_msg_error(cpl_func, "Plugin initialization failed");
110  (void)cpl_error_set_where(cpl_func);
111  return 1;
112  }
113 
114  if (cpl_pluginlist_append(list, plugin)) {
115  cpl_msg_error(cpl_func, "Error adding plugin to list");
116  (void)cpl_error_set_where(cpl_func);
117  return 1;
118  }
119 
120  return 0;
121 }
122 
123 /*----------------------------------------------------------------------------*/
131 /*----------------------------------------------------------------------------*/
132 static int gravity_postprocess_create(cpl_plugin * plugin)
133 {
134  cpl_recipe * recipe;
135  cpl_parameter * p;
136 
137  /* Do not create the recipe if an error code is already set */
138  if (cpl_error_get_code() != CPL_ERROR_NONE) {
139  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
140  cpl_func, __LINE__, cpl_error_get_where());
141  return (int)cpl_error_get_code();
142  }
143 
144  if (plugin == NULL) {
145  cpl_msg_error(cpl_func, "Null plugin");
146  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
147  }
148 
149  /* Verify plugin type */
150  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
151  cpl_msg_error(cpl_func, "Plugin is not a recipe");
152  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
153  }
154 
155  /* Get the recipe */
156  recipe = (cpl_recipe *)plugin;
157 
158  /* Create the parameters list in the cpl_recipe object */
159  recipe->parameters = cpl_parameterlist_new();
160  if (recipe->parameters == NULL) {
161  cpl_msg_error(cpl_func, "Parameter list allocation failed");
162  cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT);
163  }
164 
165  /* Fill the parameters list */
166 
167  /* Averaging */
168  gravi_parameter_add_average_vis (recipe->parameters);
169 
170  /* Force */
171  p = cpl_parameter_new_value ("gravity.postprocess.force-merge", CPL_TYPE_BOOL,
172  "Force merging even if inconsistent data",
173  "gravity.postprocess", FALSE);
174  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "force-merge");
175  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
176  cpl_parameterlist_append (recipe->parameters, p);
177 
178  /* Remove FT */
179  p = cpl_parameter_new_value ("gravity.postprocess.remove-ft", CPL_TYPE_BOOL,
180  "Remove FT extensions",
181  "gravity.postprocess", FALSE);
182  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "remove-ft");
183  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
184  cpl_parameterlist_append (recipe->parameters, p);
185 
186  /* Remove SC */
187  p = cpl_parameter_new_value ("gravity.postprocess.remove-sc", CPL_TYPE_BOOL,
188  "Remove SC extensions",
189  "gravity.postprocess", FALSE);
190  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "remove-sc");
191  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
192  cpl_parameterlist_append (recipe->parameters, p);
193 
194  /* Remove OPDC */
195  p = cpl_parameter_new_value ("gravity.postprocess.remove-opdc", CPL_TYPE_BOOL,
196  "Remove OPDC extensions",
197  "gravity.postprocess", FALSE);
198  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "remove-opdc");
199  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
200  cpl_parameterlist_append (recipe->parameters, p);
201 
202  /* Remove MET */
203  p = cpl_parameter_new_value ("gravity.postprocess.remove-met", CPL_TYPE_BOOL,
204  "Remove METROLOGY related extensions",
205  "gravity.postprocess", FALSE);
206  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "remove-met");
207  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
208  cpl_parameterlist_append (recipe->parameters, p);
209 
210  /* Resamp SC */
211  p = cpl_parameter_new_value ("gravity.postprocess.nbin-lambda-sc", CPL_TYPE_INT,
212  "Bin SC extensions in spectral dimension",
213  "gravity.postprocess", 0);
214  cpl_parameter_set_alias (p, CPL_PARAMETER_MODE_CLI, "nbin-lambda-sc");
215  cpl_parameter_disable (p, CPL_PARAMETER_MODE_ENV);
216  cpl_parameterlist_append (recipe->parameters, p);
217 
218  return 0;
219 }
220 
221 /*----------------------------------------------------------------------------*/
227 /*----------------------------------------------------------------------------*/
228 static int gravity_postprocess_exec(cpl_plugin * plugin)
229 {
230 
231  cpl_recipe * recipe;
232  int recipe_status;
233  cpl_errorstate initial_errorstate = cpl_errorstate_get();
234 
235 
236  /* Return immediately if an error code is already set */
237  if (cpl_error_get_code() != CPL_ERROR_NONE) {
238  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
239  cpl_func, __LINE__, cpl_error_get_where());
240  return (int)cpl_error_get_code();
241  }
242 
243  if (plugin == NULL) {
244  cpl_msg_error(cpl_func, "Null plugin");
245  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
246  }
247 
248  /* Verify plugin type */
249  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
250  cpl_msg_error(cpl_func, "Plugin is not a recipe");
251  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
252  }
253 
254  /* Get the recipe */
255  recipe = (cpl_recipe *)plugin;
256 
257  /* Verify parameter and frame lists */
258  if (recipe->parameters == NULL) {
259  cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
260  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
261  }
262  if (recipe->frames == NULL) {
263  cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
264  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
265  }
266 
267  /* Invoke the recipe */
268  recipe_status = gravity_postprocess(recipe->frames, recipe->parameters);
269 
270  /* Ensure DFS-compliance of the products */
271  if (cpl_dfs_update_product_header(recipe->frames)) {
272  if (!recipe_status){
273  recipe_status = (int)cpl_error_get_code();
274  }
275  }
276 
277  if (!cpl_errorstate_is_equal(initial_errorstate)) {
278  /* Dump the error history since recipe execution start.
279  At this point the recipe cannot recover from the error */
280  cpl_errorstate_dump(initial_errorstate, CPL_FALSE, NULL);
281  }
282 
283  return recipe_status;
284 }
285 
286 /*----------------------------------------------------------------------------*/
292 /*----------------------------------------------------------------------------*/
293 static int gravity_postprocess_destroy(cpl_plugin * plugin)
294 {
295  cpl_recipe * recipe;
296 
297  if (plugin == NULL) {
298  cpl_msg_error(cpl_func, "Null plugin");
299  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
300  }
301 
302  /* Verify plugin type */
303  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
304  cpl_msg_error(cpl_func, "Plugin is not a recipe");
305  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
306  }
307 
308  /* Get the recipe */
309  recipe = (cpl_recipe *)plugin;
310 
311  cpl_parameterlist_delete(recipe->parameters);
312 
313  return 0;
314 }
315 
316 
317 /*----------------------------------------------------------------------------*/
325 /*----------------------------------------------------------------------------*/
326 static int gravity_postprocess(cpl_frameset * frameset,
327  const cpl_parameterlist * parlist)
328 {
329  cpl_frameset *used_frameset=NULL;
330  cpl_frame * frame=NULL, *frame_merged=NULL;
331 
332  gravi_data * data_merged=NULL, * data=NULL;
333 
334  /* Message */
335  gravity_print_banner ();
336  cpl_msg_set_time_on();
337  cpl_msg_set_component_on();
338  gravi_msg_function_start(1);
339 
340  cpl_ensure_code(gravi_dfs_set_groups(frameset) == CPL_ERROR_NONE,
341  cpl_error_get_code());
342 
343  cpl_size nframe = cpl_frameset_get_size (frameset);
344  int force = gravi_param_get_bool (parlist, "gravity.postprocess.force-merge");
345 
346  /* To use this recipe the frameset must not be
347  * empty and have at least two frames */
348  if ( nframe<1 ) {
349  cpl_error_set_message (cpl_func, CPL_ERROR_ILLEGAL_INPUT,
350  "Not enough frames in the frameset");
351  goto cleanup;
352  }
353 
354  /* Check the frame have all the same TAG
355  * (cannot merge different type of frame) */
356 
357  /* Insert calibration frame into the used frameset */
358  used_frameset = cpl_frameset_new();
359 
360 
361  /* Loop on other frames to append them*/
362  for (cpl_size f = 0; f < nframe ; f++ ) {
363 
364  /* Load the frame */
365  frame = cpl_frameset_get_position (frameset, f);
366  data = gravi_data_load_frame (frame, used_frameset);
367 
368  /* Remove some data */
369  if (gravi_param_get_bool (parlist, "gravity.postprocess.remove-ft")) {
370  cpl_msg_info (cpl_func, "Erase FT");
371  gravi_data_erase_type (data, "_FT");
372  CPLCHECK_CLEAN ("Cannot delete FT");
373  }
374 
375  /* Remove some data */
376  if (gravi_param_get_bool (parlist, "gravity.postprocess.remove-sc")) {
377  cpl_msg_info (cpl_func, "Erase SC");
378  gravi_data_erase_type (data, "_SC");
379  CPLCHECK_CLEAN ("Cannot delete SC");
380  }
381 
382  if (gravi_param_get_bool (parlist, "gravity.postprocess.remove-opdc")) {
383  cpl_msg_info (cpl_func, "Erase OPDC");
384  gravi_data_erase_type (data, "OPDC");
385  CPLCHECK_CLEAN ("Cannot delete OPDC");
386  }
387 
388  if (gravi_param_get_bool (parlist, "gravity.postprocess.remove-met")) {
389  cpl_msg_info (cpl_func, "Erase MET");
390  gravi_data_erase_type (data, "METROLOGY");
391  gravi_data_erase_type (data, "VIS_MET");
392  CPLCHECK_CLEAN ("Cannot delete MET");
393  }
394 
395  if (f == 0) {
396  /* Use the first frame for merging */
397  frame_merged = frame;
398  data_merged = data; data = NULL;
399  }
400  else {
401  /* Merge for first frame */
402  gravi_data_append (data_merged, data, force);
403  FREE (gravi_data_delete, data);
404  }
405 
406  CPLCHECK_CLEAN ("Cannot append frames");
407  }
408 
409 
410  /* Co-add them if required (FIXME: and if VIS) */
411  if (gravi_param_get_bool (parlist, "gravity.postprocess.average-vis")) {
412 
413  gravi_msg_warning ("FIXME", "Average the different observations = EXPERIMENTAL");
414  gravi_average_vis (data_merged);
415 
416  CPLCHECK_CLEAN ("Cannot average VIS");
417  }
418 
419  /* Resample them if required (FIXME: and if VIS) */
420  cpl_size resamp_sc = gravi_param_get_int (parlist, "gravity.postprocess.nbin-lambda-sc");
421  if ( resamp_sc > 1) {
422 
423  gravi_msg_warning ("FIXME", "Resamp the SC data = EXPERIMENTAL");
424  gravi_vis_resamp (data_merged, resamp_sc);
425 
426  CPLCHECK_CLEAN ("Cannot resamp SC");
427  }
428 
429 
430  /* Recompute the TIME column from the MJD column
431  * in all OIFITS tables to follow standard */
432  gravi_vis_mjd_to_time (data_merged);
433 
434  /* Save the output data file based on the first frame of the frameset */
435  gravi_data_save_new (data_merged, frameset, NULL, parlist,
436  used_frameset, frame_merged, "gravity_postprocess",
437  NULL, "POSTPROCESSED");
438 
439  CPLCHECK_CLEAN ("Cannot save the POSTPROCESSED product");
440 
441  /* Terminate the function */
442  goto cleanup;
443 
444 cleanup:
445  /* Deallocation of all variables */
446  cpl_msg_info(cpl_func,"Memory cleanup");
447 
448  FREE (gravi_data_delete,data);
449  FREE (gravi_data_delete,data_merged);
450  FREE (cpl_frameset_delete,used_frameset);
451 
452  gravi_msg_function_exit(1);
453  return (int)cpl_error_get_code();
454 }