GRAVI Pipeline Reference Manual  1.2.3
gravi_calib.c
1 /* $Id: gravi_calib.c,v 1.10 2012/03/23 15:10:40 nazouaoui Exp $
2  *
3  * This file is part of the GRAVI Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
36 /*
37  * History :
38  * 11/01/2019 : Move global parameter from gravi_calib.h
39  * initialize sigma with 0
40  * add brackets to 'for' statements
41  */
42 /*----------------------------------------------------------------------------
43  DEBUG
44  -----------------------------------------------------------------------------*/
45 
46 #define INFO_DEBUG 0
47 /*-----------------------------------------------------------------------------
48  Includes
49  -----------------------------------------------------------------------------*/
50 
51 #ifdef HAVE_CONFIG_H
52 #include <config.h>
53 #endif
54 
55 #include <cpl.h>
56 #include <string.h>
57 #include <stdio.h>
58 #include <time.h>
59 #include <math.h>
60 #include <complex.h>
61 
62 #include "gravi_data.h"
63 #include "gravi_dfs.h"
64 #include "gravi_pfits.h"
65 #include "gravi_cpl.h"
66 
67 #include "gravi_utils.h"
68 
69 #include "gravi_preproc.h"
70 #include "gravi_calib.h"
71 
72 /*-----------------------------------------------------------------------------
73  ekw 11/01/2019 : Global parameter from gravi_calib.h
74  -----------------------------------------------------------------------------*/
75 
76 static double met_Sep_2016[64] = {-0.000338233 ,-0.000282107 ,0.000345582 ,0.000381979 ,0.000326764 ,-0.000225664 ,5.35555e-05 ,0.00060938 ,-0.000205619 ,-0.000265577 ,0.00017356 ,0.000265687 ,0.000295987 ,3.05255e-05 ,-0.000498407 ,0.000232075 ,0.00502058 ,0.002045925 ,-0.000110657 ,-0.000403592 ,-0.000065043 ,-0.000433645 ,-0.000626545 ,3.29765e-05 ,-0.001694525 ,-0.0011684 ,6.2795e-06 ,-0.001584115 ,-0.000735553 ,-0.000868538 ,-0.000985087 ,-0.0008204 ,0.000768577 ,0.000848342 ,-0.000134943 ,-0.000385157 ,-8.68822e-05 ,-0.000757366 ,0.000446051 ,-0.000231723 ,0.000790425 ,-0.000638897 ,0.000503496 ,-7.78205e-05 ,0.000287366 ,0.000243789 ,0.000083288 ,-0.000125138 ,-0.000147337 ,4.14224e-05 ,0.000123082 ,-0.00117179 ,4.54785e-05 ,-0.000186707 ,0.000682836 ,0.00090649 ,0.000357256 ,-0.002133845 ,-0.00151895 ,-0.00150048 ,-0.00266423 ,-0.0030716 ,0.000599228 ,-0.001078583};
77 static double met_Mar_2017[64] = {0.000481474 ,0.000678173 ,0.00022232 ,9.04735e-05 ,0.001030283 ,0.000400478 ,0.000199579 ,0.000610902 ,0.000870061 ,0.001148845 ,0.000706636 ,0.000491999 ,0.00093158 ,0.001224225 ,0.000652115 ,0.001117025 ,0.00481452 ,0.002359475 ,0.000641491 ,0.000126103 ,0.000158341 ,0.001455785 ,0.000227113 ,0.000366087 ,-0.000372672 ,-0.000814455 ,0.000403834 ,-0.00072791 ,-0.000422227 ,0.000413887 ,-0.000024651 ,0.000106683 ,0.00092596 ,0.000327427 ,0.000775269 ,0.000906505 ,0.000108337 ,-0.000214467 ,0.001249965 ,0.000694693 ,0.000718101 ,0.00083926 ,0.00138818 ,0.00131215 ,0.001113065 ,0.00134113 ,0.000972572 ,0.00073247 ,-3.35943e-05 ,0.000312545 ,0.000365923 ,0.000510906 ,0.000955084 ,0.000808904 ,0.000403238 ,-0.000150186 ,0.000200673 ,-0.00078668 ,-0.00078133 ,-0.00039955 ,-0.00226333 ,-0.00177287 ,0.000435995 ,-0.000397403};
78 static double met_Jun_2017[64] = {0.00029923 ,5.42185e-05 ,-0.000023823 ,0.000187444 ,0.000650675 ,0.000135629 ,-0.000047364 ,0.000479489 ,0.000463565 ,0.000536066 ,0.000528847 ,0.000874895 ,0.000702853 ,0.000438988 ,0.000183642 ,0.000536044 ,0.004829985 ,0.00215068 ,5.94134e-05 ,-0.000164468 ,-4.89301e-05 ,0.000400028 ,-0.000277333 ,-0.000169422 ,-0.00136967 ,-0.000998661 ,0.000244959 ,-0.00123726 ,-0.000404182 ,-0.000511517 ,-0.000398515 ,-0.000314167 ,0.000546602 ,0.000547136 ,0.000388259 ,0.000108443 ,-0.000266431 ,-0.000734324 ,0.000946866 ,0.0001005 ,0.000739434 ,0.000530113 ,0.000821013 ,0.000748435 ,0.00094666 ,0.00102861 ,-0.00013052 ,0.000576223 ,-0.000657692 ,-0.000293389 ,0.000146337 ,-0.000338176 ,0.000241386 ,0.00005168 ,0.000495891 ,-1.76273e-05 ,8.84135e-05 ,-0.00153857 ,-0.000850848 ,-0.000986058 ,-0.002700175 ,-0.002620785 ,0.000440797 ,-0.00063843};
79 static double met_Jul_2017[64] = {8.19405e-05 ,-0.000061716 ,0.000067827 ,0.000234624 ,0.000521161 ,0.000031451 ,0.000211139 ,0.000578157 ,0.000316435 ,0.000431433 ,0.000431228 ,0.000761259 ,0.000576742 ,0.000299923 ,0.00034315 ,0.000561323 ,0.004834095 ,0.00208259 ,-0.000155192 ,-0.000120373 ,-0.000328756 ,0.000265791 ,-0.000249398 ,-0.000310416 ,-0.001047503 ,-0.000884526 ,0.000431645 ,-0.000890023 ,-0.000330709 ,-0.000540866 ,-0.0002067 ,-0.000334498 ,0.00035945 ,0.000683828 ,0.000464533 ,0.000166613 ,-0.000276387 ,-0.00088093 ,0.001283446 ,0.00016521 ,0.000578248 ,0.000334565 ,0.000762716 ,0.000521964 ,0.000757276 ,0.000959739 ,0.000035001 ,0.000442429 ,-0.0006928 ,-0.000105863 ,-0.00006137 ,-0.000454825 ,-0.000157343 ,-5.68635e-05 ,0.000296553 ,-3.80711e-05 ,0.000384616 ,-0.00149983 ,-0.000650183 ,-0.00076473 ,-0.002494945 ,-0.00246317 ,0.000794264 ,-0.000337915};
80 static double met_Aug_2017[64] = {0.000225917 ,0.00033629 ,1.70956e-05 ,0.000178987 ,0.00103744 ,0.000275415 ,0.000107582 ,0.000678724 ,0.000402821 ,0.000713535 ,0.000518528 ,0.000607008 ,0.000506748 ,0.000640514 ,0.000300349 ,0.000522693 ,0.005585285 ,0.00305222 ,0.000771553 ,0.001006485 ,0.000618719 ,0.00130769 ,0.000419028 ,0.000491215 ,-0.000815277 ,-0.000646205 ,0.000640702 ,-0.000881775 ,-0.000200994 ,-0.000258582 ,-0.000465663 ,-0.00004838 ,0.000591627 ,0.000615608 ,0.000197913 ,0.000352664 ,-0.000113565 ,-0.000637229 ,0.001241975 ,-6.659e-06 ,0.000649339 ,0.000641595 ,0.000673658 ,0.000657426 ,0.001263897 ,0.001258565 ,0.000050901 ,0.000318844 ,-9.23866e-05 ,0.00065074 ,0.000793493 ,0.000600095 ,0.001049984 ,0.000858337 ,0.000786737 ,0.000737051 ,0.000576774 ,-0.001384295 ,-0.000642989 ,-0.0008569 ,-0.00208747 ,-0.00247888 ,0.00108834 ,-0.000190064};
81 
82 /*-----------------------------------------------------------------------------
83  Private prototypes
84  -----------------------------------------------------------------------------*/
85 
86 cpl_error_code gravi_fit_profile (cpl_vector * values_x0,
87  cpl_vector * values_y0,
88  cpl_vector * values_sigma,
89  cpl_image * mean_img,
90  int ref_x, int ref_y,
91  int size_profile,
92  const char * resolution);
93 
94 cpl_image * gravi_create_profile_image (cpl_image * mean_img,
95  cpl_vector * values_x0,
96  cpl_vector * values_y0,
97  cpl_vector * values_sigma,
98  cpl_size iy_min, cpl_size iy_max,
99  const char * resolution);
100 
101 /*-----------------------------------------------------------------------------
102  Functions code
103  -----------------------------------------------------------------------------*/
104 
105 /*----------------------------------------------------------------------------*/
120 /*----------------------------------------------------------------------------*/
121 
122 gravi_data * gravi_compute_dark (gravi_data * raw_data)
123 {
124  gravi_msg_function_start(1);
125  cpl_ensure (raw_data, CPL_ERROR_NULL_INPUT, NULL);
126 
127  /* Create the output DARK or SKY map */
128  gravi_data * dark_map = gravi_data_new (0);
129 
130  /* Dump full header of RAW data */
131  cpl_propertylist * dark_header = gravi_data_get_header (dark_map);
132  cpl_propertylist * raw_header = gravi_data_get_header (raw_data);
133  cpl_propertylist_append (dark_header, raw_header);
134 
135  /* Check if this is a SKY or a DARK */
136  const char * dpr_type = gravi_pfits_get_dpr_type (raw_header);
137  int isSky = strstr (dpr_type, "SKY")?1:0;
138 
139  /* The dark file must contains all the shutter close */
140  if ( isSky==0 && !gravi_data_check_shutter_closed (raw_data) ) {
141  gravi_pfits_add_check (dark_header, "DARK has some shutter OPEN !!");
142  }
143 
144  /* The sky file must contains all the shutter open */
145  if ( isSky==1 && !gravi_data_check_shutter_open (raw_data) ) {
146  gravi_pfits_add_check (dark_header, "SKY has some shutter CLOSED !!");
147  }
148 
149  /*
150  * Compute the SC DARK
151  */
152  if (!gravi_data_has_extension (raw_data, GRAVI_IMAGING_DATA_SC_EXT)) {
153  cpl_msg_warning (cpl_func,"The DARK data has no IMAGING_DATA_SC");
154  }
155  else
156  {
157  cpl_msg_info (cpl_func, "Computing the %s of SC",isSky?"SKY":"DARK");
158 
159  /* Copy IMAGING_DETECTOR into product */
160  gravi_data_copy_ext (dark_map, raw_data, GRAVI_IMAGING_DETECTOR_SC_EXT);
161 
162  /* Load the IMAGING_DATA table or image list */
163  cpl_imagelist * imglist = gravi_data_get_cube (raw_data, GRAVI_IMAGING_DATA_SC_EXT);
164 
165  /* Compute the median image of the imagelist */
166  cpl_image * median_img = cpl_imagelist_collapse_median_create (imglist);
167  CPLCHECK_NUL ("Cannot compute the median dark");
168 
169  /* Compute the std of each pixels
170  * FIXME: look at cpl_imagelist function */
171  cpl_imagelist * temp_imglist = cpl_imagelist_new ();
172  for (cpl_size i = 0; i < cpl_imagelist_get_size(imglist); i ++){
173  cpl_image * image = cpl_imagelist_get (imglist, i);
174  cpl_imagelist_set (temp_imglist, cpl_image_power_create (image, 2), i);
175  }
176  cpl_image * mean_img = cpl_imagelist_collapse_create (imglist);
177  cpl_image_power (mean_img, 2);
178  cpl_image * stdev_img = cpl_imagelist_collapse_create (temp_imglist);
179  cpl_image_subtract (stdev_img, mean_img);
180  cpl_image_power (stdev_img, 0.5);
181  FREE (cpl_imagelist_delete, temp_imglist);
182  FREE (cpl_image_delete, mean_img);
183  CPLCHECK_NUL ("Cannot compute the STD of the DARK");
184 
185  /* Compute the QC parameters RMS and MEDIAN */
186  cpl_msg_info (cpl_func, "Compute QC parameters");
187  double mean_qc = cpl_image_get_median (median_img);
188  double darkrms = cpl_image_get_median (stdev_img);
189  cpl_propertylist_update_double (dark_header, isSky?QC_MEANSKY_SC:QC_MEANDARK_SC, mean_qc);
190  cpl_propertylist_update_double (dark_header, isSky?QC_SKYRMS_SC:QC_DARKRMS_SC, darkrms);
191 
192  /* Verbose */
193  cpl_msg_info (cpl_func, "QC_MEDIAN%s_SC = %e",isSky?"SKY":"DARK", mean_qc);
194  cpl_msg_info (cpl_func, "QC_%sRMS_SC = %e",isSky?"SKY":"DARK", darkrms);
195 
196  /* Put the data in the output table : dark_map */
197  cpl_propertylist * img_plist = gravi_data_get_plist (raw_data, GRAVI_IMAGING_DATA_SC_EXT);
198  img_plist = cpl_propertylist_duplicate (img_plist);
199  gravi_data_add_img (dark_map, img_plist, GRAVI_IMAGING_DATA_SC_EXT, median_img);
200 
201  img_plist = cpl_propertylist_duplicate (img_plist);
202  gravi_data_add_img (dark_map, img_plist, GRAVI_IMAGING_ERR_SC_EXT, stdev_img);
203  CPLCHECK_NUL ("Cannot set the SC data");
204 
205  } /* End SC case */
206 
207  /*
208  * Compute the FT DARK
209  */
210  if (!gravi_data_has_extension (raw_data, GRAVI_IMAGING_DATA_FT_EXT)) {
211  cpl_msg_warning (cpl_func,"The DARK data has no IMAGING_DATA_FT");
212  }
213  else
214  {
215  cpl_msg_info(cpl_func, "Computing the %s of FT",isSky?"SKY":"DARK");
216 
217  /* Copy IMAGING_DETECTOR into product */
218  gravi_data_copy_ext (dark_map, raw_data, GRAVI_IMAGING_DETECTOR_FT_EXT);
219 
220  /* Load the IMAGING_DATA table as DOUBLE */
221  cpl_msg_info (cpl_func,"Load data");
222  cpl_table * table_ft = gravi_data_get_table (raw_data, GRAVI_IMAGING_DATA_FT_EXT);
223  cpl_table_cast_column (table_ft, "PIX", "PIX", CPL_TYPE_DOUBLE);
224  cpl_imagelist * imglist = gravi_imagelist_wrap_column (table_ft, "PIX");
225  CPLCHECK_NUL ("Cannot load the FT data");
226 
227  /* Compute the median image of the imagelist */
228  cpl_msg_info (cpl_func,"Compute mean and median");
229  cpl_image * median_img = cpl_imagelist_collapse_median_create (imglist);
230  cpl_image * mean_img = cpl_imagelist_collapse_create (imglist);
231  CPLCHECK_NUL ("Cannot compute the MEAN dark");
232 
233  /* Compute the std of each pixels */
234  cpl_msg_info (cpl_func,"Compute std");
235  cpl_imagelist_subtract_image (imglist, mean_img);
236  cpl_imagelist_power (imglist, 2.0);
237  cpl_image * stdev_img = cpl_imagelist_collapse_create (imglist);
238  cpl_image_power (stdev_img, 0.5);
239 
240  /* We power back the data, because we are working in-place */
241  cpl_imagelist_power (imglist, 0.5);
242  cpl_imagelist_add_image (imglist, mean_img);
243 
244  /* Delete data */
245  cpl_msg_info (cpl_func,"Delete data");
246  FREE (gravi_imagelist_unwrap_images, imglist);
247  FREE (cpl_image_delete, mean_img);
248  CPLCHECK_NUL ("Cannot compute the STD of the DARK");
249 
250  /* Compute the QC parameters RMS and MEDIAN */
251  cpl_msg_info (cpl_func, "Compute QC parameters");
252  double mean_qc = cpl_image_get_mean (median_img);
253  double darkrms = cpl_image_get_mean (stdev_img);
254  cpl_propertylist_update_double (dark_header, isSky?QC_MEANSKY_FT:QC_MEANDARK_FT, mean_qc);
255  cpl_propertylist_update_double (dark_header, isSky?QC_SKYRMS_FT:QC_DARKRMS_FT, darkrms);
256  CPLCHECK_NUL ("Cannot compute the QC");
257 
258  /* Verbose */
259  cpl_msg_info (cpl_func, "QC_MEDIAN%s_FT = %e",isSky?"SKY":"DARK", mean_qc);
260  cpl_msg_info (cpl_func, "QC_%sRMS_FT = %e",isSky?"SKY":"DARK", darkrms);
261 
262  /* Create the output DARK table, with a single row */
263  cpl_table * median_table = cpl_table_extract (table_ft, 0, 1);
264  cpl_array * median_array = gravi_array_wrap_image (median_img);
265  cpl_table_set_array (median_table, "PIX", 0, median_array);
266  FREE (cpl_array_unwrap, median_array);
267  FREE (cpl_image_delete, median_img);
268  CPLCHECK_NUL("Cannot set median in table");
269 
270  /* Put median dark in the output gravi_data */
271  gravi_data_add_table (dark_map, NULL, GRAVI_IMAGING_DATA_FT_EXT, median_table);
272  CPLCHECK_NUL("Cannot save median in gravi_data");
273 
274  /* Create the output DARK RMS table, with a single row */
275  cpl_table * stdev_table = cpl_table_extract (table_ft, 0, 1);
276  cpl_array * stdev_array = gravi_array_wrap_image (stdev_img);
277  cpl_table_set_array (stdev_table, "PIX", 0, stdev_array);
278  FREE (cpl_array_unwrap, stdev_array);
279  FREE (cpl_image_delete, stdev_img);
280  CPLCHECK_NUL("Cannot set rms in table");
281 
282  /* Put median dark in the output gravi_data */
283  gravi_data_add_table (dark_map, NULL, GRAVI_IMAGING_ERR_FT_EXT, stdev_table);
284  CPLCHECK_NUL("Cannot save median in gravi_data");
285 
286  } /* End case FT */
287 
288  /*
289  * Compute the METROLOGY DARK
290  */
291  if (( isSky==1 )||(!gravi_data_has_extension (raw_data, GRAVI_METROLOGY_EXT))) {
292 
293  if ( isSky==0 )
294  cpl_msg_warning (cpl_func,"The DARK data has no METROLOGY");
295 
296  }
297  else
298  {
299 
300  cpl_msg_info(cpl_func, "Computing the %s of METROLOGY",isSky?"SKY":"DARK");
301 
302  /* Load the IMAGING_DATA table as DOUBLE */
303  cpl_msg_info (cpl_func,"Load data");
304  cpl_table * table_met = gravi_data_get_table (raw_data, GRAVI_METROLOGY_EXT);
305  cpl_table_cast_column (table_met, "VOLT", "VOLT", CPL_TYPE_DOUBLE);
306  cpl_imagelist * imglist = gravi_imagelist_wrap_column (table_met, "VOLT");
307  CPLCHECK_NUL ("Cannot load the VOLT data");
308 
309  /* Compute the median image of the imagelist */
310  cpl_msg_info (cpl_func,"Compute mean and median");
311  cpl_image * median_img = cpl_imagelist_collapse_median_create (imglist);
312  cpl_image * mean_img = cpl_imagelist_collapse_create (imglist);
313  CPLCHECK_NUL ("Cannot compute the MEAN dark");
314 
315  /* Compute the std of each pixels */
316  cpl_msg_info (cpl_func,"Compute std");
317  cpl_imagelist_subtract_image (imglist, mean_img);
318  cpl_imagelist_power (imglist, 2.0);
319  cpl_image * stdev_img = cpl_imagelist_collapse_create (imglist);
320  cpl_image_power (stdev_img, 0.5);
321 
322  /* We power back the data, because we are working in-place */
323  cpl_imagelist_power (imglist, 0.5);
324  cpl_imagelist_add_image (imglist, mean_img);
325 
326  /* Delete data */
327  cpl_msg_info (cpl_func,"Delete data");
328  FREE (gravi_imagelist_unwrap_images, imglist);
329  FREE (cpl_image_delete, mean_img);
330  CPLCHECK_NUL ("Cannot compute the STD of the DARK");
331 
332  /* Compute the QC parameters RMS and MEDIAN */
333  cpl_msg_info (cpl_func, "Compute QC parameters");
334  double mean_qc = cpl_image_get_mean (median_img);
335  double darkrms = cpl_image_get_mean (stdev_img);
336  cpl_propertylist_update_double (dark_header, QC_MEANDARK_MET, mean_qc);
337  cpl_propertylist_update_double (dark_header, QC_DARKRMS_MET, darkrms);
338  CPLCHECK_NUL ("Cannot compute the QC");
339 
340  /* Verbose */
341  cpl_msg_info (cpl_func, "QC_MEDIAN%s_MET = %e","DARK", mean_qc);
342  cpl_msg_info (cpl_func, "QC_%sRMS_MET = %e","DARK", darkrms);
343 
344  /* Create the output DARK table, with a single row */
345  cpl_table * median_table = cpl_table_extract (table_met, 0, 1);
346  cpl_array * median_array = gravi_array_wrap_image (median_img);
347 
348  /* Put to zero the dark of the fiber coupler unit */
349  for (cpl_size diode = 64; diode < 80; diode++){
350  cpl_array_set(median_array, diode, 0);
351  }
352 
353  /* Check on time */
354  double time_mjd_obs= cpl_propertylist_get_double (raw_header, "MJD-OBS");
355 
356  /* If the data is too old, we are using an old dataset */
357  for (cpl_size diode = 0; diode < 64; diode++)
358  {
359  if (time_mjd_obs < 57747) cpl_array_set(median_array, diode, met_Sep_2016[diode]); /* 25 decembre 2016 */
360  else if (time_mjd_obs < 57851) cpl_array_set(median_array, diode, met_Mar_2017[diode]); /* 8 April */
361  else if (time_mjd_obs < 57924) cpl_array_set(median_array, diode, met_Jun_2017[diode]); /* 20 Juin */
362  else if (time_mjd_obs < 57990) cpl_array_set(median_array, diode, met_Jul_2017[diode]); /* 25 Juillet */
363  else if (time_mjd_obs < 58208.01) cpl_array_set(median_array, diode, met_Aug_2017[diode]);
364  }
365 
366  cpl_table_set_array (median_table, "VOLT", 0, median_array);
367  FREE (cpl_array_unwrap, median_array);
368  FREE (cpl_image_delete, median_img);
369  CPLCHECK_NUL("Cannot set median in table");
370 
371  /* Put median dark in the output gravi_data */
372  gravi_data_add_table (dark_map, NULL, GRAVI_METROLOGY_EXT, median_table);
373  CPLCHECK_NUL("Cannot save median in gravi_data");
374 
375  /* Create the output DARK RMS table, with a single row */
376  cpl_table * stdev_table = cpl_table_extract (table_met, 0, 1);
377  cpl_array * stdev_array = gravi_array_wrap_image (stdev_img);
378  cpl_table_set_array (stdev_table, "VOLT", 0, stdev_array);
379  FREE (cpl_array_unwrap, stdev_array);
380  FREE (cpl_image_delete, stdev_img);
381  CPLCHECK_NUL("Cannot set rms in table");
382 
383  /* Put median dark in the output gravi_data */
384  gravi_data_add_table (dark_map, NULL, GRAVI_METROLOGY_ERR_EXT, stdev_table);
385  CPLCHECK_NUL("Cannot save median in gravi_data");
386 
387  } /* End case METROLOGY */
388 
389  /*
390  * Compute the ACQ DARK
391  */
392  if (!gravi_data_has_extension (raw_data, GRAVI_IMAGING_DATA_ACQ_EXT)) {
393  cpl_msg_warning (cpl_func,"The DARK data has no IMAGING_DATA_ACQ");
394  }
395  else
396  {
397  cpl_msg_info (cpl_func, "Computing the %s of ACQ",isSky?"SKY":"DARK");
398 
399  /* Load the IMAGING_DATA table or image list */
400  cpl_imagelist * imglist = gravi_data_get_cube (raw_data, GRAVI_IMAGING_DATA_ACQ_EXT);
401 
402  /* Compute the median image of the imagelist */
403  cpl_image * median_img = cpl_imagelist_collapse_median_create (imglist);
404  CPLCHECK_NUL ("Cannot compute the median dark");
405 
406  /* Compute the QC parameters RMS and MEDIAN */
407  cpl_msg_info (cpl_func, "Compute QC parameters");
408  double mean_qc = cpl_image_get_median (median_img);
409 
410  /* Verbose */
411  cpl_msg_info (cpl_func, "QC_MEDIAN%s_ACQ = %e",isSky?"SKY":"DARK", mean_qc);
412  cpl_propertylist_update_double (dark_header, isSky?"ESO QC MEDIANSKY ACQ":"ESO QC MEDIANDARK ACQ",
413  mean_qc);
414 
415  /* Put the data in the output table : dark_map */
416  cpl_propertylist * img_plist = gravi_data_get_plist (raw_data, GRAVI_IMAGING_DATA_ACQ_EXT);
417  img_plist = cpl_propertylist_duplicate (img_plist);
418  gravi_data_add_img (dark_map, img_plist, GRAVI_IMAGING_DATA_ACQ_EXT, median_img);
419  CPLCHECK_NUL("Cannot save median in gravi_data");
420  }
421 
422  gravi_msg_function_exit(1);
423  return dark_map;
424 }
425 
426 /*---------------------------------------------------------------------------*/
438 /*---------------------------------------------------------------------------*/
439 
440 gravi_data * gravi_average_dark (gravi_data ** data, cpl_size ndata)
441 {
442  gravi_msg_function_start(0);
443  cpl_ensure (data, CPL_ERROR_NULL_INPUT, NULL);
444 
445  gravi_msg_warning ("FIXME","Averaging DARK or SKY is experimental");
446 
447  gravi_data * output_data = gravi_data_duplicate (data[0]);
448 
449  /* Average the IMAGING_DATA of SC */
450  cpl_msg_info (cpl_func,"Average IMAGING_DATA of SC");
451 
452  cpl_image * darksc_image = gravi_data_get_img (output_data, GRAVI_IMAGING_DATA_SC_EXT);
453  for (int file = 1; file < ndata; file++) {
454  cpl_image_add (darksc_image, gravi_data_get_img (data[file], GRAVI_IMAGING_DATA_SC_EXT));
455  }
456  cpl_image_divide_scalar (darksc_image, ndata);
457  CPLCHECK_NUL ("Cannot average DARK of SC");
458 
459  /* Average the IMAGING_ERR of SC */
460  cpl_msg_info (cpl_func,"Average IMAGING_ERR of SC");
461 
462  cpl_image * stdevsc_image = gravi_data_get_img (output_data, GRAVI_IMAGING_ERR_SC_EXT);
463  for (int file = 1; file < ndata; file++) {
464  cpl_image_add (stdevsc_image, gravi_data_get_img (data[file], GRAVI_IMAGING_ERR_SC_EXT));
465  }
466  cpl_image_divide_scalar (stdevsc_image, ndata);
467  CPLCHECK_NUL ("Cannot average DARKERR of SC");
468 
469  /* Average the IMAGING_DATA of FT */
470  cpl_msg_info (cpl_func,"Average IMAGING_DATA of FT");
471 
472  cpl_array * darkft_array;
473  darkft_array = cpl_table_get_data_array (gravi_data_get_table (output_data, GRAVI_IMAGING_DATA_FT_EXT), "PIX")[0];
474  for (int file = 1; file < ndata; file++) {
475  cpl_array_add (darkft_array, cpl_table_get_data_array (gravi_data_get_table (data[file], GRAVI_IMAGING_DATA_FT_EXT), "PIX")[0]);
476  }
477  cpl_array_divide_scalar (darkft_array, ndata);
478  CPLCHECK_NUL ("Cannot average DARK of FT");
479 
480  /* Average the IMAGING_ERR of FT */
481  cpl_msg_info (cpl_func,"Average IMAGING_ERR of FT");
482 
483  cpl_array * errft_array;
484  errft_array = cpl_table_get_data_array (gravi_data_get_table (output_data, GRAVI_IMAGING_ERR_FT_EXT), "PIX")[0];
485  for (int file = 1; file < ndata; file++) {
486  cpl_array_add (errft_array, cpl_table_get_data_array (gravi_data_get_table (data[file], GRAVI_IMAGING_ERR_FT_EXT), "PIX")[0]);
487  }
488  cpl_array_divide_scalar (errft_array, ndata);
489  CPLCHECK_NUL ("Cannot average DARK of FT");
490 
491  gravi_msg_function_exit(0);
492  return output_data;
493 }
494 
495 
496 /*----------------------------------------------------------------------------*/
517 /*----------------------------------------------------------------------------*/
518 
519 cpl_error_code gravi_fit_profile (cpl_vector * values_x0,
520  cpl_vector * values_y0,
521  cpl_vector * values_sigma,
522  cpl_image * mean_img,
523  int ref_x, int ref_y,
524  int size_profile,
525  const char * resolution)
526 {
527  gravi_msg_function_start(0);
528  cpl_ensure_code (values_x0, CPL_ERROR_NULL_INPUT);
529  cpl_ensure_code (values_y0, CPL_ERROR_NULL_INPUT);
530  cpl_ensure_code (values_sigma, CPL_ERROR_NULL_INPUT);
531  cpl_ensure_code (mean_img, CPL_ERROR_NULL_INPUT);
532  cpl_ensure_code (resolution, CPL_ERROR_NULL_INPUT);
533 
534  /* Compute middle of size_profile */
535  int middle = floor (size_profile / 2);
536 
537  /* Get the image dimensions */
538  int nx = cpl_image_get_size_x (mean_img);
539  int ny = cpl_image_get_size_y (mean_img);
540 
541  cpl_ensure_code (ref_x > 0 && ref_x < nx, CPL_ERROR_ILLEGAL_INPUT);
542  cpl_ensure_code (ref_y > 0 && ref_y < ny, CPL_ERROR_ILLEGAL_INPUT);
543 
544  /* y0: best fit profile position (y0_ is of previous channel)
545  * sigma: best fit profile width (sigma_ is of previous channel) */
546  double y0, y0_ = ref_y;
547  double sigma, sigma_ = 0;
548 
549  /* Loop on spectral channels. Here fit all the
550  * points under the reference point */
551  int coord_x = ref_x;
552  while (coord_x <= nx) {
553 
554  /* Extract the corresponding column in the 2D image */
555  cpl_vector * line_data;
556  line_data = cpl_vector_new_from_image_column (mean_img,
557  coord_x);
558 
559  /* Construction of the vector of references points y_data and
560  * its values z_data for the gaussian fit */
561  cpl_vector * y_data, * z_data;
562  y_data = cpl_vector_new (size_profile);
563 
564  if (! (strcmp(resolution, "LOW") && strcmp(resolution, "MED")) ) {
565  /* Case LOW and MED */
566  for (cpl_size i = 0; i < size_profile; i++){
567  cpl_vector_set (y_data, i, ref_y - size_profile/2 + i);
568  }
569  z_data = cpl_vector_extract (line_data,
570  ref_y - size_profile/2,
571  ref_y - size_profile/2+size_profile-1, 1);
572  }
573  else {
574  /* Case HIGH */
575  if ((ref_y - middle) <= 0 ) {
576  for (cpl_size i = 0; i < size_profile; i++){
577  cpl_vector_set (y_data, i, i);
578  }
579  z_data = cpl_vector_extract (line_data, 0,
580  size_profile - 1, 1);
581  }
582  else if ((ref_y - middle + size_profile) >= ny ){
583  for (cpl_size i = 0; i < size_profile; i++){
584  cpl_vector_set (y_data, i, ny - size_profile + i);
585  }
586  z_data = cpl_vector_extract (line_data,
587  ny - size_profile,
588  ny - 1, 1);
589  }
590  else {
591  for (cpl_size i = 0; i < size_profile; i++){
592  cpl_vector_set(y_data, i, ref_y + (i - middle));
593  }
594  z_data = cpl_vector_extract (line_data, ref_y - middle,
595  ref_y - middle + size_profile - 1, 1);
596  }
597  }
598 
599  /* Fit a Gaussian to the extracted data */
600  cpl_errorstate prestate = cpl_errorstate_get();
601  double area, mse, offset = 0;
602  cpl_vector_fit_gaussian (y_data, NULL, z_data, NULL,
603  CPL_FIT_ALL, &y0, &sigma, &area,
604  &offset, &mse, NULL, NULL);
605 
606  /* If the fit fail, we keep the value of the
607  * previous spectral channel */
608  if (cpl_error_get_code() == CPL_ERROR_CONTINUE){
609  cpl_errorstate_set (prestate);
610  if (coord_x == ref_x) {
611  y0 = ref_y;
612  sigma = 1;
613  }else{
614  y0 = y0_;
615  sigma = sigma_;
616  }
617  cpl_msg_warning (cpl_func, "Cannot fit profile of channel %d, new y0=%e", coord_x, y0);
618  }
619 
620  CPLCHECK_MSG ("Error during the gaussian fit");
621 
622  /* If more than 1 pixel shift, we keep the value of the
623  * previous spectral channel */
624  if ((fabs(y0 - y0_) >= 1) && (coord_x != ref_x)) {
625  cpl_msg_warning (cpl_func, "Too much difference of channel %d with previous (%e pixels)", coord_x, y0 - y0_);
626  y0 = y0_;
627  sigma = sigma_;
628  }
629 
630  /* In HIGH Resolution, we update ref_y in order to
631  * follow the curvature when extracting the data */
632  if (! (strcmp(resolution, "HIGH"))) {
633  ref_y = floor(y0);
634  }
635 
636  /* Record x0 and sigma to compare to next channel */
637  y0_ = y0;
638  sigma_ = sigma;
639 
640  /* Save best fit y0 and sigma into output vectors */
641  cpl_vector_set (values_x0, (coord_x - 1), coord_x - 1);
642  cpl_vector_set (values_y0, (coord_x - 1), y0);
643  cpl_vector_set (values_sigma, (coord_x - 1), sigma);
644 
645  /* Increment spectral channel number */
646  coord_x ++;
647 
648  FREE (cpl_vector_delete, line_data);
649  FREE (cpl_vector_delete, z_data);
650  FREE (cpl_vector_delete, y_data);
651 
652  } /* End fit the column ref_x -> nx */
653 
654 
655  /* Reset coord_x to the middle of the spectra */
656  coord_x = ref_x - 1;
657 
658  /* In HIGH, we use the best-fit position for this channel */
659  if (! (strcmp(resolution, "HIGH"))) {
660  ref_y = cpl_vector_get (values_y0, (ref_x - 1));
661  }
662 
663 
664  /* Loop on spectral channels. Here fit all the
665  * points under the reference point */
666  while (coord_x > 0) {
667 
668  /* Extract the corresponding column in the 2D image */
669  cpl_vector * line_data;
670  line_data = cpl_vector_new_from_image_column (mean_img,
671  coord_x);
672 
673  /* Construction of the vector of references points y_data and
674  * its values z_data for the gaussian fit */
675  cpl_vector * y_data, * z_data;
676  y_data = cpl_vector_new(size_profile);
677 
678  if (! (strcmp(resolution, "LOW") && strcmp(resolution, "MED")) ) {
679  /* Case LOW and MED */
680  for (cpl_size i = 0; i < size_profile; i++){
681  cpl_vector_set(y_data, i, ref_y - size_profile/2 + i);
682  }
683  z_data = cpl_vector_extract (line_data,
684  ref_y - size_profile/2,
685  ref_y - size_profile/2 +size_profile-1, 1);
686  }
687  else {
688  /* Case HIGH */
689  if ((ref_y - middle) <= 0 ){
690  for (cpl_size i = 0; i < size_profile; i++){
691  cpl_vector_set(y_data, i, i);
692  }
693  z_data = cpl_vector_extract (line_data, 0,
694  size_profile - 1, 1);
695  }
696  else if ((ref_y - middle + size_profile) >= ny ){
697  for (cpl_size i = 0; i < size_profile; i++){
698  cpl_vector_set(y_data, i, ny - size_profile + i);
699  }
700  z_data = cpl_vector_extract (line_data,
701  ny - size_profile,
702  ny - 1, 1);
703  }
704  else{
705  for (cpl_size i = 0; i < size_profile; i++){
706  cpl_vector_set(y_data, i, ref_y + (i - middle));
707  }
708  z_data = cpl_vector_extract (line_data, ref_y - middle,
709  ref_y - middle + size_profile - 1, 1);
710  }
711  }
712 
713  /* Fit a Gaussian to the extracted data */
714  cpl_errorstate prestate = cpl_errorstate_get();
715  double area, mse, offset = 0;
716  cpl_vector_fit_gaussian (y_data, NULL, z_data, NULL,
717  CPL_FIT_ALL, &y0, &sigma, &area,
718  &offset, &mse, NULL, NULL);
719 
720  /* If the fit fail, we keep the value of the
721  * previous spectral channel */
722  if (cpl_error_get_code() == CPL_ERROR_CONTINUE){
723  cpl_errorstate_set (prestate);
724  if (coord_x == ref_x-1){
725  y0 = ref_y;
726  sigma = 1;
727  } else {
728  y0 = y0_;
729  sigma = sigma_;
730  }
731  cpl_msg_warning (cpl_func, "Cannot fit profile of channel %d, new y0=%e", coord_x, y0);
732  }
733 
734  CPLCHECK_MSG ("Error during the gaussian fit");
735 
736  /* If more than 1 pixel shift, we keep the value of the
737  * previous spectral channel */
738  if ((fabs(y0 - y0_) >= 1) && (coord_x != ref_x-1)) {
739  cpl_msg_warning (cpl_func, "Too much difference of channel %d with previous (%e pixels)", coord_x, y0 - y0_);
740  y0 = y0_;
741  sigma = sigma_;
742  }
743 
744  /* In HIGH Resolution, we update ref_y in order to
745  * follow the curvature when extracting the data */
746  if (! (strcmp(resolution, "HIGH"))) {
747  ref_y = floor(y0);
748  }
749 
750  /* Record x0 and sigma to compare to next channel */
751  y0_ = y0;
752  sigma_ = sigma;
753 
754  /* Save best fit y0 and sigma into output vectors */
755  cpl_vector_set (values_x0, (coord_x - 1), coord_x);
756  cpl_vector_set (values_y0, (coord_x - 1), y0 );
757  cpl_vector_set (values_sigma, (coord_x - 1), sigma);
758 
759  /* Increment spectral channel number */
760  coord_x --;
761 
762  FREE (cpl_vector_delete, line_data);
763  FREE (cpl_vector_delete, z_data);
764  FREE (cpl_vector_delete, y_data);
765 
766  } /* End fit the column ref_x -> 0 */
767 
768 
769  gravi_msg_function_exit(0);
770  return CPL_ERROR_NONE;
771 }
772 
773 /*----------------------------------------------------------------------------*/
795 /*----------------------------------------------------------------------------*/
796 
797 cpl_image * gravi_create_profile_image (cpl_image * mean_img,
798  cpl_vector * values_x0,
799  cpl_vector * values_y0,
800  cpl_vector * values_sigma,
801  cpl_size iy_min,
802  cpl_size iy_max,
803  const char * mode)
804 {
805  int nv = 0;
806  gravi_msg_function_start(0);
807  cpl_ensure (values_x0, CPL_ERROR_NULL_INPUT, NULL);
808  cpl_ensure (values_y0, CPL_ERROR_NULL_INPUT, NULL);
809  cpl_ensure (values_sigma, CPL_ERROR_NULL_INPUT, NULL);
810  cpl_ensure (mean_img, CPL_ERROR_NULL_INPUT, NULL);
811  cpl_ensure (mode, CPL_ERROR_NULL_INPUT, NULL);
812 
813  /* Get the image dimensions */
814  cpl_size nx = cpl_image_get_size_x (mean_img);
815  cpl_size ny = cpl_image_get_size_y (mean_img);
816 
817  cpl_msg_info (cpl_func, "iy_min = %lld, iy_max = %lld, ny = %lld",
818  iy_min, iy_max, ny);
819 
820  cpl_ensure (iy_min >= 0 && iy_min < ny, CPL_ERROR_ILLEGAL_INPUT, NULL);
821  cpl_ensure (iy_max >= 0 && iy_max < ny, CPL_ERROR_ILLEGAL_INPUT, NULL);
822 
823  /* Filter the profile params with a polynomial fit in spectral direction
824  * FIXME: could be a running median instead of fit, surely more stable.
825  * However the values contains some 'accidents' or modulations that
826  * may not be real and are efficiently removed by polynomial fit. */
827  cpl_matrix * matrix = cpl_matrix_wrap (1, nx, cpl_vector_get_data(values_x0));
828  cpl_size power;
829 
830  /* Fit the y0 -- apply a median filter first */
831  cpl_polynomial * y0_poly = cpl_polynomial_new(1);
832  power = 4;
833  cpl_vector * temp_median = cpl_vector_filter_median_create(values_y0,3);
834  cpl_polynomial_fit (y0_poly, matrix, NULL, temp_median, NULL,
835  CPL_FALSE, NULL, &power);
836  cpl_vector_delete (temp_median);
837  CPLCHECK_NUL ("Cannot fit the y0");
838 
839  /* Fit the sigma */
840  cpl_polynomial * sigma_poly = cpl_polynomial_new(1);
841  power = 5;
842  cpl_polynomial_fit (sigma_poly, matrix, NULL, values_sigma,
843  NULL, CPL_FALSE, NULL, &power);
844  CPLCHECK_NUL ("Cannot fit the sigma");
845 
846  /* Compute the new y0 and sigma from these fits */
847  cpl_vector * valuesfit_y0 = cpl_vector_new(nx);
848  cpl_vector * valuesfit_sig = cpl_vector_new(nx);
849 
850  for (cpl_size ix = 0; ix < nx; ix++){
851  double result;
852  result = cpl_polynomial_eval_1d (y0_poly, cpl_vector_get(values_x0, ix), NULL);
853  cpl_vector_set(valuesfit_y0, ix, result);
854  result = cpl_polynomial_eval_1d (sigma_poly, cpl_vector_get(values_x0, ix), NULL);
855  cpl_vector_set(valuesfit_sig, ix, result);
856  }
857  FREE (cpl_polynomial_delete, y0_poly);
858  FREE (cpl_polynomial_delete, sigma_poly);
859  cpl_matrix_unwrap (matrix);
860 
861  /*
862  * Allocate image profile
863  */
864 
865  cpl_image * region_img;
866  region_img = cpl_image_new (nx, ny, CPL_TYPE_DOUBLE);
867  cpl_image_fill_window (region_img, 1,1,nx,ny, 0.0);
868 
869  /* Loop on spectral direction */
870  for (cpl_size ix = 0; ix < nx; ix++){
871 
872  double sum_flux = 0, sum_flux2 = 0;
873 
874  /* Loop on spatial direction */
875  for (cpl_size iy = iy_min; iy <= iy_max; iy++ ){
876 
877  double result;
878 
879  /* We use the measured profile */
880  if (!strcmp (mode, "PROFILE")) {
881  result = cpl_image_get (mean_img, ix+1, iy+1, &nv);
882  }
883  /* We use a fixed 5 pixel boxcar profile */
884  else if (!strcmp (mode, "BOX")) {
885  result = ( (fabs(iy - cpl_vector_get (valuesfit_y0, ix)) < 3 ) ? 1.0 : 0.0);
886  }
887  /* We use a Gaussian profile */
888  else if (!strcmp (mode, "GAUSS")) {
889  result = (iy - cpl_vector_get (valuesfit_y0, ix)) /
890  cpl_vector_get (valuesfit_sig, ix);
891  result = exp( - pow (result, 2) / 2);
892  } else {
893  cpl_msg_error (cpl_func, "BUG, report to DRS team");
894  return NULL;
895  }
896 
897  /* Fill the profile image of this region */
898  cpl_image_set (region_img, ix + 1, iy + 1, result);
899 
900  /* Compute normalization coefficients */
901  sum_flux += result;
902  sum_flux2 += result * result;
903  } /* End loop on spatial direction */
904 
905  /* Keep only effective part of the profile
906  * Force pixel <1e-7 to zero */
907  for (cpl_size iy = iy_min; iy <= iy_max; iy++ ) {
908  double img_j = cpl_image_get (region_img, ix+1, iy+1, &nv);
909  if (img_j / sum_flux < 1e-7)
910  cpl_image_set (region_img, ix+1, iy+1, 0.0);
911  }
912 
913  /* When we use a complex profile, we have to normalize
914  * it to make it flux conservative at extraction */
915  if (!strcmp (mode, "PROFILE") || !strcmp (mode, "GAUSS") ) {
916 
917  double sum_flux = 0.0;
918  double sum_flux2 = 0.0;
919  for (cpl_size iy = iy_min; iy <= iy_max; iy++ ) {
920  double img_j = cpl_image_get (region_img, ix+1, iy+1, &nv);
921  sum_flux += img_j;
922  sum_flux2 += img_j * img_j;
923  }
924 
925  for (cpl_size iy = iy_min; iy <= iy_max; iy++ ) {
926  double img_j = cpl_image_get (region_img, ix+1, iy+1, &nv);
927  cpl_image_set (region_img, ix+1, iy+1, img_j * sum_flux / sum_flux2);
928  }
929  }
930 
931  } /* End loop on columns = spectral direction */
932 
933  /* Deallocated of the fitted variables */
934  FREE (cpl_vector_delete, valuesfit_y0);
935  FREE (cpl_vector_delete, valuesfit_sig);
936 
937  gravi_msg_function_exit(0);
938  return region_img;
939 }
940 
941 /*----------------------------------------------------------------------------*/
974 /*----------------------------------------------------------------------------*/
975 
976 gravi_data * gravi_compute_profile(gravi_data ** flats_data,
977  gravi_data * dark_map, gravi_data * bad_map,
978  int nflat, const cpl_parameterlist * params)
979 {
980  /* Verbose */
981  gravi_msg_function_start(1);
982  cpl_ensure (flats_data, CPL_ERROR_NULL_INPUT, NULL);
983  cpl_ensure (dark_map, CPL_ERROR_NULL_INPUT, NULL);
984 
985  /* Ensure all beam open once */
986  cpl_ensure (gravi_data_check_shutter_beam (flats_data, nflat),
987  CPL_ERROR_ILLEGAL_INPUT, NULL);
988 
989  /* Construct the product */
990  gravi_data * out_data = gravi_data_new (0);
991  cpl_propertylist * out_header = gravi_data_get_header (out_data);
992 
993  /* Dump full header of first RAW data */
994  gravi_data * raw0 = flats_data[0];
995  cpl_propertylist_append (out_header, gravi_data_get_header (flats_data[0]));
996 
997  /* Copy IMAGING_DETECTOR into product */
998  gravi_data_copy_ext (out_data, raw0, GRAVI_IMAGING_DETECTOR_FT_EXT);
999  gravi_data_copy_ext (out_data, raw0, GRAVI_IMAGING_DETECTOR_SC_EXT);
1000 
1001 
1002  /*
1003  * (1) Compute the FLAT of the FT
1004  */
1005  if (!gravi_data_has_extension (flats_data[0], GRAVI_IMAGING_DATA_FT_EXT)) {
1006  cpl_msg_warning (cpl_func,"The FLAT data has no IMAGING_DATA_FT");
1007  }
1008  else
1009  {
1010  cpl_msg_info (cpl_func, "Computing the FLAT of FT");
1011 
1012  /* Get the dark of FT */
1013  cpl_table * darkft_table;
1014  darkft_table = gravi_data_get_table (dark_map, GRAVI_IMAGING_DATA_FT_EXT);
1015  CPLCHECK_NUL ("Cannot get DARK data of FT");
1016 
1017  /* Get the DARK as an image */
1018  cpl_image * darkft_img;
1019  darkft_img = gravi_image_from_column (darkft_table, "PIX", 0);
1020 
1021  /* Collapse each FLAT file */
1022  cpl_imagelist *imglist_ft = cpl_imagelist_new ();
1023  for (int file = 0; file < nflat; file++) {
1024 
1025  cpl_table * dataft_table;
1026  dataft_table = gravi_data_get_table (flats_data[file], GRAVI_IMAGING_DATA_FT_EXT);
1027  CPLCHECK_NUL ("Cannot get data");
1028 
1029  cpl_imagelist * imglistft_tmp;
1030  imglistft_tmp = gravi_imagelist_wrap_column (dataft_table, "PIX");
1031 
1032  /* Remove DARK */
1033  cpl_imagelist_subtract_image (imglistft_tmp, darkft_img);
1034 
1035  /* Collapse the DITs of this FLAT */
1036  cpl_imagelist_set (imglist_ft, cpl_imagelist_collapse_create (imglistft_tmp), file);
1037  gravi_imagelist_unwrap_images (imglistft_tmp);
1038  }
1039 
1040  /* Collapse the FLAT files together */
1041  cpl_image * flatft_img = cpl_imagelist_collapse_create (imglist_ft);
1042  cpl_imagelist_delete (imglist_ft);
1043 
1044  /* Create the flat_table */
1045  cpl_table * flatft_table = cpl_table_extract (gravi_data_get_table (flats_data[nflat-1],
1046  GRAVI_IMAGING_DATA_FT_EXT), 0, 1);
1047 
1048  cpl_array * flatft_array = gravi_array_wrap_image (flatft_img);
1049  cpl_table_set_array (flatft_table, "PIX", 0, flatft_array);
1050 
1051  /* Remove median image and array */
1052  FREE (cpl_array_unwrap, flatft_array);
1053  FREE (cpl_image_delete, flatft_img);
1054  FREE (cpl_image_delete, darkft_img);
1055 
1056  /* Set the FLAT as IMAGING_DATA into product */
1057  gravi_data_add_table (out_data, NULL, GRAVI_IMAGING_DATA_FT_EXT, flatft_table);
1058 
1059  } /* End FLAT of FT */
1060 
1061  /*
1062  * (2) General data used by FLAT and PROFILE for SC
1063  */
1064 
1065  /* Get that the detector_table */
1066  cpl_table * detector_table;
1067  detector_table = gravi_data_get_table (flats_data[0], GRAVI_IMAGING_DETECTOR_SC_EXT);
1068  cpl_ensure (detector_table, CPL_ERROR_ILLEGAL_INPUT, NULL);
1069 
1070  /* Get that the header of first file and of dark */
1071  cpl_propertylist * dark_header = gravi_data_get_header (dark_map);
1072  cpl_propertylist * flat0_header = gravi_data_get_header (flats_data[0]);
1073 
1074  /* Get necessary information */
1075  int det_startx = gravi_pfits_get_window_start (flat0_header);
1076  const char * resolution = gravi_pfits_get_resolution (flat0_header);
1077  const char * pola_mode = gravi_pfits_get_pola_mode(flat0_header, GRAVI_SC);
1078  int nb_region = cpl_table_get_nrow (detector_table);
1079 
1080  /* Get the DARK and BAD data */
1081  cpl_image * dark_img, * bad_img;
1082  dark_img = gravi_data_get_img (dark_map, GRAVI_IMAGING_DATA_SC_EXT);
1083  bad_img = gravi_data_get_img (bad_map, GRAVI_IMAGING_DATA_SC_EXT);
1084  cpl_ensure (dark_img && bad_img, CPL_ERROR_ILLEGAL_INPUT, NULL);
1085 
1086  /*
1087  * (3) Compute the FLAT for the SC
1088  */
1089  cpl_msg_info (cpl_func, "Computing the FLAT of SC");
1090 
1091  /* Imagelist to store the collapsed FLATs */
1092  cpl_imagelist * temp_imglist = cpl_imagelist_new ();
1093 
1094  for (int file = 0; file < nflat; file++){
1095 
1096  /* Extract necessary parameters and construct the output table */
1097  cpl_imagelist * data_imglist;
1098  data_imglist = gravi_data_get_cube (flats_data[file], GRAVI_IMAGING_DATA_SC_EXT);
1099 
1100  /* Extract data with DARK, BADPIX in [ADU] */
1101  data_imglist = cpl_imagelist_duplicate (data_imglist);
1102  cpl_imagelist_subtract_image (data_imglist, dark_img);
1103  gravi_remove_badpixel_sc (data_imglist, bad_img);
1104 
1105  /* Collapse the DITs of this FLAT */
1106  cpl_image * collapsed_img = cpl_imagelist_collapse_median_create (data_imglist);
1107  FREE (cpl_imagelist_delete, data_imglist);
1108 
1109  /* Save this FLAT in the imagelist to collapse them */
1110  cpl_imagelist_set (temp_imglist, collapsed_img,
1111  cpl_imagelist_get_size (temp_imglist));
1112 
1113  CPLCHECK_NUL ("Error");
1114  } /* End loop on FLATs*/
1115 
1116  /* Collapse the FLATs together */
1117  cpl_image * allflat_img;
1118  allflat_img = cpl_imagelist_collapse_create (temp_imglist);
1119  FREE (cpl_imagelist_delete, temp_imglist);
1120 
1121  CPLCHECK_NUL ("Cannot collapse FLATs");
1122 
1123  /* Get the extension (illumated part) of the FLAT */
1124  int * ext_dim = gravi_image_extract_dimension (allflat_img);
1125  int fullstartx = ext_dim[0] + det_startx - 2;
1126 
1127  /* Crop the FLAT into these new dimensions
1128  * FIXME: would be better to no crop FLAT */
1129  cpl_image * flatsc_img;
1130  flatsc_img = cpl_image_extract (allflat_img, ext_dim[0], 1,
1131  ext_dim[0] + ext_dim[1] - 1,
1132  cpl_image_get_size_y (allflat_img));
1133  FREE (cpl_image_delete, allflat_img);
1134 
1135  /* Set the cropped FLAT of SC in output data
1136  * STARTX is in FITS convention (start 1) while
1137  * FULLSTARTX seems in C convention (start 0) */
1138  cpl_propertylist * flat_plist = cpl_propertylist_new ();
1139  cpl_propertylist_update_int (flat_plist, PROFILE_FULLSTARTX, fullstartx);
1140  cpl_propertylist_update_int (flat_plist, PROFILE_STARTX, ext_dim[0]);
1141  cpl_propertylist_update_int (flat_plist, PROFILE_NX, ext_dim[1]);
1142 
1143  gravi_data_add_img (out_data, flat_plist, GRAVI_IMAGING_DATA_SC_EXT, flatsc_img);
1144  CPLCHECK_NUL ("Cannot set FLAT");
1145 
1146 
1147  /*
1148  * (4) Compute the PROFILE for the SC
1149  */
1150  cpl_msg_info (cpl_func, "Computing the PROFILE of SC");
1151 
1152 
1153  /* List of image to store each FLAT */
1154  cpl_image ** median_img = cpl_calloc (4, sizeof(cpl_image *));
1155 
1156  /* kernel for median filter. Use more
1157  * pixels (~15) in HIGH, to skip the cluster */
1158  cpl_size size_kernel = !strcmp (resolution, "HIGH") ? 15 : 5;
1159  cpl_msg_info (cpl_func, "Median filtering over %lld spectral pixels", size_kernel);
1160  cpl_mask * kernel = cpl_mask_new (size_kernel, 1);
1161  cpl_mask_not (kernel);
1162 
1163  for (int file = 0; file < nflat; file++){
1164  /* Get the primary header of each file */
1165  cpl_propertylist * flat_header;
1166  flat_header = gravi_data_get_header (flats_data[file]);
1167 
1168  /* Extract necessary parameters and construct the output table */
1169  cpl_imagelist * data_imglist;
1170  data_imglist = gravi_data_get_cube (flats_data[file], GRAVI_IMAGING_DATA_SC_EXT);
1171  cpl_ensure (flat_header && data_imglist, CPL_ERROR_NULL_INPUT, NULL);
1172 
1173  /* Extract data with DARK, BADPIX in [ADU] */
1174  data_imglist = cpl_imagelist_duplicate (data_imglist);
1175  cpl_imagelist_subtract_image (data_imglist, dark_img);
1176  gravi_remove_badpixel_sc (data_imglist, bad_img);
1177 
1178  /* Collapse the DITs of this FLAT */
1179  cpl_image * collapsed_img = cpl_imagelist_collapse_median_create (data_imglist);
1180  FREE (cpl_imagelist_delete, data_imglist);
1181 
1182  /* Create a filtered version of this FLAT */
1183  cpl_image * filtered_img = cpl_image_duplicate (collapsed_img);
1184  cpl_image_filter_mask (filtered_img, collapsed_img, kernel,
1185  CPL_FILTER_MEDIAN, CPL_BORDER_FILTER);
1186  FREE (cpl_image_delete, collapsed_img);
1187 
1188  /* Crop it */
1189  cpl_image * crop_img;
1190  crop_img = cpl_image_extract (filtered_img, ext_dim[0], 1,
1191  ext_dim[0] + ext_dim[1] - 1,
1192  cpl_image_get_size_y (filtered_img));
1193  FREE (cpl_image_delete, filtered_img);
1194 
1195  /* Save this filtered version in the median_img[]
1196  * According to the beam open */
1197  int id = gravi_get_shutter_id (flat_header);
1198  median_img[id] = crop_img;
1199 
1200  CPLCHECK_NUL ("Error");
1201  } /* End loop on FLATs*/
1202 
1203  FREE (cpl_mask_delete, kernel);
1204 
1205  /* Get the image dimensions after crop */
1206  int nx = cpl_image_get_size_x (median_img[0]);
1207  int ny = cpl_image_get_size_y (median_img[0]);
1208 
1209  /* Get the profile width parameter */
1210  int profile_width = gravi_param_get_int (params, "gravity.calib.profile-width");
1211  cpl_ensure (profile_width > 0, CPL_ERROR_ILLEGAL_INPUT, NULL);
1212 
1213  /* Compute the size_profile. For LOW and MEDIUM it is automatic.
1214  * For HIGH it comes from the option */
1215  int n_darkline;
1216  cpl_propertylist * flat_header;
1217  int window_mode;
1218  flat_header = gravi_data_get_header (flats_data[0]);
1219  int size_profile;
1220  /* if the windowing mode if after the 05/12/2016 */
1221  if ( cpl_propertylist_get_float(flat_header, "MJD-OBS") > 57728 ){
1222  n_darkline= 0;
1223  window_mode = 1;
1224  cpl_msg_info(cpl_func, "Windowing after MJD = 57728");
1225  if (! (strcmp(resolution, "LOW") && strcmp(resolution, "MED")) && !strcmp(pola_mode, "COMB")) {
1226  size_profile = ny/nb_region;
1227  cpl_msg_info(cpl_func, "Use a computed size_profile of %d", size_profile);
1228  }
1229  else{
1230  size_profile = (fmod(profile_width, 2) == 0) ? profile_width + 1 : profile_width;
1231  cpl_msg_info(cpl_func, "Use a given size profile of %d", size_profile);
1232  }
1233  }
1234  /* else keep old windowing for backward compatibility */
1235  else {
1236  n_darkline= 1;
1237  window_mode = 0;
1238  if (! (strcmp(resolution, "LOW") && strcmp(resolution, "MED"))) {
1239  size_profile = (ny-(nb_region+1)*n_darkline)/nb_region;
1240  cpl_msg_info(cpl_func, "Use a computed size_profile of %d", size_profile);
1241  }
1242  else{
1243  size_profile = (fmod(profile_width, 2) == 0) ? profile_width + 1 : profile_width;
1244  cpl_msg_info(cpl_func, "Use a given size profile of %d", size_profile);
1245  }
1246  }
1247 
1248  /* Construction of the PROFILE_DATA table */
1249  cpl_table * profile_table = cpl_table_new (1);
1250 
1251  /* Create the DATA# columns with their dimension */
1252  cpl_array * dimension = cpl_array_new (2, CPL_TYPE_INT);
1253  cpl_array_set (dimension, 0, nx);
1254  cpl_array_set (dimension, 1, ny);
1255  for (int region = 0; region < nb_region; region ++){
1256  const char * data = GRAVI_DATA[region];
1257  cpl_table_new_column_array (profile_table, data,
1258  CPL_TYPE_DOUBLE, nx * ny);
1259  cpl_table_set_column_dimensions (profile_table, data, dimension);
1260  }
1261  FREE (cpl_array_delete, dimension);
1262 
1263  /* Construction of the PROFILE_PARAMS table */
1264  cpl_table * params_table = cpl_table_new (nb_region);
1265  cpl_table_new_column_array (params_table, "CENTERY", CPL_TYPE_DOUBLE, nx);
1266  cpl_table_new_column_array (params_table, "WIDTH", CPL_TYPE_DOUBLE, nx);
1267 
1268 
1269  /* Construction of a map used to zero the badpixels
1270  * Mask is: good pixels = 1 ; bad pixels = 0 */
1271  cpl_image * mask_img = NULL;
1272 
1273  if ( !gravi_param_get_bool (params,
1274  "gravity.calib.force-badpix-to-zero") )
1275  {
1276  cpl_msg_info (cpl_func, "Bad pixels are *not*"
1277  "forced to zero in profiles");
1278  cpl_image_fill_window (mask_img, 1, 1, nx, ny, 1.0);
1279  }
1280  else {
1281  cpl_msg_info (cpl_func, "Bad pixels are "
1282  "forced to zero in profiles");
1283  mask_img = cpl_image_extract (bad_img, ext_dim[0], 1,
1284  ext_dim[0] + ext_dim[1] - 1, ny);
1285  cpl_image_threshold (mask_img, 0.5, 0.5, 1.0, 0.0);
1286  }
1287 
1288  /* Update mask from FLAT itself in LOW,
1289  * FIXME: to decide what we zero exactly. */
1290  if ( !strcmp (resolution, "LOW") )
1291  {
1292  cpl_msg_info (cpl_func, "Pixels with low FLAT values"
1293  " are forced to zero in profiles.");
1294 
1295  double threshold = cpl_propertylist_get_double (dark_header, QC_DARKRMS_SC);
1296  cpl_image * mask2_img = cpl_image_duplicate (flatsc_img);
1297  cpl_image_threshold (mask2_img, threshold, threshold, 0.0, 1.0);
1298  cpl_image_multiply (mask_img, mask2_img);
1299  FREE (cpl_image_delete, mask2_img);
1300  }
1301 
1302  /* Define the mode */
1303  const char * mode = gravi_param_get_string_default (params,
1304  "gravity.calib.profile-mode", "AUTO");
1305 
1306  if (!strcmp(mode, "AUTO")) {
1307  if (!strcmp(resolution, "LOW")) mode = "PROFILE";
1308  if (!strcmp(resolution, "MED")) mode = "PROFILE";
1309  if (!strcmp(resolution, "HIGH")) mode = "BOX";
1310  }
1311 
1312  cpl_msg_info (cpl_func, "Profile computed with mode: %s (%s)", mode, resolution);
1313 
1314  /* Loop on regions */
1315  for (int region = 0; region < nb_region ; region++) {
1316 
1317  int tel_1 = gravi_region_get_tel (detector_table, region, 0);
1318  int tel_2 = gravi_region_get_tel (detector_table, region, 1);
1319  CPLCHECK_NUL ("Cannot get the telescope from region");
1320 
1321  /* Compute the mean image between the first
1322  * and second telescope */
1323  cpl_image * mean_img;
1324  mean_img = cpl_image_add_create (median_img[tel_1],
1325  median_img[tel_2]);
1326  cpl_image_divide_scalar (mean_img, 2);
1327 
1328  /* Get the reference coordinates of this region. That is
1329  * a point where the spectra is supposed to be for sure
1330  * In unit of the full detector window */
1331  int ref0_x = gravi_table_get_value (detector_table, "CENTER", region, 0);
1332  int ref0_y = gravi_table_get_value (detector_table, "CENTER", region, 1);
1333 
1334  /* Convert the reference coordinates in unit of the cropped data
1335  * Actually we force ref_x and ref_y in LOW and MED. */
1336  int ref_x, ref_y;
1337 
1338  /* if the windowing mode if after the 05/12/2016 */
1339  if ( window_mode == 1 ){
1340  if (! (strcmp(resolution, "LOW") && strcmp(resolution, "MED")) && !strcmp(pola_mode, "COMB") ) {
1341  ref_x = nx / 2;
1342  ref_y = (region+1)*(size_profile)-size_profile/2;
1343  } else {
1344  //ref_x = ref0_x - det_startx - (1 + ext_dim[0]);
1345  ref_x = ref0_x - (1 + ext_dim[0]);
1346  ref_y = ref0_y;
1347  }
1348  }
1349  else {
1350  if (! (strcmp(resolution, "LOW") && strcmp(resolution, "MED")) ) {
1351  ref_x = nx / 2;
1352  ref_y = (region+1)*(size_profile+n_darkline)-size_profile/2;
1353  } else {
1354  //ref_x = ref0_x - det_startx - (1 + ext_dim[0]);
1355  ref_x = ref0_x - (1 + ext_dim[0]);
1356  ref_y = ref0_y;
1357  }
1358  }
1359 
1360  /*
1361  * Fit Gaussian to profile of each spectral channel
1362  */
1363  cpl_vector * values_x0 = cpl_vector_new (nx);
1364  cpl_vector * values_y0 = cpl_vector_new (nx);
1365  cpl_vector * values_sigma = cpl_vector_new (nx);
1366 
1367  gravi_fit_profile (values_x0, values_y0, values_sigma,
1368  mean_img, ref_x, ref_y, size_profile,
1369  resolution);
1370  CPLCHECK_NUL ("Cannot fit data into profile params");
1371 
1372  /* Fill the PROFILE_PARAM table for this region */
1373  cpl_array * values_arr;
1374  values_arr = cpl_array_wrap_double (cpl_vector_get_data (values_y0), nx);
1375  cpl_table_set_array (params_table, "CENTERY", region, values_arr);
1376  cpl_array_unwrap (values_arr);
1377 
1378  values_arr = cpl_array_wrap_double (cpl_vector_get_data(values_sigma), nx);
1379  cpl_table_set_array (params_table, "WIDTH", region, values_arr);
1380  cpl_array_unwrap (values_arr);
1381 
1382 
1383  /*
1384  * Create and compute the profile image of this region
1385  */
1386 
1387  /* Define the spatial pixels over which we fill the profile */
1388  cpl_size iy_min = 0, iy_max = ny-1;
1389  if (! (strcmp(resolution, "LOW") && strcmp(resolution, "MED")) ){
1390  iy_min = ref_y - size_profile/2;
1391  iy_max = ref_y + size_profile/2-1;
1392  }
1393 
1394  /* Fill the profile image */
1395  cpl_image * region_img;
1396  region_img = gravi_create_profile_image (mean_img, values_x0,
1397  values_y0, values_sigma,
1398  iy_min, iy_max, mode);
1399  FREE (cpl_vector_delete, values_x0);
1400  FREE (cpl_vector_delete, values_y0);
1401  FREE (cpl_vector_delete, values_sigma);
1402  CPLCHECK_NUL ("Cannot build profile image");
1403 
1404  /* Force some pixel to zero with computed mask */
1405  cpl_image_multiply (region_img, mask_img);
1406 
1407  /* Fill the profile_table for this region */
1408  cpl_array * array = gravi_array_wrap_image (region_img);
1409  cpl_table_set_array (profile_table, GRAVI_DATA[region], 0, array);
1410  FREE (cpl_array_unwrap, array);
1411  FREE (cpl_image_delete, region_img);
1412 
1413  /* Free the mean image of the 2 flats of this region */
1414  FREE (cpl_image_delete, mean_img);
1415 
1416  } /* End loop on regions */
1417 
1418  /* Save the PROFILE_DATA table */
1419  cpl_propertylist * profile_plist = cpl_propertylist_new ();
1420  cpl_propertylist_copy_property (profile_plist, flat_plist, PROFILE_FULLSTARTX);
1421  cpl_propertylist_copy_property (profile_plist, flat_plist, PROFILE_STARTX);
1422  cpl_propertylist_copy_property (profile_plist, flat_plist, PROFILE_NX);
1423  gravi_data_add_table (out_data, profile_plist,
1424  GRAVI_PROFILE_DATA_EXT, profile_table);
1425 
1426  /* Save the PROFILE_PARAMS table */
1427  cpl_propertylist * params_plist = cpl_propertylist_duplicate (profile_plist);
1428  gravi_data_add_table (out_data, params_plist,
1429  GRAVI_PROFILE_PARAMS_EXT, params_table);
1430 
1431 
1432  /*
1433  * (5) Add the QC parameter of the lateral positioning of the first region
1434  */
1435 
1436  cpl_propertylist * main_header = gravi_data_get_header (out_data);
1437  for (int reg = 0; reg < nb_region; reg += 12) {
1438  double qc_value;
1439  char qc_name[100];
1440  cpl_size idx;
1441 
1442  /* Median lateral position */
1443  qc_value = cpl_array_get_median (cpl_table_get_array (params_table, "CENTERY", reg));
1444  qc_value = floor (qc_value * 1e6) * 1e-6;
1445  sprintf (qc_name, "ESO QC PROFILE_CENTER SC%i MED", reg+1);
1446  cpl_propertylist_append_double (main_header, qc_name, qc_value);
1447  cpl_propertylist_set_comment (main_header, qc_name, "[pixel] position of region");
1448  cpl_msg_info (cpl_func,"%s = %f [pixel]", qc_name, qc_value);
1449 
1450  /* Median width */
1451  qc_value = cpl_array_get_median (cpl_table_get_array (params_table, "WIDTH", reg));
1452  qc_value = floor (qc_value * 1e6) * 1e-6;
1453  sprintf (qc_name, "ESO QC PROFILE_WIDTH SC%i MED", reg+1);
1454  cpl_propertylist_append_double (main_header, qc_name, qc_value);
1455  cpl_propertylist_set_comment (main_header, qc_name, "[pixel] width of region");
1456  cpl_msg_info (cpl_func,"%s = %f [pixel]", qc_name, qc_value);
1457 
1458  /* Lateral position in the left */
1459  idx = 1 * ext_dim[1] / 6;
1460  qc_value = cpl_array_get (cpl_table_get_array (params_table, "CENTERY", reg), idx, NULL);
1461  qc_value = floor (qc_value * 1e6) * 1e-6;
1462  sprintf (qc_name, "ESO QC PROFILE_CENTER SC%i LEFT", reg+1);
1463  cpl_propertylist_append_double (main_header, qc_name, qc_value);
1464  cpl_propertylist_set_comment (main_header, qc_name, "[pixel] at STARTX+NX/6");
1465  cpl_msg_info (cpl_func,"%s = %f [pixel] for x=%lld", qc_name, qc_value, idx);
1466 
1467  /* Lateral position in the right */
1468  idx = 5 * ext_dim[1] / 6;
1469  qc_value = cpl_array_get (cpl_table_get_array (params_table, "CENTERY", reg), idx, NULL);
1470  qc_value = floor (qc_value * 1e6) * 1e-6;
1471  sprintf (qc_name, "ESO QC PROFILE_CENTER SC%i RIGHT", reg+1);
1472  cpl_propertylist_append_double (main_header, qc_name, qc_value);
1473  cpl_propertylist_set_comment (main_header, qc_name, "[pixel] at STARTX+5*NX/6");
1474  cpl_msg_info (cpl_func,"%s = %f [pixel] for x=%lld", qc_name, qc_value, idx);
1475 
1476  CPLCHECK_NUL ("Cannot compute QC parameters");
1477  }
1478 
1479  /* Deallocation of the variables */
1480  FREE (cpl_image_delete, mask_img);
1481  FREELOOP (cpl_image_delete, median_img, 4);
1482  FREE (cpl_free, ext_dim);
1483 
1484  /* Verbose */
1485  gravi_msg_function_exit(1);
1486  return out_data;
1487 }
1488 
1489 /*----------------------------------------------------------------------------*/
1503 /*----------------------------------------------------------------------------*/
1504 
1505 cpl_propertylist * gravi_compute_gain (gravi_data ** flats_data,
1506  int nrawgain,
1507  gravi_data * dark_map)
1508 {
1509  int nv;
1510  const cpl_size maxdeg = 1, mindeg = 0;
1511  const cpl_size slope_deg = 1;
1512 
1513  /* Verbose */
1514  gravi_msg_function_start(1);
1515  cpl_ensure (flats_data, CPL_ERROR_NULL_INPUT, NULL);
1516  cpl_ensure (dark_map, CPL_ERROR_NULL_INPUT, NULL);
1517 
1518  /* Ensure all beam open once */
1519  cpl_ensure (gravi_data_check_shutter_beam (flats_data, nrawgain),
1520  CPL_ERROR_ILLEGAL_INPUT, NULL);
1521 
1522  /* Get a sequence of 4 single-shutter open files
1523  * FIXME: remove this part isn't it ?? */
1524  cpl_msg_info (cpl_func, "Search for 4-shutter sequence");
1525 
1526  gravi_data ** gain_file = cpl_calloc(4,sizeof(gravi_data*));
1527 
1528  for (int i = 0; i < nrawgain; i++){
1529  cpl_propertylist * flat_header = gravi_data_get_header (flats_data[i]);
1530 
1531  if (gravi_check_shutter (flat_header, 1,0,0,0)) {
1532  gain_file[0] = (flats_data[i]);
1533  }
1534  if (gravi_check_shutter (flat_header, 0,1,0,0)) {
1535  gain_file[1] = (flats_data[i]);
1536  }
1537  if (gravi_check_shutter (flat_header, 0,0,1,0)) {
1538  gain_file[2] = (flats_data[i]);
1539  }
1540  if (gravi_check_shutter (flat_header, 0,0,0,1)) {
1541  gain_file[3] = (flats_data[i]);
1542  }
1543  }
1544 
1545  /*
1546  * Create the output. It is a plist only since we
1547  * don't create a gravi_data
1548  */
1549  cpl_propertylist * output_plist = cpl_propertylist_new();
1550 
1551 
1552  /*
1553  *
1554  * Compute the gain for SC
1555  *
1556  */
1557  if (!gravi_data_has_extension (gain_file[0], GRAVI_IMAGING_DATA_SC_EXT)) {
1558  cpl_msg_warning (cpl_func,"The FLAT data has no IMAGING_DATA_SC");
1559  }
1560  else
1561  {
1562  cpl_msg_info (cpl_func, "Computing the gain of SC");
1563 
1564  /* Get the size of the image */
1565  cpl_image * image = gravi_data_get_img (gain_file[0], GRAVI_IMAGING_DATA_SC_EXT);
1566  cpl_size size = cpl_image_get_size_x (image) * cpl_image_get_size_y (image);
1567 
1568  /* Build matrix and vector to fit var = f(mean) */
1569  cpl_vector * vector_var = cpl_vector_new (4*size);
1570  cpl_matrix * matrix_mean = cpl_matrix_new (1, 4*size);
1571 
1572  /* Get a pointer to the dark image */
1573  cpl_msg_info (cpl_func,"DARK and BADPIX of SC are not used");
1574 
1575  /* Loop on files to fill the variance and mean vectors */
1576  for (int file = 0; file < 4; file++) {
1577 
1578  /* Get the imagelist and property lists */
1579  cpl_imagelist * data_imglist;
1580  data_imglist = gravi_data_get_cube (gain_file[file], GRAVI_IMAGING_DATA_SC_EXT);
1581 
1582  /* Compute <image> and <image^2> */
1583  cpl_imagelist * imglist1 = cpl_imagelist_new ();
1584  cpl_imagelist * imglist2 = cpl_imagelist_new ();
1585  for (cpl_size i = 0; i < cpl_imagelist_get_size (data_imglist); i ++){
1586  image = cpl_image_cast (cpl_imagelist_get (data_imglist, i), CPL_TYPE_DOUBLE);
1587  cpl_imagelist_set (imglist1, image, i);
1588  cpl_imagelist_set (imglist2, cpl_image_power_create (image, 2), i);
1589  }
1590  cpl_image * image1_mean = cpl_imagelist_collapse_create (imglist1);
1591  cpl_image * image2_mean = cpl_imagelist_collapse_create (imglist2);
1592  FREE (cpl_imagelist_delete, imglist1);
1593  FREE (cpl_imagelist_delete, imglist2);
1594 
1595  /* Loop on pixels to fill the vector and matrix structures
1596  * used latter in the PTC fit */
1597  cpl_size nx = cpl_image_get_size_x (image1_mean);
1598  cpl_size ny = cpl_image_get_size_y (image1_mean);
1599  for (cpl_size i = 0; i < nx; i ++) {
1600  for (cpl_size j = 0; j < ny; j ++) {
1601  cpl_vector_set (vector_var, file * size + (j + ny*i),
1602  cpl_image_get (image2_mean, i+1, j+1, &nv) -
1603  pow (cpl_image_get (image1_mean, i+1, j+1, &nv), 2));
1604  cpl_matrix_set (matrix_mean, 0, file * size + (j + ny*i),
1605  cpl_image_get (image1_mean, i+1, j+1, &nv));
1606  }
1607  }
1608  /* End loop on pixels */
1609 
1610  /* Deallocation of variables */
1611  FREE (cpl_image_delete, image1_mean);
1612  FREE (cpl_image_delete, image2_mean);
1613  }
1614  /* End loop on files */
1615 
1616  /* Fit the curve variance versus the median,
1617  thus the output gains are in ADU/e */
1618  cpl_polynomial * fit_slope = cpl_polynomial_new (1);
1619  cpl_polynomial_fit (fit_slope, matrix_mean, NULL, vector_var, NULL,
1620  CPL_FALSE, &mindeg, &maxdeg);
1621 
1622  /* Get the slope */
1623  double slope = cpl_polynomial_get_coeff (fit_slope, &slope_deg);
1624  cpl_msg_info (cpl_func, "mean gain SC = %.4f [ADU/e-] Mean gain of detector", slope);
1625  cpl_propertylist_append_double (output_plist, QC_MEANGAIN_SC, slope);
1626  cpl_propertylist_set_comment (output_plist, QC_MEANGAIN_SC, "[ADU/e-] Mean gain of SC detector" );
1627 
1628  /* Delete */
1629  cpl_vector_delete (vector_var);
1630  cpl_matrix_delete (matrix_mean);
1631  cpl_polynomial_delete (fit_slope);
1632  }
1633  /* End SC case */
1634 
1635  /*
1636  *
1637  * Compute the gain for FT
1638  *
1639  */
1640  if (!gravi_data_has_extension (gain_file[0], GRAVI_IMAGING_DATA_SC_EXT)) {
1641  cpl_msg_warning (cpl_func,"The FLAT data has no IMAGING_DATA_FT");
1642  }
1643  else
1644  {
1645  cpl_msg_info (cpl_func, "Computing the gain of FT");
1646 
1647  /* Get DARK image */
1648  cpl_table * dark_table;
1649  dark_table = gravi_data_get_table (dark_map, GRAVI_IMAGING_DATA_FT_EXT);
1650  cpl_image * dark_img = gravi_image_from_column (dark_table, "PIX", 0);
1651 
1652  /* Get the size of the image */
1653  cpl_size size = cpl_table_get_column_depth (dark_table, "PIX");
1654 
1655  /* Build matrix and vector to fit var = f(mean) */
1656  cpl_vector * vector_var = cpl_vector_new (4 * size);
1657  cpl_matrix * matrix_mean = cpl_matrix_new (1, 4 * size);
1658 
1659  /* Compute the gain for each file */
1660  for (int file = 0; file < 4; file++) {
1661 
1662  /* Get the tables */
1663  cpl_table * data_table;
1664  data_table = gravi_data_get_table (gain_file[file], GRAVI_IMAGING_DATA_FT_EXT);
1665  cpl_table_cast_column (data_table, "PIX", "PIX", CPL_TYPE_DOUBLE);
1666  cpl_imagelist * data_imglist = gravi_imagelist_from_column (data_table,"PIX");
1667  CPLCHECK_NUL ("Cannot get data");
1668 
1669  /* Remove DARK */
1670  cpl_imagelist_subtract_image (data_imglist, dark_img);
1671 
1672  /* Compute MEAN */
1673  cpl_image * mean_img = cpl_imagelist_collapse_create (data_imglist);
1674 
1675  /* Compute VARIANCE */
1676  cpl_imagelist_subtract_image (data_imglist, mean_img);
1677  cpl_imagelist_power (data_imglist, 2.0);
1678  cpl_image * var_img = cpl_imagelist_collapse_create (data_imglist);
1679 
1680  FREE (cpl_imagelist_delete, data_imglist);
1681 
1682  /* Loop on pixels to fill the vector and matrix structures
1683  * used latter in the PTC fit */
1684  cpl_size nx = cpl_image_get_size_x (mean_img);
1685  cpl_size ny = cpl_image_get_size_y (mean_img);
1686  for (cpl_size i = 0; i < nx; i ++) {
1687  for (cpl_size j = 0; j < ny; j ++) {
1688  cpl_vector_set (vector_var, file * size + (j + ny*i),
1689  cpl_image_get (var_img, i+1, j+1, &nv));
1690  cpl_matrix_set (matrix_mean, 0, file * size + (j + ny*i),
1691  cpl_image_get (mean_img, i+1, j+1, &nv));
1692  }
1693  }
1694  /* End loop on pixels */
1695 
1696  /* Deallocation of variables */
1697  FREE (cpl_image_delete, var_img);
1698  FREE (cpl_image_delete, mean_img);
1699  } /* End loop on files */
1700 
1701  /* Fit the curve variance versus the median,
1702  thus the output gains are in ADU/e */
1703  cpl_polynomial * fit_slope = cpl_polynomial_new (1);
1704  cpl_polynomial_fit (fit_slope, matrix_mean, NULL, vector_var, NULL,
1705  CPL_FALSE, &mindeg, &maxdeg);
1706 
1707  /* Get the slope */
1708  double slope = cpl_polynomial_get_coeff (fit_slope, &slope_deg);
1709  cpl_msg_info (cpl_func, "mean gain FT = %.4f [ADU/e-] Mean gain of detector", slope);
1710  cpl_propertylist_append_double (output_plist, QC_MEANGAIN_FT, slope);
1711  cpl_propertylist_set_comment (output_plist, QC_MEANGAIN_FT, "[ADU/e-] Mean gain of FT detector");
1712 
1713  /* Delete */
1714  FREE (cpl_vector_delete, vector_var);
1715  FREE (cpl_matrix_delete, matrix_mean);
1716  FREE (cpl_polynomial_delete, fit_slope);
1717  FREE (cpl_image_delete, dark_img);
1718 
1719  } /* End FT case*/
1720 
1721  cpl_free (gain_file);
1722 
1723  /* Verbose */
1724  gravi_msg_function_exit(1);
1725  return output_plist;
1726 }
1727 
1728 /*----------------------------------------------------------------------------*/
1758 /*----------------------------------------------------------------------------*/
1759 
1760 gravi_data * gravi_compute_badpix (gravi_data * dark_map,
1761  gravi_data ** flats_data,
1762  int nflat,
1763  const cpl_parameterlist * params)
1764 {
1765  /* Verbose */
1766  gravi_msg_function_start(1);
1767  cpl_ensure (dark_map, CPL_ERROR_NULL_INPUT, NULL);
1768  cpl_ensure (params, CPL_ERROR_NULL_INPUT, NULL);
1769  cpl_ensure (nflat==4 || nflat==0, CPL_ERROR_ILLEGAL_INPUT, NULL);
1770 
1771  /* Construction of the bad pixels map */
1772  gravi_data * bad_map = gravi_data_new(0);
1773  cpl_propertylist * bad_header = gravi_data_get_header (bad_map);
1774 
1775  /* Dump full header into product */
1776  cpl_propertylist * dark_header = gravi_data_get_header (dark_map);
1777  cpl_propertylist_append (bad_header, dark_header);
1778 
1779  /*
1780  * Compute bad pixels of FT
1781  */
1782 
1783  if (!gravi_data_has_extension (dark_map, GRAVI_IMAGING_DATA_FT_EXT)) {
1784  cpl_msg_warning (cpl_func,"The DARK map has no IMAGING_DATA_FT");
1785  }
1786  else
1787  {
1788  cpl_msg_info (cpl_func,"Compute BADPIXEL of FT");
1789 
1790  /* Copy necessary tables */
1791  gravi_data_copy_ext (bad_map, dark_map, GRAVI_IMAGING_DETECTOR_FT_EXT);
1792 
1793  /* This is the FT */
1794  cpl_table * dark_table = gravi_data_get_table (dark_map, GRAVI_IMAGING_DATA_FT_EXT);
1795 
1796  /* Verify it has one single row */
1797  cpl_size n_row = cpl_table_get_nrow (dark_table);
1798  cpl_ensure (n_row == 1, CPL_ERROR_ILLEGAL_INPUT, NULL);
1799 
1800  /* Get the rms factor for dark bad pixel threshold */
1801  int bad_dark_factor = gravi_param_get_int (params, "gravity.calib.bad-dark-threshold");
1802 
1803  CPLCHECK_NUL ("Cannot get data");
1804 
1805  /* Get the dark rms and the mean dark */
1806  double dark_rms = cpl_propertylist_get_double (dark_header, QC_DARKRMS_FT);
1807  double dark_mean = cpl_propertylist_get_double (dark_header, QC_MEANDARK_FT);
1808 
1809  /* Compute the specified range are declared as bad pixels */
1810  double range_max = dark_mean + bad_dark_factor * dark_rms;
1811 
1812  /* Compute the number of bad pixel */
1813  int count_bp_dark = 0;
1814 
1815  /* Create output table */
1816  cpl_table * bad_table = cpl_table_extract (dark_table, 0, 1);
1817 
1818  /* Loop on pixels */
1819  cpl_size npix = cpl_table_get_column_depth (dark_table, "PIX");
1820  cpl_array * dark_array = cpl_table_get_data_array (dark_table, "PIX")[0];
1821  cpl_array * bad_array = cpl_table_get_data_array (bad_table, "PIX")[0];
1822 
1823  for (cpl_size pix = 0; pix < npix; pix++) {
1824  cpl_array_set (bad_array, pix, 0);
1825  if (cpl_array_get (dark_array, pix, NULL) > range_max) {
1826  cpl_array_set (bad_array, pix, BADPIX_DARK);
1827  count_bp_dark ++;
1828  }
1829  }
1830 
1831  /* Set QC parameter */
1832  cpl_propertylist_append_int (bad_header, QC_BADPIX_FT, count_bp_dark);
1833  cpl_msg_info (cpl_func, "QC_BADPIX_FT = %d", count_bp_dark);
1834 
1835  /* Set the badpixel map of FT in output data */
1836  gravi_data_add_table (bad_map, NULL, GRAVI_IMAGING_DATA_FT_EXT, bad_table);
1837 
1838  } /* End FT case */
1839 
1840  /*
1841  * Compute bad pixels of SC
1842  */
1843 
1844  if (!gravi_data_has_extension (dark_map, GRAVI_IMAGING_DATA_SC_EXT)) {
1845  cpl_msg_warning (cpl_func,"The DARK map has no IMAGING_DATA_SC");
1846  }
1847  else
1848  {
1849  cpl_msg_info (cpl_func,"Compute BADPIXEL of SC");
1850 
1851  /* Copy necessary tables */
1852  gravi_data_copy_ext (bad_map, dark_map, GRAVI_IMAGING_DETECTOR_SC_EXT);
1853 
1854  /* This is the SC */
1855  cpl_image * dark_img = gravi_data_get_img (dark_map, GRAVI_IMAGING_DATA_SC_EXT);
1856  cpl_image * std_img = gravi_data_get_img (dark_map, GRAVI_IMAGING_ERR_SC_EXT);
1857  cpl_size nx = cpl_image_get_size_x (dark_img);
1858  cpl_size ny = cpl_image_get_size_y (dark_img);
1859  CPLCHECK_NUL ("Cannot get the SC data");
1860 
1861  /* Compute an image with only the high-frequency of dark */
1862  cpl_image * darkhf_img = cpl_image_cast (dark_img, CPL_TYPE_DOUBLE);
1863  cpl_mask * kernel = cpl_mask_new (9, 9);
1864  cpl_mask_not (kernel);
1865  cpl_image_filter_mask (darkhf_img, dark_img, kernel, CPL_FILTER_MEDIAN, CPL_BORDER_FILTER);
1866  FREE (cpl_mask_delete, kernel);
1867 
1868  cpl_image_subtract (darkhf_img, dark_img);
1869  cpl_image_multiply_scalar (darkhf_img, -1.0);
1870  CPLCHECK_NUL ("Cannot create darkhf");
1871 
1872  /* Get the rms factor for dark bad pixel threshold */
1873  int bad_dark_factor = gravi_param_get_int (params, "gravity.calib.bad-dark-threshold");
1874 
1875  /* Accepted range for DARK mean */
1876  double dark_rms = cpl_propertylist_get_double (dark_header, QC_DARKRMS_SC);
1877  double dark_mean = cpl_image_get_mean (darkhf_img);
1878  double dark_max = dark_mean + 2*bad_dark_factor * dark_rms;
1879  double dark_min = dark_mean - 2*bad_dark_factor * dark_rms;
1880 
1881  /* Accepted range for dark STD */
1882  double dark_rms_std = cpl_image_get_stdev (std_img);
1883  double std_max = bad_dark_factor * dark_rms_std;
1884  double std_min = 0.05 * dark_rms_std;
1885 
1886  /* Cannot detect bad pix on LOW mode */
1887  if (nx < 60) {
1888  cpl_msg_warning (cpl_func, "Don't detect SC bapixels in mode LOW");
1889  dark_max = 50000; //7; can't find a good param for slpit and combined
1890  dark_min = -5000;
1891  std_max *= 5;
1892  std_min *= 5; // FIXME: weird ??
1893  }
1894 
1895  /* If we provide FLATs, we also use them. flat_img
1896  * will be filled with a filtered version of the FLAT */
1897  cpl_image * flat_img = NULL, * flatmed_img = NULL, * flatmask_img = NULL;
1898 
1899  if (!nflat) {
1900  cpl_msg_info (cpl_func,"No FLATs provided, detect only DARK");
1901  }
1902  else {
1903  cpl_ensure (flats_data, CPL_ERROR_NULL_INPUT, NULL);
1904  cpl_msg_info (cpl_func,"FLATs used to detect badpix");
1905 
1906  /* Init co-add FLAT image */
1907  flat_img = cpl_image_new (nx,ny,CPL_TYPE_DOUBLE);
1908  cpl_image_fill_window (flat_img, 1, 1, nx, ny, 0.0);
1909 
1910  /* Co-add FLATs, remove a self-estimate bias value
1911  * (the normal bias-pixels maybe wrong because of defocus),
1912  * assuming only half of the detector is illuminated */
1913  for (int f = 0; f<nflat; f++) {
1914  cpl_image * img = cpl_imagelist_collapse_median_create (
1915  gravi_data_get_cube (flats_data[f],
1916  GRAVI_IMAGING_DATA_SC_EXT));
1917  cpl_image_subtract (img, dark_img);
1918 
1919  double bias = gravi_image_get_quantile (img, 0.25);
1920  cpl_image_subtract_scalar (img, bias);
1921  cpl_image_add (flat_img, img);
1922  cpl_image_delete (img);
1923  CPLCHECK_NUL ("Cannot add flats");
1924  } /* End co-add flats */
1925 
1926  /* Create kernel for horizontal median filtering
1927  * Note that the large cluster is ~11 pixel wide */
1928  cpl_size kernel_x = 5;
1929  if (nx > 100) kernel_x = 11;
1930  if (nx > 1000) kernel_x = 31;
1931  cpl_msg_info (cpl_func,"Kernel of (%lld,%i) pixels for median filtering", kernel_x, 1);
1932  cpl_mask * kernel = cpl_mask_new (kernel_x, 1);
1933  cpl_mask_not (kernel);
1934 
1935  /* Run the median filter */
1936  flatmed_img = cpl_image_duplicate (flat_img);
1937  cpl_image_filter_mask (flatmed_img, flat_img, kernel, CPL_FILTER_MEDIAN, CPL_BORDER_FILTER);
1938  FREE (cpl_mask_delete, kernel);
1939 
1940  /* Compute a map of illuminated pixels at 100 [adu] */
1941  double threshold = 100.0;
1942  flatmask_img = cpl_image_duplicate (flatmed_img);
1943  cpl_image_threshold (flatmask_img, threshold, threshold, 0, 1.0);
1944 
1945  double fraction = cpl_image_get_flux (flatmask_img) / (nx*ny) * 100.0;
1946  cpl_msg_info (cpl_func, "Fraction of detector illuminated = %.2f%% (>%.1f adu)",
1947  fraction, threshold);
1948 
1949  /* Add this image of illuminated part in BAD map */
1950  gravi_data_add_img (bad_map, NULL, GRAVI_IMAGING_MASK_SC_EXT,
1951  flatmask_img);
1952 
1953  } /* End case FLAT provided */
1954 
1955 
1956  /* Compute the number of bad pixel */
1957  int count_bp_dark = 0;
1958  int count_bp_rms = 0;
1959  int count_bp_flat = 0;
1960  int count_bp = 0;
1961 
1962  /* Create the badpixel image */
1963  cpl_image * bad_img = cpl_image_new (nx, ny, CPL_TYPE_INT);
1964 
1965  /* Loop on pixels */
1966  for (cpl_size i = 0; i < nx; i++){
1967  for (cpl_size j = 0; j < ny; j++){
1968  int nv = 0, is_bad = 0, flag = 0;
1969 
1970  /* Tag the bad pixel on its mean value ... */
1971  double dij = cpl_image_get (darkhf_img, i + 1, j + 1, &nv);
1972  if ( (dij > dark_max)||
1973  (dij < dark_min)) {
1974  flag += BADPIX_DARK;
1975  count_bp_dark ++;
1976  is_bad = 1;
1977  }
1978  /* ... and its variance */
1979  double stdij = cpl_image_get (std_img, i + 1, j + 1, &nv);
1980  if ( (stdij > std_max) ||
1981  (stdij < std_min) ) {
1982  flag += BADPIX_RMS;
1983  count_bp_rms ++;
1984  is_bad = 1;
1985  }
1986  /* ... and the flat */
1987  if (flat_img) {
1988  double flat = cpl_image_get (flat_img, i + 1, j + 1, &nv);
1989  double med = cpl_image_get (flatmed_img, i + 1, j + 1, &nv);
1990  double mask = cpl_image_get (flatmask_img, i + 1, j + 1, &nv);
1991  if ( flat < 0.5 * med && mask && i>4 && i<nx-4 ) {
1992  flag += BADPIX_FLAT;
1993  count_bp_flat ++;
1994  is_bad = 1;
1995  } }
1996 
1997  /* Set this tag */
1998  count_bp += is_bad;
1999  cpl_image_set (bad_img, i + 1, j + 1, flag);
2000 
2001  CPLCHECK_NUL ("Cannot compute bad pixel");
2002  }
2003  } /* End loop on pixels */
2004 
2005  /* Set the badpixel map of SC in IMAGING_DATA_SC */
2006  gravi_data_add_img (bad_map, NULL, GRAVI_IMAGING_DATA_SC_EXT, bad_img);
2007 
2008  /* Update QC parameter of main header */
2009  cpl_propertylist_append_int (bad_header, QC_BADPIX_SC, count_bp);
2010  cpl_msg_info (cpl_func, "QC_BADPIX_SC (total) = %d (%.2f%%)",
2011  count_bp, (100.0 * count_bp) / (nx*ny));
2012  cpl_propertylist_append_int (bad_header, QC_BADPIX_DARK_SC, count_bp_dark);
2013  cpl_msg_info (cpl_func, "QC_BADPIX_DARK_SC = %d", count_bp_dark);
2014  cpl_propertylist_append_int (bad_header, QC_BADPIX_RMS_SC, count_bp_rms);
2015  cpl_msg_info (cpl_func, "QC_BADPIX_RMS_SC = %d", count_bp_rms);
2016  cpl_propertylist_append_int (bad_header, QC_BADPIX_FLAT_SC, count_bp_flat);
2017  cpl_msg_info (cpl_func, "QC_BADPIX_FLAT_SC = %d", count_bp_flat);
2018 
2019  FREE (cpl_image_delete, darkhf_img);
2020  FREE (cpl_image_delete, flat_img);
2021  FREE (cpl_image_delete, flatmed_img);
2022  } /* End SC case*/
2023 
2024  /*
2025  * Compute bad pixels of ACQ
2026  */
2027 
2028  if (!gravi_data_has_extension (dark_map, GRAVI_IMAGING_DATA_ACQ_EXT)) {
2029  cpl_msg_warning (cpl_func,"The DARK map has no IMAGING_DATA_ACQ");
2030  }
2031  else
2032  {
2033  cpl_msg_info (cpl_func,"Compute BADPIXEL of ACQ");
2034 
2035  /* This is the SC */
2036  cpl_image * dark_img = gravi_data_get_img (dark_map, GRAVI_IMAGING_DATA_ACQ_EXT);
2037  CPLCHECK_NUL ("Cannot get dark");
2038 
2039  /* Compute an image with only the high-frequency of dark */
2040  cpl_image * darkhf_img = cpl_image_cast (dark_img, CPL_TYPE_DOUBLE);
2041  cpl_mask * kernel = cpl_mask_new (21, 21);
2042  cpl_mask_not (kernel);
2043  cpl_image_filter_mask (darkhf_img, dark_img, kernel, CPL_FILTER_MEDIAN, CPL_BORDER_FILTER);
2044  FREE (cpl_mask_delete, kernel);
2045 
2046  cpl_image_subtract (darkhf_img, dark_img);
2047  cpl_image_multiply_scalar (darkhf_img, -1.0);
2048  cpl_image_abs (darkhf_img);
2049  CPLCHECK_NUL ("Cannot create darkhf");
2050 
2051  /* Get the STD of the dark_hf */
2052  double dark_std = cpl_image_get_median (darkhf_img);
2053  cpl_msg_info (cpl_func, "DARK_ACQ_STD = %e", dark_std);
2054 
2055  /* Detect bad pixel as >5 STD */
2056  double threshold = 20 * dark_std + 1e-10;
2057  cpl_image_threshold (darkhf_img, threshold, threshold, 0, 1);
2058  cpl_image * bad_img = cpl_image_cast (darkhf_img, CPL_TYPE_INT);
2059 
2060  /* Remove pixel in the middle of a square pattern */
2061  for (cpl_size x=2; x<cpl_image_get_size_x (bad_img);x++) {
2062  for (cpl_size y=2; y<cpl_image_get_size_y (bad_img);y++) {
2063  int nv = 0;
2064  if (cpl_image_get (bad_img,x,y-1,&nv) &&
2065  cpl_image_get (bad_img,x,y+1,&nv) &&
2066  cpl_image_get (bad_img,x-1,y,&nv) &&
2067  cpl_image_get (bad_img,x+1,y,&nv)) {
2068  cpl_image_set (bad_img,x,y, 1);
2069  }
2070  }
2071  }
2072 
2073  /* Set QC parameter */
2074  cpl_size count_bp = 0;
2075  int nv = 0;
2076 
2077  /* EKW 10/01/2019 Attention, I added brackets to avoid . [-Wmisleading-indentat */
2078  for (cpl_size x=0; x<cpl_image_get_size_x (bad_img);x++) {
2079  for (cpl_size y=0; y<cpl_image_get_size_y (bad_img);y++) {
2080  if (cpl_image_get (bad_img,x+1,y+1,&nv)) count_bp++;
2081  }
2082  }
2083 
2084  cpl_propertylist_append_int (bad_header, "ESO QC BADPIX ACQ", count_bp);
2085 
2086  /* Set the badpixel map of ACQ in IMAGING_DATA_ACQ */
2087  gravi_data_add_img (bad_map, NULL, GRAVI_IMAGING_DATA_ACQ_EXT, bad_img);
2088 
2089  FREE (cpl_image_delete, darkhf_img);
2090  }
2091 
2092  /* Verbose */
2093  gravi_msg_function_exit(1);
2094  return bad_map;
2095 }
2096 
2097 
2098 /*----------------------------------------------------------------------------*/
2119 /*----------------------------------------------------------------------------*/
2120 
2121 gravi_data * gravi_compute_biasmask (gravi_data * dark_map,
2122  gravi_data ** flats_data,
2123  int nflat,
2124  const cpl_parameterlist * params)
2125 {
2126  /* Verbose */
2127  gravi_msg_function_start(1);
2128  cpl_ensure (dark_map, CPL_ERROR_NULL_INPUT, NULL);
2129  cpl_ensure (params, CPL_ERROR_NULL_INPUT, NULL);
2130  cpl_ensure (nflat==4 || nflat==0, CPL_ERROR_ILLEGAL_INPUT, NULL);
2131 
2132  /* Construction of the bad pixels map */
2133  gravi_data * biasmask_map = gravi_data_new(0);
2134  cpl_propertylist * biasmask_header = gravi_data_get_header (biasmask_map);
2135 
2136  /* Dump full header into product */
2137  cpl_propertylist * dark_header = gravi_data_get_header (dark_map);
2138  cpl_propertylist_append (biasmask_header, dark_header);
2139 
2140  /* Copy necessary tables */
2141  gravi_data_copy_ext (biasmask_map, dark_map, GRAVI_IMAGING_DETECTOR_SC_EXT);
2142 
2143  /* This is the SC */
2144  cpl_image * dark_img = gravi_data_get_img (dark_map, GRAVI_IMAGING_DATA_SC_EXT);
2145  cpl_size nx = cpl_image_get_size_x (dark_img);
2146  cpl_size ny = cpl_image_get_size_y (dark_img);
2147  CPLCHECK_NUL ("Cannot get the SC data");
2148 
2149  /* Init co-add FLAT image */
2150  cpl_image * flat_img = cpl_image_new (nx,ny,CPL_TYPE_DOUBLE);
2151  cpl_image_fill_window (flat_img, 1, 1, nx, ny, 0.0);
2152 
2153  /* Co-add FLATs, remove a self-estimate bias value
2154  * (the normal bias-pixels maybe wrong because of defocus),
2155  * assuming only half of the detector is illuminated */
2156  for (int f = 0; f<nflat; f++) {
2157  cpl_image * img = cpl_imagelist_collapse_median_create (
2158  gravi_data_get_cube (flats_data[f],
2159  GRAVI_IMAGING_DATA_SC_EXT));
2160  cpl_image_subtract (img, dark_img);
2161 
2162  double bias = gravi_image_get_quantile (img, 0.25);
2163  cpl_image_subtract_scalar (img, bias);
2164  cpl_image_add (flat_img, img);
2165  cpl_image_delete (img);
2166  CPLCHECK_NUL ("Cannot add flats");
2167  } /* End co-add flats */
2168 
2169  /* Create kernel for horizontal median filtering
2170  * Note that the large cluster is ~11 pixel wide */
2171  cpl_size kernel_x = 5;
2172  if (nx > 100) kernel_x = 11;
2173  if (nx > 1000) kernel_x = 31;
2174  cpl_msg_info (cpl_func,"Kernel of (%lld,%i) pixels for median filtering", kernel_x, 1);
2175  cpl_mask * kernel = cpl_mask_new (kernel_x, 1);
2176  cpl_mask_not (kernel);
2177 
2178  /* Run the median filter */
2179  cpl_image * flatmed_img = cpl_image_duplicate (flat_img);
2180  cpl_image_filter_mask (flatmed_img, flat_img, kernel, CPL_FILTER_MEDIAN, CPL_BORDER_FILTER);
2181  FREE (cpl_mask_delete, kernel);
2182 
2183  /* Compute a map of illuminated pixels at 100 [adu] */
2184  double threshold = 100.0;
2185  cpl_image * biasmask_img = cpl_image_duplicate (flatmed_img);
2186  cpl_image_threshold (biasmask_img, threshold, threshold, 1.0, 0.0);
2187 
2188  double fraction = cpl_image_get_flux (biasmask_img) / (nx*ny) * 100.0;
2189  cpl_msg_info (cpl_func, "Fraction of detector in bias mask = %.2f%% (<%.1f adu)",
2190  fraction, threshold);
2191 
2192  /* Add this image of illuminated part in BAD map */
2193  gravi_data_add_img (biasmask_map, NULL, GRAVI_BIAS_MASK_SC_EXT,
2194  biasmask_img);
2195 
2196  FREE (cpl_image_delete, flat_img);
2197  FREE (cpl_image_delete, flatmed_img);
2198 
2199  /* Verbose */
2200  gravi_msg_function_exit(1);
2201  return biasmask_map;
2202 }
2203 
2204 /*----------------------------------------------------------------------------*/
2224 /*----------------------------------------------------------------------------*/
2225 
2226 gravi_data * gravi_compute_piezotf (gravi_data * data,
2227  const cpl_parameterlist * params)
2228 {
2229  /* Verbose */
2230  gravi_msg_function_start(1);
2231  cpl_ensure (data, CPL_ERROR_NULL_INPUT, NULL);
2232  cpl_ensure (params, CPL_ERROR_NULL_INPUT, NULL);
2233 
2234  /* Construction of the piezo tf product */
2235  gravi_data * piezo_tf = gravi_data_new(0);
2236  cpl_propertylist * piezotf_header = gravi_data_get_header (piezo_tf);
2237 
2238  /* Dump full header into product */
2239  cpl_propertylist * data_header = gravi_data_get_header (data);
2240  cpl_propertylist_append (piezotf_header, data_header);
2241 
2242  /* Copy necessary tables */
2243  gravi_data_copy_ext (piezo_tf, data, GRAVI_OPDC_EXT);
2244 
2245  /* Get the OPDC table */
2246  cpl_table * opdc = gravi_data_get_table (piezo_tf, GRAVI_OPDC_EXT);
2247 
2248  /* Get the necessary variables */
2249  int nbase = 6;
2250  int ntel = 4;
2251  int nresp = 5;
2252  int nsmooth = 201;
2253  char qc_name[100];
2254  char name[100];
2255  double phase;
2256  int base1,base2,base3,sign1,sign2,sign3;
2257  int ndit_small, dit_matrix;
2258  double twoPi = 2.0 * CPL_MATH_PI;
2259 
2260 
2261  /* DO THE COMPUTATION*/
2262 
2263  // Read data array from OPDC table
2264 
2265  cpl_size ndit = cpl_table_get_nrow (opdc);
2266  ndit_small=ndit-nsmooth-(nresp+1);
2267  cpl_msg_info (cpl_func, "Preparing matrix inversion with NDIT = %lld",ndit);
2268  cpl_array** opd = cpl_table_get_data_array (opdc, "OPD");
2269  cpl_array** piezo = cpl_table_get_data_array (opdc, "PIEZO_DL_OFFSET");
2270  CPLCHECK_NUL ("Cannot read the OPDC data");
2271 
2272  // create temporary arrays
2273 
2274  cpl_array * phase_array = cpl_array_new(nbase, CPL_TYPE_DOUBLE);
2275  cpl_array * piezo_array = cpl_array_new(ntel, CPL_TYPE_DOUBLE);
2276  cpl_matrix * phase_matrix = cpl_matrix_new (ndit_small*nbase,1);
2277  cpl_matrix * piezo_matrix = cpl_matrix_new (ndit_small*nbase, nresp*ntel);
2278  cpl_matrix * piezo_header_resp = cpl_matrix_new (nresp*ntel,1);
2279 
2280  // Unwrap phase of OPD
2281  for (cpl_size dit = 0 ; dit < ndit-1 ; dit ++)
2282  for (cpl_size base = 0 ; base < nbase; base ++)
2283  {
2284  phase=cpl_array_get(opd[dit+1],base, NULL)-cpl_array_get(opd[dit],base, NULL);
2285  phase-= twoPi * floor( phase / twoPi );
2286  if (phase > CPL_MATH_PI) phase -= twoPi;
2287  cpl_array_set(opd[dit+1],base, phase+cpl_array_get(opd[dit],base, NULL));
2288  }
2289 
2290  // filter OPDs, Commands and create Matrixes
2291 
2292  for (cpl_size dit = 0 ; dit < ndit-nsmooth; dit ++)
2293  {
2294  // Filter OPDs
2295  cpl_array_fill_window (phase_array, 0, nbase, 0.0);
2296  for (cpl_size smooth = 0 ; smooth < nsmooth; smooth ++)
2297  cpl_array_add( phase_array, opd[smooth+dit] );
2298  cpl_array_multiply_scalar (phase_array, -1.0 / nsmooth);
2299  cpl_array_add( phase_array, opd[ (int) (nsmooth/2+dit) ] );
2300 
2301  // Filter PIEZO commands
2302  cpl_array_fill_window_double (piezo_array, 0, ntel, 0.0);
2303  for (cpl_size smooth = 0 ; smooth < nsmooth; smooth ++)
2304  cpl_array_add( piezo_array, piezo[smooth+dit] );
2305  cpl_array_multiply_scalar (piezo_array, -1.0 / nsmooth);
2306  cpl_array_add( piezo_array, piezo[ (int) (nsmooth/2+dit) ] );
2307 
2308  // Store values into phase matrix for SVD inversion
2309  if (( dit-(nresp+1) >= 0 )&( dit-(nresp+1) < ndit_small))
2310  for (cpl_size base = 0 ; base < nbase; base ++)
2311  {
2312  int dit_matrix =dit-(nresp+1)+base*ndit_small;
2313  cpl_matrix_set(phase_matrix,dit_matrix,0,cpl_array_get(phase_array, base, NULL));
2314  }
2315 
2316  // Store values into piezo matrix for SVD inversion
2317  for (cpl_size tel = 0 ; tel < ntel; tel ++)
2318  {
2319  switch (tel)
2320  {
2321  default:
2322  case 0:
2323  base1=0;
2324  sign1=-1;
2325  base2=1;
2326  sign2=-1;
2327  base3=2;
2328  sign3=-1;
2329  break;
2330  case 1:
2331  base1=0;
2332  sign1=1;
2333  base2=3;
2334  sign2=-1;
2335  base3=4;
2336  sign3=-1;
2337  break;
2338  case 2:
2339  base1=1;
2340  sign1=1;
2341  base2=3;
2342  sign2=1;
2343  base3=5;
2344  sign3=-1;
2345  break;
2346  case 3:
2347  base1=2;
2348  sign1=1;
2349  base2=4;
2350  sign2=1;
2351  base3=5;
2352  sign3=1;
2353  break;
2354  }
2355 
2356  for (cpl_size resp = 0 ; resp < nresp; resp ++)
2357  if (( dit-(nresp+1)+(1+resp) >= 0 )&( dit-(nresp+1)+(1+resp) < ndit_small))
2358  {
2359  dit_matrix =dit-(nresp+1)+(1+resp)+base1*ndit_small;
2360  cpl_matrix_set(piezo_matrix,dit_matrix,resp*ntel+tel,sign1*cpl_array_get(piezo_array, tel, NULL));
2361  dit_matrix =dit-(nresp+1)+(1+resp)+base2*ndit_small;
2362  cpl_matrix_set(piezo_matrix,dit_matrix,resp*ntel+tel,sign2*cpl_array_get(piezo_array, tel, NULL));
2363  dit_matrix =dit-(nresp+1)+(1+resp)+base3*ndit_small;
2364  cpl_matrix_set(piezo_matrix,dit_matrix,resp*ntel+tel,sign3*cpl_array_get(piezo_array, tel, NULL));
2365  }
2366  }
2367  }
2368  CPLCHECK_NUL ("Cannot create matrix for SVD inversion");
2369 
2370  // resolve SVD
2371  cpl_msg_info (cpl_func, "Doing SVD inversion" );
2372  cpl_matrix * piezo_resp = cpl_matrix_solve_normal(piezo_matrix,phase_matrix); // coef_vis is 20x1
2373  cpl_matrix * residuals_fit = cpl_matrix_product_create(piezo_matrix,piezo_resp);
2374  cpl_matrix_subtract (residuals_fit,phase_matrix); //residuals_fit is (ndit-5)*nbase
2375  CPLCHECK_NUL ("Failed to do SVD inversion");
2376 
2377  // Get piezo response from header
2378 
2379  for (cpl_size tel = 0 ; tel < ntel; tel ++)
2380  for (cpl_size resp = 0 ; resp < nresp; resp ++)
2381  {
2382  sprintf (name, "ESO FT KAL P%lld_RESP%lld", tel+1, resp+1);
2383  cpl_matrix_set( piezo_header_resp, resp*ntel+ tel, 0, cpl_propertylist_get_double (piezotf_header, name));
2384  }
2385 
2386  // output QC parameters
2387 
2388  sprintf (qc_name, "ESO QC FT KAL P_FIT");
2389  cpl_propertylist_update_double (piezotf_header, qc_name, cpl_matrix_get_stdev( residuals_fit ) );
2390  cpl_propertylist_set_comment (piezotf_header, qc_name, "Fitting standard deviation [rad]");
2391  cpl_msg_info (cpl_func, "Fit standard deviation = %e [rad]", cpl_matrix_get_stdev( residuals_fit ) );
2392 
2393 
2394  sprintf (name, "ESO FT RATE");
2395  double sampling = cpl_propertylist_get_double (piezotf_header, name);
2396 
2397  for (cpl_size tel = 0 ; tel < ntel; tel ++)
2398  {
2399  for (cpl_size resp = 0 ; resp < nresp; resp ++)
2400  {
2401  sprintf (qc_name, "ESO QC FT KAL P%lld_RESP%lld", tel+1, resp+1);
2402  cpl_propertylist_update_double (piezotf_header, qc_name, cpl_matrix_get( piezo_resp, resp*ntel+ tel,0 ) );
2403  cpl_propertylist_set_comment (piezotf_header, qc_name, "Kalman piezo response");
2404 
2405  cpl_msg_info (cpl_func, "QC FT KAL P%lld_RESP%lld = %5.5g [rad/Volts]", tel+1, resp+1, cpl_matrix_get( piezo_resp, resp*ntel+ tel,0 ));
2406 
2407  }
2408 
2409  // Get gain in rad/Volts
2410 
2411  double QC_gain=0.0;
2412  for (cpl_size resp = 0 ; resp < nresp; resp ++)
2413  QC_gain+=cpl_matrix_get( piezo_resp, resp*ntel + tel,0 );
2414 
2415  sprintf (qc_name, "ESO QC FT KAL P%lld_GAIN", tel+1);
2416  cpl_propertylist_update_double (piezotf_header, qc_name, QC_gain );
2417  cpl_propertylist_set_comment (piezotf_header, qc_name, "Open loop gain [rad/Volts]");
2418 
2419  // Get pur delay in milliseconds
2420 
2421  double QC_delay=0.0;
2422  for (cpl_size resp = 0 ; resp < nresp; resp ++)
2423  QC_delay+=(resp+1)*cpl_matrix_get( piezo_resp, resp*ntel + tel,0 )*sampling;
2424  QC_delay/=QC_gain;
2425 
2426  sprintf (qc_name, "ESO QC FT KAL P%lld_DELAY", tel+1);
2427  cpl_propertylist_update_double (piezotf_header, qc_name, QC_delay );
2428  cpl_propertylist_set_comment (piezotf_header, qc_name, "Open loop latency [ms]");
2429 
2430  // Get standard deviation
2431  double QC_std=0.0;
2432  for (cpl_size resp = 0 ; resp < nresp; resp ++)
2433  QC_std+=(cpl_matrix_get( piezo_resp, resp*ntel + tel,0 ) - cpl_matrix_get( piezo_header_resp, resp*ntel+ tel, 0))
2434  * (cpl_matrix_get( piezo_resp, resp*ntel + tel,0 ) - cpl_matrix_get( piezo_header_resp, resp*ntel+ tel, 0));
2435 
2436  sprintf (qc_name, "ESO QC FT KAL P%lld_STDEV", tel+1);
2437  cpl_propertylist_update_double (piezotf_header, qc_name, sqrt(QC_std)/sqrt(nresp) );
2438  cpl_propertylist_set_comment (piezotf_header, qc_name, "Stdev of RTC [radians]");
2439 
2440  }
2441  CPLCHECK_NUL ("Failed to generate and store QC parameters");
2442 
2443  // delete disposable arrays
2444  cpl_array_delete(phase_array);
2445  cpl_array_delete(piezo_array);
2446  cpl_matrix_delete(phase_matrix);
2447  cpl_matrix_delete(piezo_matrix);
2448  cpl_matrix_delete(piezo_header_resp);
2449  cpl_matrix_delete(piezo_resp);
2450  cpl_matrix_delete(residuals_fit);
2451 
2452 
2453  /* Verbose */
2454  gravi_msg_function_exit(1);
2455  return piezo_tf;
2456 }
2457 
2458 /*----------------------------------------------------------------------------*/
2459 
2460 
gravi_data * gravi_data_new(int nb_ext)
Create an empty gravi_data.
Definition: gravi_data.c:102
double gravi_image_get_quantile(const cpl_image *img, double thr)
Compute the quantile of an image.
Definition: gravi_cpl.c:2112
cpl_error_code gravi_data_copy_ext(gravi_data *output, gravi_data *input, const char *name)
Copy extensions from one data to another.
Definition: gravi_data.c:1325
int gravi_data_has_extension(gravi_data *raw_calib, const char *ext_name)
Check if data has extension with given EXTNAME.
Definition: gravi_data.c:1443
cpl_error_code gravi_pfits_add_check(cpl_propertylist *header, char *msg)
Add a QC.CHECK keyword to the header.
Definition: gravi_pfits.c:1398
cpl_imagelist * gravi_imagelist_from_column(cpl_table *table_data, const char *data_x)
Create an imagelist from a column array in table.
Definition: gravi_cpl.c:1719
cpl_table * gravi_data_get_table(gravi_data *self, const char *extname)
Return a pointer on a table extension by its EXTNAME.
Definition: gravi_data.c:1716
cpl_error_code gravi_remove_badpixel_sc(cpl_imagelist *imglist_sc, cpl_image *bad_img)
Remove the badpixel of the SC.
int * gravi_image_extract_dimension(cpl_image *img_profile)
Compute startx and nx of the illuminated part of the image.
Definition: gravi_utils.c:727
gravi_data * gravi_compute_piezotf(gravi_data *data, const cpl_parameterlist *params)
Create piezo transfer function for Kalman Calibration & monitoring.
Definition: gravi_calib.c:2226
gravi_data * gravi_average_dark(gravi_data **data, cpl_size ndata)
Average several DARK calibration map.
Definition: gravi_calib.c:440
cpl_propertylist * gravi_compute_gain(gravi_data **flats_data, int nrawgain, gravi_data *dark_map)
Compute mean detector gain.
Definition: gravi_calib.c:1505
gravi_data * gravi_compute_biasmask(gravi_data *dark_map, gravi_data **flats_data, int nflat, const cpl_parameterlist *params)
Create BIASMASK for SC from raw FLATs and raw DARK.
Definition: gravi_calib.c:2121
cpl_array * gravi_array_wrap_image(cpl_image *img)
Wrap the data of na image into an array.
Definition: gravi_cpl.c:1776
cpl_error_code gravi_fit_profile(cpl_vector *values_x0, cpl_vector *values_y0, cpl_vector *values_sigma, cpl_image *mean_img, int ref_x, int ref_y, int size_profile, const char *resolution)
Fit the profile parameters in an image.
Definition: gravi_calib.c:519
cpl_error_code gravi_imagelist_unwrap_images(cpl_imagelist *imglist)
Unwrap an imagelist an all its images.
Definition: gravi_cpl.c:1496
cpl_image * gravi_image_from_column(cpl_table *table_data, const char *data_x, cpl_size row)
Create an image from a column array in table.
Definition: gravi_cpl.c:1687
cpl_propertylist * gravi_data_get_plist(gravi_data *self, const char *extname)
Get the propertylist from EXTNAME.
Definition: gravi_data.c:1684
gravi_data * gravi_compute_badpix(gravi_data *dark_map, gravi_data **flats_data, int nflat, const cpl_parameterlist *params)
Identify the bad pixels in the DARK map and create the BAD map.
Definition: gravi_calib.c:1760
cpl_imagelist * gravi_data_get_cube(gravi_data *self, const char *extname)
Return a pointer on an IMAGE extension by its EXTNAME.
Definition: gravi_data.c:1751
int gravi_region_get_tel(cpl_table *imaging_detector, int region, int beam)
Return the telescope id (0,1,2,3) in a beam of a region.
Definition: gravi_utils.c:512
cpl_error_code gravi_data_add_table(gravi_data *self, cpl_propertylist *plist, const char *extname, cpl_table *table)
Add a BINTABLE extension in gravi_data.
Definition: gravi_data.c:1909
gravi_data * gravi_data_duplicate(const gravi_data *self)
Create a copy of the gravi data.
Definition: gravi_data.c:236
cpl_error_code gravi_data_add_img(gravi_data *self, cpl_propertylist *plist, const char *extname, cpl_image *image)
Add an IMAGE (single image) extension in gravi_data.
Definition: gravi_data.c:2013
gravi_data * gravi_compute_dark(gravi_data *raw_data)
Compute the DARK calibration map.
Definition: gravi_calib.c:122
cpl_image * gravi_create_profile_image(cpl_image *mean_img, cpl_vector *values_x0, cpl_vector *values_y0, cpl_vector *values_sigma, cpl_size iy_min, cpl_size iy_max, const char *resolution)
Create the profile image from image and/or fitted params.
Definition: gravi_calib.c:797
cpl_imagelist * gravi_imagelist_wrap_column(cpl_table *table_data, const char *data_x)
Wrap a column array of a table into an imagelist.
Definition: gravi_cpl.c:1526
gravi_data * gravi_compute_profile(gravi_data **flats_data, gravi_data *dark_map, gravi_data *bad_map, int nflat, const cpl_parameterlist *params)
Computes the spatial profile of each spectrum for optimal extraction purpose.
Definition: gravi_calib.c:976