GRAVI Pipeline Reference Manual  1.2.3
gravi_vis.c
1 /* $Id: gravi_vis.c,v 1.10 2014/11/12 15:10:40 nazouaoui Exp $
2  *
3  * This file is part of the GRAVI Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
35 /*
36  * History
37  * 21.11.2018 memory leak in gravi_average_self_visphi
38  * changes marked as 'EKW'
39  * 11/01/2019 Fix Warning parameter 'ret'
40  */
41 /*-----------------------------------------------------------------------------
42  Includes
43  -----------------------------------------------------------------------------*/
44 
45 #ifdef HAVE_CONFIG_H
46 #include <config.h>
47 #endif
48 
49 #include <cpl.h>
50 #include <string.h>
51 #include <stdio.h>
52 #include <math.h>
53 #include <time.h>
54 #include <complex.h>
55 #include <string.h>
56 
57 #include "gravi_data.h"
58 #include "gravi_dfs.h"
59 #include "gravi_pfits.h"
60 #include "gravi_cpl.h"
61 
62 #include "gravi_utils.h"
63 
64 #include "gravi_vis.h"
65 #include "gravi_eop.h"
66 #include "gravi_tf.h"
67 
68 /*-----------------------------------------------------------------------------
69  Private prototypes
70  -----------------------------------------------------------------------------*/
71 
72 double gravi_randn (void);
73 
74 cpl_error_code gravi_array_online_variance(cpl_array * data, cpl_array * mean, cpl_array * variance, int n);
75 cpl_error_code gravi_array_online_variance_res(cpl_array ** data,
76  int n, int rephase);
77 cpl_error_code gravi_flux_average_bootstrap(cpl_table * oi_flux_avg,
78  cpl_table * oi_flux,
79  int nboot);
80 cpl_error_code gravi_t3_average_bootstrap(cpl_table * oi_t3_avg,
81  cpl_table * oi_vis,
82  cpl_table * oi_flux,
83  int nboot,
84  int use_vFactor,
85  int use_pFactor);
86 cpl_error_code gravi_vis_average_bootstrap (cpl_table * oi_vis_avg,
87  cpl_table * oi_vis2_avg,
88  cpl_table * oi_vis,
89  int nboot,
90  const char * phase_ref,
91  int use_vFactor,
92  int use_pFactor,
93  int use_debiasing);
94 
95 cpl_error_code gravi_vis_flag_nan (cpl_table * oi_table);
96 
97 cpl_error_code gravi_vis_average_amp (cpl_table *oi_table, const char *name, const char *err, int nbase);
98 cpl_error_code gravi_vis_average_phi (cpl_table *oi_table, const char *name, const char *err, int nbase);
99 cpl_error_code gravi_vis_average_value (cpl_table *oi_table, const char *name, const char *err, int nbase);
100 cpl_error_code gravi_vis_resamp_amp (cpl_table * oi_table, const char * name, const char * err,
101  cpl_size nsamp, cpl_size nwave_new);
102 cpl_error_code gravi_vis_resamp_phi (cpl_table * oi_table, const char * name, const char * err,
103  cpl_size nsamp, cpl_size nwave_new);
104 cpl_error_code gravi_vis_smooth_amp (cpl_table * oi_table, const char * name, const char * err,
105  cpl_size nsamp);
106 cpl_error_code gravi_vis_smooth_phi (cpl_table * oi_table, const char * name, const char * err,
107  cpl_size nsamp);
108 
109 cpl_error_code gravi_vis_fit_amp (cpl_table * oi_table, const char * name,
110  const char * err, cpl_size maxdeg);
111 
112 cpl_error_code gravi_vis_compute_column_mean (cpl_table * out_table,
113  cpl_table * in_table,
114  const char * name, int ntel);
115 
116 cpl_error_code gravi_vis_flag_median (cpl_table * oi_table, const char * data, const char *flag, double value);
117 
118 /*-----------------------------------------------------------------------------
119  Function code
120  -----------------------------------------------------------------------------*/
121 
122 /*----------------------------------------------------------------------------*/
126 /*----------------------------------------------------------------------------*/
127 
128 double gravi_randn (void)
129 {
130  int nsamp = 50;
131  double samp[] = {0.97446388, 0.78404357, 2.24226141, 1.85507201, 1.10792943,
132  1.34028771, -0.15399594, 0.07316682, 1.60898976, 0.33111245,
133  0.76767625, -2.1252529 , 0.3898138 , 2.1218198 , 0.51703696,
134  0.38451722, 1.07581416, -0.61435275, 1.91926679, 1.10020069,
135  1.82407999, 1.07367663, 0.46105875, 0.45497282, 1.65549611,
136  1.21647974, -0.32725523, -0.36477508, 0.43947414, 1.0242778 ,
137  2.05617949, 1.06163165, 1.24564147, 2.36249995, 0.20676319,
138  1.30886256, 0.7122533 , 2.28503709, 0.7134141 , -0.19104819,
139  2.9925884 , 0.95761567, 2.11770457, 0.34763896, 0.30040327,
140  2.3535165 , 1.65839907, 1.89819461, 1.67480833, 1.11174145};
141 
142  /* FIXME: build a better normal random generator !! */
143  return samp[rand()%nsamp];
144 }
145 
146 
147 /*-----------------------------------------------------------------------------*/
148 
149 cpl_error_code gravi_array_online_variance(cpl_array * data, cpl_array * mean, cpl_array * variance, int n)
150 {
151  cpl_ensure_code (data, CPL_ERROR_NULL_INPUT);
152  cpl_ensure_code (mean, CPL_ERROR_NULL_INPUT);
153  cpl_ensure_code (variance, CPL_ERROR_NULL_INPUT);
154  cpl_ensure_code (n>=0, CPL_ERROR_ILLEGAL_INPUT);
155 
156  double delta = 0.0;
157  double rdata = 0.0;
158  double rmean = 0.0;
159  double n1 = 1. / ( (double)n + 1.0 );
160  double n2 = (double)n / ( (double)n + 1.0 );
161 
162  cpl_size size = cpl_array_get_size (data);
163 
164  /* delta = (x - mean)/(n+1)
165  mean = mean + delta
166  M2 = M2*n/(n+1) + delta*(x - mean) */
167 
168  cpl_size w;
169  int nv = 0.0;
170  for (w = 0; w < size; w ++) {
171  /* delta = (x - mean)/(n+1) */
172  rdata = cpl_array_get (data, w, &nv);
173  rmean = cpl_array_get (mean, w, &nv);
174  delta = ( rdata - rmean ) * n1;
175  /* mean = mean + delta */
176  rmean = rmean + delta;
177  cpl_array_set (mean, w, rmean);
178  /* M2 = M2*n/(n+1) + delta*(x - mean) */
179  cpl_array_set (variance, w,
180  cpl_array_get (variance, w, &nv) * n2 + delta * ( rdata - rmean ));
181  }
182 
183  int code;
184  if ( (code=cpl_error_get_code()) ) {
185  return cpl_error_set_message(cpl_func, code, "Cannot do online variance");
186  }
187 
188  return CPL_ERROR_NONE;
189 }
190 
191 /*-----------------------------------------------------------------------------*/
207 /*-----------------------------------------------------------------------------*/
208 
209 cpl_error_code gravi_array_online_variance_res(cpl_array ** data,
210  int n, int rephase)
211 {
212  gravi_msg_function_start(0);
213  cpl_ensure_code (data, CPL_ERROR_NULL_INPUT);
214  cpl_ensure_code (n>=0, CPL_ERROR_ILLEGAL_INPUT);
215 
216  cpl_size size = cpl_array_get_size (data[0]);
217  cpl_msg_debug(cpl_func,"Start function");
218 
219  /* If first boot, we init the runnning mean and running variance
220  * and we store in the first array */
221  if ( n == 0 ) {
222  cpl_array_add (data[2], data[0]);
223  }
224 
225  /* Recenter phase around the mean phase before computing its VARIANCE */
226  if (rephase) {
227  for (int w=0; w<size; w++ ) {
228  cpl_array_set (data[0], w,
229  carg( cexp (1*I* (cpl_array_get(data[0], w, NULL) -
230  cpl_array_get(data[2], w, NULL))) ) );
231  }
232  }
233 
234  /* Run the gravi_online */
235  gravi_array_online_variance(data[0], data[1], data[3], n);
236 
237  /* Free the current boot to prepare for next integration */
238  cpl_array_fill_window (data[0], 0, size, 0.0);
239 
240  CPLCHECK_MSG ("Error in online variance");
241 
242  gravi_msg_function_exit(0);
243  return CPL_ERROR_NONE;
244 }
245 
246 /*-----------------------------------------------------------------------------*/
260 /*-----------------------------------------------------------------------------*/
261 
262 cpl_error_code gravi_flux_average_bootstrap(cpl_table * oi_flux_avg,
263  cpl_table * oi_flux,
264  int nboot)
265 {
266  gravi_msg_function_start(0);
267  cpl_ensure_code (oi_flux_avg, CPL_ERROR_ILLEGAL_OUTPUT);
268  cpl_ensure_code (oi_flux, CPL_ERROR_NULL_INPUT);
269  cpl_ensure_code (nboot>0, CPL_ERROR_ILLEGAL_INPUT);
270 
271  /* parameters */
272  int nv = 0, ntel = 4;
273  cpl_size nrow = cpl_table_get_nrow (oi_flux) / ntel;
274  cpl_size nwave = cpl_table_get_column_depth (oi_flux, "FLUX");
275 
276  /* Pointer to columns, to speed-up */
277  cpl_array ** pFLUX = cpl_table_get_data_array (oi_flux, "FLUX");
278  cpl_array ** pFLUXERR = cpl_table_get_data_array (oi_flux, "FLUXERR");
279  double * pINTTIME = cpl_table_get_data_double (oi_flux, "INT_TIME");
280  double * pMJD = cpl_table_get_data_double (oi_flux, "MJD");
281  CPLCHECK_MSG ("Cannot get the data");
282 
283  /* Loop on tel */
284  for (cpl_size tel = 0; tel < ntel; tel++) {
285 
286  /* Tel for base and base for closure */
287  cpl_size nvalid = 0;
288 
289  /* Arrays to store the final, integrated quantities
290  * 0: current boot, 1: running_mean, 2: first boot, 3: variance */
291  cpl_array **flux_res = gravi_array_new_list (4, CPL_TYPE_DOUBLE, nwave);
292  double total_exptime = 0.0, mjd_avg = 0.0;
293 
294  /*
295  * (0) Optimize the number of segment
296  */
297 
298  /* Get the number of non-rejected frames */
299  int * flag = cpl_malloc( sizeof(int) * nrow );
300  for ( int row=0 ; row<nrow; row++ ) {
301  flag[row] = 0; nvalid++;
302  }
303 
304  /* Build an optimal number of segment and nrow_per_segment */
305  cpl_size nrow_per_seg = CPL_MAX(nvalid / CPL_MIN (nrow, 100), 1);
306  cpl_size nseg = nvalid / nrow_per_seg;
307 
308  /* Ensure there are at least 5 samples to bootstrap on,
309  * if no add montecarlo samples */
310  cpl_size nsamp = 5, nmontecarlo = CPL_MAX (nsamp - nseg, 0);
311 
312  cpl_msg_info ("Stat", "%6lld valid frames over %6lld (%5.1f%%), make %4lld seg. of %5lld (miss %lld), add %lld MonteCarlo",
313  nvalid, nrow, (double)nvalid/(double)nrow*100.0,
314  nseg, nrow_per_seg, nvalid - nseg*nrow_per_seg, nmontecarlo);
315 
316  /* Case we have at least one valid frame */
317  if ( nvalid > 0 ) {
318 
319  /*
320  * (1) Pre-integration over segment, to bootstrap on less statistic
321  */
322 
323  cpl_array **flux = gravi_array_new_list (nseg + nmontecarlo, CPL_TYPE_DOUBLE, nwave);
324 
325  /* Loop on segment */
326  cpl_size row = -1;
327  for ( int seg = 0 ; seg < nseg + nmontecarlo; seg ++ ) {
328  cpl_msg_debug(cpl_func,"pre integration of seg %d start with row %lld", seg, row);
329 
330  /* Find nrow_per_seg valid frame to integrate in this segment */
331  cpl_size r = 0;
332  while ( r < nrow_per_seg ) {
333  row = (row + 1) % nrow;
334  if ( flag[row] ) {continue;} else {r++;}
335 
336  /* Get indices */
337  cpl_size rtel = row * ntel + tel;
338 
339  /* Compute the total integration time.
340  * Do not integrate for the MonteCarlo samples */
341  if (seg < nseg) {
342  total_exptime += pINTTIME[rtel];
343  mjd_avg += pMJD[rtel] * pINTTIME[rtel];
344  }
345 
346  /* fast-no-CPL integration: get pointers on data */
347  double * tflux = cpl_array_get_data_double (flux[seg]);
348 
349  /* Loop on wave */
350  for ( int w=0; w<nwave; w++ ) {
351  double FLUX = cpl_array_get (pFLUX[rtel], w, NULL);
352  double FLUXERR = cpl_array_get (pFLUXERR[rtel], w, NULL);
353 
354  /* Add noise if this is a Monte Carlo sample. */
355  if ( seg > nseg-1 ) {
356  FLUX += 2 * FLUXERR * gravi_randn();
357  }
358 
359  /* flux = < FLUX > over the frames in [e] */
360  tflux[w] += FLUX;
361 
362  } /* End loop on wave */
363  } /* End loop on rows in this segment */
364  }/* End loop on segments */
365 
366  /*
367  * (2) Compute the variance by bootstraping on the segments
368  */
369 
370  /* Loop on bootstramp to compute the avg and the
371  * variance by the bootstraping methode */
372  srand(1);
373 
374  for ( int boot = 0 ; boot < nboot ; boot ++ ) {
375  cpl_msg_debug(cpl_func,"Bootstrap %d over %d", boot+1, nboot);
376 
377  /* Integrate nseg segment randomly selected.
378  * This loop is vectorialized in spectral direction */
379  for (int rowb = 0; rowb < nseg; rowb ++){
380 
381  /* For the first bootstrap, we use all observed samples
382  * For the others, we also includes the possible montecarlo
383  * FIXME: verify the uniformity of rand for small nrows */
384  int rows;
385  if (boot == 0 ) rows = rowb;
386  else rows = rand()%(nseg+nmontecarlo);
387 
388  /* Integrate the flux of selected segments */
389  cpl_array_add (flux_res[0], flux[rows]);
390  }
391  /* End loop on selected segments */
392 
393  /* Compute the VARIANCE over the bootstraped samples with the 'online_variance' algorithm.
394  * See https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance */
395  gravi_array_online_variance_res (flux_res, boot, 0);
396 
397  CPLCHECK_MSG("while computing the variances over the bootstrap");
398  }
399  /* End Loop on bootstrap */
400 
401  /* Free list of segments and running mean */
402  FREELOOP (cpl_array_delete, flux, nseg + nmontecarlo);
403 
404  /* Convert variance from bootstrap to RMS */
405  cpl_msg_debug(cpl_func,"Put the RMS over bootstrap");
406  cpl_array_power (flux_res[3], 0.5);
407  CPLCHECK_MSG("while converting variance -> rms");
408 
409  /* Normalise integration */
410  mjd_avg /= total_exptime;
411 
412  }
413 
414  /*
415  * Step (1,2) in case there are no valid frames at all
416  */
417  if (nvalid == 0) {
418  cpl_msg_debug (cpl_func,"Not valid frames, force zero and infinit RMS");
419  cpl_array_fill_window (flux_res[3], 0, nwave, 1e10);
420  mjd_avg = cpl_table_get_column_mean (oi_flux, "MJD");
421  }
422 
423  /*
424  * (3) Save the results on the oi_t3_avg tables
425  */
426 
427  /* Save the cloture amplitude on the oi_T3 tables */
428  cpl_table_set_array (oi_flux_avg, "FLUX", tel, flux_res[2]);
429  cpl_table_set_array (oi_flux_avg, "FLUXERR", tel, flux_res[3]);
430  CPLCHECK_MSG("filling FLUX and FLUXERR");
431 
432  /* Flag the data with >100% error */
433  gravi_vis_flag_relative_threshold (oi_flux_avg, "FLUXERR", "FLUX", "FLAG", 1.0);
434  CPLCHECK_MSG("cannot flag baddata data");
435 
436  /* Compute the total integration time */
437  cpl_msg_debug(cpl_func,"Total integration time = %.3f s", total_exptime);
438  cpl_table_set_double (oi_flux_avg, "INT_TIME", tel, total_exptime);
439  cpl_table_set_double (oi_flux_avg, "MJD", tel, mjd_avg);
440  cpl_table_set (oi_flux_avg, "NVALID", tel, nvalid);
441  cpl_table_set (oi_flux_avg, "NDIT", tel, nrow);
442 
443  /* Set the TARGET_ID and STA_INDEX */
444  cpl_table_set_int (oi_flux_avg, "TARGET_ID", tel, cpl_table_get_int (oi_flux, "TARGET_ID", tel, &nv));
445  cpl_table_set_int (oi_flux_avg, "STA_INDEX", tel, cpl_table_get_int (oi_flux, "STA_INDEX", tel, &nv));
446 
447  FREELOOP (cpl_array_delete, flux_res, 4);
448  cpl_free(flag);
449 
450  } /* End loop on tel */
451 
452  gravi_msg_function_exit(0);
453  return CPL_ERROR_NONE;
454 }
455 
456 /*-----------------------------------------------------------------------------*/
471 /*-----------------------------------------------------------------------------*/
472 
473 cpl_error_code gravi_t3_average_bootstrap(cpl_table * oi_t3_avg,
474  cpl_table * oi_vis,
475  cpl_table * oi_flux,
476  int nboot,
477  int use_vFactor,
478  int use_pFactor)
479 {
480  gravi_msg_function_start(0);
481  cpl_ensure_code (oi_t3_avg, CPL_ERROR_ILLEGAL_OUTPUT);
482  cpl_ensure_code (oi_vis, CPL_ERROR_NULL_INPUT);
483  cpl_ensure_code (oi_flux, CPL_ERROR_NULL_INPUT);
484  cpl_ensure_code (nboot>0, CPL_ERROR_ILLEGAL_INPUT);
485 
486  /* Tel for base and base for closure
487  * Assume all the observations are made with the same array geometry.
488  * sta_index_t3[0] <-> tel1[clo[j][0]] = tel1[clo[j][2]]
489  * sta_index_t3[1] <-> tel1[clo[j][1]] = tel2[clo[j][0]]
490  * sta_index_t3[2] <-> tel2[clo[j][2]] = tel2[clo[j][1]]
491  */
492  int nv = 0, nbase = 6, ntel = 4, nclo = 4;
493 
494  cpl_size nrow = cpl_table_get_nrow (oi_vis) / nbase;
495  cpl_size nwave = cpl_table_get_column_depth (oi_vis, "VISDATA");
496 
497  /* Pointer to column, to speed-up */
498  cpl_array ** pVISDATA = cpl_table_get_data_array (oi_vis, "VISDATA");
499  cpl_array ** pVISERR = cpl_table_get_data_array (oi_vis, "VISERR");
500  cpl_array ** pFLUX = cpl_table_get_data_array (oi_flux, "FLUX");
501  double * pINTTIME = cpl_table_get_data_double (oi_vis, "INT_TIME");
502  double * pMJD = cpl_table_get_data_double (oi_vis, "MJD");
503  double * pUCOORD = cpl_table_get_data_double (oi_vis, "UCOORD");
504  double * pVCOORD = cpl_table_get_data_double (oi_vis, "VCOORD");
505  cpl_array ** pVFACTOR = use_vFactor?cpl_table_get_data_array (oi_vis, "V_FACTOR"):NULL;
506  double * pPFACTOR = use_pFactor?cpl_table_get_data_double (oi_vis, "P_FACTOR"):NULL;
507  CPLCHECK_MSG ("Cannot get the data");
508 
509  /* Loop on closure */
510  for (cpl_size closure = 0; closure < nclo; closure++) {
511 
512  cpl_size nvalid = 0;
513  int base0 = GRAVI_CLO_BASE[closure][0];
514  int base1 = GRAVI_CLO_BASE[closure][1];
515  int base2 = GRAVI_CLO_BASE[closure][2];
516  int ctel0 = GRAVI_CLO_TEL[closure][0];
517  int ctel1 = GRAVI_CLO_TEL[closure][1];
518  int ctel2 = GRAVI_CLO_TEL[closure][2];
519 
520  /* Arrays to store the final, integrated quantities
521  * 0: current boot, 1: running_mean, 2: first boot, 3: variance */
522  cpl_array **t3Amp_res = gravi_array_new_list (4, CPL_TYPE_DOUBLE, nwave);
523  cpl_array **t3Phi_res = gravi_array_new_list (4, CPL_TYPE_DOUBLE, nwave);
524  double total_exptime = 0.0, mjd_avg = 0.0;
525  double u1Coord = 0.0, v1Coord = 0.0, u2Coord = 0.0, v2Coord = 0.0;
526 
527  /*
528  * (0) Optimize the number of segment
529  */
530 
531  /* Get the number of non-rejected frames */
532  int * flag = cpl_table_get_data_int (oi_vis, "REJECTION_FLAG");
533  cpl_ensure_code (flag, CPL_ERROR_ILLEGAL_INPUT);
534  int * flagclo = cpl_malloc( sizeof(int) * nrow );
535  for ( int row=0 ; row<nrow; row++ ) {
536  flagclo[row] = flag[row * nbase + base0] + flag[row * nbase + base1] + flag[row * nbase + base2];
537  if ( flagclo[row] == 0 ) nvalid++;
538  }
539 
540  /* Build an optimal number of segment and nrow_per_segment */
541  cpl_size nrow_per_seg = CPL_MAX (nvalid / CPL_MIN (nrow, 100), 1);
542  cpl_size nseg = nvalid / nrow_per_seg;
543 
544  /* Ensure there are at least 5 samples to bootstrap on,
545  * if no add montecarlo samples */
546  cpl_size nsamp = 5, nmontecarlo = CPL_MAX (nsamp - nseg, 0);
547 
548  cpl_msg_info ("Stat", "%6lld valid frames over %6lld (%5.1f%%), make %4lld seg. of %5lld (miss %lld), add %lld MonteCarlo",
549  nvalid, nrow, (double)nvalid/(double)nrow*100.0,
550  nseg, nrow_per_seg, nvalid - nseg*nrow_per_seg, nmontecarlo);
551 
552  /* Case we have at least one valid frame */
553  if ( nvalid > 0 ) {
554 
555  /*
556  * (1) Pre-integration over segment, to bootstrap on less statistic
557  */
558 
559  cpl_array **bisp = gravi_array_new_list (nseg + nmontecarlo, CPL_TYPE_DOUBLE_COMPLEX, nwave);
560  cpl_array **F012 = gravi_array_new_list (nseg + nmontecarlo, CPL_TYPE_DOUBLE, nwave);
561 
562  /* Loop on segment */
563  cpl_size row = -1;
564  for ( int seg = 0 ; seg < nseg + nmontecarlo ; seg ++ ) {
565  cpl_msg_debug(cpl_func,"pre integration of seg %d start with row %lld", seg, row);
566 
567  /* Find nrow_per_seg valid frame to integrate in this segment */
568  cpl_size r = 0;
569  while ( r < nrow_per_seg ) {
570  row = (row + 1) % nrow;
571  if ( flagclo[row] ) {continue;} else {r++;}
572 
573  /* Get indices */
574  cpl_size rbase0 = row * nbase + base0;
575  cpl_size rbase1 = row * nbase + base1;
576  cpl_size rbase2 = row * nbase + base2;
577 
578  /* Compute the total integration time.
579  * Do not integrate for the MonteCarlo samples */
580  if (seg < nseg) {
581  total_exptime += pINTTIME[rbase0];
582  mjd_avg += pMJD[rbase0] * pINTTIME[rbase0];
583  u1Coord += pUCOORD[rbase0] * pINTTIME[rbase0];
584  v1Coord += pVCOORD[rbase0] * pINTTIME[rbase0];
585  u2Coord += pUCOORD[rbase1] * pINTTIME[rbase0];
586  v2Coord += pVCOORD[rbase1] * pINTTIME[rbase0];
587  }
588 
589  /* fast-no-CPL integration: get pointers on data */
590  double complex * tbisp = cpl_array_get_data_double_complex (bisp[seg]);
591  double *tF012 = cpl_array_get_data_double (F012[seg]);
592  CPLCHECK_MSG ("Cannot get data");
593 
594  double PFACTOR0 = (use_pFactor?pPFACTOR[rbase0]:1.0);
595  double PFACTOR1 = (use_pFactor?pPFACTOR[rbase1]:1.0);
596  double PFACTOR2 = (use_pFactor?pPFACTOR[rbase2]:1.0);
597  CPLCHECK_MSG ("Cannot get FACTOR data");
598 
599  /* Loop on wave */
600  for ( int w=0; w<nwave; w++ ) {
601  double complex Vis0 = cpl_array_get_complex (pVISDATA[rbase0], w, NULL);
602  double complex Vis1 = cpl_array_get_complex (pVISDATA[rbase1], w, NULL);
603  double complex Vis2 = cpl_array_get_complex (pVISDATA[rbase2], w, NULL);
604  double complex VisErr0 = cpl_array_get_complex (pVISERR[rbase0], w, NULL);
605  double complex VisErr1 = cpl_array_get_complex (pVISERR[rbase1], w, NULL);
606  double complex VisErr2 = cpl_array_get_complex (pVISERR[rbase2], w, NULL);
607  double F0 = cpl_array_get (pFLUX[row * ntel + ctel0], w, NULL);
608  double F1 = cpl_array_get (pFLUX[row * ntel + ctel1], w, NULL);
609  double F2 = cpl_array_get (pFLUX[row * ntel + ctel2], w, NULL);
610  double VFACTOR0 = (use_vFactor?cpl_array_get (pVFACTOR[row * nbase + base0], w, NULL):1.0);
611  double VFACTOR1 = (use_vFactor?cpl_array_get (pVFACTOR[row * nbase + base1], w, NULL):1.0);
612  double VFACTOR2 = (use_vFactor?cpl_array_get (pVFACTOR[row * nbase + base2], w, NULL):1.0);
613 
614  /* Add noise if this is a Monte Carlo sample.
615  * APPROX: Noise is only added to the coherent fluxes */
616  if ( seg > nseg-1 ) {
617  Vis0 += 1.*I * cimag(VisErr0) * gravi_randn();
618  Vis0 += 1. * creal(VisErr0) * gravi_randn();
619  Vis1 += 1.*I * cimag(VisErr1) * gravi_randn();
620  Vis1 += 1. * creal(VisErr1) * gravi_randn();
621  Vis2 += 1.*I * cimag(VisErr2) * gravi_randn();
622  Vis2 += 1. * creal(VisErr2) * gravi_randn();
623  }
624 
625  /* bisp = < v1*v2*conj(v3) > over the frames in [e^3] */
626  tbisp[w] += Vis0 * Vis1 * conj (Vis2);
627 
628  /* F012 = < F0*F1*F2 > over the frames in [e^3]
629  * corrected from expected visibility losses */
630  tF012[w] += F0 * F1 * F2 *
631  sqrt (CPL_MAX (VFACTOR0 * VFACTOR1 * VFACTOR2 *
632  PFACTOR0 * PFACTOR1 * PFACTOR2, 0.0));
633 
634  } /* End loop on wave */
635  } /* End loop on rows in this segment */
636  }/* End loop on segments */
637 
638  /*
639  * (2) Compute the variance by bootstraping on the segments
640  */
641 
642  /* Loop on bootstramp to compute the avg and the
643  * variance by the bootstraping methode */
644  srand(1);
645 
646  for ( int boot = 0 ; boot < nboot ; boot ++ ) {
647  cpl_msg_debug(cpl_func,"Bootstrap %d over %d", boot+1, nboot);
648 
649  /* Init the integration of nseg segment randomly selected */
650  cpl_array * bisp_boot = gravi_array_init_double_complex (nwave, 0.0 + I*0.0);
651  cpl_array * f012_boot = gravi_array_init_double (nwave, 0.0);
652 
653  /* Integrate nseg segment randomly selected.
654  * This loop is vectorialized in spectral direction */
655  for (int rowb = 0; rowb < nseg; rowb ++){
656 
657  /* For the first bootstrap, we use all observed samples
658  * For the others, we also includes the possible montecarlo
659  * FIXME: verify the uniformity of rand for small nrows */
660  int rows;
661  if (boot == 0 ) rows = rowb;
662  else rows = rand()%(nseg+nmontecarlo);
663 
664  /* Integrate the bispectre and flux of selected segments */
665  cpl_array_add (bisp_boot, bisp[rows]);
666  cpl_array_add (f012_boot, F012[rows]);
667  }
668  /* End loop on selected segments */
669 
670  /* Make sure the geometric flux is not null */
671  gravi_array_threshold_min (f012_boot, 0.0, 1e-15);
672 
673  /* Compute the argument and the module of the bispectrum */
674  FREE (cpl_array_delete, t3Amp_res[0]);
675  t3Amp_res[0] = cpl_array_duplicate(bisp_boot);
676  cpl_array_abs (t3Amp_res[0]);
677  cpl_array_divide (t3Amp_res[0], f012_boot);
678 
679  FREE (cpl_array_delete, t3Phi_res[0]);
680  t3Phi_res[0] = cpl_array_duplicate(bisp_boot);
681  cpl_array_arg (t3Phi_res[0]);
682 
683  /* Compute the VARIANCE over the bootstraped samples with the 'online_variance' algorithm.
684  * See https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance */
685  gravi_array_online_variance_res (t3Amp_res, boot, 0);
686  gravi_array_online_variance_res (t3Phi_res, boot, 1);
687 
688  FREE (cpl_array_delete, bisp_boot);
689  FREE (cpl_array_delete, f012_boot);
690  CPLCHECK_MSG("while computing the variances over the bootstrap");
691  }
692  /* End Loop on bootstrap */
693 
694  /* Free list of segments and running mean */
695  FREELOOP (cpl_array_delete, bisp, nseg + nmontecarlo);
696  FREELOOP (cpl_array_delete, F012, nseg + nmontecarlo);
697 
698  /* Convert variance from bootstrap to RMS */
699  cpl_msg_debug(cpl_func,"Put the RMS over bootstrap");
700  cpl_array_power (t3Phi_res[3], 0.5);
701  cpl_array_power (t3Amp_res[3], 0.5);
702  CPLCHECK_MSG("while converting variance -> rms");
703 
704  /* Normalizase integration */
705  mjd_avg /= total_exptime;
706  u1Coord /= total_exptime;
707  v1Coord /= total_exptime;
708  u2Coord /= total_exptime;
709  v2Coord /= total_exptime;
710 
711  }
712 
713  /*
714  * Step (1,2) in case there are no valid frames at all
715  */
716  if (nvalid == 0) {
717  cpl_msg_debug (cpl_func,"Not valid frames, force zero and infinit RMS");
718  cpl_array_fill_window (t3Amp_res[3], 0, nwave, 1e10);
719  cpl_array_fill_window (t3Phi_res[3], 0, nwave, 1e10);
720  mjd_avg = cpl_table_get_column_mean (oi_vis, "MJD");
721  }
722 
723  /*
724  * (3) Save the results on the oi_t3_avg tables
725  */
726 
727  /* Save the cloture amplitude on the oi_T3 tables */
728  cpl_table_set_array (oi_t3_avg, "T3AMP", closure, t3Amp_res[2]);
729  cpl_table_set_array (oi_t3_avg, "T3AMPERR", closure, t3Amp_res[3]);
730  CPLCHECK_MSG("filling T3AMP");
731 
732  /* Save the cloture phase on the oi_T3 tables */
733  gravi_table_set_array_phase (oi_t3_avg, "T3PHI", closure, t3Phi_res[2]);
734  gravi_table_set_array_phase (oi_t3_avg, "T3PHIERR", closure, t3Phi_res[3]);
735  CPLCHECK_MSG("filling T3PHI");
736 
737  /* Flag the data with >100% error or >60deg error */
738  gravi_vis_flag_threshold (oi_t3_avg, "T3PHIERR", "FLAG", 60.0);
739  gravi_vis_flag_threshold (oi_t3_avg, "T3AMPERR", "FLAG", 1.0);
740  gravi_vis_flag_median (oi_t3_avg, "T3PHIERR", "FLAG", 5.0);
741  CPLCHECK_MSG("cannot flag baddata data");
742 
743  /* Compute the total integration time and MJD */
744  cpl_msg_debug(cpl_func,"Total integration time = %.3f s", total_exptime);
745  cpl_table_set_double (oi_t3_avg, "INT_TIME", closure, total_exptime);
746  cpl_table_set_double (oi_t3_avg, "MJD", closure, mjd_avg);
747  cpl_table_set_double (oi_t3_avg, "U1COORD", closure, u1Coord);
748  cpl_table_set_double (oi_t3_avg, "V1COORD", closure, v1Coord);
749  cpl_table_set_double (oi_t3_avg, "U2COORD", closure, u2Coord);
750  cpl_table_set_double (oi_t3_avg, "V2COORD", closure, v2Coord);
751  cpl_table_set (oi_t3_avg, "NVALID", closure, nvalid);
752  cpl_table_set (oi_t3_avg, "NDIT", closure, nrow);
753 
754  /* Set the TARGET_ID */
755  cpl_table_set_int (oi_t3_avg, "TARGET_ID", closure, cpl_table_get_int (oi_vis, "TARGET_ID", base0, &nv));
756 
757  /* Set STA_INDEX */
758  cpl_array * sta_index = cpl_array_new (3, CPL_TYPE_INT);
759  cpl_array_set_int (sta_index, 0, cpl_table_get_int (oi_flux,"STA_INDEX", ctel0, &nv));
760  cpl_array_set_int (sta_index, 1, cpl_table_get_int (oi_flux,"STA_INDEX", ctel1, &nv));
761  cpl_array_set_int (sta_index, 2, cpl_table_get_int (oi_flux,"STA_INDEX", ctel2, &nv));
762  cpl_table_set_array (oi_t3_avg, "STA_INDEX", closure, sta_index);
763  FREE (cpl_array_delete, sta_index);
764 
765  /* Free the aggregate flags for closures */
766  FREELOOP (cpl_array_delete, t3Phi_res, 4);
767  FREELOOP (cpl_array_delete, t3Amp_res, 4);
768  FREE (cpl_free, flagclo);
769 
770  } /* End loop on closure */
771 
772  gravi_msg_function_exit(0);
773  return CPL_ERROR_NONE;
774 }
775 
776 /*-----------------------------------------------------------------------------*/
795 /*-----------------------------------------------------------------------------*/
796 
797 cpl_error_code gravi_vis_average_bootstrap (cpl_table * oi_vis_avg,
798  cpl_table * oi_vis2_avg,
799  cpl_table * oi_vis,
800  int nboot,
801  const char * phase_ref,
802  int use_vFactor,
803  int use_pFactor,
804  int use_debiasing)
805 {
806  gravi_msg_function_start(0);
807  cpl_ensure_code (oi_vis_avg, CPL_ERROR_ILLEGAL_OUTPUT);
808  cpl_ensure_code (oi_vis2_avg, CPL_ERROR_ILLEGAL_OUTPUT);
809  cpl_ensure_code (oi_vis, CPL_ERROR_NULL_INPUT);
810  cpl_ensure_code (nboot>0, CPL_ERROR_ILLEGAL_INPUT);
811 
812  /* Parameters */
813  int nv = 0, nbase = 6;
814  cpl_size nrow = cpl_table_get_nrow (oi_vis) / nbase;
815  cpl_size nwave = cpl_table_get_column_depth (oi_vis, "VISDATA");
816 
817  /* Pointer to columns, to speed-up */
818  cpl_array ** pVISDATA = cpl_table_get_data_array (oi_vis, "VISDATA");
819  cpl_array ** pVISERR = cpl_table_get_data_array (oi_vis, "VISERR");
820  cpl_array ** pFNORM = cpl_table_get_data_array (oi_vis, "F1F2");
821  double * pINTTIME = cpl_table_get_data_double (oi_vis, "INT_TIME");
822  double * pMJD = cpl_table_get_data_double (oi_vis, "MJD");
823  double * pUCOORD = cpl_table_get_data_double (oi_vis, "UCOORD");
824  double * pVCOORD = cpl_table_get_data_double (oi_vis, "VCOORD");
825  cpl_array ** pVFACTOR = use_vFactor?cpl_table_get_data_array (oi_vis, "V_FACTOR"):NULL;
826  double * pPFACTOR = use_pFactor?cpl_table_get_data_double (oi_vis, "P_FACTOR"):NULL;
827  CPLCHECK_MSG ("Cannot get the data");
828 
829  /* Get the reference phase */
830  cpl_array ** pPHASEREF = NULL;
831  if (phase_ref && strcmp (phase_ref, "NONE"))
832  pPHASEREF = cpl_table_get_data_array (oi_vis, phase_ref);
833  CPLCHECK_MSG ("Cannot get the reference phase data (column missing?)");
834 
835  /* Loop on base */
836  for (cpl_size base = 0; base < nbase; base ++) {
837 
838  /* Tel for base */
839  cpl_size nvalid = 0;
840 
841  /* Arrays to store the final, integrated quantities
842  * 0: current boot, 1: running_mean, 2: first boot, 3: variance */
843  cpl_array **visR_res = gravi_array_new_list (4, CPL_TYPE_DOUBLE, nwave);
844  cpl_array **visI_res = gravi_array_new_list (4, CPL_TYPE_DOUBLE, nwave);
845  cpl_array **vis2_res = gravi_array_new_list (4, CPL_TYPE_DOUBLE, nwave);
846  cpl_array **visAmp_res = gravi_array_new_list (4, CPL_TYPE_DOUBLE, nwave);
847  cpl_array **visPhi_res = gravi_array_new_list (4, CPL_TYPE_DOUBLE, nwave);
848  double total_exptime = 0.0, mjd_avg = 0.0;
849  double uCoord = 0.0, vCoord = 0.0;
850 
851  /*
852  * (0) Optimize the number of segment
853  */
854 
855  /* Get the number of non-rejected frames */
856  int * flag = cpl_table_get_data_int (oi_vis, "REJECTION_FLAG");
857  cpl_ensure_code (flag, CPL_ERROR_ILLEGAL_INPUT);
858  for ( int row=0 ; row<nrow; row++ ) if ( flag[row * nbase + base] == 0 ) nvalid++;
859 
860  /* Build an optimal number of segment and nrow_per_segment */
861  cpl_size nrow_per_seg = CPL_MAX (nvalid / CPL_MIN (nrow, 100), 1);
862  cpl_size nseg = nvalid / nrow_per_seg;
863 
864  /* Ensure there are at least 5 samples to bootstrap on,
865  * if no add montecarlo samples */
866  cpl_size nsamp = 5, nmontecarlo = CPL_MAX (nsamp - nseg, 0);
867 
868  cpl_msg_info ("Stat", "%6lld valid frames over %6lld (%5.1f%%), make %4lld seg. of %5lld (miss %lld), add %lld MonteCarlo",
869  nvalid, nrow, (double)nvalid/(double)nrow*100.0,
870  nseg, nrow_per_seg, nvalid - nseg*nrow_per_seg, nmontecarlo);
871 
872  /* Case we have at least one valid frame */
873  if ( nvalid > 0 ) {
874 
875  /*
876  * (1) Pre-integration over segment, to bootstrap on less statistic
877  */
878  cpl_array **visR = gravi_array_new_list (nseg + nmontecarlo, CPL_TYPE_DOUBLE, nwave);
879  cpl_array **visI = gravi_array_new_list (nseg + nmontecarlo, CPL_TYPE_DOUBLE, nwave);
880  cpl_array **POWER = gravi_array_new_list (nseg + nmontecarlo, CPL_TYPE_DOUBLE, nwave);
881  cpl_array **F12 = gravi_array_new_list (nseg + nmontecarlo, CPL_TYPE_DOUBLE, nwave);
882  cpl_array **F1F2 = gravi_array_new_list (nseg + nmontecarlo, CPL_TYPE_DOUBLE, nwave);
883 
884  /* Loop on segment */
885  cpl_size row = -1;
886  for ( int seg = 0 ; seg < nseg + nmontecarlo ; seg ++ ) {
887  cpl_msg_debug(cpl_func,"pre integration of seg %d start with row %lld", seg, row);
888 
889  /* Find nrow_per_seg valid frame to integrate in this segment */
890  cpl_size r = 0;
891  while ( r < nrow_per_seg ) {
892  row = (row + 1) % nrow;
893  if ( flag[row * nbase + base] ) {continue;} else {r++;}
894 
895  /* Get indices */
896  cpl_size rbase = row * nbase + base;
897 
898  /* Compute the total integration time.
899  * Do not integrate for the MonteCarlo samples */
900  if (seg < nseg) {
901  total_exptime += pINTTIME[rbase];
902  mjd_avg += pMJD[rbase] * pINTTIME[rbase];
903  uCoord += pUCOORD[rbase] * pINTTIME[rbase];
904  vCoord += pVCOORD[rbase] * pINTTIME[rbase];
905  }
906 
907  /* fast-no-CPL integration: get pointers on data */
908  double *tR = cpl_array_get_data_double (visR[seg]);
909  double *tI = cpl_array_get_data_double (visI[seg]);
910  double *tP = cpl_array_get_data_double (POWER[seg]);
911  double *tF1F2 = cpl_array_get_data_double (F1F2[seg]);
912  double *tF12 = cpl_array_get_data_double (F12[seg]);
913  CPLCHECK_MSG ("Cannot get data");
914 
915  double PFACTOR = (use_pFactor?pPFACTOR[rbase]:1.0);
916  CPLCHECK_MSG ("Cannot get FACTOR data");
917 
918  /* Loop on wave */
919  for (int w = 0; w < nwave; w++) {
920  double VFACTOR = use_vFactor?cpl_array_get (pVFACTOR[rbase], w, NULL):1.0;
921  double PHASEREF = pPHASEREF?cpl_array_get (pPHASEREF[rbase], w, NULL):0.0;
922  double FNORM = cpl_array_get (pFNORM[rbase], w, NULL);
923  double complex Vis = cpl_array_get_complex (pVISDATA[rbase], w, NULL);
924  double complex VErr = cpl_array_get_complex (pVISERR[rbase], w, NULL);
925  double mR = creal (Vis);
926  double mI = cimag (Vis);
927  double eR = creal (VErr);
928  double eI = cimag (VErr);
929  CPLCHECK_MSG ("Cannot get data");
930 
931  /* Add noise if this is a Monte Carlo sample.
932  * APPROX: Noise is only added to the coherent flux */
933  if ( seg > nseg-1 ) {
934  mR += 2 * eR * gravi_randn();
935  mI += 2 * eI * gravi_randn();
936  }
937 
938  if (PHASEREF) {
939  /* Integrate <R> and <I> rephased */
940  tR[w] += cos(PHASEREF) * mR - sin(PHASEREF) * mI;
941  tI[w] += cos(PHASEREF) * mI + sin(PHASEREF) * mR;
942  } else {
943  /* Integrate directly without rephasing */
944  tR[w] += mR;
945  tI[w] += mI;
946  }
947 
948  /* Compute the flux <F1F2> and <sqrt(|F1F2|)> x sign(F1F2)
949  * corrected by the vFactor if needed */
950  tF1F2[w] += FNORM * VFACTOR * PFACTOR;
951  tF12[w] += sqrt( CPL_MAX (FNORM * VFACTOR * PFACTOR, 0.0) );
952 
953  /* Integrate < R2 + I2 - sR2 - sI2 > */
954  if (use_debiasing) {
955  tP[w] += mR*mR + mI*mI - eR*eR - eI*eI;
956  } else {
957  tP[w] += mR*mR + mI*mI;
958  }
959  } /* End loop on wave */
960  } /* End loop on rows in this segment */
961  }/* End loop on segments */
962 
963 
964  /*
965  * (2) Compute the variance by bootstraping on the segments
966  */
967 
968  /* Loop on bootstramp to compute the avg and the
969  * variance by the bootstraping methode */
970  srand(1);
971 
972  for ( int boot = 0 ; boot < nboot ; boot ++ ) {
973  cpl_msg_debug (cpl_func,"Bootstrap %d over %d", boot+1, nboot);
974 
975  /* Init the itegration of nseg segments */
976  cpl_array * F12_boot = gravi_array_init_double (nwave, 0.0);
977  cpl_array * F1F2_boot = gravi_array_init_double (nwave, 0.0);
978 
979  /* Integrate nseg segments randomly selected.
980  * This loop is vectorialized in spectral direction */
981  for (int rowb = 0; rowb < nseg; rowb ++) {
982 
983  /* For the first bootstrap, we use all observed samples
984  * For the others, we also includes the possible montecarlo
985  * FIXME: verify the uniformity of rand for small nrows */
986  int rows;
987  if (boot == 0 ) rows = rowb;
988  else rows = rand()%(nseg+nmontecarlo);
989 
990  /* Integrate the selected segments */
991  cpl_array_add (visR_res[0], visR[rows]);
992  cpl_array_add (visI_res[0], visI[rows]);
993  cpl_array_add (vis2_res[0], POWER[rows]);
994  cpl_array_add (F12_boot, F12[rows]);
995  cpl_array_add (F1F2_boot, F1F2[rows]);
996  }
997  /* End loop to integrate nseg segments randomly selected
998  * We now have a random realisation of a high SNR dataset
999  * to which we can apply the estimators */
1000 
1001  /* Make sure the geometric flux is not null */
1002  gravi_array_threshold_min (F1F2_boot, 0.0, 1e-15);
1003 
1004  /* Energy estimator
1005  vis2 = POWER / F1F2 */
1006  cpl_array_divide (vis2_res[0], F1F2_boot);
1007  CPLCHECK_MSG("while computing the energie integration");
1008 
1009  /* Norm of coherent integration
1010  visAmp = sqrt(visR^2 + visR^2) / F12 */
1011  FREE (cpl_array_delete, visAmp_res[0]);
1012  visAmp_res[0] = gravi_array_compute_norm2 (visR_res[0], visI_res[0]);
1013  cpl_array_power (visAmp_res[0], 0.5);
1014  cpl_array_divide (visAmp_res[0], F12_boot);
1015  CPLCHECK_MSG("while computing the norm of the coherent integration");
1016 
1017  /* Phase of coherent integration
1018  visPhi = arctan( visI, visR ) */
1019  FREE (cpl_array_delete, visPhi_res[0]);
1020  visPhi_res[0] = gravi_array_wrap_complex (visR_res[0], visI_res[0]);
1021  cpl_array_arg (visPhi_res[0]);
1022 
1023  /* Compute the VARIANCE over the bootstraped samples with the 'online_variance' algorithm.
1024  * See https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance */
1025  gravi_array_online_variance_res (vis2_res, boot, 0);
1026  gravi_array_online_variance_res (visAmp_res, boot, 0);
1027  gravi_array_online_variance_res (visR_res, boot, 0);
1028  gravi_array_online_variance_res (visI_res, boot, 0);
1029  gravi_array_online_variance_res (visPhi_res, boot, 1);
1030  CPLCHECK_MSG("while computing the variances over the bootstrap");
1031 
1032  /* Delete these temporary integrations */
1033  FREE (cpl_array_delete, F12_boot);
1034  FREE (cpl_array_delete, F1F2_boot);
1035  CPLCHECK_MSG("while freeing during bootstrap");
1036  }
1037  /* End bootstrap */
1038 
1039  /* Free list of segments */
1040  FREELOOP (cpl_array_delete, visR, nseg + nmontecarlo);
1041  FREELOOP (cpl_array_delete, visI, nseg + nmontecarlo);
1042  FREELOOP (cpl_array_delete, POWER, nseg + nmontecarlo);
1043  FREELOOP (cpl_array_delete, F12, nseg + nmontecarlo);
1044  FREELOOP (cpl_array_delete, F1F2, nseg + nmontecarlo);
1045 
1046  /* Convert variance from bootstrap to RMS */
1047  cpl_msg_debug (cpl_func,"Put the RMS over bootstrap");
1048  cpl_array_power (vis2_res[3], 0.5);
1049  cpl_array_power (visAmp_res[3], 0.5);
1050  cpl_array_power (visPhi_res[3], 0.5);
1051  cpl_array_power (visR_res[3], 0.5);
1052  cpl_array_power (visI_res[3], 0.5);
1053  CPLCHECK_MSG ("while converting variance -> rms");
1054 
1055  /* Normalize integration */
1056  mjd_avg /= total_exptime;
1057  uCoord /= total_exptime;
1058  vCoord /= total_exptime;
1059 
1060  }
1061 
1062  /*
1063  * Step (1,2) in case there are no valid frames at all
1064  */
1065  if (nvalid == 0) {
1066  cpl_msg_debug (cpl_func,"Not valid frames, force zero and infinit RMS");
1067  cpl_array_fill_window (vis2_res[3], 0, nwave, 1e10);
1068  cpl_array_fill_window (visR_res[3], 0, nwave, 1e10);
1069  cpl_array_fill_window (visI_res[3], 0, nwave, 1e10);
1070  cpl_array_fill_window (visAmp_res[3], 0, nwave, 1e10);
1071  cpl_array_fill_window (visPhi_res[3], 0, nwave, 1e10);
1072  mjd_avg = cpl_table_get_column_mean (oi_vis, "MJD");
1073  }
1074 
1075  /*
1076  * (3) Save the results on the oi_vis_avg tables
1077  */
1078 
1079  cpl_table_set_array (oi_vis2_avg, "VIS2DATA", base, vis2_res[2]);
1080  cpl_table_set_array (oi_vis2_avg, "VIS2ERR", base, vis2_res[3]);
1081  CPLCHECK_MSG("filling VIS2");
1082 
1083  gravi_table_set_array_double_complex (oi_vis_avg, "VISDATA", base, visR_res[2], visI_res[2]);
1084  gravi_table_set_array_double_complex (oi_vis_avg, "VISERR", base, visR_res[3], visI_res[3]);
1085  CPLCHECK_MSG("filling VISDATA");
1086 
1087  cpl_table_set_array (oi_vis_avg, "RVIS", base, visR_res[2]);
1088  cpl_table_set_array (oi_vis_avg, "RVISERR", base, visR_res[3]);
1089  CPLCHECK_MSG("filling RVIS");
1090 
1091  cpl_table_set_array (oi_vis_avg, "IVIS", base, visI_res[2]);
1092  cpl_table_set_array (oi_vis_avg, "IVISERR", base, visI_res[3]);
1093  CPLCHECK_MSG("filling IVIS");
1094 
1095  cpl_table_set_array (oi_vis_avg, "VISAMP", base, visAmp_res[2]);
1096  cpl_table_set_array (oi_vis_avg, "VISAMPERR", base, visAmp_res[3]);
1097  CPLCHECK_MSG("filling VISAMP");
1098 
1099  gravi_table_set_array_phase (oi_vis_avg, "VISPHI", base, visPhi_res[2]);
1100  gravi_table_set_array_phase (oi_vis_avg, "VISPHIERR", base, visPhi_res[3]);
1101  CPLCHECK_MSG("filling VISPHI");
1102 
1103  /* Flag the data with >100% error or >60deg error */
1104  gravi_vis_flag_threshold (oi_vis2_avg, "VIS2ERR", "FLAG", 1.0);
1105  gravi_vis_flag_median (oi_vis2_avg, "VIS2ERR", "FLAG", 5.0);
1106  gravi_vis_flag_threshold (oi_vis_avg, "VISAMPERR", "FLAG", 1.0);
1107  gravi_vis_flag_median (oi_vis_avg, "VISPHIERR", "FLAG", 5.0);
1108  CPLCHECK_MSG("cannot flag baddata data");
1109 
1110  /* Compute the total integration time */
1111  cpl_table_set (oi_vis_avg, "INT_TIME", base, total_exptime);
1112  cpl_table_set (oi_vis2_avg, "INT_TIME", base, total_exptime);
1113  cpl_table_set (oi_vis_avg, "MJD", base, mjd_avg);
1114  cpl_table_set (oi_vis2_avg, "MJD", base, mjd_avg);
1115  cpl_table_set (oi_vis_avg, "UCOORD", base, uCoord);
1116  cpl_table_set (oi_vis_avg, "VCOORD", base, vCoord);
1117  cpl_table_set (oi_vis2_avg, "UCOORD", base, uCoord);
1118  cpl_table_set (oi_vis2_avg, "VCOORD", base, vCoord);
1119  CPLCHECK_MSG("cannot fill time");
1120 
1121  /* Set some statistics */
1122  cpl_table_set (oi_vis2_avg, "NVALID", base, nvalid);
1123  cpl_table_set (oi_vis2_avg, "NDIT", base, nrow);
1124  cpl_table_set (oi_vis_avg, "NVALID", base, nvalid);
1125  cpl_table_set (oi_vis_avg, "NDIT", base, nrow);
1126  CPLCHECK_MSG("cannot fill nvalid");
1127 
1128  /* Set the TARGET_ID and STA_INDEX */
1129  cpl_table_set_int (oi_vis_avg, "TARGET_ID", base, cpl_table_get_int (oi_vis, "TARGET_ID", base, &nv));
1130  cpl_table_set_int (oi_vis2_avg, "TARGET_ID", base, cpl_table_get_int (oi_vis, "TARGET_ID", base, &nv));
1131  cpl_table_set_array (oi_vis_avg, "STA_INDEX", base, cpl_table_get_array (oi_vis, "STA_INDEX", base));
1132  cpl_table_set_array (oi_vis2_avg, "STA_INDEX", base, cpl_table_get_array (oi_vis, "STA_INDEX", base));
1133 
1134  /* Free variance */
1135  FREELOOP (cpl_array_delete, vis2_res, 4);
1136  FREELOOP (cpl_array_delete, visAmp_res, 4);
1137  FREELOOP (cpl_array_delete, visPhi_res, 4);
1138  FREELOOP (cpl_array_delete, visR_res, 4);
1139  FREELOOP (cpl_array_delete, visI_res, 4);
1140 
1141  }
1142  /* End loop on bases */
1143 
1144  gravi_msg_function_exit(0);
1145  return CPL_ERROR_NONE;
1146 }
1147 
1148 double gdAbacusErrPhi(double x)
1173 {
1174  const double Asymptot = CPL_MATH_PI / sqrt(3.0);
1175  double c[8] = {2.7191808010909,
1176  -17.1901043936273,
1177  45.0654103760899,
1178  -63.4441678243197,
1179  52.3098941426378,
1180  -25.8090699917488,
1181  7.84352873962491,
1182  -1.57308595820081};
1183 
1184  double x2, x3, x4, x5, x6, x7, z;
1185  if (x > Asymptot) {
1186  return (1E10);
1187  }
1188  if (x > 1.74413) {
1189  return (0.691 / (Asymptot - x));
1190  }
1191  if (x < 0.1) {
1192  return (x);
1193  }
1194  x2 = x*x;
1195  x3 = x2*x;
1196  x4 = x2*x2;
1197  x5 = x3*x2;
1198  x6 = x3*x3;
1199  x7 = x6*x;
1200  z = c[0] * x7 + c[1] * x6 + c[2] * x5 + c[3] * x4 + c[4] * x3 + c[5] * x2 + c[6] * x + c[7];
1201  return pow(10, z);
1202 }
1203 
1204 /*-----------------------------------------------------------------------------*/
1215 /*-----------------------------------------------------------------------------*/
1216 
1217 cpl_error_code gravi_average_self_visphi(cpl_table * oi_vis_avg,
1218  cpl_table * oi_vis,
1219  cpl_array * wavenumber,
1220  const char * phase_ref, int* cmin, int* cmax, int nrange)
1221 {
1222  gravi_msg_function_start(0);
1223  cpl_ensure_code(oi_vis_avg, CPL_ERROR_ILLEGAL_OUTPUT);
1224  cpl_ensure_code(oi_vis, CPL_ERROR_NULL_INPUT);
1225 
1226  /* Parameters */
1227  int nbase = 6;
1228  cpl_size nrow = cpl_table_get_nrow(oi_vis) / nbase;
1229  cpl_size nwave = cpl_table_get_column_depth(oi_vis, "VISDATA");
1230 
1231  int use_crange = (nrange > 0);
1232  if (use_crange) {
1233  for (int i = 0; i < nrange; ++i) {
1234  if (cmax[i] - cmin[i] < 1) {
1235  use_crange = 0;
1236  break;
1237  }
1238  if (cmax[i] > nwave - 1) {
1239  use_crange = 0;
1240  break;
1241  }
1242  if (cmin[i] < 0) {
1243  use_crange = 0;
1244  break;
1245  }
1246  }
1247  if (use_crange) {
1248  for (int i = 0; i < nrange; ++i) cpl_msg_info("Reference Channel", "Part %02d [%3d:%3d]", i + 1, cmin[i], cmax[i]);
1249  } else {
1250  cpl_msg_info("Warning (SELF_VISPHI)", "Invalid Ranges Found, continuing with default Method.");
1251  }
1252  }
1253 
1254  /* Pointer to columns, to speed-up */
1255  cpl_array ** pVISDATA = cpl_table_get_data_array(oi_vis, "VISDATA");
1256  cpl_array ** pVISERR = cpl_table_get_data_array(oi_vis, "VISERR");
1257  CPLCHECK_MSG("Cannot get the data");
1258 
1259  /* Get the reference phase unless it has already been removed*/
1260  cpl_array ** pPHASEREF = NULL;
1261  if (phase_ref && strcmp(phase_ref, "NONE"))
1262  pPHASEREF = cpl_table_get_data_array(oi_vis, phase_ref);
1263  CPLCHECK_MSG("Cannot get the reference phase data (column missing?)");
1264 
1265  /* Loop on base */
1266  for (cpl_size base = 0; base < nbase; base++) {
1267 
1268  /* number of valid rows for base */
1269  cpl_size nvalid = 0;
1270 
1271  /* Arrays to store the final, integrated quantities
1272  * 0: running_mean, 2: variance */
1273  cpl_array **visPhi_res = gravi_array_new_list(2, CPL_TYPE_DOUBLE, nwave);
1274 
1275  /* Get the number of non-rejected frames */
1276  int * flag = cpl_table_get_data_int(oi_vis, "REJECTION_FLAG");
1277  cpl_ensure_code(flag, CPL_ERROR_ILLEGAL_INPUT);
1278  for (int row = 0; row < nrow; row++) if (flag[row * nbase + base] == 0) nvalid++;
1279 
1280  cpl_msg_info("Stat (SELF_VISPHI)", "%6lld valid frames over %6lld (%5.1f%%)",
1281  nvalid, nrow, (double) nvalid / (double) nrow * 100.0);
1282 
1283  /*
1284  * return in case there are no valid frames at all
1285  */
1286  if (nvalid == 0) {
1287  cpl_msg_debug(cpl_func, "No valid frames, force zero and infinite RMS");
1288  cpl_array_fill_window(visPhi_res[1], 0, nwave, 1e10);
1289  gravi_table_set_array_phase(oi_vis_avg, "VISPHI", base, visPhi_res[0]);
1290  gravi_table_set_array_phase(oi_vis_avg, "VISPHIERR", base, visPhi_res[1]);
1291  CPLCHECK_MSG("filling VISPHI");
1292  /* Free variance */
1293  FREELOOP(cpl_array_delete, visPhi_res, 2);
1294  break;
1295  }
1296 
1297  cpl_array **Vis = gravi_array_new_list(nvalid, CPL_TYPE_DOUBLE_COMPLEX, nwave);
1298  cpl_array **EVis = gravi_array_new_list(nvalid, CPL_TYPE_DOUBLE_COMPLEX, nwave);
1299 
1300  cpl_array **W1 = gravi_array_new_list(nvalid, CPL_TYPE_DOUBLE_COMPLEX, nwave);
1301  cpl_array **EW1 = gravi_array_new_list(nvalid, CPL_TYPE_DOUBLE_COMPLEX, nwave);
1302  /* Loop on row */
1303 
1304  for (cpl_size currentRow = 0, validRowIndex = -1; currentRow < nrow; currentRow++) {
1305  if (flag[currentRow * nbase + base]) {
1306  continue;
1307  } else {
1308  /* Get indices */
1309  cpl_size rbase = currentRow * nbase + base;
1310  validRowIndex++;
1311 
1312  /* fast-no-CPL integration: get pointers on data */
1313  double complex *ptrC = cpl_array_get_data_double_complex(Vis[validRowIndex]);
1314  double complex *ptrEC = cpl_array_get_data_double_complex(EVis[validRowIndex]);
1315  CPLCHECK_MSG("Cannot get data");
1316 
1317  /* Loop on wave */
1318  for (int w = 0; w < nwave; w++) {
1319  double PHASEREF = pPHASEREF ? cpl_array_get(pPHASEREF[rbase], w, NULL) : 0.0;
1320  double complex vis = cpl_array_get_double_complex(pVISDATA[rbase], w, NULL);
1321  double complex viserr = cpl_array_get_double_complex(pVISERR[rbase], w, NULL);
1322  CPLCHECK_MSG("Cannot get data");
1323  if (PHASEREF) {
1324  /* rephase <R> and <I> */
1325  ptrC[w] = (cos(PHASEREF) * creal(vis) - sin(PHASEREF) * cimag(vis)) +
1326  I * (cos(PHASEREF) * cimag(vis) + sin(PHASEREF) * creal(vis));
1327  } else {
1328  /* no rephasing */
1329  ptrC[w] = vis;
1330  }
1331  ptrEC[w] = viserr;
1332  } /* End loop on wave */
1333  } /* End frame not flagged */
1334  } /* End loop on rows */
1335 
1336  for (int irow = 0; irow < nvalid; irow++) {
1337 
1338  /* Normalize the phasor to avoid bad pixels */
1339  /*
1340  cpl_array * norm = cpl_array_duplicate(Vis[irow]);
1341  cpl_array_abs(norm);
1342  cpl_array_divide(Vis[irow], norm);
1343  */
1344 
1345  /* Compute and remove the mean group delay in [m] */
1346  double mean_delay = 0.0;
1347  gravi_array_get_group_delay_loop(&Vis[irow], wavenumber, &mean_delay, 1, 2e-3, CPL_FALSE); /*search at 20 lambda max if recentered with FT*/
1348  gravi_array_multiply_phasor(Vis[irow], -2 * I * CPL_MATH_PI * mean_delay, wavenumber);
1349 
1350  /* Compute and remove the mean phase in [rad] */
1351  double mean_phase = carg(cpl_array_get_mean_complex(Vis[irow]));
1352  cpl_array_multiply_scalar_complex(Vis[irow], cexp(-I * mean_phase));
1353  }
1354 
1355  cpl_array *CRef = cpl_array_new(nwave, CPL_TYPE_DOUBLE_COMPLEX);
1356  cpl_array *ECRef = cpl_array_new(nwave, CPL_TYPE_DOUBLE_COMPLEX);
1357  double complex *pCRef = cpl_array_get_data_double_complex(CRef);
1358  double complex *pECRef = cpl_array_get_data_double_complex(ECRef);
1359 
1360  /* Production of the reference channel: mean of all R and I parts
1361  * of all channels except the one considered ( eq 2.3) */
1362  for (int irow = 0; irow < nvalid; irow++) {
1363  /*reset sum to 0 */
1364  double complex totalVis = 0.0 + I * 0.0;
1365  double complex totalEVis = 0.0 + I * 0.0;
1366  /* fast-no-CPL integration: get pointers on data */
1367  double complex *pVis = cpl_array_get_data_double_complex(Vis[irow]);
1368  double complex *pEVis = cpl_array_get_data_double_complex(EVis[irow]);
1369 
1370  double complex *pW1 = cpl_array_get_data_double_complex(W1[irow]);
1371  double complex *pEW1 = cpl_array_get_data_double_complex(EW1[irow]);
1372 
1373  CPLCHECK_MSG("Cannot get data");
1374  if (use_crange) {
1375  /* sum all Vis for this row */
1376  int nchans = 0;
1377  for (int i = 0; i < nrange; ++i) {
1378  for (int w = cmin[i]; w < cmax[i]; ++w) {
1379  totalVis += pVis[w];
1380  totalEVis += pEVis[w];
1381  nchans++;
1382  }
1383  }
1384  for (int w = 0; w < nwave; w++) {
1385  cpl_array_set_double_complex(CRef, w, totalVis / nchans);
1386  cpl_array_set_double_complex(ECRef, w, totalEVis / nchans);
1387  }
1388  } else {
1389  /* sum all Vis for this row */
1390  for (int w = 0; w < nwave; w++) {
1391  totalVis += pVis[w];
1392  totalEVis += pEVis[w];
1393  }
1394  /* then construct Cref by substracting current R and I
1395  * at that Wlen and make the arithmetic mean. The code permits
1396  * to avoid not only the channel itself removed but the 2*radius
1397  * around (not activated). */
1398  int iw = 0;
1399  int radius = 0;
1400  int divider = nwave - (2 * radius) - 1;
1401  for (; iw < radius; iw++) {
1402  cpl_array_set_double_complex(CRef, iw, totalVis / nwave);
1403  cpl_array_set_double_complex(ECRef, iw, totalEVis / nwave);
1404  }
1405  for (; iw < nwave - radius; iw++) {
1406  double complex tmp = 0.0 + I * 0.0;
1407  double complex Etmp = 0.0 + I * 0.0;
1408  for (int j = iw; j < iw + 2 * radius + 1; ++j) tmp += pVis[iw];
1409  cpl_array_set_double_complex(CRef, iw, (totalVis - tmp) / divider);
1410  for (int j = iw; j < iw + 2 * radius + 1; ++j) Etmp += pEVis[iw];
1411  cpl_array_set_double_complex(ECRef, iw, (totalEVis - Etmp) / divider);
1412  }
1413  for (; iw < radius; iw++) {
1414  cpl_array_set_double_complex(CRef, iw, totalVis / nwave);
1415  cpl_array_set_double_complex(ECRef, iw, totalEVis / nwave);
1416  }
1417  }
1418  /* Now the interspectrum is C*~C_Ref. Store in w1. */
1419  for (int w = 0; w < nwave; w++) {
1420  pW1[w] = pVis[w] * conj(pCRef[w]);
1421 
1422  /* Please have a look to the F. Millour thesis
1423  (http://tel.archives-ouvertes.fr/tel-00134268),
1424  pp.91-92 (eq. 4.55 to 4.58) */
1425  pEW1[w] = (creal(pEVis[w]) * pow(creal(pCRef[w]), 2) +
1426  creal(pECRef[w]) * pow(creal(pVis[w]), 2) +
1427  cimag(pVis[w]) * pow(cimag(pCRef[w]), 2) +
1428  cimag(pECRef[w]) + pow(cimag(pVis[w]), 2)) +
1429 
1430  I * (
1431  cimag(pVis[w]) * pow(creal(pCRef[w]), 2) +
1432  cimag(pECRef[w]) * pow(creal(pVis[w]), 2) +
1433  creal(pVis[w]) * pow(cimag(pCRef[w]), 2) +
1434  creal(pECRef[w]) + pow(cimag(pVis[w]), 2)
1435  );
1436  }
1437  }
1438 
1439  FREE(cpl_array_delete, CRef);
1440  FREE(cpl_array_delete, ECRef);
1441  FREELOOP(cpl_array_delete, Vis, nvalid);
1442  FREELOOP(cpl_array_delete, EVis, nvalid);
1443 
1444  double *pPhi = cpl_array_get_data_double(visPhi_res[0]);
1445  double *pPhiErr = cpl_array_get_data_double(visPhi_res[1]);
1446 
1447  /* Compute mean VisPhi as average of selected frames. */
1448  for (int w = 0; w < nwave; w++) {
1449  cpl_array *cpxVisVect = cpl_array_new(nvalid, CPL_TYPE_DOUBLE_COMPLEX);
1450 
1451  /* The W1 vector */
1452  for (int irow = 0; irow < nvalid; irow++) {
1453  const double complex *pW1 = cpl_array_get_data_double_complex_const(W1[irow]);
1454  cpl_array_set_double_complex(cpxVisVect, irow, pW1[w]);
1455  }
1456  /* The Phase Herself */
1457  /* average re and im */
1458  double complex w1Avg;
1459  w1Avg = cpl_array_get_mean_complex(cpxVisVect);
1460 
1461  /* store */
1462  pPhi[w] = atan2(cimag(w1Avg), creal(w1Avg));
1463  /* WE USE THE STATISTICAL ERROR FOR BINNING */
1464  w1Avg = conj(w1Avg);
1465  cpl_array *Vect = cpl_array_new(nvalid, CPL_TYPE_DOUBLE);
1466  for (int irow = 0; irow < nvalid; irow++) {
1467  const double complex *tW1 = cpl_array_get_data_double_complex_const(W1[irow]);
1468  /* add w1*conj(w1Avg) to vector*/
1469  cpl_array_set_double(Vect, irow, atan2(cimag(tW1[w] * w1Avg), creal(tW1[w] * w1Avg)));
1470  }
1471  double x = cpl_array_get_stdev(Vect);
1472  /* Err on Phi must be corrected with an abacus*/
1473  pPhiErr[w] = gdAbacusErrPhi(x / sqrt(nvalid));
1474  /* START EKW 21/11/2018 */
1475  FREE(cpl_array_delete, cpxVisVect);
1476  FREE(cpl_array_delete, Vect);
1477  /* END EKW 21/11/2018 */
1478  }
1479 
1480  gravi_table_set_array_phase(oi_vis_avg, "VISPHI", base, visPhi_res[0]);
1481  gravi_table_set_array_phase(oi_vis_avg, "VISPHIERR", base, visPhi_res[1]);
1482  CPLCHECK_MSG("filling VISPHI");
1483  /* Free variance */
1484  FREELOOP(cpl_array_delete, visPhi_res, 2);
1485  /* START EKW 21/11/2018 */
1486  FREELOOP(cpl_array_delete, EW1 , nvalid);
1487  FREELOOP(cpl_array_delete, W1 , nvalid);
1488  /* END EKW 21/11/2018 */
1489 
1490  }
1491 
1492  /* End loop on bases */
1493 
1494  gravi_msg_function_exit(0);
1495  return CPL_ERROR_NONE;
1496 }
1497 
1498 /*----------------------------------------------------------------------------*/
1507 /*----------------------------------------------------------------------------*/
1508 
1509 gravi_data * gravi_compute_vis (gravi_data * p2vmred_data,
1510  const cpl_parameterlist * parlist,
1511  cpl_size * current_frame)
1512 {
1513  gravi_msg_function_start(1);
1514  cpl_ensure (p2vmred_data, CPL_ERROR_NULL_INPUT, NULL);
1515  cpl_ensure (parlist, CPL_ERROR_NULL_INPUT, NULL);
1516 
1517  int nv, nbase = 6, ntel = 4;
1518 
1519  /*
1520  * Compute the limit of integration
1521  */
1522 
1523  /* Get the current position and the maximum nb to integrate */
1524  cpl_size max_frame = gravi_param_get_int (parlist, "gravity.vis.max-frame");
1525  cpl_msg_info (cpl_func,"Average %lli frames starting from %lli",
1526  max_frame, *current_frame);
1527 
1528 
1529 
1530  /* Get the SC DIT */
1531  cpl_propertylist * p2vmred_header = gravi_data_get_header (p2vmred_data);
1532  double dit_sc = gravi_pfits_get_dit_sc (p2vmred_header) * 1e6;
1533 
1534  /* Get the vis_SC table for first polarisation */
1535  int npol_sc = gravi_pfits_get_pola_num (p2vmred_header, GRAVI_SC);
1536  cpl_table * vis_SC = gravi_data_get_oi_vis (p2vmred_data, GRAVI_SC, 0, npol_sc);
1537  cpl_size nrow = cpl_table_get_nrow (vis_SC) / nbase;
1538 
1539  /* Get first and last frame for this integration */
1540  cpl_size sframe = *current_frame;
1541  cpl_size eframe = CPL_MIN (*current_frame + max_frame - 1, nrow-1);
1542 
1543  /* Check if we reached the end of the file, or increment */
1544  if (eframe >= nrow-1) *current_frame = -1;
1545  else *current_frame += max_frame;
1546 
1547  /* Compute start and end-time */
1548  double start_time, end_time;
1549  start_time = cpl_table_get (vis_SC, "TIME", sframe*nbase, &nv) - dit_sc/2;
1550  end_time = cpl_table_get (vis_SC, "TIME", eframe*nbase, &nv) + dit_sc/2;
1551 
1552  /* Compute verbose */
1553  cpl_msg_info (cpl_func,"Integrate frames: first = %lli last = %lli", sframe, eframe);
1554  cpl_msg_info (cpl_func,"start = %f end = %f [s]", start_time*1e-6, end_time*1e-6);
1555 
1556 
1557  /*
1558  * Prepare the output
1559  */
1560 
1561  cpl_msg_info(cpl_func, "Construction of the averaged output data");
1562 
1563  gravi_data * vis_data = gravi_data_new (0);
1564  cpl_propertylist * vis_header = gravi_data_get_header (vis_data);
1565  cpl_propertylist_append (vis_header, p2vmred_header);
1566 
1567  /* Copy the oi tables needed in output data.
1568  * This will duplicate all OI_WAVELENGTH tables */
1569  gravi_data_copy_ext (vis_data, p2vmred_data, GRAVI_OI_ARRAY_EXT);
1570  gravi_data_copy_ext (vis_data, p2vmred_data, GRAVI_OI_TARGET_EXT);
1571  gravi_data_copy_ext (vis_data, p2vmred_data, GRAVI_OI_WAVELENGTH_EXT);
1572 
1573  CPLCHECK_NUL ("Cannot get tables for output data");
1574 
1575  /*
1576  * Start with FT
1577  */
1578 
1579  if (gravi_data_has_type (p2vmred_data, "_FT") <= 0 ) {
1580  cpl_msg_info (cpl_func, "P2VMRED data has no FT extensions");
1581  }
1582  else {
1583  /* Reduction parameters */
1584  int v_factor_flag_ft = 0;
1585  int p_factor_flag_ft = 0;
1586  int debiasing_flag_ft = gravi_param_get_bool (parlist, "gravity.vis.debias-ft");
1587  int nboot_ft = gravi_param_get_int (parlist, "gravity.vis.nboot");
1588  const char * phase_ref_ft = "SELF_REF";
1589  CPLCHECK_NUL("Cannot get parameters");
1590 
1591  cpl_msg_info (cpl_func, "Bias subtraction of V2 for FT is %s",debiasing_flag_ft?"ENABLE":"DISABLE");
1592  cpl_msg_info (cpl_func, "Reference phase for FT is %s",phase_ref_ft);
1593 
1594 
1595  /*
1596  * Loop on polarisations
1597  */
1598  int npol_ft = gravi_pfits_get_pola_num (p2vmred_header, GRAVI_FT);
1599  for (int pol = 0; pol < npol_ft; pol++) {
1600  cpl_msg_info (cpl_func, "Start FT polarisation %d over %d",pol+1, npol_ft);
1601 
1602  /* Get the input table of FT */
1603  cpl_table * vis_FT = gravi_data_get_oi_vis (p2vmred_data, GRAVI_FT, pol, npol_ft);
1604  cpl_table * flux_FT = gravi_data_get_oi_flux (p2vmred_data, GRAVI_FT, pol, npol_ft);
1605  int nwave_ft = cpl_table_get_column_depth (vis_FT, "VISDATA");
1606  CPLCHECK_NUL ("Cannot get data");
1607 
1608  /* Create averated product */
1609  cpl_table * oi_vis2_FT = gravi_table_oi_create (nwave_ft, 1, GRAVI_OI_VIS2_EXT);
1610  gravi_table_new_column (oi_vis2_FT, "NDIT", NULL, CPL_TYPE_INT);
1611  gravi_table_new_column (oi_vis2_FT, "NVALID", NULL, CPL_TYPE_INT);
1612 
1613  cpl_table * oi_vis_FT = gravi_table_oi_create (nwave_ft, 1, GRAVI_OI_VIS_EXT);
1614  gravi_table_new_column (oi_vis_FT, "NDIT", NULL, CPL_TYPE_INT);
1615  gravi_table_new_column (oi_vis_FT, "NVALID", NULL, CPL_TYPE_INT);
1616 
1617  cpl_table * oi_T3_FT = gravi_table_oi_create (nwave_ft, 1, GRAVI_OI_T3_EXT);
1618  gravi_table_new_column (oi_T3_FT, "NDIT", NULL, CPL_TYPE_INT);
1619  gravi_table_new_column (oi_T3_FT, "NVALID", NULL, CPL_TYPE_INT);
1620 
1621  cpl_table * oi_flux_FT = gravi_table_oi_create (nwave_ft, 1, GRAVI_OI_FLUX_EXT);
1622  gravi_table_new_column (oi_flux_FT, "NDIT", NULL, CPL_TYPE_INT);
1623  gravi_table_new_column (oi_flux_FT, "NVALID", NULL, CPL_TYPE_INT);
1624  CPLCHECK_NUL ("Cannot create product");
1625 
1626  /* Keep only selected rows */
1627  vis_FT = gravi_table_extract_time_interval (vis_FT, start_time, end_time);
1628  flux_FT = gravi_table_extract_time_interval (flux_FT, start_time, end_time);
1629 
1630  /*
1631  * Compute OIVIS2 and OIVIS for FT
1632  */
1633  cpl_msg_info (cpl_func, "Compute OIVIS2 and OIVIS for FT");
1634 
1635  gravi_vis_average_bootstrap (oi_vis_FT, oi_vis2_FT, vis_FT,
1636  nboot_ft,
1637  phase_ref_ft,
1638  v_factor_flag_ft,
1639  p_factor_flag_ft,
1640  debiasing_flag_ft);
1641  CPLCHECK_NUL("Cannot average the FT frames");
1642 
1643  /*
1644  * Compute OIT3 for FT
1645  */
1646  cpl_msg_info (cpl_func, "Compute OIT3 for FT");
1647 
1648  gravi_t3_average_bootstrap (oi_T3_FT, vis_FT, flux_FT,
1649  nboot_ft,
1650  v_factor_flag_ft,
1651  p_factor_flag_ft);
1652  CPLCHECK_NUL("Cannot average t3 of FT");
1653 
1654  /*
1655  * Compute OI_FLUX for FT
1656  */
1657  cpl_msg_info (cpl_func, "Compute OI_FLUX for FT");
1658 
1659  gravi_flux_average_bootstrap (oi_flux_FT, flux_FT,
1660  nboot_ft);
1661  CPLCHECK_NUL("Cannot average flux of FT");
1662 
1663  /*
1664  * Add tables in the vis_data
1665  */
1666  cpl_propertylist * vis_plist = gravi_data_get_oi_vis_plist (p2vmred_data, GRAVI_FT, pol, npol_ft);
1667  cpl_propertylist * oivis_plist = cpl_propertylist_new();
1668  cpl_propertylist_copy_property (oivis_plist, vis_plist, "DATE-OBS");
1669  cpl_propertylist_copy_property (oivis_plist, vis_plist, "EXTVER");
1670  cpl_propertylist_copy_property (oivis_plist, vis_plist, "ARRNAME");
1671  cpl_propertylist_copy_property (oivis_plist, vis_plist, "INSNAME");
1672  cpl_propertylist_update_string (oivis_plist, "AMPTYP","absolute");
1673  cpl_propertylist_update_string (oivis_plist, "PHITYP","differential");
1674  cpl_propertylist_update_int (oivis_plist, "PHIORDER",1);
1675  gravi_data_add_table (vis_data, oivis_plist, GRAVI_OI_VIS_EXT, oi_vis_FT);
1676 
1677  cpl_propertylist * oivis2_plist = cpl_propertylist_new();
1678  cpl_propertylist_copy_property (oivis2_plist, vis_plist, "DATE-OBS");
1679  cpl_propertylist_copy_property (oivis2_plist, vis_plist, "EXTVER");
1680  cpl_propertylist_copy_property (oivis2_plist, vis_plist, "ARRNAME");
1681  cpl_propertylist_copy_property (oivis2_plist, vis_plist, "INSNAME");
1682  gravi_data_add_table (vis_data, oivis2_plist, GRAVI_OI_VIS2_EXT, oi_vis2_FT);
1683 
1684  cpl_propertylist * oit3_plist = cpl_propertylist_new();
1685  cpl_propertylist_copy_property (oit3_plist, vis_plist, "DATE-OBS");
1686  cpl_propertylist_copy_property (oit3_plist, vis_plist, "EXTVER");
1687  cpl_propertylist_copy_property (oit3_plist, vis_plist, "ARRNAME");
1688  cpl_propertylist_copy_property (oit3_plist, vis_plist, "INSNAME");
1689  gravi_data_add_table (vis_data, oit3_plist, GRAVI_OI_T3_EXT, oi_T3_FT);
1690 
1691  cpl_propertylist * oiflux_plist = cpl_propertylist_new();
1692  cpl_propertylist_copy_property (oiflux_plist, vis_plist, "DATE-OBS");
1693  cpl_propertylist_copy_property (oiflux_plist, vis_plist, "EXTVER");
1694  cpl_propertylist_copy_property (oiflux_plist, vis_plist, "ARRNAME");
1695  cpl_propertylist_copy_property (oiflux_plist, vis_plist, "INSNAME");
1696  cpl_propertylist_update_string (oiflux_plist, "CALSTAT", "U");
1697  cpl_propertylist_set_comment (oiflux_plist, "CALSTAT", "Uncalibrated flux per telescope");
1698  gravi_data_add_table (vis_data, oiflux_plist, GRAVI_OI_FLUX_EXT, oi_flux_FT);
1699 
1700  CPLCHECK_NUL ("Cannot add tables");
1701 
1702  FREE (cpl_table_delete, vis_FT);
1703  FREE (cpl_table_delete, flux_FT);
1704 
1705  } /* end loop on pol */
1706  } /* End FT */
1707 
1708  /*
1709  * Then with SC
1710  */
1711 
1712  if (gravi_data_has_type (p2vmred_data, "_SC") <= 0 ) {
1713  cpl_msg_info (cpl_func, "P2VMRED data has no SC extensions");
1714  }
1715  else {
1716 
1717  /* Read the object separation. Actually should
1718  * be the mode maybe more than the separation */
1719  double SOBJ_X = gravi_pfits_get_sobj_x (p2vmred_header);
1720  double SOBJ_Y = gravi_pfits_get_sobj_y (p2vmred_header);
1721  double SOBJ_R = sqrt(SOBJ_X*SOBJ_X + SOBJ_Y*SOBJ_Y);
1722 
1723  /* Reference phase requested by user, deal with AUTO */
1724  const char * phase_ref_sc = gravi_param_get_string (parlist, "gravity.vis.phase-ref-sc");
1725 
1726  if ( !strcmp (phase_ref_sc,"AUTO") && SOBJ_R > 0.001)
1727  {
1728  phase_ref_sc = "IMAGING_REF";
1729  if (cpl_table_has_column (vis_SC, phase_ref_sc) == 0) {
1730  phase_ref_sc = "PHASE_REF";
1731  cpl_msg_warning (cpl_func, "No table 'IMAGING_REF', changing mode to phase_ref_sc='PHASE_REF'");
1732  CPLCHECK_NUL("tototo....");
1733  }
1734  }
1735  else if ( !strcmp (phase_ref_sc,"AUTO") && SOBJ_R < 0.001)
1736  phase_ref_sc = "PHASE_REF";
1737 
1738  cpl_msg_info (cpl_func, "Reference phase for SC is %s",phase_ref_sc);
1739 
1740  /* Output phase requested by user, deal with AUTO */
1741  const char * output_phase_sc = gravi_param_get_string (parlist, "gravity.vis.output-phase-sc");
1742 
1743  if ( !strcmp (output_phase_sc,"AUTO") && SOBJ_R > 0.001)
1744  output_phase_sc = "ABSOLUTE";
1745  else if ( !strcmp (output_phase_sc,"AUTO") && SOBJ_R < 0.001)
1746  output_phase_sc = "DIFFERENTIAL";
1747 
1748  /*
1749  force SELF_REF if SELF_VISPHI unless NONE has been selected (for what purpose?)
1750  */
1751  if (!strcmp(output_phase_sc, "SELF_VISPHI")) {
1752  if (strcmp(phase_ref_sc, "NONE")) {
1753  phase_ref_sc = "SELF_REF"; /*we are robust to phase_ref_sc=NONE but SELF_REF is better */
1754  cpl_msg_info(cpl_func, "Reference phase for SC forced to %s due to option SELF_VISPHI", phase_ref_sc);
1755  } else cpl_msg_info(cpl_func, "Reference phase for SC is %s", phase_ref_sc);
1756  } else cpl_msg_info(cpl_func, "Reference phase for SC is %s", phase_ref_sc);
1757 
1758  cpl_msg_info (cpl_func, "Output phase for SC is %s",output_phase_sc);
1759 
1760  /* Other reduction parameters */
1761  int v_factor_flag_sc = strstr (gravi_param_get_string (parlist, "gravity.vis.vis-correction-sc"),"VFACTOR") ? 1 : 0;
1762  int p_factor_flag_sc = strstr (gravi_param_get_string (parlist, "gravity.vis.vis-correction-sc"),"PFACTOR") ? 1 : 0;
1763  int debiasing_flag_sc = gravi_param_get_bool (parlist, "gravity.vis.debias-sc");
1764  int nboot_sc = gravi_param_get_int (parlist, "gravity.vis.nboot");
1765  const char* rangeString = gravi_param_get_string (parlist, "gravity.vis.output-phase-channels");
1766 
1767  cpl_msg_info (cpl_func, "Bias subtraction of V2 for SC is %s",debiasing_flag_sc?"ENABLE":"DISABLE");
1768  /*
1769  force VFACTOR and PFACTOR to 0 for SELF_VISPHI by precaution.
1770  */
1771  if (!strcmp(output_phase_sc, "SELF_VISPHI")) {
1772  v_factor_flag_sc= 0;
1773  p_factor_flag_sc= 0;
1774  cpl_msg_info(cpl_func, "vFactor correction for SC is %s due to option SELF_VISPHI", v_factor_flag_sc ? "ENABLE" : "DISABLE");
1775  cpl_msg_info(cpl_func, "pFactor correction for SC is %s due to option SELF_VISPHI", p_factor_flag_sc ? "ENABLE" : "DISABLE");
1776  } else {
1777  cpl_msg_info (cpl_func, "vFactor correction for SC is %s",v_factor_flag_sc?"ENABLE":"DISABLE");
1778  cpl_msg_info (cpl_func, "pFactor correction for SC is %s",p_factor_flag_sc?"ENABLE":"DISABLE");
1779  }
1780 
1781  CPLCHECK_NUL("Cannot get parameters");
1782 
1783  /*
1784  * Loop on polarisations
1785  */
1786  int npol_sc = gravi_pfits_get_pola_num (p2vmred_header, GRAVI_SC);
1787  for (int pol = 0; pol < npol_sc; pol++) {
1788  cpl_msg_info (cpl_func, "Start SC polarisation %d over %d",pol+1, npol_sc);
1789 
1790  /* Get the input table of SC */
1791  cpl_table * vis_SC = gravi_data_get_oi_vis (p2vmred_data, GRAVI_SC, pol, npol_sc);
1792  cpl_table * flux_SC = gravi_data_get_oi_flux (p2vmred_data, GRAVI_SC, pol, npol_sc);
1793  cpl_table * oi_wavelengthsc = gravi_data_get_oi_wave (p2vmred_data, GRAVI_SC, pol, npol_sc);
1794  CPLCHECK_NUL ("Cannot get data");
1795 
1796  /* Compute the wavenumber for SC in [m^-1] */
1797  cpl_array * wavenumber_sc;
1798  int nwave_sc = cpl_table_get_column_depth (vis_SC, "VISDATA");
1799  wavenumber_sc = cpl_array_new (nwave_sc, CPL_TYPE_DOUBLE);
1800  for (cpl_size wave = 0; wave < nwave_sc; wave ++){
1801  cpl_array_set (wavenumber_sc, wave, 1./cpl_table_get (oi_wavelengthsc, "EFF_WAVE", wave, &nv));
1802  }
1803 
1804  CPLCHECK_NUL ("Cannot build the wave and wavenumber");
1805 
1806  /* Create averaged tables */
1807  cpl_table * oi_vis2_SC = gravi_table_oi_create (nwave_sc, 1, GRAVI_OI_VIS2_EXT);
1808  gravi_table_new_column (oi_vis2_SC, "NDIT", NULL, CPL_TYPE_INT);
1809  gravi_table_new_column (oi_vis2_SC, "NVALID", NULL, CPL_TYPE_INT);
1810 
1811  cpl_table * oi_vis_SC = gravi_table_oi_create (nwave_sc, 1, GRAVI_OI_VIS_EXT);
1812  gravi_table_new_column (oi_vis_SC, "NDIT", NULL, CPL_TYPE_INT);
1813  gravi_table_new_column (oi_vis_SC, "NVALID", NULL, CPL_TYPE_INT);
1814  gravi_table_new_column (oi_vis_SC, "GDELAY", "m", CPL_TYPE_DOUBLE);
1815  gravi_table_new_column (oi_vis_SC, "PHASE", "rad", CPL_TYPE_DOUBLE);
1816 
1817  cpl_table * oi_T3_SC = gravi_table_oi_create (nwave_sc, 1, GRAVI_OI_T3_EXT);
1818  gravi_table_new_column (oi_T3_SC, "NDIT", NULL, CPL_TYPE_INT);
1819  gravi_table_new_column (oi_T3_SC, "NVALID", NULL, CPL_TYPE_INT);
1820 
1821  cpl_table * oi_flux_SC = gravi_table_oi_create (nwave_sc, 1, GRAVI_OI_FLUX_EXT);
1822  gravi_table_new_column (oi_flux_SC, "NDIT", NULL, CPL_TYPE_INT);
1823  gravi_table_new_column (oi_flux_SC, "NVALID", NULL, CPL_TYPE_INT);
1824  gravi_table_new_column (oi_flux_SC, "LKDT_MET_FC", "mjd", CPL_TYPE_DOUBLE);
1825 
1826  CPLCHECK_NUL("Cannot create columns in averaged OIFITS...");
1827 
1828  /* Keep only selected rows */
1829  vis_SC = gravi_table_extract_time_interval (vis_SC, start_time, end_time);
1830  flux_SC = gravi_table_extract_time_interval (flux_SC, start_time, end_time);
1831 
1832  /*
1833  * Compute OIVIS2 and OIVIS for SC
1834  */
1835  cpl_msg_info (cpl_func, "Compute OIVIS2 and OIVIS for SC");
1836 
1837  gravi_vis_average_bootstrap (oi_vis_SC, oi_vis2_SC, vis_SC,
1838  nboot_sc,
1839  phase_ref_sc,
1840  v_factor_flag_sc,
1841  p_factor_flag_sc,
1842  debiasing_flag_sc);
1843  CPLCHECK_NUL("Cannot average the SC frames");
1844 
1845  /* Compute other columns */
1846  gravi_vis_compute_column_mean (oi_vis_SC, vis_SC, "OPD_MET_FC", 6);
1847  gravi_vis_compute_column_mean (oi_vis_SC, vis_SC, "PHASE_REF_COEFF", 6);
1848  gravi_vis_compute_column_mean (oi_vis_SC, vis_SC, "E_U", 6);
1849  gravi_vis_compute_column_mean (oi_vis_SC, vis_SC, "E_V", 6);
1850  gravi_vis_compute_column_mean (oi_vis_SC, vis_SC, "E_W", 6);
1851  gravi_vis_compute_column_mean (oi_vis_SC, vis_SC, "E_AZ", 6);
1852  gravi_vis_compute_column_mean (oi_vis_SC, vis_SC, "E_ZD", 6);
1853  CPLCHECK_NUL("Cannot compute means.");
1854 
1855  /* Re compute the astrometric phase from VISDATA to deal with absolute phase
1856  * VISDATA as well as (R,I) remains unchanged. The goal is to split
1857  * the differential phase, calibratable so far, from the absolute phase */
1858  if ( !strcmp (output_phase_sc,"DIFFERENTIAL") ) {
1859  for (int base = 0; base < nbase; base++) {
1860 
1861  /* We duplicate VISDATA, to keep the value untouched in the table */
1862  cpl_array * visData_sc, * visErr_sc;
1863  visData_sc = cpl_array_cast (cpl_table_get_array (oi_vis_SC, "VISDATA", base),
1864  CPL_TYPE_DOUBLE_COMPLEX);
1865 
1866  /* Normalize the phasor to avoid bad pixels */
1867  visErr_sc = cpl_array_duplicate (visData_sc);
1868  cpl_array_abs (visErr_sc);
1869  cpl_array_divide (visData_sc, visErr_sc);
1870 
1871  /* Compute and remove the mean group delay in [m] */
1872  double mean_delay = 0.0;
1873  gravi_array_get_group_delay_loop (&visData_sc, wavenumber_sc, &mean_delay, 1, 2e-3, CPL_FALSE);
1874  gravi_array_multiply_phasor (visData_sc, - 2*I*CPL_MATH_PI * mean_delay, wavenumber_sc);
1875 
1876  /* Save this delay [m] */
1877  cpl_table_set (oi_vis_SC, "GDELAY", base, mean_delay);
1878 
1879  /* Compute and remove the mean phase in [rad] */
1880  double mean_phase = carg (cpl_array_get_mean_complex (visData_sc));
1881  cpl_array_multiply_scalar_complex (visData_sc, cexp(- I * mean_phase));
1882 
1883  /* Save this phase [rad] */
1884  cpl_table_set (oi_vis_SC, "PHASE", base, mean_phase);
1885 
1886  /* Set back the phase in [deg] */
1887  cpl_array_arg (visData_sc);
1888  gravi_table_set_array_phase (oi_vis_SC, "VISPHI", base, visData_sc);
1889  cpl_array_delete (visData_sc);
1890  cpl_array_delete (visErr_sc);
1891 
1892  CPLCHECK_NUL("when computing the astrometric phase");
1893 
1894  } /* End loop on base */
1895  }
1896  if (!strcmp(output_phase_sc, "SELF_VISPHI")) {
1897  int* cmin=NULL;
1898  int* cmax=NULL;
1899  int nrange=0;
1900  /* rather complex C way to analyse strings defining clusters of wavelengths like " [ 12345 : 12346 , 678 : 680 , 822:864]" */
1901  if (strcmp(rangeString, "UNKNOWN")) {
1902  /*find number of ranges (they are separated by ',')*/
1903  char *str, *str1 ;
1904  int l=strlen(rangeString)+1;
1905  int j=0;
1906  int i=0;
1907  str=(char*)malloc(l);
1908  strncpy(str,rangeString,l); /*for future use*/
1909  for (i = 0; i<l; ++i) if (str[i]!=' ' && str[i]!='\t') str[j++]=str[i]; //remove ALL blanks.
1910  str[j]='\0';
1911  l=strlen(str)+1;
1912  str1=(char*)malloc(l);
1913  strncpy(str1,str,l); /*for future use, as strtok destroys its arguments*/
1914 
1915  char *token;
1916  token = strtok(str, "[,]");
1917  while (token) {
1918  nrange++;
1919  token = strtok(NULL, "[,]");
1920  }
1921  if (nrange > 1) {
1922  cmin=(int*) calloc(nrange,sizeof(int));
1923  cmax=(int*) calloc(nrange,sizeof(int));
1924 
1925  char *str2, *subtoken;
1926  char *saveptr1, *saveptr2;
1927  for (j = 0; ; j++, str1 = NULL) {
1928  token = strtok_r(str1, "[,]" , &saveptr1);
1929  if (token == NULL)
1930  break;
1931  for (str2 = token, i=0; i<2 ; str2 = NULL, ++i) {
1932  subtoken = strtok_r(str2, ":", &saveptr2);
1933  if (subtoken == NULL)
1934  break;
1935  /* int ret=sscanf(subtoken,"%d", (i==0)?&(cmin[j]):&(cmax[j])); */
1936  }
1937  }
1938  }
1939  }
1940 
1941  gravi_average_self_visphi(oi_vis_SC, vis_SC, wavenumber_sc, phase_ref_sc, cmin, cmax, nrange);
1942  }
1943  /*
1944  * Compute OIT3 for SC
1945  */
1946  cpl_msg_info (cpl_func, "Compute OIT3 for SC");
1947 
1948  gravi_t3_average_bootstrap (oi_T3_SC, vis_SC, flux_SC,
1949  nboot_sc,
1950  v_factor_flag_sc,
1951  p_factor_flag_sc);
1952  CPLCHECK_NUL("Cannot average t3 of SC");
1953 
1954  /*
1955  * Compute OI_FLUX for SC
1956  */
1957  cpl_msg_info (cpl_func, "Compute OI_FLUX for SC");
1958 
1959  gravi_flux_average_bootstrap (oi_flux_SC, flux_SC,
1960  nboot_sc);
1961  CPLCHECK_NUL("Cannot average flux of SC");
1962 
1963  /* Compute other columns */
1964  gravi_vis_compute_column_mean (oi_flux_SC, flux_SC, "OPD_MET_FC", 4);
1965  gravi_vis_compute_column_mean (oi_flux_SC, flux_SC, "FT_POS", 4);
1966  gravi_vis_compute_column_mean (oi_flux_SC, flux_SC, "SC_POS", 4);
1967  gravi_vis_compute_column_mean (oi_flux_SC, flux_SC, "OPL_AIR", 4);
1968  CPLCHECK_NUL ("Cannot compute mean columns");
1969 
1970  /* Save the FC metrology lock date */
1971  for (int tel = 0; tel < ntel; tel++){
1972  double lockdate = gravi_pfits_get_metfc_lockmjd (p2vmred_header, tel);
1973  cpl_table_set (oi_flux_SC, "LKDT_MET_FC", tel, lockdate);
1974  }
1975 
1976  /*
1977  * Add tables in the vis_data
1978  */
1979  cpl_propertylist * vis_plist = gravi_data_get_oi_vis_plist (p2vmred_data, GRAVI_SC, pol, npol_sc);
1980  cpl_propertylist * oivis_plist = cpl_propertylist_new();
1981  cpl_propertylist_copy_property (oivis_plist, vis_plist, "DATE-OBS");
1982  cpl_propertylist_copy_property (oivis_plist, vis_plist, "EXTVER");
1983  cpl_propertylist_copy_property (oivis_plist, vis_plist, "ARRNAME");
1984  cpl_propertylist_copy_property (oivis_plist, vis_plist, "INSNAME");
1985  cpl_propertylist_update_string (oivis_plist, "AMPTYP","absolute");
1986  if ( !strcmp (output_phase_sc,"DIFFERENTIAL") )
1987  cpl_propertylist_update_string (oivis_plist, "PHITYP","differential");
1988  if ( !strcmp (output_phase_sc,"ABSOLUTE") )
1989  cpl_propertylist_update_string (oivis_plist, "PHITYP","absolute");
1990  cpl_propertylist_update_int (oivis_plist, "PHIORDER",1);
1991  gravi_data_add_table (vis_data, oivis_plist, GRAVI_OI_VIS_EXT, oi_vis_SC);
1992 
1993  cpl_propertylist * oivis2_plist = cpl_propertylist_new();
1994  cpl_propertylist_copy_property (oivis2_plist, vis_plist, "DATE-OBS");
1995  cpl_propertylist_copy_property (oivis2_plist, vis_plist, "EXTVER");
1996  cpl_propertylist_copy_property (oivis2_plist, vis_plist, "ARRNAME");
1997  cpl_propertylist_copy_property (oivis2_plist, vis_plist, "INSNAME");
1998  gravi_data_add_table (vis_data, oivis2_plist, GRAVI_OI_VIS2_EXT, oi_vis2_SC);
1999 
2000  cpl_propertylist * oit3_plist = cpl_propertylist_new();
2001  cpl_propertylist_copy_property (oit3_plist, vis_plist, "DATE-OBS");
2002  cpl_propertylist_copy_property (oit3_plist, vis_plist, "EXTVER");
2003  cpl_propertylist_copy_property (oit3_plist, vis_plist, "ARRNAME");
2004  cpl_propertylist_copy_property (oit3_plist, vis_plist, "INSNAME");
2005  gravi_data_add_table (vis_data, oit3_plist, GRAVI_OI_T3_EXT, oi_T3_SC);
2006 
2007  cpl_propertylist * oiflux_plist = cpl_propertylist_new();
2008  cpl_propertylist_copy_property (oiflux_plist, vis_plist, "DATE-OBS");
2009  cpl_propertylist_copy_property (oiflux_plist, vis_plist, "EXTVER");
2010  cpl_propertylist_copy_property (oiflux_plist, vis_plist, "ARRNAME");
2011  cpl_propertylist_copy_property (oiflux_plist, vis_plist, "INSNAME");
2012  cpl_propertylist_update_string (oiflux_plist, "CALSTAT", "U");
2013  cpl_propertylist_set_comment (oiflux_plist, "CALSTAT", "Uncalibrated flux per telescope");
2014  gravi_data_add_table (vis_data, oiflux_plist, GRAVI_OI_FLUX_EXT, oi_flux_SC);
2015 
2016  CPLCHECK_NUL ("Cannot add tables");
2017 
2018  /* Delete waves */
2019  FREE (cpl_array_delete, wavenumber_sc);
2020  CPLCHECK_NUL ("Cannot delete wavenumber");
2021 
2022  FREE (cpl_table_delete, vis_SC);
2023  FREE (cpl_table_delete, flux_SC);
2024 
2025  } /* end loop on pol */
2026  } /* End SC */
2027 
2028 
2029  /* Add the night-obs in the main HEADER. This is the MJD of the begining
2030  of the night, similar for all files from noon to noon (~local time) */
2031  double mjd_obs = cpl_table_get_column_mean (gravi_data_get_table (vis_data, GRAVI_OI_VIS_EXT), "MJD");
2032  cpl_propertylist_update_int (vis_header, GRAVI_NIGHT_OBS, (int)floor(mjd_obs-0.625));
2033 
2034  gravi_msg_function_exit(1);
2035  return vis_data;
2036 }
2037 
2038 
2039 /*----------------------------------------------------------------------------*/
2046 /*----------------------------------------------------------------------------*/
2047 
2048 cpl_error_code gravi_compute_vis_qc (gravi_data * vis_data)
2049 {
2050  gravi_msg_function_start(1);
2051  cpl_ensure_code (vis_data, CPL_ERROR_NULL_INPUT);
2052 
2053  int nv, nbase = 6, ntel=4, nclo=4;
2054  char qc_name[100];
2055 
2056  /*
2057  * Prepare the output
2058  */
2059 
2060  cpl_propertylist * vis_header = gravi_data_get_header (vis_data);
2061  cpl_propertylist * plist = gravi_data_get_header (vis_data);
2062 
2063 
2064  /*
2065  * Start with FT
2066  */
2067  if (gravi_data_has_type (vis_data, "_FT") <= 0 ) {
2068  cpl_msg_info (cpl_func, "VIS data has no FT extensions");
2069  }
2070  else {
2071 
2072  /* Loop on polarisations */
2073  int npol_ft = gravi_pfits_get_pola_num (vis_header, GRAVI_FT);
2074  for (int pol = 0; pol < npol_ft; pol++) {
2075  cpl_msg_info (cpl_func, "Start FT polarisation %d over %d",pol+1, npol_ft);
2076 
2077  /* Loop on bases to compute OIVIS2 and OIVIS for FT
2078  */
2079  cpl_msg_info (cpl_func, "Compute QC OIVIS2 and OIVIS for FT");
2080 
2081  cpl_table * oi_vis2_FT = gravi_data_get_oi_vis2 (vis_data, GRAVI_FT, pol, npol_ft);
2082  cpl_table * oi_vis_FT = gravi_data_get_oi_vis (vis_data, GRAVI_FT, pol, npol_ft);
2083 
2084  for (int base = 0; base < nbase; base++) {
2085 
2086  /* Add the QC parameters for FT */
2087  sprintf (qc_name, "ESO QC ACCEPTED_RATIO_FT%s_P%d", GRAVI_BASE_NAME[base], pol+1);
2088  double ratio = gravi_table_get_column_mean (oi_vis_FT, "NVALID", base, nbase) /
2089  gravi_table_get_column_mean (oi_vis_FT, "NDIT", base, nbase);
2090  cpl_propertylist_update_double (plist, qc_name, round(100 * ratio * 1e2) / 1e2);
2091  cpl_propertylist_set_comment (plist, qc_name, "[%] of accepted frames");
2092 
2093  sprintf (qc_name, "ESO QC VISPHIERR_FT%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2094  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis_FT, "VISPHIERR", base, nbase));
2095  cpl_propertylist_set_comment (plist, qc_name, "[deg] mean over lbd");
2096 
2097  sprintf (qc_name, "ESO QC VIS2_FT%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2098  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis2_FT, "VIS2DATA", base, nbase));
2099  cpl_propertylist_set_comment (plist, qc_name, "mean over lbd");
2100 
2101  sprintf (qc_name, "ESO QC VIS2ERR_FT%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2102  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis2_FT, "VIS2ERR", base, nbase));
2103  cpl_propertylist_set_comment (plist, qc_name, "mean over lbd");
2104 
2105  sprintf (qc_name, "ESO QC VISAMP_FT%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2106  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis_FT, "VISAMP", base, nbase));
2107  cpl_propertylist_set_comment (plist, qc_name, "mean over lbd");
2108 
2109  sprintf (qc_name, "ESO QC VISAMPERR_FT%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2110  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis_FT, "VISAMPERR", base, nbase));
2111  cpl_propertylist_set_comment (plist, qc_name, "mean over lbd");
2112 
2113  CPLCHECK_MSG("Cannot compute QC parameter for OI_VIS for FT");
2114  } /* End loop on base */
2115 
2116  /*
2117  * Loop on triplet to compute OIT3 for FT
2118  */
2119  cpl_msg_info (cpl_func, "Compute QC OIT3 for FT");
2120 
2121  cpl_table * oi_T3_FT = gravi_data_get_oi_t3 (vis_data, GRAVI_FT, pol, npol_ft);
2122 
2123  for (int clo = 0; clo < nclo; clo++){
2124 
2125  sprintf (qc_name, "ESO QC T3PHI_FT%s_P%d AVG", GRAVI_CLO_NAME[clo], pol+1);
2126  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_T3_FT, "T3PHI", clo, nclo));
2127  cpl_propertylist_set_comment (plist, qc_name, "[deg] mean over lbd");
2128 
2129  sprintf (qc_name, "ESO QC T3PHIERR_FT%s_P%d AVG", GRAVI_CLO_NAME[clo], pol+1);
2130  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_T3_FT, "T3PHIERR", clo, nclo));
2131  cpl_propertylist_set_comment (plist, qc_name, "[deg] mean over lbd");
2132 
2133  CPLCHECK_MSG("Cannot compute QC parameter for OI_T3 for FT");
2134  } /* End loop on triplets */
2135 
2136  /*
2137  * Loop on beams to compute OI_FLUX for FT
2138  */
2139  cpl_msg_info (cpl_func, "Compute QC OI_FLUX for FT");
2140 
2141  cpl_table * oi_flux_FT = gravi_data_get_oi_flux (vis_data, GRAVI_FT, pol, npol_ft);
2142 
2143  for (int tel = 0; tel < ntel; tel++){
2144 
2145  sprintf (qc_name, "ESO QC FLUX_FT%d_P%d AVG", tel+1, pol+1);
2146  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_flux_FT, "FLUX", tel, ntel));
2147  cpl_propertylist_set_comment (plist, qc_name, "[e/total_int_time] mean over lbd");
2148 
2149  sprintf (qc_name, "ESO QC FLUXERR_FT%d_P%d AVG", tel+1, pol+1);
2150  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_flux_FT, "FLUXERR", tel, ntel));
2151  cpl_propertylist_set_comment (plist, qc_name, "[e/total_int_time] mean over lbd");
2152 
2153  sprintf (qc_name, "ESO QC FLUXRATE_FT%d_P%d SUM", tel+1, pol+1);
2154  double flux_rate = cpl_array_get_mean (cpl_table_get_array (oi_flux_FT, "FLUX", tel)) *
2155  cpl_array_get_size (cpl_table_get_array (oi_flux_FT, "FLUX", tel)) / cpl_table_get_double (oi_flux_FT, "INT_TIME", tel, &nv);
2156  cpl_propertylist_update_double (plist, qc_name, flux_rate);
2157  cpl_propertylist_set_comment (plist, qc_name, "[e/s] sum over lbd");
2158 
2159  CPLCHECK_MSG("Cannot compute QC parameter for OI_FLUX for FT");
2160  } /* End loop on beams */
2161 
2162  } /* end loop on pol */
2163  } /* End FT */
2164 
2165 
2166 
2167  /*
2168  * Then with SC
2169  */
2170  if (gravi_data_has_type (vis_data, "_SC") <= 0 ) {
2171  cpl_msg_info (cpl_func, "VIS data has no SC extensions");
2172  }
2173  else {
2174 
2175  /* Loop on polarisations */
2176  int npol_sc = gravi_pfits_get_pola_num (vis_header, GRAVI_SC);
2177  for (int pol = 0; pol < npol_sc; pol++) {
2178 
2179  /*
2180  * Loop on bases to compute OIVIS2 and OIVIS for SC
2181  */
2182  cpl_msg_info (cpl_func, "Compute QC OIVIS2 and OIVIS for SC");
2183 
2184  cpl_table * oi_vis2_SC = gravi_data_get_oi_vis2 (vis_data, GRAVI_SC, pol, npol_sc);
2185  cpl_table * oi_vis_SC = gravi_data_get_oi_vis (vis_data, GRAVI_SC, pol, npol_sc);
2186 
2187  for (int base = 0; base < nbase; base++) {
2188  /* FIXME: repair these QC parameters, for instance by computing them in P2VMRED */
2189 
2190  // sprintf (qc_name, "ESO QC VFACTOR%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2191  // double vmean = gravi_table_get_column_mean (vis_SC, "V_FACTOR_WL", base, nbase);
2192  // cpl_propertylist_update_double (plist, qc_name, vmean);
2193  // cpl_propertylist_set_comment (plist, qc_name, "mean v-factor");
2194  //
2195  // sprintf (qc_name, "ESO QC PFACTOR%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2196  // double pmean = gravi_table_get_column_mean (vis_SC, "P_FACTOR", base, nbase);
2197  // cpl_propertylist_update_double (plist, qc_name, pmean);
2198  // cpl_propertylist_set_comment (plist, qc_name, "mean p-factor");
2199 
2200  sprintf (qc_name, "ESO QC ACCEPTED_RATIO_SC%s_P%d", GRAVI_BASE_NAME[base], pol+1);
2201  double ratio = gravi_table_get_column_mean (oi_vis_SC, "NVALID", base, nbase) /
2202  gravi_table_get_column_mean (oi_vis_SC, "NDIT", base, nbase);
2203  cpl_propertylist_update_double (plist, qc_name, round(100 * ratio * 1e2) / 1e2);
2204  cpl_propertylist_set_comment (plist, qc_name, "[%] of accepted frames");
2205 
2206  sprintf (qc_name, "ESO QC GD_SC%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2207  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis_SC, "GDELAY", base, nbase));
2208  cpl_propertylist_set_comment (plist, qc_name, "[m] mean Group-Delay");
2209 
2210  sprintf (qc_name, "ESO QC VIS2_SC%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2211  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis2_SC, "VIS2DATA", base, nbase));
2212  cpl_propertylist_set_comment (plist, qc_name, "mean over lbd");
2213 
2214  sprintf (qc_name, "ESO QC VIS2ERR_SC%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2215  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis2_SC, "VIS2ERR", base, nbase));
2216  cpl_propertylist_set_comment (plist, qc_name, "mean over lbd");
2217 
2218  sprintf (qc_name, "ESO QC VISPHI_SC%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2219  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis_SC, "VISPHI", base, nbase));
2220  cpl_propertylist_set_comment (plist, qc_name, "[deg] mean over lbd");
2221 
2222  sprintf (qc_name, "ESO QC VISPHIERR_SC%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2223  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis_SC, "VISPHIERR", base, nbase));
2224  cpl_propertylist_set_comment (plist, qc_name, "[deg] mean over lbd");
2225 
2226  sprintf (qc_name, "ESO QC VISAMP_SC%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2227  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis_SC, "VISAMP", base, nbase));
2228  cpl_propertylist_set_comment (plist, qc_name, "mean over lbd");
2229 
2230  sprintf (qc_name, "ESO QC VISAMPERR_SC%s_P%d AVG", GRAVI_BASE_NAME[base], pol+1);
2231  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_vis_SC, "VISAMPERR", base, nbase));
2232  cpl_propertylist_set_comment (plist, qc_name, "mean over lbd");
2233 
2234  double coeff2 = gravi_table_get_value (oi_vis_SC, "PHASE_REF_COEFF", base, 2);
2235  sprintf (qc_name, "ESO QC PHASE_REF_COEFF2 SC%s_P%d", GRAVI_BASE_NAME[base], pol+1);
2236  cpl_propertylist_update_double (plist, qc_name, coeff2);
2237  cpl_propertylist_set_comment (plist, qc_name, "[rad] 2sd order of FT phase");
2238 
2239  CPLCHECK_MSG("Cannot set QC parameter for OI_VIS for SC");
2240  } /* End loop on base */
2241 
2242  /*
2243  * Loop on triplet to compute OIT3 for SC
2244  */
2245  cpl_msg_info (cpl_func, "Compute QC OIT3 for SC");
2246 
2247  cpl_table * oi_T3_SC = gravi_data_get_oi_t3 (vis_data, GRAVI_SC, pol, npol_sc);
2248 
2249  for (int clo = 0; clo < nclo; clo++){
2250 
2251  sprintf (qc_name, "ESO QC T3PHI_SC%s_P%d AVG", GRAVI_CLO_NAME[clo], pol+1);
2252  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_T3_SC, "T3PHI", clo, nclo));
2253  cpl_propertylist_set_comment (plist, qc_name, "[deg] mean over lbd");
2254 
2255  sprintf (qc_name, "ESO QC T3PHIERR_SC%s_P%d AVG", GRAVI_CLO_NAME[clo], pol+1);
2256  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_T3_SC, "T3PHIERR", clo, nclo));
2257  cpl_propertylist_set_comment (plist, qc_name, "[deg] mean over lbd");
2258 
2259  CPLCHECK_MSG("Cannot set QC parameter for OI_T3 for SC");
2260  }/* End loop on triplets */
2261 
2262  /*
2263  * Loop on beams to compute OI_FLUX for SC
2264  */
2265  cpl_msg_info (cpl_func, "Compute QC OI_FLUX for SC");
2266 
2267  cpl_table * oi_flux_SC = gravi_data_get_oi_flux (vis_data, GRAVI_SC, pol, npol_sc);
2268 
2269  for (int tel = 0; tel < ntel; tel++){
2270 
2271  sprintf (qc_name, "ESO QC FLUX_SC%d_P%d AVG", tel+1, pol+1);
2272  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_flux_SC, "FLUX", tel, ntel));
2273  cpl_propertylist_set_comment (plist, qc_name, "[e/total_int_time] mean over lbd");
2274 
2275  sprintf (qc_name, "ESO QC FLUXERR_SC%d_P%d AVG", tel+1, pol+1);
2276  cpl_propertylist_update_double (plist, qc_name, gravi_table_get_column_mean (oi_flux_SC, "FLUXERR", tel, ntel));
2277  cpl_propertylist_set_comment (plist, qc_name, "[e/total_int_time] mean over lbd");
2278 
2279  sprintf (qc_name, "ESO QC FLUXRATE_SC%d_P%d SUM", tel+1, pol+1);
2280  double flux_rate = cpl_array_get_mean (cpl_table_get_array (oi_flux_SC, "FLUX", tel)) *
2281  cpl_array_get_size(cpl_table_get_array (oi_flux_SC, "FLUX", tel)) / cpl_table_get_double (oi_flux_SC, "INT_TIME", tel, &nv);
2282  cpl_propertylist_update_double (plist, qc_name, flux_rate);
2283  cpl_propertylist_set_comment (plist, qc_name, "[e/s] sum over lbd");
2284 
2285  double ftpos_mean = cpl_table_get (oi_flux_SC, "FT_POS", tel, NULL);
2286  sprintf (qc_name, "ESO QC FT_POS SC%d_P%d", tel+1, pol+1);
2287  cpl_propertylist_update_double (plist, qc_name, ftpos_mean);
2288  cpl_propertylist_set_comment (plist, qc_name, "[V]");
2289 
2290  double oplair_mean = cpl_table_get (oi_flux_SC, "OPL_AIR", tel, NULL);
2291  sprintf (qc_name, "ESO QC OPL_AIR SC%d_P%d", tel+1, pol+1);
2292  cpl_propertylist_update_double (plist, qc_name, oplair_mean);
2293  cpl_propertylist_set_comment (plist, qc_name, "[m]");
2294 
2295  CPLCHECK_MSG("Cannot set QC parameter for OI_FLUX for SC");
2296  } /* End loop on beams */
2297 
2298 
2299  } /* end loop on pol */
2300  } /* End SC */
2301 
2302  gravi_msg_function_exit(1);
2303  return CPL_ERROR_NONE;
2304 }
2305 
2306 /*----------------------------------------------------------------------------*/
2315 /*----------------------------------------------------------------------------*/
2316 
2317 cpl_error_code gravi_normalize_sc_to_ft (gravi_data * vis_data)
2318 {
2319  gravi_msg_function_start(1);
2320  cpl_ensure_code (vis_data, CPL_ERROR_NULL_INPUT);
2321 
2322  cpl_propertylist * hdr_data = gravi_data_get_header (vis_data);
2323  int pol, npol = gravi_pfits_get_pola_num (hdr_data, GRAVI_SC);
2324  double qFactor;
2325 
2326  if ( npol != gravi_pfits_get_pola_num (hdr_data, GRAVI_FT)) {
2327  return cpl_error_set_message(cpl_func,CPL_ERROR_ILLEGAL_INPUT,"polarisation of SC and FT shall be compatible");
2328  }
2329 
2330  /* Loop on polarisation. Assume the same polarisation
2331  * splitting for both SC and FT */
2332  for ( pol= 0 ; pol < npol ; pol++ ) {
2333 
2334  cpl_table * oi_wave_sc = gravi_data_get_oi_wave (vis_data, GRAVI_SC, pol, npol);
2335  cpl_table * oi_wave_ft = gravi_data_get_oi_wave (vis_data, GRAVI_FT, pol, npol);
2336 
2337  /*
2338  * Get data for VIS2 and VISAMP
2339  */
2340  cpl_table * oi_vis2_sc = gravi_data_get_oi_vis2 (vis_data, GRAVI_SC, pol, npol);
2341  cpl_table * oi_vis2_ft = gravi_data_get_oi_vis2 (vis_data, GRAVI_FT, pol, npol);
2342  cpl_table * oi_vis_sc = gravi_data_get_oi_vis (vis_data, GRAVI_SC, pol, npol);
2343  cpl_table * oi_vis_ft = gravi_data_get_oi_vis (vis_data, GRAVI_FT, pol, npol);
2344 
2345  CPLCHECK_MSG("Cannot get data");
2346 
2347  /* Loop on baselines */
2348  for (cpl_size base = 0; base < cpl_table_get_nrow (oi_vis2_ft); base ++) {
2349 
2350  /* Get the VIS2 of FT */
2351  const cpl_array * vis2_ft = cpl_table_get_array (oi_vis2_ft, "VIS2DATA", base);
2352 
2353  /* Create the rebin VIS2 of SC */
2354  cpl_array * vis2_lr = gravi_array_rebin (cpl_table_get_array (oi_vis2_sc, "VIS2DATA", base),
2355  cpl_table_get_array (oi_vis2_sc, "VIS2ERR", base),
2356  oi_wave_sc, oi_wave_ft);
2357 
2358  /* Compute the mean visibility loss over the band.
2359  * FIXME: shall use vis2_ft = (a.lbd+b) * vis2_lr, which could be done with cpl_matrix */
2360  qFactor = cpl_array_get_mean (vis2_lr) / cpl_array_get_mean (vis2_ft);
2361  cpl_msg_info (cpl_func, "vis2 %lli: qFactor = %f", base, qFactor);
2362 
2363  /* Divide the SC data */
2364  cpl_array_divide_scalar (cpl_table_get_data_array (oi_vis2_sc, "VIS2DATA")[base], qFactor);
2365  cpl_array_divide_scalar (cpl_table_get_data_array (oi_vis2_sc, "VIS2ERR")[base], qFactor);
2366 
2367  /* Get the VISAMP of FT */
2368  const cpl_array * vis_ft = cpl_table_get_array (oi_vis_ft, "VISAMP", base);
2369 
2370  /* Create the rebin VISAMP of SC */
2371  cpl_array * vis_lr = gravi_array_rebin (cpl_table_get_array (oi_vis_sc, "VISAMP", base),
2372  cpl_table_get_array (oi_vis_sc, "VISAMPERR", base),
2373  oi_wave_sc, oi_wave_ft);
2374 
2375  /* Compute the mean visibility loss over the band. */
2376  qFactor = cpl_array_get_mean (vis_lr) / cpl_array_get_mean (vis_ft);
2377  cpl_msg_info (cpl_func, "visAmp %lli: qFactor = %f", base, qFactor);
2378 
2379  /* Divide the SC data */
2380  cpl_array_divide_scalar (cpl_table_get_data_array (oi_vis_sc, "VISAMP")[base], qFactor);
2381  cpl_array_divide_scalar (cpl_table_get_data_array (oi_vis_sc, "VISAMPERR")[base], qFactor);
2382 
2383  FREE (cpl_array_delete, vis2_lr);
2384  FREE (cpl_array_delete, vis_lr);
2385  } /* End loop on baselines */
2386 
2387  } /* End loop on pol */
2388 
2389  gravi_msg_function_exit(1);
2390  return CPL_ERROR_NONE;
2391 }
2392 
2393 /*----------------------------------------------------------------------------*/
2401 /*----------------------------------------------------------------------------*/
2402 
2403 cpl_error_code gravi_vis_mjd_to_time (gravi_data * vis_data)
2404 {
2405  gravi_msg_function_start(1);
2406  cpl_ensure_code (vis_data, CPL_ERROR_NULL_INPUT);
2407  char date[90];
2408 
2409  /* Loop on extension */
2410  int next = gravi_data_get_size (vis_data);
2411  for (int ext = 0; ext < next; ext ++) {
2412 
2413  const char * extname = gravi_data_get_extname (vis_data,ext);
2414  if (!strcmp (extname, "OI_VIS") ||
2415  !strcmp (extname, "OI_VIS2") ||
2416  !strcmp (extname, "OI_T3") ||
2417  !strcmp (extname, "OI_FLUX")) {
2418 
2419  /* Get the DATE-OBS in format YYYY-MM-DDT00:00:00.000 */
2420  cpl_propertylist * plist = gravi_data_get_plist_x (vis_data, ext);
2421  sprintf (date, "%.10sT00:00:00.000", cpl_propertylist_get_string (plist, "DATE-OBS"));
2422 
2423  /* Get the MJD of this DATE-OBS */
2424  double mjd0 = gravi_convert_to_mjd (date);
2425  cpl_msg_debug (cpl_func, "DATE-OBS = %s -> mjd = %.3f", date, mjd0);
2426 
2427  /* Compute TIME in [s] following the OIFITS standard */
2428  cpl_table * oi_table = gravi_data_get_table_x (vis_data, ext);
2429  cpl_size nrow = cpl_table_get_nrow (oi_table);
2430  for (cpl_size row = 0; row < nrow; row++) {
2431  double mjd = cpl_table_get (oi_table, "MJD", row, NULL);
2432  cpl_table_set (oi_table, "TIME", row, (mjd-mjd0) * 24 * 3600);
2433  }
2434 
2435  /* Set units */
2436  cpl_table_set_column_unit (oi_table, "TIME", "s");
2437  }
2438  } /* End loop on extensions */
2439 
2440  gravi_msg_function_exit(1);
2441  return CPL_ERROR_NONE;
2442 }
2443 
2444 
2445 /*----------------------------------------------------------------------------*/
2455 /*----------------------------------------------------------------------------*/
2456 
2457 cpl_error_code gravi_flat_flux (gravi_data * vis_data, gravi_data * p2vm_map)
2458 {
2459  gravi_msg_function_start(1);
2460  cpl_ensure_code (vis_data, CPL_ERROR_NULL_INPUT);
2461  cpl_ensure_code (p2vm_map, CPL_ERROR_NULL_INPUT);
2462 
2463  int num_used_tf = 1;
2464  gravi_data **used_tf_data = &p2vm_map;
2465 
2466  /* Get the header */
2467  cpl_propertylist * hdr_data = gravi_data_get_header (vis_data);
2468 
2469  /* For each type of data SC / FT */
2470  int ntype_data = 2;
2471  for (int type_data = 0; type_data < ntype_data ; type_data ++) {
2472 
2473  /* Loop on polarisation */
2474  int npol = gravi_pfits_get_pola_num (hdr_data, type_data);
2475  for (int pol= 0 ; pol < npol ; pol++ ) {
2476 
2477  /* Calibrate the FLUX as a real quantity */
2478  double delta_t = 10000.0;
2479  gravi_apply_tf_amp (vis_data, NULL, used_tf_data, num_used_tf,
2480  GRAVI_OI_FLUX_EXT,
2481  GRAVI_INSNAME(type_data, pol, npol),
2482  "FLUX", "FLUXERR", 4, delta_t);
2483 
2484  CPLCHECK_MSG("Cannot apply normalize flux");
2485 
2486  }
2487  /* End loop on polarisation */
2488  }
2489  /* End loop on data_type */
2490 
2491  gravi_msg_function_exit(1);
2492  return CPL_ERROR_NONE;
2493 }
2494 
2495 /*----------------------------------------------------------------------------*/
2505 /*----------------------------------------------------------------------------*/
2506 
2507 cpl_error_code gravi_vis_average_amp (cpl_table *oi_table, const char *name, const char *err, int nbase)
2508 {
2509  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
2510  cpl_ensure_code (name, CPL_ERROR_NULL_INPUT);
2511  cpl_ensure_code (err, CPL_ERROR_NULL_INPUT);
2512 
2513  cpl_size nwave = cpl_table_get_column_depth (oi_table, name);
2514  cpl_array * weight = gravi_array_init_double (nwave, 0.0);
2515  cpl_array * value = gravi_array_init_double (nwave, 0.0);
2516 
2517  /* Loop on base */
2518  int nrow = cpl_table_get_nrow (oi_table) / nbase;
2519  for (cpl_size base = 0; base < nbase ; base++) {
2520  cpl_array_fill_window (weight, 0, nwave, 0.0);
2521  cpl_array_fill_window (value, 0, nwave, 0.0);
2522 
2523  /* Loop on row and wave */
2524  for (cpl_size row = 0; row < nrow ; row++) {
2525  const cpl_array * rval = cpl_table_get_array (oi_table, name, base + row*nbase);
2526  const cpl_array * rerr = cpl_table_get_array (oi_table, err, base + row*nbase);
2527  const cpl_array * flag = cpl_table_get_array (oi_table, "FLAG", base + row*nbase);
2528  for (cpl_size wave = 0; wave < nwave; wave++) {
2529  double w = pow (cpl_array_get (rerr, wave, NULL), -2);
2530  if (cpl_array_get (flag, wave, NULL)) w = 10e-20;
2531  double v = cpl_array_get (rval, wave, NULL);
2532  cpl_array_set (weight, wave, cpl_array_get (weight, wave, NULL) + w);
2533  cpl_array_set (value, wave, cpl_array_get (value, wave, NULL) + v * w);
2534  }
2535  }
2536  CPLCHECK_MSG("Cannot average amp");
2537 
2538  /* Set the mean */
2539  cpl_array_divide (value, weight);
2540  cpl_table_set_array (oi_table, name, base, value);
2541  /* Set the variance of the mean */
2542  cpl_array_power (weight, -0.5);
2543  cpl_table_set_array (oi_table, err, base, weight);
2544 
2545 
2546 
2547  } /* End loop on base */
2548 
2549  FREE (cpl_array_delete, weight);
2550  FREE (cpl_array_delete, value);
2551  return CPL_ERROR_NONE;
2552 }
2553 
2554 /*----------------------------------------------------------------------------*/
2565 /*----------------------------------------------------------------------------*/
2566 
2567 cpl_error_code gravi_vis_average_phi (cpl_table *oi_table, const char *name, const char *err, int nbase)
2568 {
2569  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
2570  cpl_ensure_code (name, CPL_ERROR_NULL_INPUT);
2571  cpl_ensure_code (err, CPL_ERROR_NULL_INPUT);
2572 
2573  cpl_size nwave = cpl_table_get_column_depth (oi_table, name);
2574  cpl_array * weight = gravi_array_init_double (nwave, 0.0);
2575  cpl_array * value = gravi_array_init_double_complex (nwave, 0.0 + I*0.0);
2576 
2577  /* Loop on base */
2578  int nrow = cpl_table_get_nrow (oi_table) / nbase;
2579  for (cpl_size base = 0; base < nbase ; base++) {
2580  cpl_array_fill_window (weight, 0, nwave, 0.0);
2581  cpl_array_fill_window_complex (value, 0, nwave, 0.0);
2582 
2583  /* Loop on row and wave */
2584  for (cpl_size row = 0; row < nrow ; row++) {
2585  const cpl_array * rval = cpl_table_get_array (oi_table, name, base + row*nbase);
2586  const cpl_array * rerr = cpl_table_get_array (oi_table, err, base + row*nbase);
2587  const cpl_array * flag = cpl_table_get_array (oi_table, "FLAG", base + row*nbase);
2588  for (cpl_size wave = 0; wave < nwave; wave++) {
2589  double w = pow (cpl_array_get (rerr, wave, NULL), -2);
2590  if (cpl_array_get (flag, wave, NULL)) w = 10e-20;
2591  double complex v = cexp (1.*I * cpl_array_get (rval, wave, NULL) * CPL_MATH_RAD_DEG);
2592  cpl_array_set (weight, wave, cpl_array_get (weight, wave, NULL) + w);
2593  cpl_array_set_complex (value, wave, cpl_array_get_complex (value, wave, NULL) + v * w);
2594  }
2595  }
2596  CPLCHECK_MSG("Cannot average phi");
2597 
2598  /* Set the mean */
2599  gravi_table_set_array_phase (oi_table, name, base, value);
2600  /* Set the variance of the mean */
2601  cpl_array_power (weight, -0.5);
2602  cpl_table_set_array (oi_table, err, base, weight);
2603 
2604  } /* End loop on base */
2605 
2606  FREE (cpl_array_delete, weight);
2607  FREE (cpl_array_delete, value);
2608  return CPL_ERROR_NONE;
2609 }
2610 
2611 /*----------------------------------------------------------------------------*/
2621 /*----------------------------------------------------------------------------*/
2622 
2623 cpl_error_code gravi_vis_average_value (cpl_table *oi_table, const char *name, const char *err, int nbase)
2624 {
2625  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
2626  cpl_ensure_code (name, CPL_ERROR_NULL_INPUT);
2627  int nv = 0;
2628 
2629  /* Loop on base */
2630  int nrow = cpl_table_get_nrow (oi_table) / nbase;
2631  for (cpl_size base = 0; base < nbase ; base++) {
2632  double weight = 0.0;
2633  double value = 0.0;
2634 
2635  /* Loop on row */
2636  for (cpl_size row = 0; row < nrow ; row++) {
2637  double w = (err!=NULL ? pow (cpl_table_get (oi_table, err, base + row*nbase, &nv), -2.0) : 1.0);
2638  value += cpl_table_get (oi_table, name, base + row*nbase, &nv) * w;
2639  weight += w;
2640  }
2641  CPLCHECK_MSG("Cannot average value");
2642 
2643  /* Set the mean */
2644  cpl_table_set (oi_table, name, base, value / weight);
2645  } /* End loop on base */
2646 
2647  return CPL_ERROR_NONE;
2648 }
2649 
2650 /*----------------------------------------------------------------------------*/
2659 /*----------------------------------------------------------------------------*/
2660 cpl_error_code gravi_force_uncertainties (gravi_data * oi_data,
2661  const cpl_parameterlist * parlist)
2662 {
2663  gravi_msg_function_start(1);
2664  cpl_ensure_code (oi_data, CPL_ERROR_NULL_INPUT);
2665  cpl_ensure_code (parlist, CPL_ERROR_NULL_INPUT);
2666 
2667  double err;
2668 
2669  /* Get header */
2670  cpl_propertylist * header = gravi_data_get_header (oi_data);
2671 
2672  /* VISAMPERR SC */
2673  err = gravi_param_get_double_default (parlist, "gravity.postprocess.visamperr-sc", -1.0);
2674  if ( err > 0) {
2675  cpl_msg_info (cpl_func,"Force VISAMPERR to %.e for all observations", err);
2676 
2677  int npol = gravi_pfits_get_pola_num (header, GRAVI_SC);
2678  for (int pol = 0; pol < npol; pol++) {
2679  cpl_table * oi_table = gravi_data_get_oi_vis (oi_data, GRAVI_SC, pol, npol);
2680  cpl_array ** array = cpl_table_get_data_array (oi_table, "VISAMPERR");
2681  for (cpl_size row = 0; row<cpl_table_get_nrow (oi_table); row++) {
2682  cpl_array_fill_window (array[row], 0, CPL_SIZE_MAX, err);
2683  }
2684  }
2685  }
2686 
2687  /* VISPHIERR SC */
2688  err = gravi_param_get_double_default (parlist, "gravity.postprocess.visphierr-sc", -1.0);
2689  if ( err > 0) {
2690  cpl_msg_info (cpl_func,"Force VISPHIERR to %.e for all observations", err);
2691 
2692  int npol = gravi_pfits_get_pola_num (header, GRAVI_SC);
2693  for (int pol = 0; pol < npol; pol++) {
2694  cpl_table * oi_table = gravi_data_get_oi_vis (oi_data, GRAVI_SC, pol, npol);
2695  cpl_array ** array = cpl_table_get_data_array (oi_table, "VISPHIERR");
2696  for (cpl_size row = 0; row<cpl_table_get_nrow (oi_table); row++) {
2697  cpl_array_fill_window (array[row], 0, CPL_SIZE_MAX, err);
2698  }
2699  }
2700  }
2701 
2702  /* FLUXERR SC */
2703  err = gravi_param_get_double_default (parlist, "gravity.postprocess.fluxerr-sc", -1.0);
2704  if ( err > 0) {
2705  cpl_msg_info (cpl_func,"Force FLUXERR to %.e for all observations", err);
2706 
2707  int npol = gravi_pfits_get_pola_num (header, GRAVI_SC);
2708  for (int pol = 0; pol < npol; pol++) {
2709  cpl_table * oi_table = gravi_data_get_oi_flux (oi_data, GRAVI_SC, pol, npol);
2710  cpl_array ** array = cpl_table_get_data_array (oi_table, "FLUXERR");
2711  for (cpl_size row = 0; row<cpl_table_get_nrow (oi_table); row++) {
2712  cpl_array_fill_window (array[row], 0, CPL_SIZE_MAX, err);
2713  }
2714  }
2715  }
2716 
2717  /* VIS2ERR SC */
2718  err = gravi_param_get_double_default (parlist, "gravity.postprocess.vis2err-sc", -1.0);
2719  if ( err > 0) {
2720  cpl_msg_info (cpl_func,"Force VIS2ERR to %.e for all observations", err);
2721 
2722  int npol = gravi_pfits_get_pola_num (header, GRAVI_SC);
2723  for (int pol = 0; pol < npol; pol++) {
2724  cpl_table * oi_table = gravi_data_get_oi_vis2 (oi_data, GRAVI_SC, pol, npol);
2725  cpl_array ** array = cpl_table_get_data_array (oi_table, "VIS2ERR");
2726  for (cpl_size row = 0; row<cpl_table_get_nrow (oi_table); row++) {
2727  cpl_array_fill_window (array[row], 0, CPL_SIZE_MAX, err);
2728  }
2729  }
2730  }
2731 
2732  gravi_msg_function_exit(1);
2733  return CPL_ERROR_NONE;
2734 }
2735 
2736 
2737 /*----------------------------------------------------------------------------*/
2748 /*----------------------------------------------------------------------------*/
2749 
2750 cpl_error_code gravi_average_vis (gravi_data * oi_data)
2751 {
2752  gravi_msg_function_start(1);
2753  cpl_ensure_code (oi_data, CPL_ERROR_NULL_INPUT);
2754 
2755  gravi_msg_warning ("FIXME", "Average the different observation is EXPERIMENTAL");
2756 
2757  cpl_msg_warning (cpl_func, "FIXME: Weightening of UVCOORD and MJD is not done properly yet !");
2758  cpl_msg_warning (cpl_func, "FIXME: Integration of INT_TIME is not done properly yet !");
2759 
2760  int nbase = 6;
2761  cpl_table * oi_table;
2762 
2763  /* Create output data */
2764  cpl_propertylist * header = gravi_data_get_header (oi_data);
2765 
2766  /* Loop on oidata, type_data and polarisation */
2767  for (int type_data = 0; type_data < 2 ; type_data ++) {
2768 
2769  if (!gravi_data_has_type (oi_data, type_data == GRAVI_SC ? "_SC" : "_FT")) {
2770  cpl_msg_info (cpl_func, "OI_VIS has no %s, skip", GRAVI_TYPE(type_data));
2771  continue;
2772  }
2773 
2774  int npol = gravi_pfits_get_pola_num (header, type_data);
2775  for (int pol = 0 ; pol < npol ; pol++ ) {
2776 
2777  cpl_size nrow = cpl_table_get_nrow (gravi_data_get_oi_vis (oi_data, type_data, pol, npol))/nbase;
2778  if (nrow == 1) {
2779  cpl_msg_info (cpl_func, "OI_VIS %s has only one observation, skip", GRAVI_TYPE(type_data));
2780  continue;
2781  }
2782 
2783  /* OI_VIS */
2784  oi_table = gravi_data_get_oi_vis (oi_data, type_data, pol, npol);
2785  gravi_vis_average_value (oi_table, "TIME", NULL, 6);
2786  gravi_vis_average_value (oi_table, "MJD", NULL, 6);
2787  gravi_vis_average_value (oi_table, "INT_TIME", NULL, 6);
2788  gravi_vis_average_value (oi_table, "UCOORD", NULL, 6);
2789  gravi_vis_average_value (oi_table, "VCOORD", NULL, 6);
2790  gravi_vis_average_amp (oi_table, "VISAMP", "VISAMPERR", 6);
2791  gravi_vis_average_phi (oi_table, "VISPHI", "VISPHIERR", 6);
2792  gravi_vis_average_amp (oi_table, "RVIS", "RVISERR", 6);
2793  gravi_vis_average_amp (oi_table, "IVIS", "IVISERR", 6);
2794  // gravi_vis_average_amp (oi_table, "VISDATA", "VISERR"); // FIXME: to be done !!
2795  gravi_msg_warning ("FIXME", "VISDATA are not averaged !!!!");
2796  cpl_table_erase_window (oi_table, 6, CPL_SIZE_MAX);
2797 
2798  CPLCHECK_MSG ("Cannot co-add OI_VIS");
2799 
2800  /* OI_VIS2 */
2801  oi_table = gravi_data_get_oi_vis2 (oi_data, type_data, pol, npol);
2802  gravi_vis_average_value (oi_table, "TIME", NULL, 6);
2803  gravi_vis_average_value (oi_table, "MJD", NULL, 6);
2804  gravi_vis_average_value (oi_table, "INT_TIME", NULL, 6);
2805  gravi_vis_average_value (oi_table, "UCOORD", NULL, 6);
2806  gravi_vis_average_value (oi_table, "VCOORD", NULL, 6);
2807  gravi_vis_average_amp (oi_table, "VIS2DATA", "VIS2ERR", 6);
2808  cpl_table_erase_window (oi_table, 6, CPL_SIZE_MAX);
2809 
2810  CPLCHECK_MSG ("Cannot co-add OI_VIS2");
2811 
2812  /* OI_FLUX */
2813  oi_table = gravi_data_get_oi_flux (oi_data, type_data, pol, npol);
2814  gravi_vis_average_value (oi_table, "TIME", NULL, 4);
2815  gravi_vis_average_value (oi_table, "MJD", NULL, 4);
2816  gravi_vis_average_value (oi_table, "INT_TIME", NULL, 4);
2817  gravi_vis_average_amp (oi_table, "FLUX", "FLUXERR", 4);
2818  cpl_table_erase_window (oi_table, 4, CPL_SIZE_MAX);
2819 
2820  CPLCHECK_MSG ("Cannot co-add OI_FLUX");
2821 
2822  /* OI_T3 */
2823  oi_table = gravi_data_get_oi_t3 (oi_data, type_data, pol, npol);
2824  gravi_vis_average_value (oi_table, "TIME", NULL, 4);
2825  gravi_vis_average_value (oi_table, "MJD", NULL, 4);
2826  gravi_vis_average_value (oi_table, "INT_TIME", NULL, 4);
2827  gravi_vis_average_value (oi_table, "U1COORD", NULL, 4);
2828  gravi_vis_average_value (oi_table, "V1COORD", NULL, 4);
2829  gravi_vis_average_value (oi_table, "U2COORD", NULL, 4);
2830  gravi_vis_average_value (oi_table, "V2COORD", NULL, 4);
2831  gravi_vis_average_amp (oi_table, "T3AMP", "T3AMPERR", 4);
2832  gravi_vis_average_phi (oi_table, "T3PHI", "T3PHIERR", 4);
2833  cpl_table_erase_window (oi_table, 4, CPL_SIZE_MAX);
2834 
2835  CPLCHECK_MSG ("Cannot co-add OI_T3");
2836 
2837  } /* End loop on polarisation */
2838  } /* End loop on type_data */
2839 
2840  gravi_msg_function_exit(1);
2841  return CPL_ERROR_NONE;
2842 }
2843 
2844 /*----------------------------------------------------------------------------*/
2853 /*----------------------------------------------------------------------------*/
2854 
2855 cpl_error_code gravi_vis_smooth_amp (cpl_table * oi_table, const char * name, const char * err,
2856  cpl_size nsamp)
2857 {
2858  gravi_msg_function_start(1);
2859  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
2860  if (nsamp < 1) return CPL_ERROR_NONE;
2861  int nv;
2862 
2863  /* Get values */
2864  cpl_size nwave = cpl_table_get_column_depth (oi_table, name);
2865  cpl_size nrow = cpl_table_get_nrow (oi_table);
2866  cpl_ensure_code (nrow > 0, CPL_ERROR_ILLEGAL_INPUT);
2867 
2868  /* Get arrays */
2869  cpl_array ** v_array = cpl_table_get_data_array (oi_table, name);
2870  cpl_array ** e_array = cpl_table_get_data_array (oi_table, err);
2871  cpl_array ** f_array = cpl_table_get_data_array (oi_table, "FLAG");
2872  CPLCHECK_MSG ("Cannot get data");
2873 
2874  /* Allocate output */
2875  cpl_array * smo_array = cpl_array_duplicate (v_array[0]);
2876  cpl_array * err_array = cpl_array_duplicate (e_array[0]);
2877 
2878  /* Loop on rows */
2879  for (cpl_size row = 0 ; row < nrow ; row ++) {
2880 
2881  /* Median filter the uncertainties, to avoid
2882  * putting all on some sample */
2883  cpl_vector * i_vector = cpl_vector_new (nwave);
2884  for (cpl_size wave = 0; wave < nwave; wave++)
2885  cpl_vector_set (i_vector, wave, cpl_array_get (e_array[row],wave,&nv));
2886  cpl_vector * o_vector;
2887  o_vector = cpl_vector_filter_median_create (i_vector, nsamp);
2888 
2889  /* Loop on waves */
2890  for (cpl_size wave = 0 ; wave < nwave ; wave ++) {
2891  double sum = 0.0, weight = 0.0;
2892 
2893  /* Loop on samples to average */
2894  for (cpl_size samp = CPL_MAX(0,wave-nsamp) ; samp < CPL_MIN(nwave,wave+nsamp) ; samp ++) {
2895  if (cpl_array_get (f_array[row],samp,&nv)) {
2896  weight += 10e-20;
2897  sum += 0.0;
2898  } else {
2899  double w = pow (cpl_vector_get (o_vector,samp), -2);
2900  sum += cpl_array_get (v_array[row],samp,&nv) * w;
2901  weight += w;
2902  }
2903  }
2904 
2905  cpl_array_set_double (smo_array, wave, sum / weight);
2906  cpl_array_set_double (err_array, wave, pow (weight, -0.5));
2907  }
2908 
2909  /* Set back */
2910  cpl_table_set_array (oi_table, name, row, smo_array);
2911  cpl_table_set_array (oi_table, err, row, err_array);
2912  CPLCHECK_MSG ("Cannot smooth amp");
2913 
2914  FREE (cpl_vector_delete, i_vector);
2915  FREE (cpl_vector_delete, o_vector);
2916  } /* End loop on rows */
2917 
2918  FREE (cpl_array_delete, smo_array);
2919  FREE (cpl_array_delete, err_array);
2920 
2921  gravi_msg_function_exit(1);
2922  return CPL_ERROR_NONE;
2923 }
2924 
2925 /*----------------------------------------------------------------------------*/
2934 /*----------------------------------------------------------------------------*/
2935 
2936 cpl_error_code gravi_vis_smooth_phi (cpl_table * oi_table, const char * name, const char * err,
2937  cpl_size nsamp)
2938 {
2939  gravi_msg_function_start(1);
2940  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
2941  if (nsamp < 1) return CPL_ERROR_NONE;
2942 
2943  /* Get values */
2944  cpl_size nwave = cpl_table_get_column_depth (oi_table, name);
2945  cpl_size nrow = cpl_table_get_nrow (oi_table);
2946  cpl_ensure_code (nrow > 0, CPL_ERROR_ILLEGAL_INPUT);
2947  int nv;
2948 
2949  /* Get arrays */
2950  cpl_array ** v_array = cpl_table_get_data_array (oi_table, name);
2951  cpl_array ** e_array = cpl_table_get_data_array (oi_table, err);
2952  cpl_array ** f_array = cpl_table_get_data_array (oi_table, "FLAG");
2953  CPLCHECK_MSG ("Cannot get data");
2954 
2955  /* Allocate output */
2956  cpl_array * smo_array = cpl_array_duplicate (v_array[0]);
2957  cpl_array * err_array = cpl_array_duplicate (e_array[0]);
2958 
2959  /* Loop on rows */
2960  for (cpl_size row = 0 ; row < nrow ; row ++) {
2961 
2962  /* Median filter the uncertainties, to avoid
2963  * putting all on some sample */
2964  cpl_vector * i_vector = cpl_vector_new (nwave);
2965  for (cpl_size wave = 0; wave < nwave; wave++)
2966  cpl_vector_set (i_vector, wave, cpl_array_get (e_array[row],wave,&nv));
2967  cpl_vector * o_vector;
2968  o_vector = cpl_vector_filter_median_create (i_vector, nsamp);
2969 
2970  /* Loop on waves */
2971  for (cpl_size wave = 0 ; wave < nwave ; wave ++) {
2972  double complex sum = 0.0 + I*0.0;
2973  double weight = 0.0;
2974 
2975  /* Loop on samples to average */
2976  for (cpl_size samp = CPL_MAX(0,wave-nsamp) ; samp < CPL_MIN(nwave,wave+nsamp) ; samp ++) {
2977  if (cpl_array_get (f_array[row],samp,&nv)) {
2978  weight += 10e-20;
2979  sum += 0.0;
2980  } else {
2981  double w = pow (cpl_vector_get (o_vector,samp), -2);
2982  sum += cexp (1.*I* cpl_array_get (v_array[row],samp,&nv) * CPL_MATH_RAD_DEG) * w;
2983  weight += w;
2984  }
2985  }
2986 
2987  cpl_array_set_double (smo_array, wave, carg (sum) * CPL_MATH_DEG_RAD);
2988  cpl_array_set_double (err_array, wave, pow (weight, -0.5));
2989  }
2990 
2991  /* Set back */
2992  cpl_table_set_array (oi_table, name, row, smo_array);
2993  cpl_table_set_array (oi_table, err, row, err_array);
2994  CPLCHECK_MSG ("Cannot smooth phi");
2995 
2996  FREE (cpl_vector_delete, i_vector);
2997  FREE (cpl_vector_delete, o_vector);
2998  } /* End loop on rows */
2999 
3000  FREE (cpl_array_delete, smo_array);
3001  FREE (cpl_array_delete, err_array);
3002 
3003  gravi_msg_function_exit(1);
3004  return CPL_ERROR_NONE;
3005 }
3006 
3007 /*----------------------------------------------------------------------------*/
3016 /*----------------------------------------------------------------------------*/
3017 
3018 cpl_error_code gravi_vis_fit_amp (cpl_table * oi_table, const char * name,
3019  const char * err, cpl_size maxdeg)
3020 {
3021  gravi_msg_function_start(1);
3022  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
3023  if (maxdeg < 0) return CPL_ERROR_NONE;
3024 
3025  /* Get values */
3026  cpl_size nwave = cpl_table_get_column_depth (oi_table, name);
3027  cpl_size nrow = cpl_table_get_nrow (oi_table);
3028  cpl_ensure_code (nrow > 0, CPL_ERROR_ILLEGAL_INPUT);
3029  int nv;
3030 
3031  /* Get arrays */
3032  cpl_array ** v_array = cpl_table_get_data_array (oi_table, name);
3033  cpl_array ** e_array = cpl_table_get_data_array (oi_table, err);
3034  cpl_array ** f_array = cpl_table_get_data_array (oi_table, "FLAG");
3035  CPLCHECK_MSG ("Cannot get data");
3036 
3037  /* Loop on rows */
3038  for (cpl_size row = 0 ; row < nrow ; row ++) {
3039 
3040  /* Create the vectors and matrix */
3041  cpl_matrix * coeff = cpl_matrix_new (nwave,maxdeg+1);
3042  cpl_matrix * rhs = cpl_matrix_new (nwave,1);
3043 
3044  /* Fill */
3045  for (cpl_size wave = 0 ; wave < nwave ; wave ++) {
3046  double weight = cpl_array_get (f_array[row],wave,&nv) ? 10e-20 :
3047  pow (cpl_array_get (e_array[row],wave,&nv), -2);
3048  double value = cpl_array_get (f_array[row],wave,&nv) ? 0.0 :
3049  cpl_array_get (v_array[row],wave,&nv);
3050  cpl_matrix_set (rhs, wave, 0, value * weight);
3051  for (cpl_size deg = 0; deg <= maxdeg; deg++)
3052  cpl_matrix_set (coeff, wave, deg, pow ((double)wave,(double)deg) * weight);
3053  CPLCHECK_MSG ("Cannot fill");
3054  }
3055 
3056  /* Solve */
3057  cpl_errorstate prev_state = cpl_errorstate_get();
3058  cpl_matrix * solve = cpl_matrix_solve_normal (coeff, rhs);
3059 
3060  /* Dump errors */
3061  if ( !cpl_errorstate_is_equal (prev_state))
3062  {
3063  cpl_errorstate_dump (prev_state, 0, NULL);
3064  cpl_msg_error (cpl_func,"%s row=%lld",name,row);
3065  }
3066  CPLCHECK_MSG ("Cannot solve matrix");
3067 
3068  /* Evaluate */
3069  for (cpl_size wave = 0 ; wave < nwave ; wave ++) {
3070  double value = 0;
3071  for (cpl_size deg = 0; deg <= maxdeg; deg++)
3072  value += cpl_matrix_get (solve, deg, 0) * pow (wave, deg);
3073  cpl_array_set (v_array[row],wave,value);
3074  CPLCHECK_MSG ("Cannot evaluate");
3075  }
3076 
3077  FREE (cpl_matrix_delete, coeff);
3078  FREE (cpl_matrix_delete, rhs);
3079  FREE (cpl_matrix_delete, solve);
3080  }
3081 
3082  gravi_msg_function_exit(1);
3083  return CPL_ERROR_NONE;
3084 }
3085 
3086 /*----------------------------------------------------------------------------*/
3100 /*----------------------------------------------------------------------------*/
3101 
3102 cpl_error_code gravi_vis_smooth (gravi_data * oi_data,
3103  cpl_size nsamp_vis,
3104  cpl_size nsamp_flx,
3105  cpl_size maxdeg)
3106 {
3107  gravi_msg_function_start(1);
3108  cpl_ensure_code (oi_data, CPL_ERROR_NULL_INPUT);
3109 
3110  cpl_table * oi_table;
3111 
3112  /* Create output data */
3113  cpl_propertylist * header = gravi_data_get_header (oi_data);
3114 
3115  int type_data = GRAVI_SC;
3116  int npol = gravi_pfits_get_pola_num (header, type_data);
3117 
3118  for (int pol = 0 ; pol < npol ; pol++ ) {
3119 
3120  /* OI_FLUX */
3121  oi_table = gravi_data_get_oi_flux (oi_data, type_data, pol, npol);
3122  gravi_vis_flag_nan (oi_table);
3123  gravi_vis_smooth_amp (oi_table, "FLUX", "FLUXERR", nsamp_flx);
3124  gravi_vis_flag_relative_threshold (oi_table, "FLUXERR", "FLUX", "FLAG", 1.0);
3125  CPLCHECK_MSG ("Cannot resamp OI_FLUX");
3126 
3127  /* OI_VIS2 */
3128  oi_table = gravi_data_get_oi_vis2 (oi_data, type_data, pol, npol);
3129  gravi_vis_flag_nan (oi_table);
3130  gravi_vis_flag_median (oi_table, "VIS2ERR", "FLAG", 5.0);
3131  gravi_vis_smooth_amp (oi_table, "VIS2DATA", "VIS2ERR", nsamp_vis);
3132  gravi_vis_fit_amp (oi_table, "VIS2DATA", "VIS2ERR", maxdeg);
3133  gravi_vis_flag_threshold (oi_table, "VIS2ERR", "FLAG", 1.);
3134  CPLCHECK_MSG ("Cannot resamp OI_VIS2");
3135 
3136  /* OI_VIS */
3137  oi_table = gravi_data_get_oi_vis (oi_data, type_data, pol, npol);
3138  gravi_vis_flag_nan (oi_table);
3139  gravi_vis_flag_median (oi_table, "VISPHIERR", "FLAG", 5.0);
3140  gravi_vis_smooth_amp (oi_table, "VISAMP", "VISAMPERR", nsamp_vis);
3141  gravi_vis_smooth_phi (oi_table, "VISPHI", "VISPHIERR", nsamp_vis);
3142  gravi_vis_fit_amp (oi_table, "VISAMP", "VISAMPERR", maxdeg);
3143  gravi_vis_fit_amp (oi_table, "VISPHI", "VISPHIERR", maxdeg);
3144  gravi_vis_smooth_amp (oi_table, "RVIS", "RVISERR", nsamp_flx);
3145  gravi_vis_smooth_amp (oi_table, "IVIS", "IVISERR", nsamp_flx);
3146  gravi_vis_flag_threshold (oi_table, "VISAMPERR", "FLAG", 1.);
3147 
3148  gravi_msg_warning ("FIXME", "VISDATA is not properly smooth !!");
3149  CPLCHECK_MSG ("Cannot resamp OI_VIS");
3150 
3151  /* OI_T3 */
3152  oi_table = gravi_data_get_oi_t3 (oi_data, type_data, pol, npol);
3153  gravi_vis_flag_nan (oi_table);
3154  gravi_vis_flag_median (oi_table, "T3PHIERR", "FLAG", 5.0);
3155  gravi_vis_smooth_amp (oi_table, "T3AMP", "T3AMPERR", nsamp_vis);
3156  gravi_vis_smooth_phi (oi_table, "T3PHI", "T3PHIERR", nsamp_vis);
3157  gravi_vis_fit_amp (oi_table, "T3AMP", "T3AMPERR", maxdeg);
3158  gravi_vis_fit_amp (oi_table, "T3PHI", "T3PHIERR", maxdeg);
3159  gravi_vis_flag_threshold (oi_table, "T3AMPERR", "FLAG", 1.0);
3160  CPLCHECK_MSG ("Cannot resamp OI_T3");
3161 
3162  } /* End loop on polarisation */
3163 
3164  gravi_msg_function_exit(1);
3165  return CPL_ERROR_NONE;
3166 }
3167 
3168 /*----------------------------------------------------------------------------*/
3178 /*----------------------------------------------------------------------------*/
3179 
3180 cpl_error_code gravi_vis_resamp_amp (cpl_table * oi_table, const char * name, const char * err,
3181  cpl_size nsamp, cpl_size nwave_new)
3182 {
3183  gravi_msg_function_start(1);
3184  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
3185 
3186  /* Loop on rows */
3187  cpl_size nrow = cpl_table_get_nrow (oi_table);
3188  cpl_ensure_code (nrow > 0, CPL_ERROR_ILLEGAL_INPUT);
3189  for (cpl_size row = 0 ; row < nrow ; row ++) {
3190 
3191  /* Loop on new waves */
3192  for (cpl_size wave = 0 ; wave < nwave_new ; wave ++) {
3193  double sum = 0.0;
3194  double weight = 0.0;
3195  for (cpl_size samp = 0 ; samp < nsamp ; samp ++) {
3196  double w = pow (gravi_table_get_value (oi_table,err,row,wave*nsamp+samp), -2.0);
3197  if (gravi_table_get_value (oi_table,"FLAG",row,wave*nsamp+samp)) w = 10e-20;
3198  sum += gravi_table_get_value (oi_table,name,row,wave*nsamp+samp) * w;
3199  weight += w;
3200  }
3201  gravi_table_set_value (oi_table,name,row,wave, sum / weight);
3202  gravi_table_set_value (oi_table,err,row,wave, pow (weight, -0.5));
3203  }
3204 
3205  } /* End loop on rows */
3206 
3207  cpl_table_set_column_depth (oi_table, name, nwave_new);
3208  cpl_table_set_column_depth (oi_table, err, nwave_new);
3209 
3210  gravi_msg_function_exit(1);
3211  return CPL_ERROR_NONE;
3212 }
3213 
3214 /*----------------------------------------------------------------------------*/
3224 /*----------------------------------------------------------------------------*/
3225 
3226 cpl_error_code gravi_vis_resamp_phi (cpl_table * oi_table, const char * name, const char * err,
3227  cpl_size nsamp, cpl_size nwave_new)
3228 {
3229  gravi_msg_function_start(1);
3230  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
3231 
3232  /* Loop on rows */
3233  cpl_size nrow = cpl_table_get_nrow (oi_table);
3234  cpl_ensure_code (nrow > 0, CPL_ERROR_ILLEGAL_INPUT);
3235  for (cpl_size row = 0 ; row < nrow ; row ++) {
3236 
3237  /* Loop on new waves */
3238  for (cpl_size wave = 0 ; wave < nwave_new ; wave ++) {
3239  double complex sum = 0.0;
3240  double weight = 0.0;
3241  for (cpl_size samp = 0 ; samp < nsamp ; samp ++) {
3242  double w = pow (gravi_table_get_value (oi_table,err,row,wave*nsamp+samp), -2.0);
3243  if (gravi_table_get_value (oi_table,"FLAG",row,wave*nsamp+samp)) w = 10e-20;
3244  sum += cexp (1.*I* gravi_table_get_value (oi_table,name,row,wave*nsamp+samp) * CPL_MATH_RAD_DEG) * w;
3245  weight += w;
3246  }
3247  gravi_table_set_value (oi_table,name,row,wave, carg (sum) * CPL_MATH_DEG_RAD);
3248  gravi_table_set_value (oi_table,err,row,wave, pow (weight, -0.5));
3249  }
3250 
3251  } /* End loop on rows */
3252 
3253  cpl_table_set_column_depth (oi_table, name, nwave_new);
3254  cpl_table_set_column_depth (oi_table, err, nwave_new);
3255 
3256  gravi_msg_function_exit(1);
3257  return CPL_ERROR_NONE;
3258 }
3259 
3260 /*----------------------------------------------------------------------------*/
3271 /*----------------------------------------------------------------------------*/
3272 
3273 cpl_error_code gravi_vis_resamp (gravi_data * oi_data, cpl_size nsamp)
3274 {
3275  gravi_msg_function_start(1);
3276  cpl_ensure_code (oi_data, CPL_ERROR_NULL_INPUT);
3277  cpl_ensure_code (nsamp>1, CPL_ERROR_ILLEGAL_INPUT);
3278 
3279  cpl_table * oi_table;
3280  cpl_size nwave, nwave_new;
3281  int nv = 0;
3282 
3283  /* Create output data */
3284  cpl_propertylist * header = gravi_data_get_header (oi_data);
3285 
3286  int type_data = GRAVI_SC;
3287  int npol = gravi_pfits_get_pola_num (header, type_data);
3288  for (int pol = 0 ; pol < npol ; pol++ ) {
3289 
3290  /* OI_WAVELENGTH table */
3291  oi_table = gravi_data_get_oi_wave (oi_data, type_data, pol, npol);
3292  nwave = cpl_table_get_nrow (oi_table);
3293 
3294  /* New number of wave sample */
3295  nwave_new = nwave / nsamp;
3296  cpl_msg_info (cpl_func, "Resamp the SC data by %lld bins: %lld -> %lld",
3297  nsamp, nwave, nwave_new);
3298  cpl_ensure_code (nwave_new > 1, CPL_ERROR_ILLEGAL_INPUT);
3299 
3300  /* Get the wavelength */
3301  for (cpl_size wave = 0 ; wave < nwave_new ; wave ++) {
3302  /* Compute effective band first */
3303  cpl_table_set (oi_table, "EFF_BAND", wave,
3304  cpl_table_get (oi_table, "EFF_WAVE", wave*nsamp+nsamp-1, &nv) -
3305  cpl_table_get (oi_table, "EFF_WAVE", wave*nsamp, &nv));
3306 
3307  /* Average them */
3308  double mean = 0.0;
3309  for (cpl_size samp = 0 ; samp < nsamp ; samp++)
3310  mean += cpl_table_get (oi_table, "EFF_WAVE", wave*nsamp+samp, &nv);
3311  cpl_table_set (oi_table, "EFF_WAVE", wave, mean / nsamp);
3312  }
3313 
3314  /* Erase last entries */
3315  cpl_table_erase_window (oi_table, nwave_new, CPL_SIZE_MAX);
3316  CPLCHECK_MSG ("Cannot resamp OI_WAVELENGTH");
3317 
3318  /* OI_FLUX */
3319  oi_table = gravi_data_get_oi_flux (oi_data, type_data, pol, npol);
3320  gravi_vis_resamp_amp (oi_table, "FLUX", "FLUXERR", nsamp, nwave_new);
3321  cpl_table_set_column_depth (oi_table, "FLAG", nwave_new);
3322  gravi_vis_flag_relative_threshold (oi_table, "FLUXERR", "FLUX", "FLAG", 1.0);
3323  CPLCHECK_MSG ("Cannot resamp OI_FLUX");
3324 
3325  /* OI_VIS2 */
3326  oi_table = gravi_data_get_oi_vis2 (oi_data, type_data, pol, npol);
3327  gravi_vis_resamp_amp (oi_table, "VIS2DATA", "VIS2ERR", nsamp, nwave_new);
3328  cpl_table_set_column_depth (oi_table, "FLAG", nwave_new);
3329  gravi_vis_flag_threshold (oi_table, "VIS2ERR", "FLAG", 1.);
3330  CPLCHECK_MSG ("Cannot resamp OI_VIS2");
3331 
3332  /* OI_VIS */
3333  oi_table = gravi_data_get_oi_vis (oi_data, type_data, pol, npol);
3334  gravi_vis_resamp_amp (oi_table, "VISAMP", "VISAMPERR", nsamp, nwave_new);
3335  gravi_vis_resamp_amp (oi_table, "VISPHI", "VISPHIERR", nsamp, nwave_new);
3336  gravi_vis_resamp_amp (oi_table, "RVIS", "RVISERR", nsamp, nwave_new);
3337  gravi_vis_resamp_amp (oi_table, "IVIS", "IVISERR", nsamp, nwave_new);
3338  cpl_table_set_column_depth (oi_table, "FLAG", nwave_new);
3339  gravi_vis_flag_threshold (oi_table, "VISAMPERR", "FLAG", 1.);
3340 
3341  gravi_msg_warning ("FIXME", "VISDATA is not properly resampled !!");
3342  cpl_table_set_column_depth (oi_table, "VISDATA", nwave_new);
3343  cpl_table_set_column_depth (oi_table, "VISERR", nwave_new);
3344  CPLCHECK_MSG ("Cannot resamp OI_VIS");
3345 
3346  /* OI_T3 */
3347  oi_table = gravi_data_get_oi_t3 (oi_data, type_data, pol, npol);
3348  gravi_vis_resamp_amp (oi_table, "T3AMP", "T3AMPERR", nsamp, nwave_new);
3349  gravi_vis_resamp_amp (oi_table, "T3PHI", "T3PHIERR", nsamp, nwave_new);
3350  cpl_table_set_column_depth (oi_table, "FLAG", nwave_new);
3351  gravi_vis_flag_threshold (oi_table, "T3AMPERR", "FLAG", 1.0);
3352  CPLCHECK_MSG ("Cannot resamp OI_T3");
3353 
3354  } /* End loop on polarisation */
3355 
3356  gravi_msg_function_exit(1);
3357  return CPL_ERROR_NONE;
3358 }
3359 
3360 /*----------------------------------------------------------------------------*/
3366 /*----------------------------------------------------------------------------*/
3367 
3368 cpl_error_code gravi_vis_flag_nan (cpl_table * oi_table)
3369 {
3370  gravi_msg_function_start(1);
3371  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
3372 
3373  cpl_size nrow = cpl_table_get_nrow (oi_table);
3374  cpl_ensure_code (nrow > 0, CPL_ERROR_ILLEGAL_INPUT);
3375 
3376  int ncols = 10;
3377  const char * names[] = {"VIS2DATA","VIS2ERR","VISAMP","VISAMPERR",
3378  "VISPHI","VISPHIERR","T3PHI","T3PHIERR",
3379  "T3AMP","T3AMPERR"};
3380 
3381  /* Loop on columns */
3382  for (int c = 0; c < ncols; c++) {
3383 
3384  if (!cpl_table_has_column (oi_table,names[c])) continue;
3385  cpl_msg_info (cpl_func,"Check column %s",names[c]);
3386 
3387  /* Get data of this columns */
3388  cpl_size nwave = cpl_table_get_column_depth (oi_table, names[c]);
3389  cpl_array ** v_array = cpl_table_get_data_array (oi_table, names[c]);
3390  cpl_array ** f_array = cpl_table_get_data_array (oi_table, "FLAG");
3391  CPLCHECK_MSG ("Cannot get data");
3392 
3393  /* Loop on rows and waves */
3394  cpl_size ninvalid = 0;
3395  for (cpl_size row = 0; row < nrow ; row ++) {
3396  for (cpl_size wave = 0 ; wave < nwave ; wave ++) {
3397 
3398  /* Get value */
3399  int nv = 0;
3400  double value = cpl_array_get (v_array[row], wave, &nv);
3401 
3402  /* Check value */
3403  if (nv || isnan (value)) {
3404  cpl_array_set (f_array[row], wave, 1.0);
3405  cpl_array_set (v_array[row], wave, 0.0);
3406  ninvalid ++;
3407  }
3408  CPLCHECK_MSG ("Cannot check data");
3409  }
3410  } /* End loop on rows and waves */
3411 
3412  /* Verbose */
3413  if (ninvalid > 0) {
3414  cpl_msg_warning (cpl_func, "Flag %lld invalid data (NAN or NULL) in %s", ninvalid, names[c]);
3415  }
3416 
3417  } /* End loop on columns */
3418 
3419  gravi_msg_function_exit(1);
3420  return CPL_ERROR_NONE;
3421 }
3422 
3423 /*----------------------------------------------------------------------------*/
3435 /*----------------------------------------------------------------------------*/
3436 
3437 cpl_error_code gravi_vis_flag_threshold (cpl_table * oi_table, const char * data, const char *flag, double value)
3438 {
3439  gravi_msg_function_start(0);
3440  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
3441  cpl_ensure_code (data, CPL_ERROR_NULL_INPUT);
3442  cpl_ensure_code (flag, CPL_ERROR_ILLEGAL_OUTPUT);
3443 
3444  /* Get pointer to speed up */
3445  int nv = 0;
3446  cpl_size nrow = cpl_table_get_nrow (oi_table);
3447  cpl_array ** pdata = cpl_table_get_data_array (oi_table, data);
3448  cpl_array ** pflag = cpl_table_get_data_array (oi_table, flag);
3449 
3450  CPLCHECK_MSG ("Cannot get data");
3451 
3452  cpl_size size = cpl_array_get_size (pdata[0]);
3453 
3454  /* Loop on row and index. Add to FLAG if data is above threshold */
3455  for ( cpl_size row = 0 ; row < nrow ; row ++ ) {
3456  if (pdata[row]==NULL) continue;
3457 
3458  for ( cpl_size indx = 0 ; indx < size ; indx ++ ) {
3459  if ( cpl_array_get (pdata[row], indx, &nv) > value ) {
3460  cpl_array_set (pflag[row], indx, cpl_array_get (pflag[row], indx, &nv) + 1 );
3461  }
3462  }
3463  }
3464 
3465  gravi_msg_function_exit(0);
3466  return CPL_ERROR_NONE;
3467 }
3468 
3469 /*----------------------------------------------------------------------------*/
3481 /*----------------------------------------------------------------------------*/
3482 
3483 cpl_error_code gravi_vis_flag_median (cpl_table * oi_table, const char * data, const char *flag, double value)
3484 {
3485  gravi_msg_function_start(0);
3486  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
3487  cpl_ensure_code (data, CPL_ERROR_NULL_INPUT);
3488  cpl_ensure_code (flag, CPL_ERROR_ILLEGAL_OUTPUT);
3489 
3490  /* Get pointer to speed up */
3491  int nv = 0;
3492  cpl_size nrow = cpl_table_get_nrow (oi_table);
3493  cpl_array ** pdata = cpl_table_get_data_array (oi_table, data);
3494  cpl_array ** pflag = cpl_table_get_data_array (oi_table, flag);
3495 
3496  CPLCHECK_MSG ("Cannot get data");
3497 
3498  cpl_size size = cpl_array_get_size (pdata[0]);
3499  cpl_vector * i_vector = cpl_vector_new (size);
3500 
3501  /* Loop on row and index. Add to FLAG if data is above threshold */
3502  for ( cpl_size row = 0 ; row < nrow ; row ++ ) {
3503  if (pdata[row]==NULL || size<100) continue;
3504 
3505  /* Set */
3506  for (cpl_size indx = 0; indx < size; indx++)
3507  cpl_vector_set (i_vector, indx, cpl_array_get (pdata[0],indx,&nv));
3508 
3509  /* Median filter over 8 pixels wide */
3510  cpl_vector * o_vector = cpl_vector_filter_median_create (i_vector, 4);
3511 
3512  /* Check whose pixel have a large values compare to this median */
3513  for ( cpl_size indx = 0 ; indx < size ; indx ++ ) {
3514  if ( cpl_array_get (pdata[row], indx, &nv) > value * cpl_vector_get (o_vector, indx)) {
3515  cpl_array_set (pflag[row], indx, cpl_array_get (pflag[row], indx, &nv) + 1 );
3516  }
3517  }
3518 
3519  FREE (cpl_vector_delete, o_vector);
3520  }
3521 
3522  FREE (cpl_vector_delete, i_vector);
3523  gravi_msg_function_exit(0);
3524  return CPL_ERROR_NONE;
3525 }
3526 
3527 
3528 /*----------------------------------------------------------------------------*/
3541 /*----------------------------------------------------------------------------*/
3542 
3543 cpl_error_code gravi_vis_flag_relative_threshold(cpl_table * oi_table, const char * err,
3544  const char * data, const char *flag, double value)
3545 {
3546  gravi_msg_function_start(0);
3547  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
3548  cpl_ensure_code (data, CPL_ERROR_NULL_INPUT);
3549  cpl_ensure_code (err, CPL_ERROR_NULL_INPUT);
3550  cpl_ensure_code (flag, CPL_ERROR_ILLEGAL_OUTPUT);
3551 
3552  /* Get pointer to speed up */
3553  int nv = 0;
3554  cpl_size row, nrow = cpl_table_get_nrow (oi_table);
3555  cpl_array ** perr = cpl_table_get_data_array (oi_table, err);
3556  cpl_array ** pdata = cpl_table_get_data_array (oi_table, data);
3557  cpl_array ** pflag = cpl_table_get_data_array (oi_table, flag);
3558 
3559  CPLCHECK_MSG ("Cannot get data");
3560 
3561  cpl_size indx, size = cpl_array_get_size (pdata[0]);
3562 
3563  /* Loop on row and index. Add to FLAG if data is above threshold */
3564  for ( row = 0 ; row < nrow ; row ++ ) {
3565  if (perr[row]==NULL) continue;
3566  cpl_array * tmp = cpl_array_duplicate (perr[row]);
3567  cpl_array_divide (tmp, pdata[row]);
3568  for ( indx = 0 ; indx < size ; indx ++ ) {
3569  if ( cpl_array_get (tmp, indx, &nv) > value) {
3570  cpl_array_set (pflag[row], indx, cpl_array_get (pflag[row], indx, &nv) + 1 );
3571  }
3572  }
3573  cpl_array_delete (tmp);
3574  }
3575 
3576  gravi_msg_function_exit(0);
3577  return CPL_ERROR_NONE;
3578 }
3579 
3580 /*----------------------------------------------------------------------------*/
3591 /*----------------------------------------------------------------------------*/
3592 
3593 cpl_error_code gravi_vis_erase_obs (cpl_table * oi_table, cpl_array *flag_array, cpl_size ntel)
3594 {
3595  gravi_msg_function_start(0);
3596  cpl_ensure_code (oi_table, CPL_ERROR_NULL_INPUT);
3597  cpl_ensure_code (flag_array, CPL_ERROR_NULL_INPUT);
3598 
3599  /* Get nrow */
3600  cpl_size nrow = cpl_table_get_nrow (oi_table) / ntel;
3601  cpl_ensure_code (nrow == cpl_array_get_size (flag_array),
3602  CPL_ERROR_ILLEGAL_INPUT);
3603 
3604  /* Loop and select */
3605  cpl_table_unselect_all (oi_table);
3606  for (cpl_size row = 0; row < nrow; row++) {
3607  if (cpl_array_get (flag_array, row, NULL) == 0) continue;
3608  cpl_table_or_selected_window (oi_table, row * ntel, ntel);
3609  }
3610 
3611  /* Delete selected */
3612  cpl_table_erase_selected (oi_table);
3613  CPLCHECK_MSG ("Cannot erase");
3614 
3615  gravi_msg_function_exit(0);
3616  return CPL_ERROR_NONE;
3617 }
3618 
3619 /*----------------------------------------------------------------------------*/
3639 /*----------------------------------------------------------------------------*/
3640 
3641 cpl_error_code gravi_vis_compute_column_mean (cpl_table * out_table,
3642  cpl_table * in_table,
3643  const char * name, int ntel)
3644 {
3645  gravi_msg_function_start(0);
3646  cpl_ensure_code (out_table, CPL_ERROR_NULL_INPUT);
3647  cpl_ensure_code (in_table, CPL_ERROR_NULL_INPUT);
3648  cpl_ensure_code (name, CPL_ERROR_NULL_INPUT);
3649  cpl_ensure_code (ntel == cpl_table_get_nrow (out_table),
3650  CPL_ERROR_ILLEGAL_OUTPUT);
3651 
3652  /* Check if column exist */
3653  if ( !cpl_table_has_column (in_table, name)) {
3654  cpl_msg_info (cpl_func, "Cannot average column %s (not existing)", name);
3655  return CPL_ERROR_NONE;
3656  }
3657 
3658  /* Cast the type into an int, to avoid warnings */
3659  int type = cpl_table_get_column_type (in_table, name);
3660  cpl_size depth = cpl_table_get_column_depth (in_table, name);
3661 
3662  /* Get the number of rows */
3663  cpl_size nrow = cpl_table_get_nrow (in_table) / ntel;
3664  cpl_ensure_code (nrow, CPL_ERROR_ILLEGAL_INPUT);
3665 
3666  /* Get the column REJECTION_FLAG */
3667  int * flag = NULL;
3668  if (cpl_table_has_column (in_table, "REJECTION_FLAG"))
3669  flag = cpl_table_get_data_int (in_table, "REJECTION_FLAG");
3670 
3671  cpl_msg_info (cpl_func, "Average column: %s (%s REJECTION_FLAG)",
3672  name, flag ? "with" : "without");
3673 
3674  switch (type) {
3675  case CPL_TYPE_DOUBLE:
3676  case CPL_TYPE_FLOAT:
3677  case CPL_TYPE_INT:
3678  /* Case scalar column */
3679  if (!cpl_table_has_column (out_table, name))
3680  cpl_table_new_column (out_table, name, CPL_TYPE_DOUBLE);
3681  for (int tel = 0; tel < ntel; tel++) {
3682  cpl_size nvalid = 0;
3683  double mean = 0.0;
3684  for (cpl_size row = 0; row < nrow; row++) {
3685  if (flag && flag[row*ntel+tel] != 0) continue;
3686  nvalid ++;
3687  mean += cpl_table_get (in_table, name, row*ntel+tel, NULL);
3688  }
3689  if (nvalid > 0) mean /= nvalid;
3690  cpl_table_set (out_table, name, tel, mean);
3691  }
3692  break;
3693 
3694  case CPL_TYPE_POINTER|CPL_TYPE_DOUBLE:
3695  case CPL_TYPE_POINTER|CPL_TYPE_FLOAT:
3696  case CPL_TYPE_POINTER|CPL_TYPE_INT:
3697  /* Case real array column */
3698  if (!cpl_table_has_column (out_table, name))
3699  cpl_table_new_column_array (out_table, name, CPL_TYPE_DOUBLE, depth);
3700  for (int tel = 0; tel < ntel; tel++) {
3701  cpl_size nvalid = 0;
3702  cpl_array * mean = gravi_array_init_double (depth, 0.0);
3703  for (cpl_size row = 0; row < nrow; row++) {
3704  if (flag && flag[row*ntel+tel] != 0) continue;
3705  nvalid ++;
3706  cpl_array_add (mean, cpl_table_get_array (in_table, name, row*ntel+tel));
3707  CPLCHECK_MSG ("Cannot add arrays...");
3708  }
3709  if (nvalid > 0) cpl_array_divide_scalar (mean, nvalid);
3710  cpl_table_set_array (out_table, name, tel, mean);
3711  FREE (cpl_array_delete, mean);
3712  }
3713  break;
3714 
3715  case CPL_TYPE_POINTER|CPL_TYPE_DOUBLE_COMPLEX:
3716  case CPL_TYPE_POINTER|CPL_TYPE_FLOAT_COMPLEX:
3717  /* Case complex array column */
3718  if (!cpl_table_has_column (out_table, name))
3719  cpl_table_new_column_array (out_table, name, CPL_TYPE_DOUBLE_COMPLEX, depth);
3720  for (int tel = 0; tel < ntel; tel++) {
3721  cpl_size nvalid = 0;
3722  cpl_array * mean = gravi_array_init_double_complex (depth, 0.0*I+0.0);
3723  for (cpl_size row = 0; row < nrow; row++) {
3724  if (flag && flag[row*ntel+tel] != 0) continue;
3725  nvalid ++;
3726  cpl_array_add (mean, cpl_table_get_array (in_table, name, row*ntel+tel));
3727  }
3728  if (nvalid > 0) cpl_array_divide_scalar (mean, nvalid);
3729  cpl_table_set_array (out_table, name, tel, mean);
3730 
3731  FREE (cpl_array_delete, mean);
3732  }
3733  break;
3734 
3735  cpl_msg_error (cpl_func, "Type column not yet supported...");
3736  cpl_error_set_message (cpl_func, CPL_ERROR_ILLEGAL_INPUT,"This type is not supported.");
3737  return CPL_ERROR_ILLEGAL_INPUT;
3738  }
3739 
3740  /* Copy units */
3741  cpl_table_set_column_unit (out_table, name, cpl_table_get_column_unit (in_table, name));
3742 
3743  gravi_msg_function_exit(0);
3744  return CPL_ERROR_NONE;
3745 }
3746 
3747 /*----------------------------------------------------------------------------*/
3759 /*----------------------------------------------------------------------------*/
3760 
3761 cpl_error_code gravi_vis_force_time (gravi_data * oi_data)
3762 {
3763  gravi_msg_function_start(1);
3764  cpl_ensure_code (oi_data, CPL_ERROR_NULL_INPUT);
3765 
3766  /* Number of extensions */
3767  cpl_size nb_ext = gravi_data_get_size (oi_data);
3768  cpl_ensure_code (nb_ext>0, CPL_ERROR_ILLEGAL_INPUT);
3769 
3770  /* Init averaging */
3771  double mean_mjd = 0.0;
3772  double mean_time = 0.0;
3773  cpl_size count = 0;
3774 
3775  /* Loop on extensions */
3776  for (int ext = 0; ext < nb_ext; ext++) {
3777 
3778  /* Check if data table */
3779  const char * extname = gravi_data_get_extname (oi_data, ext);
3780  if (!strcmp (extname, "OI_VIS") || !strcmp (extname, "OI_VIS2") ||
3781  !strcmp (extname, "OI_T3") || !strcmp (extname, "OI_FLUX")) {
3782 
3783  /* Average MJD and TIME */
3784  cpl_table * oi_table = gravi_data_get_table_x (oi_data, ext);
3785  mean_mjd += cpl_table_get_column_mean (oi_table, "MJD");
3786  mean_time += cpl_table_get_column_mean (oi_table, "TIME");
3787  count ++;
3788 
3789  CPLCHECK_MSG ("Cannot get TIME or MJD...");
3790  }
3791  }
3792 
3793  /* Compute mean */
3794  cpl_ensure_code (count>0, CPL_ERROR_ILLEGAL_INPUT);
3795  mean_mjd /= count;
3796  mean_time /= count;
3797 
3798  /* Verbose */
3799  cpl_msg_info (cpl_func, "Mean MDJ = %g [mdj]", mean_mjd);
3800  cpl_msg_info (cpl_func, "Mean TIME = %g [s]", mean_time);
3801 
3802  /* Loop on extensions */
3803  for (int ext = 0; ext < nb_ext; ext++) {
3804 
3805  /* Check if data table */
3806  const char * extname = gravi_data_get_extname (oi_data, ext);
3807  if (!strcmp (extname, "OI_VIS") || !strcmp (extname, "OI_VIS2") ||
3808  !strcmp (extname, "OI_T3") || !strcmp (extname, "OI_FLUX")) {
3809 
3810  /* Set MJD and TIME */
3811  cpl_table * oi_table = gravi_data_get_table_x (oi_data, ext);
3812  cpl_table_fill_column_window (oi_table, "MJD", 0, CPL_SIZE_MAX, mean_mjd);
3813  cpl_table_fill_column_window (oi_table, "TIME", 0, CPL_SIZE_MAX, mean_time);
3814 
3815  CPLCHECK_MSG ("Cannot set average TIME or MJD...");
3816  }
3817  }
3818 
3819  gravi_msg_function_exit(1);
3820  return CPL_ERROR_NONE;
3821 }
3822 
3823 
gravi_data * gravi_data_new(int nb_ext)
Create an empty gravi_data.
Definition: gravi_data.c:102
cpl_error_code gravi_vis_resamp(gravi_data *oi_data, cpl_size nsamp)
Re-bin the SC table by nsamp consecutive spectral bins.
Definition: gravi_vis.c:3273
cpl_error_code gravi_data_copy_ext(gravi_data *output, gravi_data *input, const char *name)
Copy extensions from one data to another.
Definition: gravi_data.c:1325
cpl_error_code gravi_vis_average_phi(cpl_table *oi_table, const char *name, const char *err, int nbase)
Average phases column of a multi-observation OIFITS table Phases are averaged with arg{<exp(i...
Definition: gravi_vis.c:2567
cpl_propertylist * gravi_data_get_plist_x(gravi_data *self, int i)
Get the propertylist of an extension by position.
Definition: gravi_data.c:1511
cpl_error_code gravi_vis_flag_nan(cpl_table *oi_table)
Flag samples of OIFITS table which are NAN or NULL.
Definition: gravi_vis.c:3368
cpl_error_code gravi_array_online_variance_res(cpl_array **data, int n, int rephase)
On-line variance of arrays.
Definition: gravi_vis.c:209
cpl_error_code gravi_vis_resamp_phi(cpl_table *oi_table, const char *name, const char *err, cpl_size nsamp, cpl_size nwave_new)
Rebin phase column of OIFITS table (arg{<exp(i.phi)>})
Definition: gravi_vis.c:3226
cpl_error_code gravi_vis_average_bootstrap(cpl_table *oi_vis_avg, cpl_table *oi_vis2_avg, cpl_table *oi_vis, int nboot, const char *phase_ref, int use_vFactor, int use_pFactor, int use_debiasing)
Average the visibility of all DITs into a final, averaged value.
Definition: gravi_vis.c:797
cpl_error_code gravi_compute_vis_qc(gravi_data *vis_data)
The function compute the QC parameters for a VIS (averaged) data.
Definition: gravi_vis.c:2048
double gravi_randn(void)
Normal distribution pseudo-random generator.
Definition: gravi_vis.c:128
cpl_error_code gravi_vis_compute_column_mean(cpl_table *out_table, cpl_table *in_table, const char *name, int ntel)
Compute the mean of a column in OIFITS table, and save the result in the specified output table...
Definition: gravi_vis.c:3641
cpl_error_code gravi_vis_average_value(cpl_table *oi_table, const char *name, const char *err, int nbase)
Average scalar column of a multi-observation OIFITS table. The averaged quantities are stored in the ...
Definition: gravi_vis.c:2623
gravi_data * gravi_compute_vis(gravi_data *p2vmred_data, const cpl_parameterlist *parlist, cpl_size *current_frame)
The function average the individual frames of a P2VMREDUCED file into a final, single observation per...
Definition: gravi_vis.c:1509
cpl_error_code gravi_vis_average_amp(cpl_table *oi_table, const char *name, const char *err, int nbase)
Average amplitudes column of a multi-observation OIFITS table The averaged quantities are stored in t...
Definition: gravi_vis.c:2507
cpl_table * gravi_table_extract_time_interval(cpl_table *table, double start, double end)
Extract rows from table based on the TIME column.
Definition: gravi_cpl.c:2826
cpl_error_code gravi_vis_fit_amp(cpl_table *oi_table, const char *name, const char *err, cpl_size maxdeg)
Smooth amp column of OIFITS table.
Definition: gravi_vis.c:3018
cpl_array ** gravi_array_new_list(int n, cpl_type type, int size)
Allocate a list of arrays, pre-filled with 0.0.
Definition: gravi_cpl.c:94
cpl_error_code gravi_array_multiply_phasor(cpl_array *input, double complex factor, cpl_array *phase)
Multiply a REAL phase to a COMPLEX array, in-place: input = input * cexp (factor * phase) ...
Definition: gravi_cpl.c:995
cpl_table * gravi_data_get_table(gravi_data *self, const char *extname)
Return a pointer on a table extension by its EXTNAME.
Definition: gravi_data.c:1716
cpl_error_code gravi_array_get_group_delay_loop(cpl_array **input, cpl_array *sigma, double *gd, cpl_size nrow, double max_width, int verbose)
Optimized computation of GDELAY for a list of arrays.
Definition: gravi_cpl.c:1187
cpl_error_code gravi_vis_erase_obs(cpl_table *oi_table, cpl_array *flag_array, cpl_size ntel)
Erase observation from an OIFITS table.
Definition: gravi_vis.c:3593
cpl_error_code gravi_vis_mjd_to_time(gravi_data *vis_data)
Recompute the TIME column of all OIFITS extension from the MJD column, following the OIFITS standard ...
Definition: gravi_vis.c:2403
cpl_error_code gravi_vis_smooth_phi(cpl_table *oi_table, const char *name, const char *err, cpl_size nsamp)
Smooth phase column of OIFITS table.
Definition: gravi_vis.c:2936
cpl_error_code gravi_vis_flag_threshold(cpl_table *oi_table, const char *data, const char *flag, double value)
Flag samples of OIFITS table based on absolute threshold.
Definition: gravi_vis.c:3437
double gravi_param_get_double_default(const cpl_parameterlist *parlist, const char *name, double def)
Get the parameter from the parameter list.
Definition: gravi_dfs.c:1045
cpl_error_code gravi_vis_smooth(gravi_data *oi_data, cpl_size nsamp_vis, cpl_size nsamp_flx, cpl_size maxdeg)
Smooth the SC table by nsamp consecutive spectral bins.
Definition: gravi_vis.c:3102
cpl_error_code gravi_average_self_visphi(cpl_table *oi_vis_avg, cpl_table *oi_vis, cpl_array *wavenumber, const char *phase_ref, int *cmin, int *cmax, int nrange)
Compute Averaged VISPHI in the manner described, e.g., in F. Millour&#39;s thesis.
Definition: gravi_vis.c:1217
cpl_error_code gravi_vis_smooth_amp(cpl_table *oi_table, const char *name, const char *err, cpl_size nsamp)
Smooth amplitude column of OIFITS table.
Definition: gravi_vis.c:2855
cpl_error_code gravi_vis_resamp_amp(cpl_table *oi_table, const char *name, const char *err, cpl_size nsamp, cpl_size nwave_new)
Rebin amplitude column of OIFITS table.
Definition: gravi_vis.c:3180
cpl_error_code gravi_average_vis(gravi_data *oi_data)
Coadd the observations together.
Definition: gravi_vis.c:2750
int gravi_data_has_type(gravi_data *self, const char *type)
Return the number of ext whose EXTNAME and INSNAME match &#39;type&#39;.
Definition: gravi_data.c:1468
cpl_table * gravi_data_get_table_x(gravi_data *self, int i)
Get the table of an extension by position.
Definition: gravi_data.c:1536
double gdAbacusErrPhi(double x)
Definition: gravi_vis.c:1148
cpl_error_code gravi_vis_flag_median(cpl_table *oi_table, const char *data, const char *flag, double value)
Flag samples of OIFITS table based on runnning median.
Definition: gravi_vis.c:3483
cpl_error_code gravi_flux_average_bootstrap(cpl_table *oi_flux_avg, cpl_table *oi_flux, int nboot)
Average the flux of all DITs into a final, averaged value.
Definition: gravi_vis.c:262
cpl_error_code gravi_vis_force_time(gravi_data *oi_data)
Force all data in OI_TABLE to have the same TIME and MJD.
Definition: gravi_vis.c:3761
cpl_error_code gravi_normalize_sc_to_ft(gravi_data *vis_data)
Align the SC visibilities on the FT visibilities.
Definition: gravi_vis.c:2317
cpl_error_code gravi_data_add_table(gravi_data *self, cpl_propertylist *plist, const char *extname, cpl_table *table)
Add a BINTABLE extension in gravi_data.
Definition: gravi_data.c:1909
cpl_error_code gravi_flat_flux(gravi_data *vis_data, gravi_data *p2vm_map)
Divide the OI_FLUX by OI_FLUX from the P2VM (no checks, no time distance...)
Definition: gravi_vis.c:2457
cpl_error_code gravi_force_uncertainties(gravi_data *oi_data, const cpl_parameterlist *parlist)
Force uncertainties.
Definition: gravi_vis.c:2660
cpl_error_code gravi_vis_flag_relative_threshold(cpl_table *oi_table, const char *err, const char *data, const char *flag, double value)
Flag samples of OIFITS table based on relative threshold.
Definition: gravi_vis.c:3543
cpl_error_code gravi_apply_tf_amp(gravi_data *science, gravi_data *science_tf, gravi_data **used_tf_data, int num_tf_data, const char *extName, const char *insName, const char *ampName, const char *ampErrName, int nbase, double delta_t)
Interpolate the TF at the time of the science observation for an amplitude quantity.
Definition: gravi_tf.c:248
cpl_error_code gravi_t3_average_bootstrap(cpl_table *oi_t3_avg, cpl_table *oi_vis, cpl_table *oi_flux, int nboot, int use_vFactor, int use_pFactor)
Average the closure-phase of all DITs into a final, averaged value.
Definition: gravi_vis.c:473
cpl_table * gravi_table_oi_create(int nwave, int nrow, const char *oi_name)
Create the oi table (oi_vis, oi_vis2, oi_t3)
Definition: gravi_utils.c:153
int gravi_data_get_size(const gravi_data *self)
Get the number of extension in a gravi_data.
Definition: gravi_data.c:802