VIRCAM Pipeline  2.3.10
vircam_genlincur.c
1 /* $Id: vircam_genlincur.c,v 1.34 2012-01-15 17:40:09 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2012-01-15 17:40:09 $
24  * $Revision: 1.34 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <cpl.h>
35 
36 #include <math.h>
37 
38 #include <casu_utils.h>
39 #include <casu_stats.h>
40 #include "vircam_utils.h"
41 #include "vircam_mods.h"
42 #include "vircam_pfits.h"
43 #include "vircam_channel.h"
44 
45 
46 #define SZCOLNAME 16
47 
48 static double nom_val = 10000;
49 
50 
51 static double linval(double inval, double *kfacs, double tolerance,
52  int niter, double *b, int norder);
53 static double getkfac(long index, long ncpts, float reset_time,
54  float read_time, float delay_time, float exptime);
55 static void getco(double *a, int nord, int m);
56 
59 /*---------------------------------------------------------------------------*/
116 /*---------------------------------------------------------------------------*/
117 
118 extern int vircam_genlincur(double **fdata, int nimages, double *exps,
119  double mindit, casu_tfits *chantab,
120  int norder, cpl_table **lchantab,
121  double **lindata, int *status) {
122 
123  const char *fctid = "vircam_genlincur";
124  int retval,i,j,nbad,oldnorder,k,ii,nullval;
125  long np;
126  double *meds,sigfit,**aco,c0,lin_nom,*temp,*polyco,pt,*work,kfac;
127  double sum,t10000,*kfacs;
128  parquet *pp;
129  cpl_table *ctab,*lc;
130  cpl_array *exparray,*medsarray,*polyfitco,*workarray;
131 
132  /* Inherited status */
133 
134  *lchantab = NULL;
135  if (*status != CASU_OK)
136  return(*status);
137 
138  /* Check that you have enough images in the list */
139 
140  if (nimages < norder+1) {
141  cpl_msg_error(fctid,
142  "Not enought images (%" CPL_SIZE_FORMAT ") for fit order (%" CPL_SIZE_FORMAT ")",
143  (cpl_size)nimages,(cpl_size)norder);
144  FATAL_ERROR
145  }
146 
147  /* Open the parquet structure for the channel table; for now we
148  only need the number of channels in each structure, rather than
149  the structure itself */
150 
151  ctab = casu_tfits_get_table(chantab);
152  retval = vircam_chan_fill(ctab,&pp,&np);
153  if (retval != CASU_OK) {
154  *status = retval;
155  return(retval);
156  }
157  vircam_chan_free(np,&pp);
158 
159  /* Create an output channel table. Copy the input channel table and then
160  massage the linearity part */
161 
162  lc = cpl_table_duplicate(ctab);
163  oldnorder = cpl_table_get_int(lc,"norder",0,&nullval);
164  if (oldnorder > norder) {
165  for (i = norder+1; i <= oldnorder; i++) {
166  char * colname = cpl_sprintf("coeff_%d", i);
167  //snprintf(colname,SZCOLNAME,"coeff_%d", i);
168  cpl_table_erase_column(lc,colname);
169  cpl_free(colname);
170  }
171  } else if (oldnorder < norder) {
172  for (i = oldnorder+1; i <= norder; i++) {
173  char * colname = cpl_sprintf("coeff_%d", i);
174  //snprintf(colname,SZCOLNAME,"coeff_%d", i);
175  if (cpl_table_has_column(lc,colname)) {
176  cpl_free(colname);
177  continue;
178  }
179  cpl_table_new_column(lc,colname,CPL_TYPE_DOUBLE);
180  cpl_free(colname);
181  }
182  }
183 
184  /* Get some memory for the fitting arrays */
185 
186  exparray = cpl_array_wrap_double(exps,nimages);
187  medsarray = cpl_array_new((cpl_size)nimages,CPL_TYPE_DOUBLE);
188  meds = cpl_array_get_data_double(medsarray);
189  aco = cpl_malloc(norder*sizeof(double *));
190  for (i = 0; i < norder; i++)
191  aco[i] = cpl_malloc(norder*sizeof(double));
192  temp = cpl_malloc(norder*sizeof(double));
193  kfacs = cpl_malloc(norder*sizeof(double));
194 
195  /* Get memory for output array of linearised stats */
196 
197  *lindata = cpl_malloc(nimages*np*sizeof(double));
198 
199  /* Loop for each channel */
200 
201  nbad = 0;
202  for (i = 0; i < np; i++) {
203 
204  /* Load the data up for this channel */
205 
206  for (j = 0; j < nimages; j++)
207  meds[j] = fdata[j][i];
208 
209  /* Do the initial fit */
210 
211  if (vircam_polyfit(exparray,medsarray,norder,1,2,2.0,2.0,&polyfitco,
212  &sigfit) != CASU_OK) {
213  nbad++;
214  cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
215  cpl_table_set_double(lc,"coeff_1",(cpl_size)i,1.0);
216  for (k = 1; k < norder; k++) {
217  char * colname = cpl_sprintf("coeff_%d",k+1);
218  //snprintf(colname,SZCOLNAME,"coeff_%d",k+1);
219  cpl_table_set_double(lc,colname,(cpl_size)i,0.0);
220  cpl_free(colname);
221  }
222  freearray(polyfitco);
223  continue;
224  }
225  polyco = cpl_array_get_data_double(polyfitco);
226 
227  /* Work out linearity */
228 
229  for (j = 0; j < nimages; j++)
230  if (meds[j] > nom_val)
231  break;
232 
233  if (j == 0) { /* The first frame already exeeds nom_val */
234  nbad++;
235  cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
236  cpl_table_set_double(lc,"coeff_1",(cpl_size)i,1.0);
237  for (k = 1; k < norder; k++) {
238  char * colname = cpl_sprintf("coeff_%d",k+1);
239  //snprintf(colname,SZCOLNAME,"coeff_%d",k+1);
240  cpl_table_set_double(lc,colname,(cpl_size)i,0.0);
241  cpl_free(colname);
242  }
243  freearray(polyfitco);
244  continue;
245  }
246 
247  t10000 = exps[j-1] + (nom_val - meds[j-1])/(meds[j] - meds[j-1]);
248  sum = 0.0;
249  for (j = 0; j < norder; j++)
250  sum += (double)(j+1)*polyco[j]*pow(t10000,(double)j);
251  lin_nom = 100.0*fabs(sum - polyco[0])/polyco[0];
252 
253  /* Get intermediate coefficients for matrix */
254 
255  for (j = 0; j < norder; j++) {
256  getco(temp,norder,j+1);
257  for (k = 0; k < norder; k++) {
258  pt = pow(mindit,(double)(j-k));
259  aco[j][k] = pt*temp[k];
260  }
261  }
262 
263  /* Solve matrix equation to do the back substitution */
264 
265  if (vircam_solve_gauss(aco,polyco,norder) != CASU_OK) {
266  nbad++;
267  cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
268  cpl_table_set_double(lc,"coeff_1",(cpl_size)i,1.0);
269  for (k = 1; k < norder; k++) {
270  char * colname = cpl_sprintf("coeff_%d",k+1);
271  //snprintf(colname,SZCOLNAME,"coeff_%d",k+1);
272  cpl_table_set_double(lc,colname,(cpl_size)i,0.0);
273  cpl_free(colname);
274  }
275  freearray(polyfitco);
276  continue;
277  }
278 
279  /* Now normalise to unit slope and write the result to the table*/
280 
281  c0 = polyco[0];
282  for (j = 0; j < norder; j++) {
283  polyco[j] /= pow(c0,(double)(j+1));
284  char * colname = cpl_sprintf("coeff_%d",j+1);
285  //snprintf(colname,SZCOLNAME,"coeff_%d",j+1);
286  cpl_table_set_double(lc,colname,(cpl_size)i,polyco[j]);
287  cpl_free(colname);
288  }
289  cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
290 
291  /* Work out how well the solution creates a linear system. Loop
292  for each input image and work out a 'linearised' median. Then
293  do a linear fit to the linearsed median vs exposure time. */
294 
295  workarray = cpl_array_new((cpl_size)nimages,CPL_TYPE_DOUBLE);
296  work = cpl_array_get_data_double(workarray);
297  for (j = 0; j < nimages; j++) {
298  kfac = mindit/exps[j];
299  kfacs[0] = 1.0;
300  for (ii = 1; ii < norder; ii++)
301  kfacs[ii] = pow((kfac+1.0),(double)(ii+1)) -
302  pow(kfac,(double)(ii+1));
303  work[j] = linval(meds[j],kfacs,0.5,10,polyco,norder);
304  (*lindata)[j*np+i] = work[j];
305  }
306  freearray(polyfitco);
307  (void)vircam_polyfit(exparray,workarray,2,0,2,2.0,2.0,&polyfitco,
308  &sigfit);
309  polyco = cpl_array_get_data_double(polyfitco);
310  sigfit *= 100.0/nom_val;
311  freearray(workarray);
312  freearray(polyfitco);
313 
314  /* Put the nominal linearity and the fit quality into the header */
315 
316  cpl_table_set_double(lc,"lin_10000_err",(cpl_size)i,sigfit);
317  cpl_table_set_double(lc,"lin_10000",(cpl_size)i,lin_nom);
318  }
319 
320  /* Tidy and get out of here */
321 
322  *lchantab = cpl_table_duplicate(lc);
323  cpl_array_unwrap(exparray);
324  freearray(medsarray);
325  freespace2(aco,norder);
326  freespace(temp);
327  freetable(lc);
328  freespace(kfacs);
329  if (nbad != 0) {
330  cpl_msg_warning(fctid,
331  "%" CPL_SIZE_FORMAT " channels have a failed solution",
332  (cpl_size)nbad);
333  WARN_RETURN
334  }
335  GOOD_STATUS
336 }
337 
338 
339 /*---------------------------------------------------------------------------*/
380 /*--------------------------------------------------------------------------*/
381 
382 extern int vircam_lincor(casu_fits *infile, casu_tfits *lchantab, int kconst,
383  int ndit, int *status) {
384  int retval,i,norder,ii;
385  long naxis[2],j,rind,aind,ncpts,np;
386  float *data,texp,reset_time,read_time,delay_time;
387  double kfac_nom,lkfac,inval,outval,*lbb,dnd,*kfacs;
388  const char *fctid = "vircam_lincor";
389  parquet *pp;
390  cpl_propertylist *plist;
391  cpl_table *lctab;
392  parquet *p;
393 
394  /* Inherited status */
395 
396  if (*status != CASU_OK)
397  return(*status);
398 
399  /* Do we even need to be here? */
400 
401  if (cpl_propertylist_has(casu_fits_get_ehu(infile),"ESO DRS LINCOR"))
402  return(*status);
403 
404  /* Open the parquet structure for the channel table */
405 
406  lctab = casu_tfits_get_table(lchantab);
407  retval = vircam_chan_fill(lctab,&p,&np);
408  if (retval != CASU_OK)
409  return(retval);
410 
411  /* Get the data array for the image */
412 
413  data = cpl_image_get_data(casu_fits_get_image(infile));
414  if (data == NULL) {
415  vircam_chan_free(np,&p);
416  cpl_msg_error(fctid,"Error mapping data in input image");
417  FATAL_ERROR
418  }
419  naxis[0] = (long)cpl_image_get_size_x(casu_fits_get_image(infile));
420  naxis[1] = (long)cpl_image_get_size_y(casu_fits_get_image(infile));
421 
422  /* Get the required parameters from the header */
423 
424  plist = casu_fits_get_ehu(infile);
425  if (vircam_pfits_get_exptime(plist,&texp) != CASU_OK) {
426  vircam_chan_free(np,&p);
427  cpl_msg_error(fctid,"No exposure time in %s",
428  casu_fits_get_fullname(infile));
429  FATAL_ERROR
430  }
431  if (vircam_pfits_get_mindit(plist,&reset_time) != CASU_OK) {
432  vircam_chan_free(np,&p);
433  cpl_msg_error(fctid,"No mindit time in %s",
434  casu_fits_get_fullname(infile));
435  FATAL_ERROR
436  }
437  read_time = reset_time;
438  if (vircam_pfits_get_ditdelay(plist,&delay_time) != CASU_OK) {
439  vircam_chan_free(np,&p);
440  cpl_msg_error(fctid,"No dit delay time in %s",
441  casu_fits_get_fullname(infile));
442  FATAL_ERROR
443  }
444 
445  /* If there is a constant k factor, then calculate it now */
446 
447  kfac_nom = (double)(read_time/texp);
448 
449  /* Factor to take the number of DITs into account */
450 
451  dnd = (double)ndit;
452 
453  /* Loop for each channel now */
454 
455  for (i = 0; i < np; i++) {
456  pp = p + i;
457  ncpts = (pp->delta_i)*(pp->delta_j);
458 
459  /* Load up the fit coefficients. If there is only one coefficient
460  this is by definition 1 and therefore we can skip this channel */
461 
462  norder = pp->norder;
463  if (norder == 1)
464  continue;
465  lbb = pp->bb;
466 
467  /* Get workspace for K array and fill it in for situation of constant
468  k factor */
469 
470  kfacs = cpl_malloc(norder*sizeof(double));
471  if (kconst) {
472  kfacs[0] = 1.0;
473  for (ii = 1; ii < norder; ii++)
474  kfacs[ii] = pow((kfac_nom+1.0),(double)(ii+1)) -
475  pow(kfac_nom,(double)(ii+1));
476  }
477 
478  /* Now for each pixel */
479 
480  for (j = 0; j < ncpts; j++) {
481 
482  /* Get the 'non-constant' k-factor and fill in the K array. */
483 
484  rind = vircam_chan_d2r(pp,j);
485  aind = vircam_chan_r2a(pp,naxis,rind);
486  if (! kconst) {
487  lkfac = getkfac(j,ncpts,reset_time,read_time,delay_time,texp);
488  kfacs[0] = 1.0;
489  for (ii = 1; ii < norder; ii++)
490  kfacs[ii] = pow((lkfac+1.0),(double)(ii+1)) -
491  pow(lkfac,(double)(ii+1));
492  }
493 
494  /* Calculate the linearised value now */
495 
496  inval = ((double)data[aind])/dnd;
497  outval = linval(inval,kfacs,0.5,10,lbb,norder);
498  data[aind] = (float)(dnd*outval);
499  }
500  freespace(kfacs);
501  }
502 
503  /* Add the linearity table to the DRS header */
504 
505  cpl_propertylist_update_string(casu_fits_get_ehu(infile),
506  "ESO DRS LINCOR",
507  casu_tfits_get_filename(lchantab));
508 
509  /* Right, get out of here */
510 
511  vircam_chan_free(np,&p);
512  GOOD_STATUS
513 }
514 
515 /*---------------------------------------------------------------------------*/
544 /*---------------------------------------------------------------------------*/
545 
546 static double linval(double inval, double *kfacs, double tolerance,
547  int niter, double *b, int norder) {
548  int jj,iter;
549  double val_old,val,tol,sum;
550 
551  val = inval;
552  iter = 0;
553  tol = tolerance + 1.0;
554  while (iter < niter && tol > tolerance) {
555  val_old = val;
556  iter++;
557  sum = 0.0;
558  for (jj = norder - 1; jj >= 1; jj--)
559  sum = (sum + b[jj]*kfacs[jj])*val;
560  sum *= val;
561  val = inval - sum;
562  tol = fabs(val - val_old);
563  if (val > 65535.0) {
564  val = 65535.0;
565  break;
566  } else if (val < -1000.0) {
567  val = -1000.0;
568  break;
569  }
570  }
571  return(val);
572 }
573 
574 
575 /*---------------------------------------------------------------------------*/
603 /*---------------------------------------------------------------------------*/
604 
605 static double getkfac(long index, long npts, float reset_time,
606  float read_time, float delay_time, float exptime) {
607  double tkfac,dt1,dt2,dt3,dt4,df;
608 
609  df = ((double)index/(double)npts);
610  dt1 = (double)exptime;
611  dt2 = (double)read_time;
612  dt3 = (double)reset_time;
613  dt4 = (double)delay_time;
614  tkfac = (dt3 + dt4 + (dt2 - dt3)*df)/dt1;
615  return(tkfac);
616 }
617 
618 /*---------------------------------------------------------------------------*/
640 /*---------------------------------------------------------------------------*/
641 
642 static void getco(double *a, int nord, int m) {
643  int i,j,start;
644 
645  for (i = 0; i < nord; i++)
646  a[i] = 0.0;
647  start = m-1;
648  a[start] = 1.0;
649  j = 1;
650  for (i = start-1; i >= 0; i--) {
651  j++;
652  a[i] = a[i+1]*(double)(m - j + 2)/(double)(j-1);
653  }
654 }
655 
656 
660 /*
661 
662 $Log: not supported by cvs2svn $
663 Revision 1.33 2010/06/03 12:15:31 jim
664 A few mods to get rid of compiler warnings
665 
666 Revision 1.32 2009/12/11 06:53:35 jim
667 Minor changes to documentation
668 
669 Revision 1.31 2009/09/09 09:45:36 jim
670 Modified to speed things up
671 
672 Revision 1.30 2009/06/08 08:08:15 jim
673 Fixed memory leak and changed clipping parameters to vircam_polyfit
674 
675 Revision 1.29 2009/05/20 12:18:42 jim
676 Modified so that if the operation is already done, then it just returns
677 
678 Revision 1.28 2009/02/20 10:50:55 jim
679 Removed superfluous declarations
680 
681 Revision 1.27 2008/10/21 08:38:48 jim
682 Final estiamte on linearity error done with vircam_polyfit now
683 
684 Revision 1.26 2008/09/29 11:26:18 jim
685 Modified linval to provide a lower limit in case the fit goes crazy
686 
687 Revision 1.25 2008/08/28 09:05:37 jim
688 Fixed bug where QC was being duplicated from master BPM on rare occasions.
689 Fixed bug where ARCFILE wasn't being written to the paf file for illum_cor
690 tables. Sky combine is done with medians
691 
692 Revision 1.24 2008/01/22 19:45:24 jim
693 New version of genlincur to take into account the equality of readout and
694 reset time
695 
696 Revision 1.23 2007/11/26 09:57:14 jim
697 Linearity correction routines now take account of ndit
698 
699 Revision 1.22 2007/11/22 12:36:15 jim
700 Modified to return linearised values in an array
701 
702 Revision 1.21 2007/11/20 09:38:57 jim
703 changed definition of fit quality to percentage error at 10000 counts
704 
705 Revision 1.20 2007/11/14 14:47:32 jim
706 Modified the qualfit definition to be back in line with DRLD
707 
708 Revision 1.19 2007/11/14 12:34:21 jim
709 Fixed header comments
710 
711 Revision 1.18 2007/11/14 10:48:29 jim
712 Major rewrite to incorporate simpler and more robust algorithm
713 
714 Revision 1.17 2007/03/29 12:19:39 jim
715 Little changes to improve documentation
716 
717 Revision 1.16 2007/03/01 12:42:41 jim
718 Modified slightly after code checking
719 
720 Revision 1.15 2006/11/27 12:08:18 jim
721 Modified lincor to get a better answer. Also modified the way the fit quality
722 is calculated
723 
724 Revision 1.14 2006/09/29 11:19:31 jim
725 changed aliases on parameter names
726 
727 Revision 1.13 2006/07/04 09:19:05 jim
728 replaced all sprintf statements with snprintf
729 
730 Revision 1.12 2006/06/09 11:26:26 jim
731 Small changes to keep lint happy
732 
733 Revision 1.11 2006/04/20 11:23:15 jim
734 Now medians the data before accumulation in the event that k is constant.
735 
736 Revision 1.10 2006/03/23 21:18:47 jim
737 Minor changes mainly to comment headers
738 
739 Revision 1.9 2006/03/22 13:36:50 jim
740 cosmetic changes to keep lint happy
741 
742 Revision 1.8 2006/03/15 10:43:41 jim
743 Fixed a few things
744 
745 Revision 1.7 2006/03/08 14:32:21 jim
746 Lots of little modifications
747 
748 Revision 1.6 2006/03/03 14:29:46 jim
749 Modified definition of vir_fits and channel table
750 
751 Revision 1.5 2006/03/01 10:31:28 jim
752 Now uses new vir_fits objects
753 
754 Revision 1.4 2006/02/18 11:45:59 jim
755 Fixed a couple of memory bugs
756 
757 Revision 1.3 2006/01/23 22:58:14 jim
758 Added vircam_lincor module
759 
760 Revision 1.2 2006/01/23 16:06:03 jim
761 Added in header comments and section to evaluate the goodness of fit
762 
763 Revision 1.1 2006/01/23 10:31:56 jim
764 New file
765 
766 
767 */
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
char * casu_fits_get_fullname(casu_fits *p)
Definition: casu_fits.c:680
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
cpl_table * casu_tfits_get_table(casu_tfits *p)
Definition: casu_tfits.c:364
char * casu_tfits_get_filename(casu_tfits *p)
Definition: casu_tfits.c:510
int vircam_genlincur(double **fdata, int nimages, double *exps, double mindit, casu_tfits *chantab, int norder, cpl_table **lchantab, double **lindata, int *status)
Generate a linearity curve for each readout channel in a list of images.
int vircam_lincor(casu_fits *infile, casu_tfits *lchantab, int kconst, int ndit, int *status)
Apply linearity curves to data.
long vircam_chan_d2r(parquet *p, long l)
long vircam_chan_r2a(parquet *p, long naxis[2], long k)
int vircam_chan_fill(cpl_table *tab, parquet **p, long *np)
void vircam_chan_free(int np, parquet **p)
int vircam_pfits_get_ditdelay(const cpl_propertylist *plist, float *ditdelay)
Get the value of dit delay time.
Definition: vircam_pfits.c:279
int vircam_pfits_get_exptime(const cpl_propertylist *plist, float *exptime)
Get the value of exposure time.
Definition: vircam_pfits.c:245
int vircam_pfits_get_mindit(const cpl_propertylist *plist, float *mindit)
Get the value of mindit time.
Definition: vircam_pfits.c:262
int vircam_solve_gauss(double **a, double *b, int m)
Definition: vircam_utils.c:456
int vircam_polyfit(const cpl_array *xarray, const cpl_array *yarray, int ncoefs, int ilim, int niter, float lclip, float hclip, cpl_array **polycf, double *sigfit)
Definition: vircam_utils.c:571