VIRCAM Pipeline 2.3.15
vircam_genlincur.c
1/* $Id: vircam_genlincur.c,v 1.34 2012-01-15 17:40:09 jim Exp $
2 *
3 * This file is part of the VIRCAM Pipeline
4 * Copyright (C) 2015 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20
21/*
22 * $Author: jim $
23 * $Date: 2012-01-15 17:40:09 $
24 * $Revision: 1.34 $
25 * $Name: not supported by cvs2svn $
26 */
27
28/* Includes */
29
30#ifdef HAVE_CONFIG_H
31#include <config.h>
32#endif
33
34#include <cpl.h>
35
36#include <math.h>
37
38#include <casu_utils.h>
39#include <casu_stats.h>
40#include "vircam_utils.h"
41#include "vircam_mods.h"
42#include "vircam_pfits.h"
43#include "vircam_channel.h"
44
45
46#define SZCOLNAME 16
47
48static double nom_val = 10000;
49
50
51static double linval(double inval, double *kfacs, double tolerance,
52 int niter, double *b, int norder);
53static double getkfac(long index, long ncpts, float reset_time,
54 float read_time, float delay_time, float exptime);
55static void getco(double *a, int nord, int m);
56
59/*---------------------------------------------------------------------------*/
116/*---------------------------------------------------------------------------*/
117
118extern int vircam_genlincur(double **fdata, int nimages, double *exps,
119 double mindit, casu_tfits *chantab,
120 int norder, cpl_table **lchantab,
121 double **lindata, int *status) {
122
123 const char *fctid = "vircam_genlincur";
124 int retval,i,j,nbad,oldnorder,k,ii,nullval;
125 long np;
126 double *meds,sigfit,**aco,c0,lin_nom,*temp,*polyco,pt,*work,kfac;
127 double sum,t10000,*kfacs;
128 parquet *pp;
129 cpl_table *ctab,*lc;
130 cpl_array *exparray,*medsarray,*polyfitco,*workarray;
131
132 /* Inherited status */
133
134 *lchantab = NULL;
135 if (*status != CASU_OK)
136 return(*status);
137
138 /* Check that you have enough images in the list */
139
140 if (nimages < norder+1) {
141 cpl_msg_error(fctid,
142 "Not enought images (%" CPL_SIZE_FORMAT ") for fit order (%" CPL_SIZE_FORMAT ")",
143 (cpl_size)nimages,(cpl_size)norder);
144 FATAL_ERROR
145 }
146
147 /* Open the parquet structure for the channel table; for now we
148 only need the number of channels in each structure, rather than
149 the structure itself */
150
151 ctab = casu_tfits_get_table(chantab);
152 retval = vircam_chan_fill(ctab,&pp,&np);
153 if (retval != CASU_OK) {
154 *status = retval;
155 return(retval);
156 }
157 vircam_chan_free(np,&pp);
158
159 /* Create an output channel table. Copy the input channel table and then
160 massage the linearity part */
161
162 lc = cpl_table_duplicate(ctab);
163 oldnorder = cpl_table_get_int(lc,"norder",0,&nullval);
164 if (oldnorder > norder) {
165 for (i = norder+1; i <= oldnorder; i++) {
166 char * colname = cpl_sprintf("coeff_%d", i);
167 //snprintf(colname,SZCOLNAME,"coeff_%d", i);
168 cpl_table_erase_column(lc,colname);
169 cpl_free(colname);
170 }
171 } else if (oldnorder < norder) {
172 for (i = oldnorder+1; i <= norder; i++) {
173 char * colname = cpl_sprintf("coeff_%d", i);
174 //snprintf(colname,SZCOLNAME,"coeff_%d", i);
175 if (cpl_table_has_column(lc,colname)) {
176 cpl_free(colname);
177 continue;
178 }
179 cpl_table_new_column(lc,colname,CPL_TYPE_DOUBLE);
180 cpl_free(colname);
181 }
182 }
183
184 /* Get some memory for the fitting arrays */
185
186 exparray = cpl_array_wrap_double(exps,nimages);
187 medsarray = cpl_array_new((cpl_size)nimages,CPL_TYPE_DOUBLE);
188 meds = cpl_array_get_data_double(medsarray);
189 aco = cpl_malloc(norder*sizeof(double *));
190 for (i = 0; i < norder; i++)
191 aco[i] = cpl_malloc(norder*sizeof(double));
192 temp = cpl_malloc(norder*sizeof(double));
193 kfacs = cpl_malloc(norder*sizeof(double));
194
195 /* Get memory for output array of linearised stats */
196
197 *lindata = cpl_malloc(nimages*np*sizeof(double));
198
199 /* Loop for each channel */
200
201 nbad = 0;
202 for (i = 0; i < np; i++) {
203
204 /* Load the data up for this channel */
205
206 for (j = 0; j < nimages; j++)
207 meds[j] = fdata[j][i];
208
209 /* Do the initial fit */
210
211 if (vircam_polyfit(exparray,medsarray,norder,1,2,2.0,2.0,&polyfitco,
212 &sigfit) != CASU_OK) {
213 nbad++;
214 cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
215 cpl_table_set_double(lc,"coeff_1",(cpl_size)i,1.0);
216 for (k = 1; k < norder; k++) {
217 char * colname = cpl_sprintf("coeff_%d",k+1);
218 //snprintf(colname,SZCOLNAME,"coeff_%d",k+1);
219 cpl_table_set_double(lc,colname,(cpl_size)i,0.0);
220 cpl_free(colname);
221 }
222 freearray(polyfitco);
223 continue;
224 }
225 polyco = cpl_array_get_data_double(polyfitco);
226
227 /* Work out linearity */
228
229 for (j = 0; j < nimages; j++)
230 if (meds[j] > nom_val)
231 break;
232
233 if (j == 0) { /* The first frame already exeeds nom_val */
234 nbad++;
235 cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
236 cpl_table_set_double(lc,"coeff_1",(cpl_size)i,1.0);
237 for (k = 1; k < norder; k++) {
238 char * colname = cpl_sprintf("coeff_%d",k+1);
239 //snprintf(colname,SZCOLNAME,"coeff_%d",k+1);
240 cpl_table_set_double(lc,colname,(cpl_size)i,0.0);
241 cpl_free(colname);
242 }
243 freearray(polyfitco);
244 continue;
245 }
246
247 t10000 = exps[j-1] + (nom_val - meds[j-1])/(meds[j] - meds[j-1]);
248 sum = 0.0;
249 for (j = 0; j < norder; j++)
250 sum += (double)(j+1)*polyco[j]*pow(t10000,(double)j);
251 lin_nom = 100.0*fabs(sum - polyco[0])/polyco[0];
252
253 /* Get intermediate coefficients for matrix */
254
255 for (j = 0; j < norder; j++) {
256 getco(temp,norder,j+1);
257 for (k = 0; k < norder; k++) {
258 pt = pow(mindit,(double)(j-k));
259 aco[j][k] = pt*temp[k];
260 }
261 }
262
263 /* Solve matrix equation to do the back substitution */
264
265 if (vircam_solve_gauss(aco,polyco,norder) != CASU_OK) {
266 nbad++;
267 cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
268 cpl_table_set_double(lc,"coeff_1",(cpl_size)i,1.0);
269 for (k = 1; k < norder; k++) {
270 char * colname = cpl_sprintf("coeff_%d",k+1);
271 //snprintf(colname,SZCOLNAME,"coeff_%d",k+1);
272 cpl_table_set_double(lc,colname,(cpl_size)i,0.0);
273 cpl_free(colname);
274 }
275 freearray(polyfitco);
276 continue;
277 }
278
279 /* Now normalise to unit slope and write the result to the table*/
280
281 c0 = polyco[0];
282 for (j = 0; j < norder; j++) {
283 polyco[j] /= pow(c0,(double)(j+1));
284 char * colname = cpl_sprintf("coeff_%d",j+1);
285 //snprintf(colname,SZCOLNAME,"coeff_%d",j+1);
286 cpl_table_set_double(lc,colname,(cpl_size)i,polyco[j]);
287 cpl_free(colname);
288 }
289 cpl_table_set_int(lc,"norder",(cpl_size)i,norder);
290
291 /* Work out how well the solution creates a linear system. Loop
292 for each input image and work out a 'linearised' median. Then
293 do a linear fit to the linearsed median vs exposure time. */
294
295 workarray = cpl_array_new((cpl_size)nimages,CPL_TYPE_DOUBLE);
296 work = cpl_array_get_data_double(workarray);
297 for (j = 0; j < nimages; j++) {
298 kfac = mindit/exps[j];
299 kfacs[0] = 1.0;
300 for (ii = 1; ii < norder; ii++)
301 kfacs[ii] = pow((kfac+1.0),(double)(ii+1)) -
302 pow(kfac,(double)(ii+1));
303 work[j] = linval(meds[j],kfacs,0.5,10,polyco,norder);
304 (*lindata)[j*np+i] = work[j];
305 }
306 freearray(polyfitco);
307 (void)vircam_polyfit(exparray,workarray,2,0,2,2.0,2.0,&polyfitco,
308 &sigfit);
309 polyco = cpl_array_get_data_double(polyfitco);
310 sigfit *= 100.0/nom_val;
311 freearray(workarray);
312 freearray(polyfitco);
313
314 /* Put the nominal linearity and the fit quality into the header */
315
316 cpl_table_set_double(lc,"lin_10000_err",(cpl_size)i,sigfit);
317 cpl_table_set_double(lc,"lin_10000",(cpl_size)i,lin_nom);
318 }
319
320 /* Tidy and get out of here */
321
322 *lchantab = cpl_table_duplicate(lc);
323 cpl_array_unwrap(exparray);
324 freearray(medsarray);
325 freespace2(aco,norder);
326 freespace(temp);
327 freetable(lc);
328 freespace(kfacs);
329 if (nbad != 0) {
330 cpl_msg_warning(fctid,
331 "%" CPL_SIZE_FORMAT " channels have a failed solution",
332 (cpl_size)nbad);
333 WARN_RETURN
334 }
335 GOOD_STATUS
336}
337
338
339/*---------------------------------------------------------------------------*/
380/*--------------------------------------------------------------------------*/
381
382extern int vircam_lincor(casu_fits *infile, casu_tfits *lchantab, int kconst,
383 int ndit, int *status) {
384 int retval,i,norder,ii;
385 long naxis[2],j,rind,aind,ncpts,np;
386 float *data,texp,reset_time,read_time,delay_time;
387 double kfac_nom,lkfac,inval,outval,*lbb,dnd,*kfacs;
388 const char *fctid = "vircam_lincor";
389 parquet *pp;
390 cpl_propertylist *plist;
391 cpl_table *lctab;
392 parquet *p;
393
394 /* Inherited status */
395
396 if (*status != CASU_OK)
397 return(*status);
398
399 /* Do we even need to be here? */
400
401 if (cpl_propertylist_has(casu_fits_get_ehu(infile),"ESO DRS LINCOR"))
402 return(*status);
403
404 /* Open the parquet structure for the channel table */
405
406 lctab = casu_tfits_get_table(lchantab);
407 retval = vircam_chan_fill(lctab,&p,&np);
408 if (retval != CASU_OK)
409 return(retval);
410
411 /* Get the data array for the image */
412
413 data = cpl_image_get_data(casu_fits_get_image(infile));
414 if (data == NULL) {
415 vircam_chan_free(np,&p);
416 cpl_msg_error(fctid,"Error mapping data in input image");
417 FATAL_ERROR
418 }
419 naxis[0] = (long)cpl_image_get_size_x(casu_fits_get_image(infile));
420 naxis[1] = (long)cpl_image_get_size_y(casu_fits_get_image(infile));
421
422 /* Get the required parameters from the header */
423
424 plist = casu_fits_get_ehu(infile);
425 if (vircam_pfits_get_exptime(plist,&texp) != CASU_OK) {
426 vircam_chan_free(np,&p);
427 cpl_msg_error(fctid,"No exposure time in %s",
428 casu_fits_get_fullname(infile));
429 FATAL_ERROR
430 }
431 if (vircam_pfits_get_mindit(plist,&reset_time) != CASU_OK) {
432 vircam_chan_free(np,&p);
433 cpl_msg_error(fctid,"No mindit time in %s",
434 casu_fits_get_fullname(infile));
435 FATAL_ERROR
436 }
437 read_time = reset_time;
438 if (vircam_pfits_get_ditdelay(plist,&delay_time) != CASU_OK) {
439 vircam_chan_free(np,&p);
440 cpl_msg_error(fctid,"No dit delay time in %s",
441 casu_fits_get_fullname(infile));
442 FATAL_ERROR
443 }
444
445 /* If there is a constant k factor, then calculate it now */
446
447 kfac_nom = (double)(read_time/texp);
448
449 /* Factor to take the number of DITs into account */
450
451 dnd = (double)ndit;
452
453 /* Loop for each channel now */
454
455 for (i = 0; i < np; i++) {
456 pp = p + i;
457 ncpts = (pp->delta_i)*(pp->delta_j);
458
459 /* Load up the fit coefficients. If there is only one coefficient
460 this is by definition 1 and therefore we can skip this channel */
461
462 norder = pp->norder;
463 if (norder == 1)
464 continue;
465 lbb = pp->bb;
466
467 /* Get workspace for K array and fill it in for situation of constant
468 k factor */
469
470 kfacs = cpl_malloc(norder*sizeof(double));
471 if (kconst) {
472 kfacs[0] = 1.0;
473 for (ii = 1; ii < norder; ii++)
474 kfacs[ii] = pow((kfac_nom+1.0),(double)(ii+1)) -
475 pow(kfac_nom,(double)(ii+1));
476 }
477
478 /* Now for each pixel */
479
480 for (j = 0; j < ncpts; j++) {
481
482 /* Get the 'non-constant' k-factor and fill in the K array. */
483
484 rind = vircam_chan_d2r(pp,j);
485 aind = vircam_chan_r2a(pp,naxis,rind);
486 if (! kconst) {
487 lkfac = getkfac(j,ncpts,reset_time,read_time,delay_time,texp);
488 kfacs[0] = 1.0;
489 for (ii = 1; ii < norder; ii++)
490 kfacs[ii] = pow((lkfac+1.0),(double)(ii+1)) -
491 pow(lkfac,(double)(ii+1));
492 }
493
494 /* Calculate the linearised value now */
495
496 inval = ((double)data[aind])/dnd;
497 outval = linval(inval,kfacs,0.5,10,lbb,norder);
498 data[aind] = (float)(dnd*outval);
499 }
500 freespace(kfacs);
501 }
502
503 /* Add the linearity table to the DRS header */
504
505 cpl_propertylist_update_string(casu_fits_get_ehu(infile),
506 "ESO DRS LINCOR",
507 casu_tfits_get_filename(lchantab));
508
509 /* Right, get out of here */
510
511 vircam_chan_free(np,&p);
512 GOOD_STATUS
513}
514
515/*---------------------------------------------------------------------------*/
544/*---------------------------------------------------------------------------*/
545
546static double linval(double inval, double *kfacs, double tolerance,
547 int niter, double *b, int norder) {
548 int jj,iter;
549 double val_old,val,tol,sum;
550
551 val = inval;
552 iter = 0;
553 tol = tolerance + 1.0;
554 while (iter < niter && tol > tolerance) {
555 val_old = val;
556 iter++;
557 sum = 0.0;
558 for (jj = norder - 1; jj >= 1; jj--)
559 sum = (sum + b[jj]*kfacs[jj])*val;
560 sum *= val;
561 val = inval - sum;
562 tol = fabs(val - val_old);
563 if (val > 65535.0) {
564 val = 65535.0;
565 break;
566 } else if (val < -1000.0) {
567 val = -1000.0;
568 break;
569 }
570 }
571 return(val);
572}
573
574
575/*---------------------------------------------------------------------------*/
603/*---------------------------------------------------------------------------*/
604
605static double getkfac(long index, long npts, float reset_time,
606 float read_time, float delay_time, float exptime) {
607 double tkfac,dt1,dt2,dt3,dt4,df;
608
609 df = ((double)index/(double)npts);
610 dt1 = (double)exptime;
611 dt2 = (double)read_time;
612 dt3 = (double)reset_time;
613 dt4 = (double)delay_time;
614 tkfac = (dt3 + dt4 + (dt2 - dt3)*df)/dt1;
615 return(tkfac);
616}
617
618/*---------------------------------------------------------------------------*/
640/*---------------------------------------------------------------------------*/
641
642static void getco(double *a, int nord, int m) {
643 int i,j,start;
644
645 for (i = 0; i < nord; i++)
646 a[i] = 0.0;
647 start = m-1;
648 a[start] = 1.0;
649 j = 1;
650 for (i = start-1; i >= 0; i--) {
651 j++;
652 a[i] = a[i+1]*(double)(m - j + 2)/(double)(j-1);
653 }
654}
655
656
660/*
661
662$Log: not supported by cvs2svn $
663Revision 1.33 2010/06/03 12:15:31 jim
664A few mods to get rid of compiler warnings
665
666Revision 1.32 2009/12/11 06:53:35 jim
667Minor changes to documentation
668
669Revision 1.31 2009/09/09 09:45:36 jim
670Modified to speed things up
671
672Revision 1.30 2009/06/08 08:08:15 jim
673Fixed memory leak and changed clipping parameters to vircam_polyfit
674
675Revision 1.29 2009/05/20 12:18:42 jim
676Modified so that if the operation is already done, then it just returns
677
678Revision 1.28 2009/02/20 10:50:55 jim
679Removed superfluous declarations
680
681Revision 1.27 2008/10/21 08:38:48 jim
682Final estiamte on linearity error done with vircam_polyfit now
683
684Revision 1.26 2008/09/29 11:26:18 jim
685Modified linval to provide a lower limit in case the fit goes crazy
686
687Revision 1.25 2008/08/28 09:05:37 jim
688Fixed bug where QC was being duplicated from master BPM on rare occasions.
689Fixed bug where ARCFILE wasn't being written to the paf file for illum_cor
690tables. Sky combine is done with medians
691
692Revision 1.24 2008/01/22 19:45:24 jim
693New version of genlincur to take into account the equality of readout and
694reset time
695
696Revision 1.23 2007/11/26 09:57:14 jim
697Linearity correction routines now take account of ndit
698
699Revision 1.22 2007/11/22 12:36:15 jim
700Modified to return linearised values in an array
701
702Revision 1.21 2007/11/20 09:38:57 jim
703changed definition of fit quality to percentage error at 10000 counts
704
705Revision 1.20 2007/11/14 14:47:32 jim
706Modified the qualfit definition to be back in line with DRLD
707
708Revision 1.19 2007/11/14 12:34:21 jim
709Fixed header comments
710
711Revision 1.18 2007/11/14 10:48:29 jim
712Major rewrite to incorporate simpler and more robust algorithm
713
714Revision 1.17 2007/03/29 12:19:39 jim
715Little changes to improve documentation
716
717Revision 1.16 2007/03/01 12:42:41 jim
718Modified slightly after code checking
719
720Revision 1.15 2006/11/27 12:08:18 jim
721Modified lincor to get a better answer. Also modified the way the fit quality
722is calculated
723
724Revision 1.14 2006/09/29 11:19:31 jim
725changed aliases on parameter names
726
727Revision 1.13 2006/07/04 09:19:05 jim
728replaced all sprintf statements with snprintf
729
730Revision 1.12 2006/06/09 11:26:26 jim
731Small changes to keep lint happy
732
733Revision 1.11 2006/04/20 11:23:15 jim
734Now medians the data before accumulation in the event that k is constant.
735
736Revision 1.10 2006/03/23 21:18:47 jim
737Minor changes mainly to comment headers
738
739Revision 1.9 2006/03/22 13:36:50 jim
740cosmetic changes to keep lint happy
741
742Revision 1.8 2006/03/15 10:43:41 jim
743Fixed a few things
744
745Revision 1.7 2006/03/08 14:32:21 jim
746Lots of little modifications
747
748Revision 1.6 2006/03/03 14:29:46 jim
749Modified definition of vir_fits and channel table
750
751Revision 1.5 2006/03/01 10:31:28 jim
752Now uses new vir_fits objects
753
754Revision 1.4 2006/02/18 11:45:59 jim
755Fixed a couple of memory bugs
756
757Revision 1.3 2006/01/23 22:58:14 jim
758Added vircam_lincor module
759
760Revision 1.2 2006/01/23 16:06:03 jim
761Added in header comments and section to evaluate the goodness of fit
762
763Revision 1.1 2006/01/23 10:31:56 jim
764New file
765
766
767*/
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
char * casu_fits_get_fullname(casu_fits *p)
Definition: casu_fits.c:680
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
char * casu_tfits_get_filename(casu_tfits *p)
Definition: casu_tfits.c:510
cpl_table * casu_tfits_get_table(casu_tfits *p)
Definition: casu_tfits.c:364
int vircam_genlincur(double **fdata, int nimages, double *exps, double mindit, casu_tfits *chantab, int norder, cpl_table **lchantab, double **lindata, int *status)
Generate a linearity curve for each readout channel in a list of images.
int vircam_lincor(casu_fits *infile, casu_tfits *lchantab, int kconst, int ndit, int *status)
Apply linearity curves to data.
long vircam_chan_d2r(parquet *p, long l)
long vircam_chan_r2a(parquet *p, long naxis[2], long k)
int vircam_chan_fill(cpl_table *tab, parquet **p, long *np)
void vircam_chan_free(int np, parquet **p)
int vircam_pfits_get_ditdelay(const cpl_propertylist *plist, float *ditdelay)
Get the value of dit delay time.
Definition: vircam_pfits.c:279
int vircam_pfits_get_exptime(const cpl_propertylist *plist, float *exptime)
Get the value of exposure time.
Definition: vircam_pfits.c:245
int vircam_pfits_get_mindit(const cpl_propertylist *plist, float *mindit)
Get the value of mindit time.
Definition: vircam_pfits.c:262
int vircam_solve_gauss(double **a, double *b, int m)
Definition: vircam_utils.c:456
int vircam_polyfit(const cpl_array *xarray, const cpl_array *yarray, int ncoefs, int ilim, int niter, float lclip, float hclip, cpl_array **polycf, double *sigfit)
Definition: vircam_utils.c:571