VIRCAM Pipeline  2.3.10
vircam_linearity_analyse.c
1 /* $Id: vircam_linearity_analyse.c,v 1.65 2012-01-16 12:32:18 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2012-01-16 12:32:18 $
24  * $Revision: 1.65 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <stdio.h>
35 #include <cpl.h>
36 #include <math.h>
37 
38 #include <casu_utils.h>
39 #include <casu_mods.h>
40 #include <casu_stats.h>
41 
42 #include "vircam_utils.h"
43 #include "vircam_pfits.h"
44 #include "vircam_dfs.h"
45 #include "vircam_mods.h"
46 #include "vircam_channel.h"
47 #include "vircam_paf.h"
48 
49 /* Function prototypes */
50 
51 static int vircam_linearity_analyse_create(cpl_plugin *) ;
52 static int vircam_linearity_analyse_exec(cpl_plugin *) ;
53 static int vircam_linearity_analyse_destroy(cpl_plugin *) ;
54 static int vircam_linearity_analyse(cpl_parameterlist *, cpl_frameset *) ;
55 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
56  cpl_parameterlist *parlist);
57 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
58  cpl_parameterlist *parlist);
59 static int vircam_linearity_analyse_domedark_groups(void);
60 static double *vircam_linearity_analyse_genstat(casu_fits *fframe, int *bpm,
61  parquet *p, int np);
62 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
63  int nchan, double *facrng,
64  double *maxdiff);
65 static void vircam_mjdsort(double **fdata, double *mjd, int n);
66 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows);
67 static void vircam_linearity_analyse_init(void);
68 static void vircam_linearity_analyse_tidy(int level);
69 
70 /* Static global variables */
71 
72 static struct {
73 
74  /* Input */
75 
76  int norder;
77  float lthr;
78  float hthr;
79  int maxbpmfr;
80  int adjust;
81  int diagnostic;
82  int extenum;
83 
84  /* Output */
85 
86  float linearity;
87  float linerror;
88  float bad_pixel_stat;
89  int bad_pixel_num;
90  float facrng;
91  float maxdiff;
92 
93 } vircam_linearity_analyse_config;
94 
95 typedef struct {
96  cpl_frameset *darks;
97  cpl_frameset *domes;
98  int ndarks;
99  int ndomes;
100  float exptime;
101  unsigned char flag;
102  casu_fits **proc;
103 } ddgrp;
104 
105 #define OK_FLAG 0
106 #define SATURATE_FLAG 1
107 
108 #define SUBSET 128
109 #define SUBSET2 (SUBSET/2)
110 
111 static struct {
112  cpl_size *labels;
113  cpl_frameset *domelist;
114  int ndomes;
115  cpl_frameset *darklist;
116  int ndarks;
117  cpl_frameset *domecheck;
118  int ndomecheck;
119  cpl_frameset *darkcheck;
120  int ndarkcheck;
121  cpl_frame *inherit;
122  cpl_frame *chanfrm;
123  casu_tfits *chantab;
124  cpl_table *lchantab;
125  cpl_array *bpm_array;
126  ddgrp *ddg;
127  int nddg;
128  casu_fits **flatlist;
129  int nflatlist;
130  cpl_propertylist *plist;
131  cpl_propertylist *elist;
132  int nx;
133  int ny;
134  cpl_propertylist *phupaf;
135  cpl_table *diag1;
136  cpl_table *diag2;
137  int nfdata;
138  int nuse;
139 } ps;
140 
141 static int isfirst;
142 static int dummy;
143 static float sat;
144 static cpl_frame *product_frame_chantab = NULL;
145 static cpl_frame *product_frame_bpm = NULL;
146 static cpl_frame *product_frame_diag1 = NULL;
147 static cpl_frame *product_frame_diag2 = NULL;
148 
149 static char vircam_linearity_analyse_description[] =
150 "vircam_linearity_analyse -- VIRCAM linearity mapping recipe.\n\n"
151 "Form master dark images from the input raw frames and use these to\n"
152 "dark correct a series of dome flat exposures Using the dark\n"
153 "corrected dome flat series, work out linearity coefficients for\n"
154 "each data channel. The program expects the following files in the SOF\n"
155 " Tag Description\n"
156 " -----------------------------------------------------------------------\n"
157 " %-21s A list of raw dome flat images\n"
158 " %-21s A list of raw dark images\n"
159 " %-21s The channel table\n"
160 " %-21s A list of raw dome flat images at the monitor exposure time\n"
161 " %-21s A list of raw dark images at the monitor exposure time\n"
162 "The first three of these are required. The last two are only required if"
163 "the light source monitoring algorithm is to be used"
164 "\n";
165 
287 /* Function code */
288 
289 /*---------------------------------------------------------------------------*/
297 /*---------------------------------------------------------------------------*/
298 
299 int cpl_plugin_get_info(cpl_pluginlist *list) {
300  cpl_recipe *recipe = cpl_calloc(1,sizeof(*recipe));
301  cpl_plugin *plugin = &recipe->interface;
302  char alldesc[SZ_ALLDESC];
303  (void)snprintf(alldesc,SZ_ALLDESC,vircam_linearity_analyse_description,
304  VIRCAM_LIN_DOME_RAW,VIRCAM_LIN_DARK_RAW,
305  VIRCAM_CAL_CHANTAB_INIT,VIRCAM_LIN_DOME_CHECK,
306  VIRCAM_LIN_DARK_CHECK);
307 
308  cpl_plugin_init(plugin,
309  CPL_PLUGIN_API,
310  VIRCAM_BINARY_VERSION,
311  CPL_PLUGIN_TYPE_RECIPE,
312  "vircam_linearity_analyse",
313  "VIRCAM linearity analysis recipe",
314  alldesc,
315  "Jim Lewis",
316  "jrl@ast.cam.ac.uk",
318  vircam_linearity_analyse_create,
319  vircam_linearity_analyse_exec,
320  vircam_linearity_analyse_destroy);
321 
322  cpl_pluginlist_append(list,plugin);
323 
324  return(0);
325 }
326 
327 /*---------------------------------------------------------------------------*/
336 /*---------------------------------------------------------------------------*/
337 
338 static int vircam_linearity_analyse_create(cpl_plugin *plugin) {
339  cpl_recipe *recipe;
340  cpl_parameter *p;
341 
342  /* Get the recipe out of the plugin */
343 
344  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
345  recipe = (cpl_recipe *)plugin;
346  else
347  return(-1);
348 
349  /* Create the parameters list in the cpl_recipe object */
350 
351  recipe->parameters = cpl_parameterlist_new();
352 
353  /* Fill in the parameters. First the polynomial order */
354 
355  p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.norder",
356  CPL_TYPE_INT,
357  "Order of polynomial fit",
358  "vircam.vircam_linearity_analyse",
359  4,1,6);
360  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"nord");
361  cpl_parameterlist_append(recipe->parameters,p);
362 
363  /* The lower threshold */
364 
365  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.lthr",
366  CPL_TYPE_DOUBLE,
367  "Lower bad pixel threshold",
368  "vircam.vircam_linearity_analyse",8.0);
369  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"lthr");
370  cpl_parameterlist_append(recipe->parameters,p);
371 
372  /* The upper threshold */
373 
374  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.hthr",
375  CPL_TYPE_DOUBLE,
376  "Upper bad pixel threshold",
377  "vircam.vircam_linearity_analyse",8.0);
378  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"hthr");
379  cpl_parameterlist_append(recipe->parameters,p);
380 
381  /* The maximum number of frames to be used in forming the bad pixel mask */
382 
383  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.maxbpmfr",
384  CPL_TYPE_INT,
385  "Maximum # frames used in bpm analysis",
386  "vircam.vircam_linearity_analyse",10);
387  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"maxbpmfr");
388  cpl_parameterlist_append(recipe->parameters,p);
389 
390  /* The flag to allow statistics of the dome flat frames to be adjusted
391  using a set of monitor exposures */
392 
393  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.adjust",
394  CPL_TYPE_BOOL,
395  "Adjust stats with monitor set",
396  "vircam.vircam_linearity_analyse",1);
397  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"adjust");
398  cpl_parameterlist_append(recipe->parameters,p);
399 
400  /* The flag to allow diagnostic curves to be written out */
401 
402  p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.diagnostic",
403  CPL_TYPE_BOOL,
404  "Write out diagnostic tables",
405  "vircam.vircam_linearity_analyse",0);
406  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"diagnostic");
407  cpl_parameterlist_append(recipe->parameters,p);
408 
409  /* Extension number of input frames to use */
410 
411  p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.extenum",
412  CPL_TYPE_INT,
413  "Extension number to be done, 0 == all",
414  "vircam.vircam_linearity_analyse",
415  0,0,16);
416  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
417  cpl_parameterlist_append(recipe->parameters,p);
418 
419  /* Get out of here */
420 
421  return(0);
422 }
423 
424 /*---------------------------------------------------------------------------*/
430 /*---------------------------------------------------------------------------*/
431 
432 static int vircam_linearity_analyse_exec(cpl_plugin *plugin) {
433  cpl_recipe *recipe;
434 
435  /* Get the recipe out of the plugin */
436 
437  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
438  recipe = (cpl_recipe *)plugin;
439  else
440  return(-1);
441 
442  return(vircam_linearity_analyse(recipe->parameters,recipe->frames));
443 }
444 
445 /*---------------------------------------------------------------------------*/
451 /*---------------------------------------------------------------------------*/
452 
453 static int vircam_linearity_analyse_destroy(cpl_plugin *plugin) {
454  cpl_recipe *recipe ;
455 
456  /* Get the recipe out of the plugin */
457 
458  if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
459  recipe = (cpl_recipe *)plugin;
460  else
461  return(-1);
462 
463  cpl_parameterlist_delete(recipe->parameters);
464  return(0);
465 }
466 
467 /*---------------------------------------------------------------------------*/
474 /*---------------------------------------------------------------------------*/
475 
476 static int vircam_linearity_analyse(cpl_parameterlist *parlist,
477  cpl_frameset *framelist) {
478  const char *fctid="vircam_linearity_analyse";
479  int i,jst,jfn,j,status,k,nbad,ngood,krem,n;
480  int ndarks,ndomes,kk,live,ngood_flats,*bpm,nalloc,adjust,ndit;
481  cpl_size nlab;
482  long np;
483  float med,mindit,*exps,badfrac,expt;
484  unsigned char *rejmask,*rejplus;
485  double *dexps,**fdata,*d,*mjds,*mjdcheck,mjd,**cdata,*cf,fac;
486  double facrng,maxdiff,*lindata;
487  casu_fits **darks,**domes,*test,*outdark,*fframe;
488  parquet *pp;
489  cpl_parameter *p;
490  cpl_propertylist *drs,*plist;
491  cpl_image *outimage;
492  cpl_frame *frame;
493 
494  /* Check validity of input frameset */
495 
496  if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
497  cpl_msg_error(fctid,"Input framelist NULL or has no input data");
498  return(-1);
499  }
500 
501  /* Initialise a few things */
502 
503  vircam_linearity_analyse_init();
504 
505  /* Get the parameters */
506 
507  p = cpl_parameterlist_find(parlist,
508  "vircam.vircam_linearity_analyse.norder");
509  vircam_linearity_analyse_config.norder = cpl_parameter_get_int(p);
510  p = cpl_parameterlist_find(parlist,
511  "vircam.vircam_linearity_analyse.lthr");
512  vircam_linearity_analyse_config.lthr = (float)cpl_parameter_get_double(p);
513  p = cpl_parameterlist_find(parlist,
514  "vircam.vircam_linearity_analyse.hthr");
515  vircam_linearity_analyse_config.hthr = (float)cpl_parameter_get_double(p);
516  p = cpl_parameterlist_find(parlist,
517  "vircam.vircam_linearity_analyse.maxbpmfr");
518  vircam_linearity_analyse_config.maxbpmfr = cpl_parameter_get_int(p);
519  p = cpl_parameterlist_find(parlist,
520  "vircam.vircam_linearity_analyse.adjust");
521  vircam_linearity_analyse_config.adjust = cpl_parameter_get_bool(p);
522  p = cpl_parameterlist_find(parlist,
523  "vircam.vircam_linearity_analyse.diagnostic");
524  vircam_linearity_analyse_config.diagnostic = cpl_parameter_get_bool(p);
525  p = cpl_parameterlist_find(parlist,
526  "vircam.vircam_linearity_analyse.extenum");
527  vircam_linearity_analyse_config.extenum = cpl_parameter_get_int(p);
528 
529  /* Sort out raw from calib frames */
530 
531  if (vircam_dfs_set_groups(framelist) != CASU_OK) {
532  cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
533  vircam_linearity_analyse_tidy(2);
534  return(-1);
535  }
536 
537  /* Get framelist labels */
538 
539  if ((ps.labels = cpl_frameset_labelise(framelist,casu_compare_tags,
540  &nlab)) == NULL) {
541  cpl_msg_error(fctid,"Cannot labelise the input frames");
542  vircam_linearity_analyse_tidy(2);
543  return(-1);
544  }
545 
546  /* Get the dome flat frames */
547 
548  if ((ps.domelist = casu_frameset_subgroup(framelist,ps.labels,nlab,
549  VIRCAM_LIN_DOME_RAW)) == NULL) {
550  cpl_msg_error(fctid,"Cannot find dome flat frames in input frameset");
551  vircam_linearity_analyse_tidy(2);
552  return(-1);
553  }
554  ps.ndomes = cpl_frameset_get_size(ps.domelist);
555  ps.inherit = cpl_frameset_get_position(ps.domelist,0);
556 
557  /* Check to make sure that NDIT == 1 */
558 
559  plist = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_position(ps.domelist,0)),0);
560  (void)vircam_pfits_get_ndit(plist,&ndit);
561  freepropertylist(plist);
562  if (ndit != 1) {
563  cpl_msg_error(fctid,
564  "NDIT=%" CPL_SIZE_FORMAT ". Recipe requires that ndit == 1",
565  (cpl_size)ndit);
566  vircam_linearity_analyse_tidy(2);
567  return(-1);
568  }
569 
570  /* Get the dark frames */
571 
572  if ((ps.darklist = casu_frameset_subgroup(framelist,ps.labels,nlab,
573  VIRCAM_LIN_DARK_RAW)) == NULL) {
574  cpl_msg_error(fctid,"Cannot find dark frames in input frameset");
575  vircam_linearity_analyse_tidy(2);
576  return(-1);
577  }
578  ps.ndarks = cpl_frameset_get_size(ps.darklist);
579 
580  /* If you are planning to adjust the stats for the dome frames by using
581  a monitor exposure set, then read that frameset in now */
582 
583  if (vircam_linearity_analyse_config.adjust) {
584  if ((ps.domecheck = casu_frameset_subgroup(framelist,ps.labels,nlab,
585  VIRCAM_LIN_DOME_CHECK)) == NULL) {
586  cpl_msg_info(fctid,"No monitor frames found in sof. No adjustments made to stats");
587  vircam_linearity_analyse_config.adjust = 0;
588  ps.ndomecheck = 0;
589  } else {
590  ps.ndomecheck = cpl_frameset_get_size(ps.domecheck);
591  if ((ps.darkcheck = casu_frameset_subgroup(framelist,ps.labels,
592  nlab,VIRCAM_LIN_DARK_CHECK)) == NULL) {
593  cpl_msg_info(fctid,"No darks for monitor frames found in sof. No adjustments made to stats");
594  vircam_linearity_analyse_config.adjust = 0;
595  ps.ndomecheck = 0;
596  freeframeset(ps.domecheck);
597  ps.ndarkcheck = 0;
598  } else {
599  ps.ndarkcheck = cpl_frameset_get_size(ps.darkcheck);
600  }
601  }
602  }
603 
604  /* Check to see if there is a channel table. If so, then read it */
605 
606  if ((ps.chanfrm = casu_frameset_subgroup_1(framelist,ps.labels,nlab,
607  VIRCAM_CAL_CHANTAB_INIT)) == NULL) {
608  cpl_msg_error(fctid,"No initial channel table found");
609  vircam_linearity_analyse_tidy(2);
610  return(-1);
611  }
612 
613  /* Group the domes and darks by exposure time */
614 
615  if (vircam_linearity_analyse_domedark_groups() != 0) {
616  vircam_linearity_analyse_tidy(2);
617  return(-1);
618  }
619 
620  /* See if the number of exposure times is too small for the order of the
621  polynomial you want to fit. If it is, the adjust the order */
622 
623  if (ps.nddg < vircam_linearity_analyse_config.norder+1) {
624  cpl_msg_warning(fctid,
625  "Number of exposure times is too small: %" CPL_SIZE_FORMAT ", order: %" CPL_SIZE_FORMAT "\nTaking fit down to order %" CPL_SIZE_FORMAT,
626  (cpl_size)(ps.nddg),
627  (cpl_size)(vircam_linearity_analyse_config.norder),
628  (cpl_size)(ps.nddg-1));
629  vircam_linearity_analyse_config.norder = ps.nddg - 1;
630  }
631 
632  /* Now, how many image extensions do we want to do? If the extension
633  number is zero, then we loop for all possible extensions. If it
634  isn't then we just do the extension specified */
635 
636  vircam_exten_range(vircam_linearity_analyse_config.extenum,
637  (const cpl_frame *)cpl_frameset_get_position(ps.ddg[0].darks,0),
638  &jst,&jfn);
639  if (jst == -1 || jfn == -1) {
640  cpl_msg_error(fctid,"Unable to continue");
641  vircam_linearity_analyse_tidy(2);
642  return(-1);
643  }
644 
645  /* Now loop for all the extensions and do the BPM analysis... */
646 
647  for (j = jst; j <= jfn; j++) {
648  cpl_msg_info(fctid,"Beginning BPM work on extension %" CPL_SIZE_FORMAT,
649  (cpl_size)j);
650  isfirst = (j == jst);
651  dummy = 0;
652  vircam_linearity_analyse_config.bad_pixel_stat = 0.0;
653  vircam_linearity_analyse_config.bad_pixel_num = 0;
654  vircam_linearity_analyse_config.linerror = 0.0;
655  vircam_linearity_analyse_config.linearity = 0.0;
656  vircam_linearity_analyse_config.facrng = 0.0;
657  vircam_linearity_analyse_config.maxdiff = 0.0;
658 
659  /* Get some standard info in case we need it for dummy products */
660 
661  test = casu_fits_load(cpl_frameset_get_position(ps.ddg[0].domes,0),
662  CPL_TYPE_FLOAT,j);
663  ps.plist = cpl_propertylist_duplicate(casu_fits_get_phu(test));
664  ps.elist = cpl_propertylist_duplicate(casu_fits_get_ehu(test));
665  ps.nx = (int)cpl_image_get_size_x(casu_fits_get_image(test));
666  ps.ny = (int)cpl_image_get_size_y(casu_fits_get_image(test));
667  casu_fits_delete(test);
668 
669  /* Load up the channel table for this detector and verify it. Get
670  saturation level for this detector from FITS header */
671 
672  ps.chantab = casu_tfits_load(ps.chanfrm,j);
673  if (ps.chantab == NULL) {
674  cpl_msg_error(fctid,
675  "Channel table extension %" CPL_SIZE_FORMAT " failed to load",
676  (cpl_size)j);
677  dummy = 1;
678  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
679  return(-1);
680  continue;
681  } else if (vircam_chantab_verify(casu_tfits_get_table(ps.chantab))
682  != CASU_OK) {
683  cpl_msg_error(fctid,
684  "Channel table extension %" CPL_SIZE_FORMAT " has errors",
685  (cpl_size)j);
686  dummy = 1;
687  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
688  return(-1);
689  continue;
690  }
692  &sat) != CASU_OK) {
693  cpl_msg_error(fctid,
694  "Channel table extension header %" CPL_SIZE_FORMAT " missing saturation info",
695  (cpl_size)j);
696  dummy = 1;
697  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
698  return(-1);
699  vircam_linearity_analyse_tidy(1);
700  continue;
701  }
702 
703  /* Get the channel structure */
704 
705  vircam_chan_fill(casu_tfits_get_table(ps.chantab),&pp,&np);
706 
707  /* If doing diagnostics, then create the tables now */
708 
709  if (vircam_linearity_analyse_config.diagnostic) {
710  ps.diag1 = vircam_linearity_analyse_diagtab_init(np,ps.ndomes);
711  if (vircam_linearity_analyse_config.adjust)
712  ps.diag2 = vircam_linearity_analyse_diagtab_init(np,ps.ndomecheck);
713  }
714 
715  /* Check DETLIVE for this extension */
716 
717  if (vircam_pfits_get_detlive((const cpl_propertylist *)ps.elist,&live)
718  != CASU_OK) {
719  cpl_msg_error(fctid,"No DET LIVE keyword in this extension");
720  dummy = 1;
721  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
722  return(-1);
723  vircam_linearity_analyse_tidy(1);
724  vircam_chan_free(np,&pp);
725  continue;
726  }
727  if (! live) {
728  cpl_msg_info(fctid,"Detector flagged dead");
729  dummy = 1;
730  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
731  return(-1);
732  vircam_linearity_analyse_tidy(1);
733  vircam_chan_free(np,&pp);
734  continue;
735  }
736 
737  /* Get the value of MINDIT so that you can work out the rough level
738  of the flats before the reset frame was subtracted off */
739 
740  if (vircam_pfits_get_mindit((const cpl_propertylist *)ps.elist,
741  &mindit) != CASU_OK) {
742  cpl_msg_error(fctid,
743  "No value of MINDIT found in extension %" CPL_SIZE_FORMAT,
744  (cpl_size)j);
745  dummy = 1;
746  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
747  return(-1);
748  vircam_chan_free(np,&pp);
749  continue;
750  }
751 
752  /* Get a representative from each exposure group and check if it
753  is saturated. If it is, then reject the group from further
754  analysis */
755 
756  ngood = 0;
757  exps = cpl_malloc(ps.nddg*sizeof(float));
758  ngood_flats = 0;
759  for (i = 0; i < ps.nddg; i++) {
760  test = casu_fits_load(cpl_frameset_get_position(ps.ddg[i].domes,0),
761  CPL_TYPE_FLOAT,j);
762  med = cpl_image_get_median((const cpl_image*)casu_fits_get_image(test));
763  med *= (1.0 + mindit/ps.ddg[i].exptime);
764  if (med > sat) {
765  ps.ddg[i].flag = SATURATE_FLAG;
766  } else {
767  ngood++;
768  exps[ngood-1] = ps.ddg[i].exptime;
769  ngood_flats += ps.ddg[i].ndomes;
770  ps.ddg[i].flag = OK_FLAG;
771  }
772  casu_fits_delete(test);
773  }
774  exps = cpl_realloc(exps,ngood*sizeof(float));
775 
776  /* Are there enough non-saturated exposures for linearity fit? */
777 
778  if (ngood < vircam_linearity_analyse_config.norder+1) {
779  cpl_msg_info(fctid,
780  "Too few unsaturated flats for linearity fit for extension %" CPL_SIZE_FORMAT,
781  (cpl_size)j);
782  dummy = 1;
783  cpl_free(exps);
784  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
785  return(-1);
786  vircam_chan_free(np,&pp);
787  continue;
788  }
789 
790  /* Sort the exposure array */
791 
792  casu_sort(&exps,ngood,1);
793 
794  /* Loop for each exposure time. When you have enough flats, then
795  you can quit this loop */
796 
797  ps.nuse = min(vircam_linearity_analyse_config.maxbpmfr,ngood_flats);
798  ps.nflatlist = 0;
799  ps.flatlist = cpl_malloc(ps.nuse*sizeof(casu_fits *));
800  for (i = ngood-1; i >= 0; i--) {
801  krem = -1;
802  for (k = 0; k <= ps.nddg; k++) {
803  if (ps.ddg[k].exptime == exps[i]) {
804  krem = k;
805  break;
806  }
807  }
808 
809  /* Load the dark frames from this exposure time */
810 
811  darks = casu_fits_load_list(ps.ddg[krem].darks,CPL_TYPE_FLOAT,j);
812  ndarks = ps.ddg[krem].ndarks;
813  if (darks == NULL) {
814  cpl_msg_error(fctid,
815  "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
816  (cpl_size)j,ps.ddg[krem].exptime);
817  continue;
818  }
819 
820  /* Form a mean dark for this exposure time. If there is only one
821  then don't bother, just load up that one frame. */
822 
823  if (ndarks == 1) {
824  outdark = casu_fits_duplicate(darks[0]);
825  } else {
826  status = CASU_OK;
827  (void)casu_imcombine(darks,NULL,ndarks,1,1,1,5.0,"EXPTIME",
828  &outimage,NULL,&rejmask,&rejplus,&drs,
829  &status);
830  freespace(rejmask);
831  freespace(rejplus);
832  freepropertylist(drs);
833  if (status != CASU_OK) {
834  cpl_msg_error(fctid,
835  "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
836  (cpl_size)j,ps.ddg[krem].exptime);
837  freefitslist(darks,ndarks);
838  continue;
839  }
840  outdark = casu_fits_wrap(outimage,darks[0],NULL,NULL);
841  }
842  freefitslist(darks,ndarks);
843 
844  /* Load the flats for this group */
845 
846  domes = casu_fits_load_list(ps.ddg[krem].domes,CPL_TYPE_FLOAT,j);
847  ndomes = ps.ddg[krem].ndomes;
848  if (domes == NULL) {
849  cpl_msg_error(fctid,
850  "Error loading domes extension %" CPL_SIZE_FORMAT ", exptime %g",
851  (cpl_size)j,ps.ddg[i].exptime);
852  freefits(outdark);
853  continue;
854  }
855 
856  /* Now loop for each flat in this group or until you have
857  filled the flats buffer */
858 
859  for (kk = 0; kk < ndomes; kk++) {
860  status = CASU_OK;
861  casu_darkcor(domes[kk],outdark,1.0,&status);
862  ps.flatlist[ps.nflatlist] = casu_fits_duplicate(domes[kk]);
863  ps.ddg[krem].proc[kk] = ps.flatlist[ps.nflatlist];
864  ps.nflatlist++;
865  if (ps.nflatlist == ps.nuse)
866  break;
867  }
868 
869  /* Tidy up a bit */
870 
871  freefitslist(domes,ndomes);
872  freefits(outdark);
873 
874  /* Do we have enough yet? */
875 
876  if (ps.nflatlist == ps.nuse)
877  break;
878  }
879  freespace(exps);
880  ps.flatlist = cpl_realloc(ps.flatlist,
881  (ps.nflatlist)*sizeof(casu_fits *));
882 
883  /* Generate a bad pixel mask now */
884 
885  status = CASU_OK;
886  (void)casu_genbpm(ps.flatlist,ps.nflatlist,NULL,
887  vircam_linearity_analyse_config.lthr,
888  vircam_linearity_analyse_config.hthr,"EXPTIME",
889  &(ps.bpm_array),&nbad,&badfrac,&status);
890  bpm = cpl_array_get_data_int(ps.bpm_array);
891 
892  /* Store away some useful info */
893 
894  vircam_linearity_analyse_config.bad_pixel_num = nbad;
895  vircam_linearity_analyse_config.bad_pixel_stat = badfrac;
896 
897  /* Right. Free the pointer for flatlist, but don't delete the
898  casu_fits structures in the list because they are copied into
899  the ddg structure */
900 
901  freespace(ps.flatlist);
902  ps.nflatlist = 0;
903 
904  /* Get an initial allocation of space to hold the stats */
905 
906  nalloc = 16;
907  fdata = cpl_malloc(nalloc*sizeof(double *));
908  dexps = cpl_malloc(nalloc*sizeof(double));
909  mjds = cpl_malloc(nalloc*sizeof(double));
910 
911  /* Loop through the ddg structure, missing out any that have
912  overexposed flats. Work out the stats for the images that
913  remain. */
914 
915  cpl_msg_info(fctid,
916  "Beginning linearity work on extension %" CPL_SIZE_FORMAT,
917  (cpl_size)j);
918  ps.nfdata = 0;
919  outdark = NULL;
920  for (i = 0; i < ps.nddg; i++) {
921  if (ps.ddg[i].flag == SATURATE_FLAG)
922  continue;
923  for (k = 0; k < ps.ddg[i].ndomes; k++) {
924 
925  /* If this particular frame wasn't processed, then you need
926  to form a dark for this group. */
927 
928  if (ps.ddg[i].proc[k] == NULL) {
929  if (outdark == NULL) {
930 
931  /* Load the dark frames from this exposure time */
932 
933  darks = casu_fits_load_list(ps.ddg[i].darks,
934  CPL_TYPE_FLOAT,j);
935  ndarks = ps.ddg[i].ndarks;
936  if (darks == NULL) {
937  cpl_msg_error(fctid,
938  "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
939  (cpl_size)j,ps.ddg[i].exptime);
940  continue;
941  }
942 
943  /* Form a mean dark for this exposure time. If there
944  is only one then don't bother, just load up that
945  one frame. */
946 
947  if (ps.ddg[i].ndarks == 1) {
948  outdark = casu_fits_duplicate(darks[0]);
949  } else {
950  status = CASU_OK;
951  (void)casu_imcombine(darks,NULL,ndarks,1,1,1,5.0,
952  "EXPTIME",&outimage,NULL,
953  &rejmask,&rejplus,
954  &drs,&status);
955  freespace(rejmask);
956  freespace(rejplus);
957  freepropertylist(drs);
958  if (status != CASU_OK) {
959  cpl_msg_error(fctid,
960  "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
961  (cpl_size)j,ps.ddg[i].exptime);
962  freefitslist(darks,ndarks);
963  continue;
964  }
965  outdark = casu_fits_wrap(outimage,darks[0],
966  NULL,NULL);
967  }
968  freefitslist(darks,ndarks);
969  }
970 
971  /* Load the flat and dark correct it */
972 
973  frame = cpl_frameset_get_position(ps.ddg[i].domes,k);
974  fframe = casu_fits_load(frame,CPL_TYPE_FLOAT,j);
975  casu_darkcor(fframe,outdark,1.0,&status);
976 
977  /* If this frame has already been corrected, then use it */
978 
979  } else {
980  fframe = ps.ddg[i].proc[k];
981  }
982 
983  /* Generate the stats for this frame and store it */
984 
985  d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
986  if (ps.nfdata >= nalloc) {
987  nalloc += 16;
988  fdata = cpl_realloc(fdata,nalloc*sizeof(double *));
989  dexps = cpl_realloc(dexps,nalloc*sizeof(double));
990  mjds = cpl_realloc(mjds,nalloc*sizeof(double));
991  }
992  (void)vircam_pfits_get_mjd(casu_fits_get_phu(fframe),&mjd);
993  mjds[ps.nfdata] = mjd;
994  dexps[ps.nfdata] = (double)(ps.ddg[i].exptime);
995  fdata[ps.nfdata] = d;
996 
997  /* If doing diagnostic curves, then add the relevant
998  information to the first table now */
999 
1000  if (ps.diag1 != NULL) {
1001  cpl_table_set_string(ps.diag1,"filename",
1002  (cpl_size)(ps.nfdata),
1003  casu_fits_get_filename(fframe));
1004  cpl_table_set_double(ps.diag1,"exptime",
1005  (cpl_size)ps.nfdata,
1006  dexps[ps.nfdata]);
1007  cpl_table_set_double(ps.diag1,"mjd",(cpl_size)(ps.nfdata),
1008  mjd);
1009  for (n = 1; n <= np; n++) {
1010  char * colname = cpl_sprintf("rawflux_%02d", n);
1011  // snprintf(colname,16,"rawflux_%02d", n);
1012  cpl_table_set_double(ps.diag1,colname,
1013  (cpl_size)(ps.nfdata),d[n-1]);
1014  cpl_free(colname);
1015  }
1016  }
1017  if (ps.ddg[i].proc[k] != NULL) {
1018  freefits(ps.ddg[i].proc[k]);
1019  } else {
1020  freefits(fframe);
1021  }
1022  ps.nfdata++;
1023  }
1024  freefits(outdark);
1025  }
1026  freefits(outdark);
1027  if (ps.diag1 != NULL)
1028  cpl_table_set_size(ps.diag1,(cpl_size)(ps.nfdata));
1029 
1030  /* Now, if we are going to tweak the stats using the monitor exposure
1031  then we should do that now */
1032 
1033  if (vircam_linearity_analyse_config.adjust) {
1034 
1035  /* Get some workspace for the data array */
1036 
1037  cdata = cpl_malloc(ps.ndomecheck*sizeof(double *));
1038 
1039  /* Get the exposure time for the monitor set and make sure
1040  that the set isn't saturated */
1041 
1042  test = casu_fits_load(cpl_frameset_get_position(ps.domecheck,0),
1043  CPL_TYPE_FLOAT,j);
1044  (void)vircam_pfits_get_exptime(casu_fits_get_phu(test),&expt);
1045  med = cpl_image_get_median((const cpl_image*)casu_fits_get_image(test));
1046  med *= (1.0 + mindit/expt);
1047  adjust = 1;
1048  if (med > sat) {
1049  cpl_msg_info(fctid,"Monitor exposures saturated. No drift adjustment made");
1050  adjust = 0;
1051  }
1052  casu_fits_delete(test);
1053 
1054  /* Ok assuming all that's going well, then form a mean dark */
1055 
1056  if (adjust) {
1057 
1058  darks = casu_fits_load_list(ps.darkcheck,CPL_TYPE_FLOAT,j);
1059  ndarks = ps.ndarkcheck;
1060  if (darks == NULL) {
1061  cpl_msg_error(fctid,
1062  "Error loading check darks extension %" CPL_SIZE_FORMAT,
1063  (cpl_size)j);
1064  continue;
1065  }
1066 
1067  /* Form a mean dark for this exposure time. If there
1068  is only one then don't bother, just load up that
1069  one frame. */
1070 
1071  if (ndarks == 1) {
1072  outdark = casu_fits_duplicate(darks[0]);
1073  } else {
1074  status = CASU_OK;
1075  (void)casu_imcombine(darks,NULL,ndarks,1,1,1,5.0,"EXPTIME",
1076  &outimage,NULL,&rejmask,&rejplus,
1077  &drs,&status);
1078  freespace(rejmask);
1079  freespace(rejplus);
1080  freepropertylist(drs);
1081  if (status != CASU_OK) {
1082  cpl_msg_error(fctid,
1083  "Combine failure extension %" CPL_SIZE_FORMAT " monitor",
1084  (cpl_size)j);
1085  freefitslist(darks,ndarks);
1086  continue;
1087  }
1088  outdark = casu_fits_wrap(outimage,darks[0],
1089  NULL,NULL);
1090  }
1091  freefitslist(darks,ndarks);
1092 
1093  /* Now, loop through the monitor domes, dark correct and then
1094  do the stats */
1095 
1096  mjdcheck = cpl_malloc(ps.ndomecheck*sizeof(double));
1097  for (i = 0; i < ps.ndomecheck; i++) {
1098  frame = cpl_frameset_get_position(ps.domecheck,i);
1099  fframe = casu_fits_load(frame,CPL_TYPE_FLOAT,j);
1100  casu_darkcor(fframe,outdark,1.0,&status);
1101  d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
1102  cdata[i] = d;
1105  mjdcheck[i] = mjd;
1106 
1107  /* If doing diagnostics then fill in the table now */
1108 
1109  if (ps.diag2 != NULL) {
1110  cpl_table_set_string(ps.diag2,"filename",(cpl_size)i,
1111  casu_fits_get_filename(fframe));
1112  cpl_table_set_double(ps.diag2,"exptime",
1113  (cpl_size)i,(double)expt);
1114  cpl_table_set_double(ps.diag2,"mjd",
1115  (cpl_size)i,mjd);
1116  for (n = 1; n <= np; n++) {
1117  char * colname = cpl_sprintf("rawflux_%02d", n);
1118  // snprintf(colname,16,"rawflux_%02d", n);
1119  cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
1120  d[n-1]);
1121  cpl_free(colname);
1122  colname = cpl_sprintf("linflux_%02d", n);
1123  //snprintf(colname,16,"linflux_%02d", n);
1124  cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
1125  d[n-1]);
1126  cpl_free(colname);
1127  }
1128  }
1129  freefits(fframe);
1130  }
1131  freefits(outdark);
1132 
1133  /* Generate the correction factors now */
1134 
1135  cf = vircam_linearity_tweakfac(cdata,mjdcheck,ps.ndomecheck,
1136  np,&facrng,&maxdiff);
1137  vircam_linearity_analyse_config.facrng = 100.0*(float)facrng;
1138  vircam_linearity_analyse_config.maxdiff = 100.0*(float)maxdiff;
1139  if (ps.diag2 != NULL) {
1140  for (i = 0; i < ps.ndomecheck; i++)
1141  cpl_table_set_double(ps.diag2,"adjust_fac",
1142  (cpl_size)i,cf[i]);
1143  }
1144 
1145  /* Ok, now do the correction for each of the linearity
1146  sequence frames */
1147 
1148  for (i = 0; i < ps.nfdata; i++) {
1149  mjd = mjds[i];
1150  krem = -1;
1151  for (k = 0; k < ps.ndomecheck; k++) {
1152  if (mjd < mjdcheck[k]) {
1153  krem = k;
1154  break;
1155  }
1156  }
1157  if (krem == -1) {
1158  fac = cf[ps.ndomecheck-1];
1159  } else if (krem == 0) {
1160  fac = cf[0];
1161  } else {
1162  fac = 0.5*(cf[krem -1] + cf[krem]);
1163  }
1164  for (k = 0; k < np; k++)
1165  fdata[i][k] /= fac;
1166  if (ps.diag1 != NULL)
1167  cpl_table_set_double(ps.diag1,"adjust_fac",
1168  (cpl_size)i,fac);
1169  }
1170 
1171  /* Get rid of some stuff now */
1172 
1173  freespace2(cdata,ps.ndomecheck);
1174  freespace(cf);
1175  freespace(mjdcheck);
1176  }
1177 
1178  cpl_free(cdata);
1179  }
1180 
1181  /* Do some intermediate tidying */
1182 
1183  freespace(mjds);
1184  vircam_chan_free(np,&pp);
1185 
1186  /* Right, there should be no images left in memory now. Do the
1187  linearity analysis now. */
1188 
1189  (void)vircam_genlincur(fdata,ps.nfdata,dexps,(double)mindit,ps.chantab,
1190  vircam_linearity_analyse_config.norder,
1191  &(ps.lchantab),&lindata,&status);
1192  if (ps.diag1 != NULL) {
1193  for (i = 0; i < ps.nfdata; i++) {
1194  for (n = 0; n < np; n++) {
1195  char * colname = cpl_sprintf("linflux_%02d", n+1);
1196  // snprintf(colname,16,"linflux_%02d", n+1);
1197  cpl_table_set_double(ps.diag1,colname,
1198  (cpl_size)i,lindata[i*np+n]);
1199  cpl_free(colname);
1200  }
1201  }
1202  }
1203  freespace2(fdata,ps.nfdata);
1204  freespace(dexps);
1205  freespace(lindata);
1206  if (status != CASU_OK) {
1207  cpl_msg_error(fctid,
1208  "Linearity curve fit failed extension %" CPL_SIZE_FORMAT,
1209  (cpl_size)j);
1210  dummy = 1;
1211  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
1212  return(-1);
1213  vircam_linearity_analyse_tidy(1);
1214  continue;
1215  }
1216 
1217  /* Get some QC1 parameters by finding the average fit error and
1218  the average percentage non-linearity */
1219 
1220  vircam_linearity_analyse_config.linearity =
1221  (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000");
1222  vircam_linearity_analyse_config.linerror =
1223  (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000_err");
1224 
1225  /* Save new linearity info */
1226 
1227  if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
1228  return(-1);
1229  }
1230 
1231  /* Get out of here */
1232 
1233  vircam_linearity_analyse_tidy(2);
1234  return(0);
1235 }
1236 
1237 
1238 /*---------------------------------------------------------------------------*/
1245 /*---------------------------------------------------------------------------*/
1246 
1247 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
1248  cpl_parameterlist *parlist) {
1249  cpl_propertylist *plist,*elist,*pafprop;
1250  cpl_image *outimg;
1251  const char *outtab = "lchantab.fits";
1252  const char *outbpm = "bpm.fits";
1253  const char *outtabpaf = "lchantab";
1254  const char *outbpmpaf = "bpm";
1255  const char *outdiag1 = "ldiag1.fits";
1256  const char *outdiag2 = "ldiag2.fits";
1257  const char *fctid = "vircam_linearity_analyse_save";
1258  const char *recipeid = "vircam_linearity_analyse";
1259  int nx,ny,nord,*bpm,i;
1260 
1261  /* Do some stuff for the first extension to set up the frame */
1262 
1263  nord = vircam_linearity_analyse_config.norder;
1264  if (isfirst) {
1265 
1266  /* Set up the output frame */
1267 
1268  product_frame_chantab = cpl_frame_new();
1269  cpl_frame_set_filename(product_frame_chantab,outtab);
1270  cpl_frame_set_tag(product_frame_chantab,VIRCAM_PRO_CHANTAB);
1271  cpl_frame_set_type(product_frame_chantab,CPL_FRAME_TYPE_TABLE);
1272  cpl_frame_set_group(product_frame_chantab,CPL_FRAME_GROUP_PRODUCT);
1273  cpl_frame_set_level(product_frame_chantab,CPL_FRAME_LEVEL_FINAL);
1274 
1275  /* Set up the PRO keywords for primary */
1276 
1277  ps.phupaf = vircam_paf_phu_items(ps.plist);
1278  plist = cpl_propertylist_duplicate(ps.plist);
1279  vircam_dfs_set_product_primary_header(plist,product_frame_chantab,
1280  framelist,parlist,
1281  (char *)recipeid,
1282  "PRO-1.15",ps.inherit,0);
1283 
1284  /* Now define the table propertylist and give it an extension name and
1285  PRO keywords */
1286 
1287  elist = cpl_propertylist_duplicate(ps.elist);
1288  cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
1289  vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
1290  framelist,parlist,(char *)recipeid,
1291  "PRO-1.15",ps.inherit);
1292 
1293  /* Add QC1 info */
1294  if (isfinite(vircam_linearity_analyse_config.linearity)){
1295  cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
1296  vircam_linearity_analyse_config.linearity);
1297  } else {
1298  cpl_propertylist_update_float(elist,"ESO QC LINEARITY", 100.);
1299  }
1300  cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
1301  "% non-linearity at 10000 ADU");
1302  cpl_propertylist_update_float(elist,"ESO QC LINERROR",
1303  vircam_linearity_analyse_config.linerror);
1304  cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
1305  "% error non-linearity at 10000 ADU");
1306  cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
1307  vircam_linearity_analyse_config.facrng);
1308  cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
1309  "total % range in screen variation");
1310  cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
1311  vircam_linearity_analyse_config.maxdiff);
1312  cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
1313  "maximum % step in screen variation");
1314  cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
1315 
1316  /* Set up a dummy table if necessary */
1317 
1318  if (dummy == 1) {
1319  casu_dummy_property(elist);
1320  if (ps.lchantab == NULL)
1321  ps.lchantab = vircam_chantab_new(nord,casu_tfits_get_table(ps.chantab));
1322  }
1323 
1324  /* And finally save the table */
1325 
1326  if (cpl_table_save(ps.lchantab,plist,elist,outtab,CPL_IO_DEFAULT)
1327  != CPL_ERROR_NONE) {
1328  cpl_msg_error(fctid,"Cannot save product table extension");
1329  freepropertylist(plist);
1330  freepropertylist(elist);
1331  return(-1);
1332  }
1333  cpl_frameset_insert(framelist,product_frame_chantab);
1334 
1335  /* Write PAF */
1336 
1337  pafprop = vircam_paf_req_items(elist);
1338  casu_merge_propertylists(pafprop,ps.phupaf);
1339  vircam_paf_append(pafprop,plist,"ESO INS FILT1 NAME");
1340  vircam_paf_append(pafprop,elist,"ESO PRO CATG");
1341  vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
1342  if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
1343  "QC file",pafprop) != CASU_OK)
1344  cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
1345  cpl_propertylist_delete(pafprop);
1346 
1347  /* Quick tidy */
1348 
1349  freepropertylist(plist);
1350  freepropertylist(elist);
1351 
1352  /* Set up the output bad pixel mask primary */
1353 
1354  product_frame_bpm = cpl_frame_new();
1355  cpl_frame_set_filename(product_frame_bpm,outbpm);
1356  cpl_frame_set_tag(product_frame_bpm,VIRCAM_PRO_BPM);
1357  cpl_frame_set_type(product_frame_bpm,CPL_FRAME_TYPE_IMAGE);
1358  cpl_frame_set_group(product_frame_bpm,CPL_FRAME_GROUP_PRODUCT);
1359  cpl_frame_set_level(product_frame_bpm,CPL_FRAME_LEVEL_FINAL);
1360 
1361  /* Set up the PRO keywords for primary header in the bpm */
1362 
1363  plist = ps.plist;
1364  vircam_dfs_set_product_primary_header(plist,product_frame_bpm,
1365  framelist,parlist,
1366  (char *)recipeid,"PRO-1.15",
1367  ps.inherit,0);
1368 
1369  /* Now save the PHU 'image' */
1370 
1371  if (cpl_image_save(NULL,outbpm,CPL_TYPE_UCHAR,plist,
1372  CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
1373  cpl_msg_error(fctid,"Cannot save product PHU");
1374  cpl_frame_delete(product_frame_bpm);
1375  return(-1);
1376  }
1377  cpl_frameset_insert(framelist,product_frame_bpm);
1378 
1379  /* Section for saving diagnostic tables. First the linearity
1380  sequence diagnostics */
1381 
1382  if (ps.diag1 != NULL) {
1383 
1384  /* Set up the output frame */
1385 
1386  product_frame_diag1 = cpl_frame_new();
1387  cpl_frame_set_filename(product_frame_diag1,outdiag1);
1388  cpl_frame_set_tag(product_frame_diag1,VIRCAM_PRO_LIN_DIAG1);
1389  cpl_frame_set_type(product_frame_diag1,CPL_FRAME_TYPE_TABLE);
1390  cpl_frame_set_group(product_frame_diag1,CPL_FRAME_GROUP_PRODUCT);
1391  cpl_frame_set_level(product_frame_diag1,CPL_FRAME_LEVEL_FINAL);
1392 
1393  /* Set up the PRO keywords for primary */
1394 
1395  plist = cpl_propertylist_duplicate(ps.plist);
1396  vircam_dfs_set_product_primary_header(plist,product_frame_diag1,
1397  framelist,parlist,
1398  (char *)recipeid,
1399  "PRO-1.15",ps.inherit,0);
1400 
1401  /* Now define the table propertylist and give it an extension name and
1402  PRO keywords */
1403 
1404  elist = cpl_propertylist_duplicate(ps.elist);
1405  vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
1406  framelist,parlist,
1407  (char *)recipeid,"PRO-1.15",
1408  ps.inherit);
1409 
1410  /* Set up a dummy property if necessary */
1411 
1412  if (dummy == 1)
1413  casu_dummy_property(elist);
1414 
1415  /* And finally save the table */
1416 
1417  if (cpl_table_save(ps.diag1,plist,elist,outdiag1,CPL_IO_DEFAULT)
1418  != CPL_ERROR_NONE) {
1419  cpl_msg_error(fctid,"Cannot save product table extension");
1420  freepropertylist(plist);
1421  freepropertylist(elist);
1422  return(-1);
1423  }
1424  cpl_frameset_insert(framelist,product_frame_diag1);
1425  freepropertylist(plist);
1426  freepropertylist(elist);
1427  }
1428 
1429  /* Now the monitor sequence diagnostics */
1430 
1431  if (ps.diag2 != NULL) {
1432 
1433  /* Set up the output frame */
1434 
1435  product_frame_diag2 = cpl_frame_new();
1436  cpl_frame_set_filename(product_frame_diag2,outdiag2);
1437  cpl_frame_set_tag(product_frame_diag2,VIRCAM_PRO_LIN_DIAG2);
1438  cpl_frame_set_type(product_frame_diag2,CPL_FRAME_TYPE_TABLE);
1439  cpl_frame_set_group(product_frame_diag2,CPL_FRAME_GROUP_PRODUCT);
1440  cpl_frame_set_level(product_frame_diag2,CPL_FRAME_LEVEL_FINAL);
1441 
1442  /* Set up the PRO keywords for primary */
1443 
1444  plist = cpl_propertylist_duplicate(ps.plist);
1445  vircam_dfs_set_product_primary_header(plist,product_frame_diag2,
1446  framelist,parlist,
1447  (char *)recipeid,
1448  "PRO-1.15",ps.inherit,0);
1449 
1450  /* Now define the table propertylist and give it an extension name
1451  and PRO keywords */
1452 
1453  elist = cpl_propertylist_duplicate(ps.elist);
1454  vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
1455  framelist,parlist,
1456  (char *)recipeid,"PRO-1.15",
1457  ps.inherit);
1458 
1459  /* Set up a dummy property if necessary */
1460 
1461  if (dummy == 1)
1462  casu_dummy_property(elist);
1463 
1464  /* And finally save the table */
1465 
1466  if (cpl_table_save(ps.diag2,plist,elist,outdiag2,CPL_IO_DEFAULT)
1467  != CPL_ERROR_NONE) {
1468  cpl_msg_error(fctid,"Cannot save product table extension");
1469  freepropertylist(plist);
1470  freepropertylist(elist);
1471  return(-1);
1472  }
1473  cpl_frameset_insert(framelist,product_frame_diag2);
1474  freepropertylist(plist);
1475  freepropertylist(elist);
1476  }
1477 
1478  /* Section for all other extensions */
1479 
1480  } else {
1481 
1482  /* Do the table extension PRO keywords */
1483 
1484  elist = cpl_propertylist_duplicate(ps.elist);
1485  cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
1486  vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
1487  framelist,parlist,
1488  (char *)recipeid,
1489  "PRO-1.15",ps.inherit);
1490 
1491  /* Add QC1 info */
1492  if (isnormal(vircam_linearity_analyse_config.linearity)){
1493  cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
1494  vircam_linearity_analyse_config.linearity);
1495  } else {
1496  cpl_propertylist_update_float(elist,"ESO QC LINEARITY", 100.);
1497  }
1498 
1499  cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
1500  "% non-linearity at 10000 ADU");
1501  cpl_propertylist_update_float(elist,"ESO QC LINERROR",
1502  vircam_linearity_analyse_config.linerror);
1503  cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
1504  "% error non-linearity at 10000 ADU");
1505  cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
1506  vircam_linearity_analyse_config.facrng);
1507  cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
1508  "total % range in screen variation");
1509  cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
1510  vircam_linearity_analyse_config.maxdiff);
1511  cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
1512  "maximum % step in screen variation");
1513  cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
1514 
1515  /* Set up a dummy table if necessary */
1516 
1517  if (dummy == 1) {
1518  casu_dummy_property(elist);
1519  if (ps.lchantab == NULL)
1520  ps.lchantab = vircam_chantab_new(nord,casu_tfits_get_table(ps.chantab));
1521  }
1522 
1523  /* And finally save the table */
1524 
1525  if (cpl_table_save(ps.lchantab,NULL,elist,outtab,CPL_IO_EXTEND)
1526  != CPL_ERROR_NONE) {
1527  cpl_msg_error(fctid,"Cannot save product table extension");
1528  freepropertylist(elist);
1529  return(-1);
1530  }
1531 
1532  /* Write PAF */
1533 
1534  pafprop = vircam_paf_req_items(elist);
1535  casu_merge_propertylists(pafprop,ps.phupaf);
1536  vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
1537  vircam_paf_append(pafprop,elist,"ESO PRO CATG");
1538  vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
1539  if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
1540  "QC file",pafprop) != CASU_OK)
1541  cpl_msg_warning(fctid,"Unable to save PAF for BPM");
1542  cpl_propertylist_delete(pafprop);
1543 
1544  /* Quick tidy */
1545 
1546  freepropertylist(elist);
1547 
1548  /* Now the diagnostic tables */
1549 
1550  if (ps.diag1 != NULL) {
1551  elist = cpl_propertylist_duplicate(ps.elist);
1552  vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
1553  framelist,parlist,
1554  (char *)recipeid,"PRO-1.15",
1555  ps.inherit);
1556 
1557  /* Set up a dummy property if necessary */
1558 
1559  if (dummy == 1)
1560  casu_dummy_property(elist);
1561 
1562  /* And finally save the table */
1563 
1564  if (cpl_table_save(ps.diag1,NULL,elist,outdiag1,CPL_IO_EXTEND)
1565  != CPL_ERROR_NONE) {
1566  cpl_msg_error(fctid,"Cannot save product table extension");
1567  freepropertylist(elist);
1568  return(-1);
1569  }
1570  freepropertylist(elist);
1571  }
1572  if (ps.diag2 != NULL) {
1573  elist = cpl_propertylist_duplicate(ps.elist);
1574  vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
1575  framelist,parlist,
1576  (char *)recipeid,"PRO-1.15",
1577  ps.inherit);
1578 
1579  /* Set up a dummy property if necessary */
1580 
1581  if (dummy == 1)
1582  casu_dummy_property(elist);
1583 
1584  /* And finally save the table */
1585 
1586  if (cpl_table_save(ps.diag2,NULL,elist,outdiag2,CPL_IO_EXTEND)
1587  != CPL_ERROR_NONE) {
1588  cpl_msg_error(fctid,"Cannot save product table extension");
1589  freepropertylist(elist);
1590  return(-1);
1591  }
1592  freepropertylist(elist);
1593  }
1594 
1595  }
1596 
1597  /* Save the bpm extension now */
1598 
1599  plist = ps.elist;
1600  nx = ps.nx;
1601  ny = ps.ny;
1602  if (dummy && ps.bpm_array == NULL) {
1603  ps.bpm_array = cpl_array_new((cpl_size)(nx*ny),CPL_TYPE_INT);
1604  bpm = cpl_array_get_data_int(ps.bpm_array);
1605  for (i = 0; i < nx*ny; i++)
1606  bpm[i] = 0;
1607  }
1608  bpm = cpl_array_get_data_int(ps.bpm_array);
1609  vircam_dfs_set_product_exten_header(plist,product_frame_bpm,
1610  framelist,parlist,(char *)recipeid,
1611  "PRO-1.15",ps.inherit);
1612  cpl_propertylist_update_float(plist,"ESO QC BAD_PIXEL_STAT",
1613  vircam_linearity_analyse_config.bad_pixel_stat);
1614  cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_STAT",
1615  "Fraction of pixels that are bad");
1616  cpl_propertylist_update_int(plist,"ESO QC BAD_PIXEL_NUM",
1617  vircam_linearity_analyse_config.bad_pixel_num);
1618  cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_NUM",
1619  "Number of pixels that are bad");
1620  cpl_propertylist_update_int(plist,"ESO PRO DATANCOM",ps.nuse);
1621  if (dummy)
1622  casu_dummy_property(plist);
1623  outimg = cpl_image_wrap_int((cpl_size)nx,(cpl_size)ny,bpm);
1624  if (cpl_image_save(outimg,outbpm,CPL_TYPE_UCHAR,plist,
1625  CPL_IO_EXTEND) != CPL_ERROR_NONE) {
1626  cpl_msg_error(fctid,"Cannot save product image extension");
1627  return(-1);
1628  }
1629 
1630  /* Write PAF */
1631 
1632  pafprop = vircam_paf_req_items(plist);
1633  casu_merge_propertylists(pafprop,ps.phupaf);
1634  vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
1635  vircam_paf_append(pafprop,ps.plist,"ESO PRO CATG");
1636  vircam_paf_append(pafprop,plist,"ESO PRO DATANCOM");
1637  if (vircam_paf_print((char *)outbpmpaf,"VIRCAM/vircam_linearity_analyse",
1638  "QC file",pafprop) != CASU_OK)
1639  cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
1640  cpl_propertylist_delete(pafprop);
1641 
1642  /* Quick tidy */
1643 
1644  cpl_image_unwrap(outimg);
1645 
1646  return(0);
1647 }
1648 
1649 /*---------------------------------------------------------------------------*/
1657 /*---------------------------------------------------------------------------*/
1658 
1659 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
1660  cpl_parameterlist *parlist) {
1661  int retval;
1662  const char *fctid = "vircam_linearity_analyse_lastbit";
1663 
1664  /* Save the new channel table and bad pixel map */
1665 
1666  cpl_msg_info(fctid,
1667  "Saving linearity table and bpm for extension %" CPL_SIZE_FORMAT,
1668  (cpl_size)jext);
1669  retval = vircam_linearity_analyse_save(framelist,parlist);
1670  if (retval != 0) {
1671  vircam_linearity_analyse_tidy(2);
1672  return(-1);
1673  }
1674 
1675  /* Do some intermediate tidying */
1676 
1677  vircam_linearity_analyse_tidy(1);
1678  return(0);
1679 }
1680 
1681 /*---------------------------------------------------------------------------*/
1685 /*---------------------------------------------------------------------------*/
1686 
1687 static int vircam_linearity_analyse_domedark_groups(void) {
1688  int i,j,found;
1689  float texp;
1690  cpl_frame *frame;
1691  cpl_propertylist *plist;
1692  const char *fctid = "vircam_linearity_analyse_domedark_groups";
1693 
1694  /* Start by getting the memory for the domedark groups */
1695 
1696  ps.ddg = cpl_calloc(ps.ndomes,sizeof(ddgrp));
1697  ps.nddg = 0;
1698 
1699  /* Loop for each of the dome frames and get its exposure time. If this
1700  doesn't exist, then signal an error and go on */
1701 
1702  for (i = 0; i < ps.ndomes; i++) {
1703  frame = cpl_frameset_get_position(ps.domelist,i);
1704  plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
1705  if (vircam_pfits_get_exptime(plist,&texp) != CASU_OK) {
1706  cpl_msg_warning(fctid,"No exposure time found in %s",
1707  cpl_frame_get_filename(frame));
1708  cpl_propertylist_delete(plist);
1709  continue;
1710  }
1711  cpl_propertylist_delete(plist);
1712 
1713  /* Search the domedark groups to see if this exposure time has already
1714  been used. If not, then create a new group. If it has then just add
1715  this frame to the correct group */
1716 
1717  found = 0;
1718  for (j = 0; j < ps.nddg; j++) {
1719  if (ps.ddg[j].exptime == texp) {
1720  found = 1;
1721  break;
1722  }
1723  }
1724  if (found) {
1725  cpl_frameset_insert(ps.ddg[j].domes,cpl_frame_duplicate(frame));
1726  ps.ddg[j].ndomes += 1;
1727  } else {
1728  ps.ddg[ps.nddg].exptime = texp;
1729  ps.ddg[ps.nddg].darks = cpl_frameset_new();
1730  ps.ddg[ps.nddg].domes = cpl_frameset_new();
1731  ps.ddg[ps.nddg].ndarks = 0;
1732  ps.ddg[ps.nddg].ndomes = 1;
1733  ps.ddg[ps.nddg].flag = OK_FLAG;
1734  cpl_frameset_insert(ps.ddg[ps.nddg].domes,
1735  cpl_frame_duplicate(frame));
1736  ps.nddg += 1;
1737  }
1738  }
1739 
1740  /* Right, now loop through all the darks and get their exposure times */
1741 
1742  for (i = 0; i < ps.ndarks; i++) {
1743  frame = cpl_frameset_get_position(ps.darklist,i);
1744  plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
1745  if (vircam_pfits_get_exptime(plist,&texp) != CASU_OK) {
1746  cpl_msg_warning(fctid,"No exposure time found in %s",
1747  cpl_frame_get_filename(frame));
1748  cpl_propertylist_delete(plist);
1749  continue;
1750  }
1751  cpl_propertylist_delete(plist);
1752 
1753  /* Search the domedark groups to see if this dark fits into one of
1754  the defined groups. If not, then ignore it. If it does, then
1755  add it into the dark frameset */
1756 
1757  found = 0;
1758  for (j = 0; j < ps.nddg; j++) {
1759  if (ps.ddg[j].exptime == texp) {
1760  found = 1;
1761  break;
1762  }
1763  }
1764  if (found) {
1765  cpl_frameset_insert(ps.ddg[j].darks,cpl_frame_duplicate(frame));
1766  ps.ddg[j].ndarks += 1;
1767  }
1768  }
1769 
1770  /* Now go through the domedark groups and ditch any that don't have any
1771  dark frames */
1772 
1773  i = 0;
1774  while (i < ps.nddg) {
1775  if (ps.ddg[i].ndarks == 0) {
1776  cpl_msg_warning(fctid,
1777  "No dark frames exist for exposure %g\nThrowing these away",
1778  ps.ddg[i].exptime);
1779  freeframeset(ps.ddg[i].darks);
1780  freeframeset(ps.ddg[i].domes);
1781  for (j = i+1; j < ps.nddg; j++)
1782  ps.ddg[j-1] = ps.ddg[j];
1783  ps.nddg -= 1;
1784  } else
1785  i++;
1786  }
1787 
1788  /* Allocate some space for casu_fits arrays for processed domes */
1789 
1790  for (i = 0; i < ps.nddg; i++) {
1791  ps.ddg[i].proc = cpl_malloc(ps.ddg[i].ndomes*sizeof(casu_fits *));
1792  for (j = 0; j < ps.ddg[i].ndomes; j++)
1793  ps.ddg[i].proc[j] = NULL;
1794  }
1795 
1796  /* Resize the output array and return so long as there is anything
1797  left. If there isn't then signal a major error */
1798 
1799  if (ps.nddg > 0) {
1800  ps.ddg = cpl_realloc(ps.ddg,ps.nddg*sizeof(ddgrp));
1801  return(0);
1802  } else {
1803  cpl_msg_error(fctid,"There are no darks defined for input domes");
1804  return(-1);
1805  }
1806 }
1807 
1808 /*---------------------------------------------------------------------------*/
1817 /*---------------------------------------------------------------------------*/
1818 
1819 static double *vircam_linearity_analyse_genstat(casu_fits *fframe, int *bpm,
1820  parquet *p, int np) {
1821  int i,ist,ifn,jst,jfn,n,jind2,iind2,jj,nx,ii;
1822  parquet *pp;
1823  double *d;
1824  float *tmp,*data;
1825 
1826  /* Get the workspace for the output result */
1827 
1828  d = cpl_malloc(np*sizeof(double));
1829 
1830  /* Get the input data array */
1831 
1832  nx = (int)cpl_image_get_size_x(casu_fits_get_image(fframe));
1833  data = cpl_image_get_data_float(casu_fits_get_image(fframe));
1834 
1835  /* Get some workspace for doing the median calculations */
1836 
1837  tmp = cpl_malloc(SUBSET*SUBSET*sizeof(float));
1838 
1839  /* Loop for each channel in the parquet structure */
1840 
1841  for (i = 0; i < np; i++) {
1842  pp = p + i;
1843 
1844  /* Take the central part of the channel */
1845 
1846  ist = ((pp->delta_i)/2 - SUBSET2);
1847  ifn = ist + SUBSET - 1;
1848  jst = ((pp->delta_j)/2 - SUBSET2);
1849  jfn = jst + SUBSET - 1;
1850 
1851  /* Put the data into the workspace and do a median */
1852 
1853  n = 0;
1854  for (jj = jst; jj <= jfn; jj++) {
1855  jind2 = (jj + pp->iymin - 1)*nx;
1856  for (ii = ist; ii <= ifn; ii++) {
1857  iind2 = jind2 + ii + pp->ixmin - 1;
1858  if (bpm[iind2] == 0)
1859  tmp[n++] = data[iind2];
1860  }
1861  }
1862  d[i] = (double)casu_med(tmp,NULL,(long)n);
1863  }
1864 
1865  /* Tidy and get out of here */
1866 
1867  freespace(tmp);
1868  return(d);
1869 }
1870 
1871 /*---------------------------------------------------------------------------*/
1883 /*---------------------------------------------------------------------------*/
1884 
1885 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
1886  int nchan, double *facrng,
1887  double *maxdiff) {
1888  int i,ist,ifn,j;
1889  double *factors,sum,midval,minfac,maxfac;
1890 
1891  /* Get some memory for the output array */
1892 
1893  factors = cpl_malloc(nim*sizeof(double));
1894 
1895  /* First sort the data into order of mjd */
1896 
1897  vircam_mjdsort(fdata,mjd,nim);
1898 
1899  /* Which index is the midpoint? */
1900 
1901  if (nim % 2 == 0) {
1902  ist = nim/2 - 1;
1903  ifn = ist + 1;
1904  } else {
1905  ist = nim/2;
1906  ifn = ist;
1907  }
1908 
1909  /* Loop for each channel */
1910 
1911  for (i = 0; i < nchan; i++) {
1912 
1913  /* Get midpoint value */
1914 
1915  midval = 0.5*(fdata[ist][i] + fdata[ifn][i]);
1916 
1917  /* Now normalise all the ith channels by this value */
1918 
1919  for (j = 0; j < nim; j++)
1920  fdata[j][i] /= midval;
1921  }
1922 
1923  /* Now loop for each image and average the values for all the channels in
1924  in image */
1925 
1926  *maxdiff = 0.0;
1927  maxfac = 0.0;
1928  minfac = 0.0;
1929  for (j = 0; j < nim; j++) {
1930  sum = 0.0;
1931  for (i = 0; i < nchan; i++)
1932  sum += fdata[j][i];
1933  factors[j] = sum/(double)nchan;
1934  if (j == 0) {
1935  maxfac = factors[j];
1936  minfac = factors[j];
1937  } else {
1938  minfac = min(minfac,factors[j]);
1939  maxfac = max(maxfac,factors[j]);
1940  *maxdiff = max(*maxdiff,fabs(factors[j]-factors[j-1]));
1941  }
1942  }
1943  *facrng = maxfac - minfac;
1944 
1945  /* Get out of here */
1946 
1947  return(factors);
1948 }
1949 
1950 /*---------------------------------------------------------------------------*/
1958 /*---------------------------------------------------------------------------*/
1959 
1960 static void vircam_mjdsort(double **fdata, double *mjd, int n) {
1961  int iii,ii,i,ifin,j;
1962  double tmpmjd,*tmpdata;
1963 
1964 
1965  iii = 2;
1966  while (iii < n)
1967  iii *= 2;
1968  iii = min(n,(3*iii)/4 - 1);
1969 
1970  while (iii > 1) {
1971  iii /= 2;
1972  ifin = n - iii;
1973  for (ii = 0; ii < ifin; ii++) {
1974  i = ii;
1975  j = i + iii;
1976  if (mjd[i] > mjd[j]) {
1977  tmpmjd = mjd[j];
1978  tmpdata = fdata[j];
1979  while (1) {
1980  mjd[j] = mjd[i];
1981  fdata[j] = fdata[i];
1982  j = i;
1983  i = i - iii;
1984  if (i < 0 || mjd[0] <= tmpmjd)
1985  break;
1986  }
1987  mjd[j] = tmpmjd;
1988  fdata[j] = tmpdata;
1989  }
1990  }
1991  }
1992 }
1993 
1994 /*---------------------------------------------------------------------------*/
2001 /*---------------------------------------------------------------------------*/
2002 
2003 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows) {
2004  int i;
2005  cpl_table *t;
2006 
2007  /* Create a new table */
2008 
2009  t = cpl_table_new(nrows);
2010 
2011  /* Add the first few columns */
2012 
2013  cpl_table_new_column(t,"filename",CPL_TYPE_STRING);
2014  cpl_table_new_column(t,"exptime",CPL_TYPE_DOUBLE);
2015  cpl_table_set_column_unit(t,"exptime","seconds");
2016  cpl_table_new_column(t,"mjd",CPL_TYPE_DOUBLE);
2017  cpl_table_set_column_unit(t,"mjd","days");
2018 
2019  /* Add columns for each of the channels' raw median flux and linearised
2020  median flux */
2021 
2022  for (i = 1; i <= np; i++) {
2023  char * colname = cpl_sprintf("rawflux_%02d", i);
2024  // (void)snprintf(colname,16,"rawflux_%02d", i);
2025  cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
2026  cpl_table_set_column_unit(t,colname,"ADU");
2027  cpl_free(colname);
2028  colname = cpl_sprintf("linflux_%02d", i);
2029  //snprintf(colname,16,"linflux_%02d", i);
2030  cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
2031  cpl_table_set_column_unit(t,colname,"ADU");
2032  cpl_free(colname);
2033  }
2034 
2035  /* Finally add the correction factors that were used */
2036 
2037  cpl_table_new_column(t,"adjust_fac",CPL_TYPE_DOUBLE);
2038 
2039  /* Right, get out of here */
2040 
2041  return(t);
2042 }
2043 
2044 /*---------------------------------------------------------------------------*/
2048 /*---------------------------------------------------------------------------*/
2049 
2050 static void vircam_linearity_analyse_init(void) {
2051  ps.labels = NULL;
2052  ps.domelist = NULL;
2053  ps.darklist = NULL;
2054  ps.domecheck = NULL;
2055  ps.darkcheck = NULL;
2056  ps.ndomes = 0;
2057  ps.ndarks = 0;
2058  ps.ndomecheck = 0;
2059  ps.ndarkcheck = 0;
2060  ps.chanfrm = NULL;
2061  ps.chantab = NULL;
2062  ps.lchantab = NULL;
2063  ps.flatlist = NULL;
2064  ps.bpm_array = NULL;
2065  ps.ddg = NULL;
2066  ps.plist = NULL;
2067  ps.elist = NULL;
2068  ps.phupaf = NULL;
2069  ps.diag1 = NULL;
2070  ps.diag2 = NULL;
2071  ps.inherit = NULL;
2072 }
2073 
2074 /*---------------------------------------------------------------------------*/
2078 /*---------------------------------------------------------------------------*/
2079 
2080 static void vircam_linearity_analyse_tidy(int level) {
2081  int i;
2082 
2083  freetfits(ps.chantab);
2084  freearray(ps.bpm_array);
2085  freefitslist(ps.flatlist,ps.nflatlist);
2086  freetable(ps.lchantab);
2087  freepropertylist(ps.plist);
2088  freepropertylist(ps.elist);
2089  freetable(ps.diag1);
2090  freetable(ps.diag2);
2091  if (level == 1)
2092  return;
2093 
2094  freespace(ps.labels);
2095  freeframeset(ps.domelist);
2096  freeframeset(ps.darklist);
2097  freeframeset(ps.domecheck);
2098  freeframeset(ps.darkcheck);
2099  freeframe(ps.chanfrm);
2100  if (ps.ddg != NULL) {
2101  for (i = 0; i < ps.nddg; i++) {
2102  freeframeset(ps.ddg[i].darks);
2103  freeframeset(ps.ddg[i].domes);
2104  freefitslist(ps.ddg[i].proc,ps.ddg[i].ndomes);
2105  }
2106  freespace(ps.ddg);
2107  }
2108  freepropertylist(ps.phupaf);
2109 }
2110 
2113 /*
2114 
2115 $Log: not supported by cvs2svn $
2116 Revision 1.64 2012/01/15 17:40:09 jim
2117 Minor modifications to take into accout the changes in cpl API for v6
2118 
2119 Revision 1.63 2010/12/09 13:20:26 jim
2120 Default polynomial order is now 4
2121 
2122 Revision 1.62 2010/07/02 07:17:35 jim
2123 Fixed typo
2124 
2125 Revision 1.61 2010/06/30 12:42:00 jim
2126 A few fixes to stop compiler compaints
2127 
2128 Revision 1.60 2010/03/09 14:29:55 jim
2129 Now modified ESO PRO DATANCOM to reflect the number of images used in
2130 the analysis
2131 
2132 Revision 1.59 2009/09/09 09:50:21 jim
2133 Modified to try and get headers right
2134 
2135 Revision 1.58 2009/07/13 08:17:57 jim
2136 Fixed bug that meant saturation level wasn't being propogated
2137 
2138 Revision 1.57 2009/06/23 05:22:26 jim
2139 Adds saturation back into the channel table header
2140 
2141 Revision 1.56 2008/12/08 06:32:42 jim
2142 Changed 'missing dark frame' error to warning
2143 
2144 Revision 1.55 2008/12/05 13:28:32 jim
2145 Fixed save routine so that the correct version of PRO CATG is written to the
2146 paf file
2147 
2148 Revision 1.54 2008/10/01 04:59:13 jim
2149 Added call to vircam_frameset_fexists to check input frameset
2150 
2151 Revision 1.53 2008/09/30 11:33:00 jim
2152 Fixed bug where saturation flag wasn't being set to OK.
2153 
2154 Revision 1.52 2008/09/29 11:23:00 jim
2155 Minor fix to docs
2156 
2157 Revision 1.51 2008/01/22 19:47:56 jim
2158 New version to implement new algorithm
2159 
2160 Revision 1.50 2007/11/26 09:58:49 jim
2161 Now fails if given observation files done with NDIT != 1
2162 
2163 Revision 1.49 2007/11/23 18:34:28 jim
2164 fixed memory allocation bug
2165 
2166 Revision 1.48 2007/11/22 12:36:55 jim
2167 Modified to create diagnostic tables
2168 
2169 Revision 1.47 2007/11/20 09:41:13 jim
2170 Added ability to alter dome sequence stats by using the monitoring exposures
2171 
2172 Revision 1.46 2007/11/14 10:42:25 jim
2173 Substantial changes to incorporate new linearity analysis algorithm and to
2174 restrict the amount of memory required to do the analysis (especially
2175 the BPM work)
2176 
2177 Revision 1.45 2007/09/07 13:32:12 jim
2178 uses a sorted framelist to ensure that the correct information is given
2179 to the output product header
2180 
2181 Revision 1.44 2007/09/06 21:37:53 jim
2182 fixed call to vircam_dfs_setup_product_ routines to use the full input
2183 frameset
2184 
2185 Revision 1.43 2007/08/29 09:20:33 jim
2186 Primary header is now derived from the same header that forms the PAF rather
2187 than starting off empty and allowing CPL to copy everything it thinks you
2188 want...
2189 
2190 Revision 1.42 2007/08/23 09:02:03 jim
2191 Modified to check domes for DETLIVE before checking darks
2192 
2193 Revision 1.41 2007/07/09 13:21:55 jim
2194 Modified to use new version of vircam_exten_range
2195 
2196 Revision 1.40 2007/06/13 08:11:27 jim
2197 Modified docs to reflect changes in DFS tags
2198 
2199 Revision 1.39 2007/04/30 09:40:17 jim
2200 Added more stuff to paf files
2201 
2202 Revision 1.38 2007/04/04 10:36:07 jim
2203 Fixed typo preventing output of main PAF. Also modified to use dfs tags
2204 
2205 Revision 1.37 2007/03/29 12:19:38 jim
2206 Little changes to improve documentation
2207 
2208 Revision 1.36 2007/03/01 12:41:49 jim
2209 Modified slightly after code checking
2210 
2211 Revision 1.35 2007/02/19 21:13:04 jim
2212 added bad pixel number QC parameter
2213 
2214 Revision 1.34 2007/02/15 11:54:09 jim
2215 Modified to make a distinction between initial channel table and one that
2216 has the proper linearity information
2217 
2218 Revision 1.33 2007/02/15 06:59:38 jim
2219 Added ability to write QC paf files
2220 
2221 Revision 1.32 2007/02/07 10:12:40 jim
2222 Removed calls to vircam_ndit_correct as this is now no longer necessary
2223 
2224 Revision 1.31 2007/02/06 13:11:12 jim
2225 Fixed entry for PRO dictionary in cpl_dfs_set_product_header
2226 
2227 Revision 1.30 2006/12/13 11:45:36 jim
2228 Fixed scaling of sigma error
2229 
2230 Revision 1.29 2006/12/11 22:47:12 jim
2231 Fixed subtle bug in the way that stats were being done.
2232 
2233 Revision 1.28 2006/11/27 12:15:08 jim
2234 changed calls to cpl_propertylist_append to cpl_propertylist_update
2235 
2236 Revision 1.27 2006/11/10 09:23:46 jim
2237 Fixed save routine so to use a new version of vircam_chantab_new
2238 
2239 Revision 1.26 2006/10/31 10:27:27 jim
2240 Fixed a few bugs and modified to make sure than an extension name appear
2241 in each fits extension
2242 
2243 Revision 1.25 2006/09/09 16:49:40 jim
2244 Header comment update
2245 
2246 Revision 1.24 2006/09/08 09:20:22 jim
2247 major upgrade to main processing routine: to deal with bad input better; to
2248 write out dummy results in the case of failure; to combine raw darks on the
2249 fly for use in dark correction, rather than using master darks;
2250 
2251 Revision 1.23 2006/08/03 13:26:44 jim
2252 fixed another typo
2253 
2254 Revision 1.22 2006/08/03 10:36:32 jim
2255 Fixed typo
2256 
2257 Revision 1.21 2006/06/20 19:06:38 jim
2258 Added correction for ndit. Now adjusts the value of norder if the number
2259 of frames given is too small for the order of polynomial requested
2260 
2261 Revision 1.20 2006/06/15 09:58:58 jim
2262 Minor changes to docs
2263 
2264 Revision 1.19 2006/06/06 13:03:42 jim
2265 Fixed scaling that was causing funny stats
2266 
2267 Revision 1.18 2006/05/27 21:40:06 jim
2268 Bad pixels are now defined by a number of sigma above or below the mean
2269 
2270 Revision 1.17 2006/05/09 09:30:47 jim
2271 Fixed _save routine so that bad pixel mask is saved with unsigned char
2272 data type
2273 
2274 Revision 1.16 2006/05/08 12:32:12 jim
2275 Changed default calling parameters for vircam_imcombine
2276 
2277 Revision 1.15 2006/05/04 11:53:15 jim
2278 Fixed the way the _save routine works to be more consistent with the
2279 standard CPL way of doing things
2280 
2281 Revision 1.14 2006/05/03 12:55:17 jim
2282 Fixed some memory leaks
2283 
2284 Revision 1.13 2006/05/02 13:26:32 jim
2285 fixed bug where the wrong amount of memory was being allocated for the dark
2286 exposure times
2287 
2288 Revision 1.12 2006/05/02 11:36:29 jim
2289 fixed illegal propertylist_delete calls
2290 
2291 Revision 1.11 2006/04/27 09:46:01 jim
2292 Modified DFS frame types to conform to new dictionary
2293 
2294 Revision 1.10 2006/04/25 13:45:57 jim
2295 Fixed to adhere to new calling sequence for vircam_dfs routines
2296 
2297 Revision 1.9 2006/04/24 12:12:59 jim
2298 Fixed --help documentation and sorted out filename extension problem
2299 (.fit -> .fits)
2300 
2301 Revision 1.8 2006/04/20 11:31:34 jim
2302 Added bad pixel masking
2303 
2304 Revision 1.7 2006/03/23 21:18:45 jim
2305 Minor changes mainly to comment headers
2306 
2307 Revision 1.6 2006/03/22 14:02:51 jim
2308 cosmetic changes to keep lint happy
2309 
2310 Revision 1.5 2006/03/22 12:13:51 jim
2311 Modified to use new vircam_mask capability
2312 
2313 Revision 1.4 2006/03/15 10:43:40 jim
2314 Fixed a few things
2315 
2316 Revision 1.3 2006/03/03 14:29:06 jim
2317 Now calls routines with vir_fits.
2318 
2319 Revision 1.2 2006/02/22 10:01:22 jim
2320 Added full documentation
2321 
2322 Revision 1.1 2006/02/18 11:49:58 jim
2323 new file
2324 
2325 
2326 */
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
casu_fits * casu_fits_load(cpl_frame *frame, cpl_type type, int nexten)
Definition: casu_fits.c:80
casu_fits ** casu_fits_load_list(cpl_frameset *f, cpl_type type, int exten)
Definition: casu_fits.c:318
char * casu_fits_get_filename(casu_fits *p)
Definition: casu_fits.c:646
void casu_fits_delete(casu_fits *p)
Definition: casu_fits.c:364
casu_fits * casu_fits_duplicate(casu_fits *in)
Definition: casu_fits.c:225
casu_fits * casu_fits_wrap(cpl_image *im, casu_fits *model, cpl_propertylist *phu, cpl_propertylist *ehu)
Definition: casu_fits.c:883
cpl_propertylist * casu_fits_get_phu(casu_fits *p)
Definition: casu_fits.c:531
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
int casu_imcombine(casu_fits **fset, casu_fits **fsetv, int nfits, int combtype, int scaletype, int xrej, float thresh, const char *expkey, cpl_image **outimage, cpl_image **outvimage, unsigned char **rejmask, unsigned char **rejplus, cpl_propertylist **drs, int *status)
Stack images into a mean or median image with rejection.
int casu_darkcor(casu_fits *infile, casu_fits *darksrc, float darkscl, int *status)
Correct input data for dark current.
Definition: casu_darkcor.c:86
int casu_genbpm(casu_fits **flatlist, int nflatlist, cpl_image *master, float lthr, float hthr, const char *expkey, cpl_array **bpm_array, int *nbad, float *badfrac, int *status)
Generate a bad pixel mask from a list of dome flats.
Definition: casu_genbpm.c:97
float casu_med(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:89
cpl_table * casu_tfits_get_table(casu_tfits *p)
Definition: casu_tfits.c:364
cpl_propertylist * casu_tfits_get_ehu(casu_tfits *p)
Definition: casu_tfits.c:473
casu_tfits * casu_tfits_load(cpl_frame *table, int nexten)
Definition: casu_tfits.c:78
cpl_frame * casu_frameset_subgroup_1(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Extract a frame of a given label from a frameset.
Definition: casu_utils.c:206
int casu_compare_tags(const cpl_frame *frame1, const cpl_frame *frame2)
Compare input tags.
Definition: casu_utils.c:96
void casu_merge_propertylists(cpl_propertylist *p1, cpl_propertylist *p2)
Merge two propertylists.
Definition: casu_utils.c:399
void casu_dummy_property(cpl_propertylist *p)
Set dummy property keyword.
Definition: casu_utils.c:445
cpl_frameset * casu_frameset_subgroup(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Extract a frameset from another frameset.
Definition: casu_utils.c:149
void casu_sort(float **a, int n, int m)
Sort a 2d array by the first column and co-sort the rest.
Definition: casu_utils.c:349
int vircam_genlincur(double **fdata, int nimages, double *exps, double mindit, casu_tfits *chantab, int norder, cpl_table **lchantab, double **lindata, int *status)
Generate a linearity curve for each readout channel in a list of images.
int vircam_chantab_verify(cpl_table *intab)
int vircam_chan_fill(cpl_table *tab, parquet **p, long *np)
void vircam_chan_free(int np, parquet **p)
cpl_table * vircam_chantab_new(int nord, cpl_table *template)
int vircam_dfs_set_groups(cpl_frameset *set)
Definition: vircam_dfs.c:115
void vircam_dfs_set_product_primary_header(cpl_propertylist *plist, cpl_frame *frame, cpl_frameset *frameset, cpl_parameterlist *parlist, char *recipeid, const char *dict, cpl_frame *inherit, int synch)
Definition: vircam_dfs.c:227
void vircam_dfs_set_product_exten_header(cpl_propertylist *plist, cpl_frame *frame, cpl_frameset *frameset, cpl_parameterlist *parlist, char *recipeid, const char *dict, cpl_frame *inherit)
Definition: vircam_dfs.c:299
int vircam_pfits_get_ndit(const cpl_propertylist *plist, int *ndit)
Get the value of NDIT.
Definition: vircam_pfits.c:583
int vircam_pfits_get_mjd(const cpl_propertylist *plist, double *mjd)
Get the value of the modified Julian date.
Definition: vircam_pfits.c:729
int vircam_pfits_get_detlive(const cpl_propertylist *plist, int *detlive)
Get the value of DET_LIVE.
Definition: vircam_pfits.c:624
int vircam_pfits_get_exptime(const cpl_propertylist *plist, float *exptime)
Get the value of exposure time.
Definition: vircam_pfits.c:245
int vircam_pfits_get_mindit(const cpl_propertylist *plist, float *mindit)
Get the value of mindit time.
Definition: vircam_pfits.c:262
int vircam_pfits_get_saturation(const cpl_propertylist *plist, float *saturation)
Get the saturation level for this detector.
Definition: vircam_pfits.c:696
const char * vircam_get_license(void)
Definition: vircam_utils.c:116
void vircam_exten_range(int inexten, const cpl_frame *fr, int *out1, int *out2)
Definition: vircam_utils.c:165