VIRCAM Pipeline 2.3.12
vircam_linearity_analyse.c
1/* $Id: vircam_linearity_analyse.c,v 1.65 2012-01-16 12:32:18 jim Exp $
2 *
3 * This file is part of the VIRCAM Pipeline
4 * Copyright (C) 2015 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20
21/*
22 * $Author: jim $
23 * $Date: 2012-01-16 12:32:18 $
24 * $Revision: 1.65 $
25 * $Name: not supported by cvs2svn $
26 */
27
28/* Includes */
29
30#ifdef HAVE_CONFIG_H
31#include <config.h>
32#endif
33
34#include <stdio.h>
35#include <cpl.h>
36#include <math.h>
37
38#include <casu_utils.h>
39#include <casu_mods.h>
40#include <casu_stats.h>
41
42#include "vircam_utils.h"
43#include "vircam_pfits.h"
44#include "vircam_dfs.h"
45#include "vircam_mods.h"
46#include "vircam_channel.h"
47#include "vircam_paf.h"
48
49/* Function prototypes */
50
51static int vircam_linearity_analyse_create(cpl_plugin *) ;
52static int vircam_linearity_analyse_exec(cpl_plugin *) ;
53static int vircam_linearity_analyse_destroy(cpl_plugin *) ;
54static int vircam_linearity_analyse(cpl_parameterlist *, cpl_frameset *) ;
55static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
56 cpl_parameterlist *parlist);
57static int vircam_linearity_analyse_save(cpl_frameset *framelist,
58 cpl_parameterlist *parlist);
59static int vircam_linearity_analyse_domedark_groups(void);
60static double *vircam_linearity_analyse_genstat(casu_fits *fframe, int *bpm,
61 parquet *p, int np);
62static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
63 int nchan, double *facrng,
64 double *maxdiff);
65static void vircam_mjdsort(double **fdata, double *mjd, int n);
66static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows);
67static void vircam_linearity_analyse_init(void);
68static void vircam_linearity_analyse_tidy(int level);
69
70/* Static global variables */
71
72static struct {
73
74 /* Input */
75
76 int norder;
77 float lthr;
78 float hthr;
79 int maxbpmfr;
80 int adjust;
81 int diagnostic;
82 int extenum;
83
84 /* Output */
85
86 float linearity;
87 float linerror;
88 float bad_pixel_stat;
89 int bad_pixel_num;
90 float facrng;
91 float maxdiff;
92
93} vircam_linearity_analyse_config;
94
95typedef struct {
96 cpl_frameset *darks;
97 cpl_frameset *domes;
98 int ndarks;
99 int ndomes;
100 float exptime;
101 unsigned char flag;
102 casu_fits **proc;
103} ddgrp;
104
105#define OK_FLAG 0
106#define SATURATE_FLAG 1
107
108#define SUBSET 128
109#define SUBSET2 (SUBSET/2)
110
111static struct {
112 cpl_size *labels;
113 cpl_frameset *domelist;
114 int ndomes;
115 cpl_frameset *darklist;
116 int ndarks;
117 cpl_frameset *domecheck;
118 int ndomecheck;
119 cpl_frameset *darkcheck;
120 int ndarkcheck;
121 cpl_frame *inherit;
122 cpl_frame *chanfrm;
123 casu_tfits *chantab;
124 cpl_table *lchantab;
125 cpl_array *bpm_array;
126 ddgrp *ddg;
127 int nddg;
128 casu_fits **flatlist;
129 int nflatlist;
130 cpl_propertylist *plist;
131 cpl_propertylist *elist;
132 int nx;
133 int ny;
134 cpl_propertylist *phupaf;
135 cpl_table *diag1;
136 cpl_table *diag2;
137 int nfdata;
138 int nuse;
139} ps;
140
141static int isfirst;
142static int dummy;
143static float sat;
144static cpl_frame *product_frame_chantab = NULL;
145static cpl_frame *product_frame_bpm = NULL;
146static cpl_frame *product_frame_diag1 = NULL;
147static cpl_frame *product_frame_diag2 = NULL;
148
149static char vircam_linearity_analyse_description[] =
150"vircam_linearity_analyse -- VIRCAM linearity mapping recipe.\n\n"
151"Form master dark images from the input raw frames and use these to\n"
152"dark correct a series of dome flat exposures Using the dark\n"
153"corrected dome flat series, work out linearity coefficients for\n"
154"each data channel. The program expects the following files in the SOF\n"
155" Tag Description\n"
156" -----------------------------------------------------------------------\n"
157" %-21s A list of raw dome flat images\n"
158" %-21s A list of raw dark images\n"
159" %-21s The channel table\n"
160" %-21s A list of raw dome flat images at the monitor exposure time\n"
161" %-21s A list of raw dark images at the monitor exposure time\n"
162"The first three of these are required. The last two are only required if"
163"the light source monitoring algorithm is to be used"
164"\n";
165
287/* Function code */
288
289/*---------------------------------------------------------------------------*/
297/*---------------------------------------------------------------------------*/
298
299int cpl_plugin_get_info(cpl_pluginlist *list) {
300 cpl_recipe *recipe = cpl_calloc(1,sizeof(*recipe));
301 cpl_plugin *plugin = &recipe->interface;
302 char alldesc[SZ_ALLDESC];
303 (void)snprintf(alldesc,SZ_ALLDESC,vircam_linearity_analyse_description,
304 VIRCAM_LIN_DOME_RAW,VIRCAM_LIN_DARK_RAW,
305 VIRCAM_CAL_CHANTAB_INIT,VIRCAM_LIN_DOME_CHECK,
306 VIRCAM_LIN_DARK_CHECK);
307
308 cpl_plugin_init(plugin,
309 CPL_PLUGIN_API,
310 VIRCAM_BINARY_VERSION,
311 CPL_PLUGIN_TYPE_RECIPE,
312 "vircam_linearity_analyse",
313 "VIRCAM linearity analysis recipe",
314 alldesc,
315 "Jim Lewis",
316 "jrl@ast.cam.ac.uk",
318 vircam_linearity_analyse_create,
319 vircam_linearity_analyse_exec,
320 vircam_linearity_analyse_destroy);
321
322 cpl_pluginlist_append(list,plugin);
323
324 return(0);
325}
326
327/*---------------------------------------------------------------------------*/
336/*---------------------------------------------------------------------------*/
337
338static int vircam_linearity_analyse_create(cpl_plugin *plugin) {
339 cpl_recipe *recipe;
340 cpl_parameter *p;
341
342 /* Get the recipe out of the plugin */
343
344 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
345 recipe = (cpl_recipe *)plugin;
346 else
347 return(-1);
348
349 /* Create the parameters list in the cpl_recipe object */
350
351 recipe->parameters = cpl_parameterlist_new();
352
353 /* Fill in the parameters. First the polynomial order */
354
355 p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.norder",
356 CPL_TYPE_INT,
357 "Order of polynomial fit",
358 "vircam.vircam_linearity_analyse",
359 4,1,6);
360 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"nord");
361 cpl_parameterlist_append(recipe->parameters,p);
362
363 /* The lower threshold */
364
365 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.lthr",
366 CPL_TYPE_DOUBLE,
367 "Lower bad pixel threshold",
368 "vircam.vircam_linearity_analyse",8.0);
369 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"lthr");
370 cpl_parameterlist_append(recipe->parameters,p);
371
372 /* The upper threshold */
373
374 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.hthr",
375 CPL_TYPE_DOUBLE,
376 "Upper bad pixel threshold",
377 "vircam.vircam_linearity_analyse",8.0);
378 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"hthr");
379 cpl_parameterlist_append(recipe->parameters,p);
380
381 /* The maximum number of frames to be used in forming the bad pixel mask */
382
383 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.maxbpmfr",
384 CPL_TYPE_INT,
385 "Maximum # frames used in bpm analysis",
386 "vircam.vircam_linearity_analyse",10);
387 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"maxbpmfr");
388 cpl_parameterlist_append(recipe->parameters,p);
389
390 /* The flag to allow statistics of the dome flat frames to be adjusted
391 using a set of monitor exposures */
392
393 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.adjust",
394 CPL_TYPE_BOOL,
395 "Adjust stats with monitor set",
396 "vircam.vircam_linearity_analyse",1);
397 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"adjust");
398 cpl_parameterlist_append(recipe->parameters,p);
399
400 /* The flag to allow diagnostic curves to be written out */
401
402 p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.diagnostic",
403 CPL_TYPE_BOOL,
404 "Write out diagnostic tables",
405 "vircam.vircam_linearity_analyse",0);
406 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"diagnostic");
407 cpl_parameterlist_append(recipe->parameters,p);
408
409 /* Extension number of input frames to use */
410
411 p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.extenum",
412 CPL_TYPE_INT,
413 "Extension number to be done, 0 == all",
414 "vircam.vircam_linearity_analyse",
415 0,0,16);
416 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
417 cpl_parameterlist_append(recipe->parameters,p);
418
419 /* Get out of here */
420
421 return(0);
422}
423
424/*---------------------------------------------------------------------------*/
430/*---------------------------------------------------------------------------*/
431
432static int vircam_linearity_analyse_exec(cpl_plugin *plugin) {
433 cpl_recipe *recipe;
434
435 /* Get the recipe out of the plugin */
436
437 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
438 recipe = (cpl_recipe *)plugin;
439 else
440 return(-1);
441
442 return(vircam_linearity_analyse(recipe->parameters,recipe->frames));
443}
444
445/*---------------------------------------------------------------------------*/
451/*---------------------------------------------------------------------------*/
452
453static int vircam_linearity_analyse_destroy(cpl_plugin *plugin) {
454 cpl_recipe *recipe ;
455
456 /* Get the recipe out of the plugin */
457
458 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
459 recipe = (cpl_recipe *)plugin;
460 else
461 return(-1);
462
463 cpl_parameterlist_delete(recipe->parameters);
464 return(0);
465}
466
467/*---------------------------------------------------------------------------*/
474/*---------------------------------------------------------------------------*/
475
476static int vircam_linearity_analyse(cpl_parameterlist *parlist,
477 cpl_frameset *framelist) {
478 const char *fctid="vircam_linearity_analyse";
479 int i,jst,jfn,j,status,k,nbad,ngood,krem,n;
480 int ndarks,ndomes,kk,live,ngood_flats,*bpm,nalloc,adjust,ndit;
481 cpl_size nlab;
482 long np;
483 float med,mindit,*exps,badfrac,expt;
484 unsigned char *rejmask,*rejplus;
485 double *dexps,**fdata,*d,*mjds,*mjdcheck,mjd,**cdata,*cf,fac;
486 double facrng,maxdiff,*lindata;
487 casu_fits **darks,**domes,*test,*outdark,*fframe;
488 parquet *pp;
489 cpl_parameter *p;
490 cpl_propertylist *drs,*plist;
491 cpl_image *outimage;
492 cpl_frame *frame;
493
494 /* Check validity of input frameset */
495
496 if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
497 cpl_msg_error(fctid,"Input framelist NULL or has no input data");
498 return(-1);
499 }
500
501 /* Initialise a few things */
502
503 vircam_linearity_analyse_init();
504
505 /* Get the parameters */
506
507 p = cpl_parameterlist_find(parlist,
508 "vircam.vircam_linearity_analyse.norder");
509 vircam_linearity_analyse_config.norder = cpl_parameter_get_int(p);
510 p = cpl_parameterlist_find(parlist,
511 "vircam.vircam_linearity_analyse.lthr");
512 vircam_linearity_analyse_config.lthr = (float)cpl_parameter_get_double(p);
513 p = cpl_parameterlist_find(parlist,
514 "vircam.vircam_linearity_analyse.hthr");
515 vircam_linearity_analyse_config.hthr = (float)cpl_parameter_get_double(p);
516 p = cpl_parameterlist_find(parlist,
517 "vircam.vircam_linearity_analyse.maxbpmfr");
518 vircam_linearity_analyse_config.maxbpmfr = cpl_parameter_get_int(p);
519 p = cpl_parameterlist_find(parlist,
520 "vircam.vircam_linearity_analyse.adjust");
521 vircam_linearity_analyse_config.adjust = cpl_parameter_get_bool(p);
522 p = cpl_parameterlist_find(parlist,
523 "vircam.vircam_linearity_analyse.diagnostic");
524 vircam_linearity_analyse_config.diagnostic = cpl_parameter_get_bool(p);
525 p = cpl_parameterlist_find(parlist,
526 "vircam.vircam_linearity_analyse.extenum");
527 vircam_linearity_analyse_config.extenum = cpl_parameter_get_int(p);
528
529 /* Sort out raw from calib frames */
530
531 if (vircam_dfs_set_groups(framelist) != CASU_OK) {
532 cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
533 vircam_linearity_analyse_tidy(2);
534 return(-1);
535 }
536
537 /* Get framelist labels */
538
539 if ((ps.labels = cpl_frameset_labelise(framelist,casu_compare_tags,
540 &nlab)) == NULL) {
541 cpl_msg_error(fctid,"Cannot labelise the input frames");
542 vircam_linearity_analyse_tidy(2);
543 return(-1);
544 }
545
546 /* Get the dome flat frames */
547
548 if ((ps.domelist = casu_frameset_subgroup(framelist,ps.labels,nlab,
549 VIRCAM_LIN_DOME_RAW)) == NULL) {
550 cpl_msg_error(fctid,"Cannot find dome flat frames in input frameset");
551 vircam_linearity_analyse_tidy(2);
552 return(-1);
553 }
554 ps.ndomes = cpl_frameset_get_size(ps.domelist);
555 ps.inherit = cpl_frameset_get_position(ps.domelist,0);
556
557 /* Check to make sure that NDIT == 1 */
558
559 plist = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_position(ps.domelist,0)),0);
560 (void)vircam_pfits_get_ndit(plist,&ndit);
561 freepropertylist(plist);
562 if (ndit != 1) {
563 cpl_msg_error(fctid,
564 "NDIT=%" CPL_SIZE_FORMAT ". Recipe requires that ndit == 1",
565 (cpl_size)ndit);
566 vircam_linearity_analyse_tidy(2);
567 return(-1);
568 }
569
570 /* Get the dark frames */
571
572 if ((ps.darklist = casu_frameset_subgroup(framelist,ps.labels,nlab,
573 VIRCAM_LIN_DARK_RAW)) == NULL) {
574 cpl_msg_error(fctid,"Cannot find dark frames in input frameset");
575 vircam_linearity_analyse_tidy(2);
576 return(-1);
577 }
578 ps.ndarks = cpl_frameset_get_size(ps.darklist);
579
580 /* If you are planning to adjust the stats for the dome frames by using
581 a monitor exposure set, then read that frameset in now */
582
583 if (vircam_linearity_analyse_config.adjust) {
584 if ((ps.domecheck = casu_frameset_subgroup(framelist,ps.labels,nlab,
585 VIRCAM_LIN_DOME_CHECK)) == NULL) {
586 cpl_msg_info(fctid,"No monitor frames found in sof. No adjustments made to stats");
587 vircam_linearity_analyse_config.adjust = 0;
588 ps.ndomecheck = 0;
589 } else {
590 ps.ndomecheck = cpl_frameset_get_size(ps.domecheck);
591 if ((ps.darkcheck = casu_frameset_subgroup(framelist,ps.labels,
592 nlab,VIRCAM_LIN_DARK_CHECK)) == NULL) {
593 cpl_msg_info(fctid,"No darks for monitor frames found in sof. No adjustments made to stats");
594 vircam_linearity_analyse_config.adjust = 0;
595 ps.ndomecheck = 0;
596 freeframeset(ps.domecheck);
597 ps.ndarkcheck = 0;
598 } else {
599 ps.ndarkcheck = cpl_frameset_get_size(ps.darkcheck);
600 }
601 }
602 }
603
604 /* Check to see if there is a channel table. If so, then read it */
605
606 if ((ps.chanfrm = casu_frameset_subgroup_1(framelist,ps.labels,nlab,
607 VIRCAM_CAL_CHANTAB_INIT)) == NULL) {
608 cpl_msg_error(fctid,"No initial channel table found");
609 vircam_linearity_analyse_tidy(2);
610 return(-1);
611 }
612
613 /* Group the domes and darks by exposure time */
614
615 if (vircam_linearity_analyse_domedark_groups() != 0) {
616 vircam_linearity_analyse_tidy(2);
617 return(-1);
618 }
619
620 /* See if the number of exposure times is too small for the order of the
621 polynomial you want to fit. If it is, the adjust the order */
622
623 if (ps.nddg < vircam_linearity_analyse_config.norder+1) {
624 cpl_msg_warning(fctid,
625 "Number of exposure times is too small: %" CPL_SIZE_FORMAT ", order: %" CPL_SIZE_FORMAT "\nTaking fit down to order %" CPL_SIZE_FORMAT,
626 (cpl_size)(ps.nddg),
627 (cpl_size)(vircam_linearity_analyse_config.norder),
628 (cpl_size)(ps.nddg-1));
629 vircam_linearity_analyse_config.norder = ps.nddg - 1;
630 }
631
632 /* Now, how many image extensions do we want to do? If the extension
633 number is zero, then we loop for all possible extensions. If it
634 isn't then we just do the extension specified */
635
636 vircam_exten_range(vircam_linearity_analyse_config.extenum,
637 (const cpl_frame *)cpl_frameset_get_position(ps.ddg[0].darks,0),
638 &jst,&jfn);
639 if (jst == -1 || jfn == -1) {
640 cpl_msg_error(fctid,"Unable to continue");
641 vircam_linearity_analyse_tidy(2);
642 return(-1);
643 }
644
645 /* Now loop for all the extensions and do the BPM analysis... */
646
647 for (j = jst; j <= jfn; j++) {
648 cpl_msg_info(fctid,"Beginning BPM work on extension %" CPL_SIZE_FORMAT,
649 (cpl_size)j);
650 isfirst = (j == jst);
651 dummy = 0;
652 vircam_linearity_analyse_config.bad_pixel_stat = 0.0;
653 vircam_linearity_analyse_config.bad_pixel_num = 0;
654 vircam_linearity_analyse_config.linerror = 0.0;
655 vircam_linearity_analyse_config.linearity = 0.0;
656 vircam_linearity_analyse_config.facrng = 0.0;
657 vircam_linearity_analyse_config.maxdiff = 0.0;
658
659 /* Get some standard info in case we need it for dummy products */
660
661 test = casu_fits_load(cpl_frameset_get_position(ps.ddg[0].domes,0),
662 CPL_TYPE_FLOAT,j);
663 ps.plist = cpl_propertylist_duplicate(casu_fits_get_phu(test));
664 ps.elist = cpl_propertylist_duplicate(casu_fits_get_ehu(test));
665 ps.nx = (int)cpl_image_get_size_x(casu_fits_get_image(test));
666 ps.ny = (int)cpl_image_get_size_y(casu_fits_get_image(test));
667 casu_fits_delete(test);
668
669 /* Load up the channel table for this detector and verify it. Get
670 saturation level for this detector from FITS header */
671
672 ps.chantab = casu_tfits_load(ps.chanfrm,j);
673 if (ps.chantab == NULL) {
674 cpl_msg_error(fctid,
675 "Channel table extension %" CPL_SIZE_FORMAT " failed to load",
676 (cpl_size)j);
677 dummy = 1;
678 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
679 return(-1);
680 continue;
681 } else if (vircam_chantab_verify(casu_tfits_get_table(ps.chantab))
682 != CASU_OK) {
683 cpl_msg_error(fctid,
684 "Channel table extension %" CPL_SIZE_FORMAT " has errors",
685 (cpl_size)j);
686 dummy = 1;
687 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
688 return(-1);
689 continue;
690 }
692 &sat) != CASU_OK) {
693 cpl_msg_error(fctid,
694 "Channel table extension header %" CPL_SIZE_FORMAT " missing saturation info",
695 (cpl_size)j);
696 dummy = 1;
697 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
698 return(-1);
699 vircam_linearity_analyse_tidy(1);
700 continue;
701 }
702
703 /* Get the channel structure */
704
705 vircam_chan_fill(casu_tfits_get_table(ps.chantab),&pp,&np);
706
707 /* If doing diagnostics, then create the tables now */
708
709 if (vircam_linearity_analyse_config.diagnostic) {
710 ps.diag1 = vircam_linearity_analyse_diagtab_init(np,ps.ndomes);
711 if (vircam_linearity_analyse_config.adjust)
712 ps.diag2 = vircam_linearity_analyse_diagtab_init(np,ps.ndomecheck);
713 }
714
715 /* Check DETLIVE for this extension */
716
717 if (vircam_pfits_get_detlive((const cpl_propertylist *)ps.elist,&live)
718 != CASU_OK) {
719 cpl_msg_error(fctid,"No DET LIVE keyword in this extension");
720 dummy = 1;
721 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
722 return(-1);
723 vircam_linearity_analyse_tidy(1);
724 vircam_chan_free(np,&pp);
725 continue;
726 }
727 if (! live) {
728 cpl_msg_info(fctid,"Detector flagged dead");
729 dummy = 1;
730 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
731 return(-1);
732 vircam_linearity_analyse_tidy(1);
733 vircam_chan_free(np,&pp);
734 continue;
735 }
736
737 /* Get the value of MINDIT so that you can work out the rough level
738 of the flats before the reset frame was subtracted off */
739
740 if (vircam_pfits_get_mindit((const cpl_propertylist *)ps.elist,
741 &mindit) != CASU_OK) {
742 cpl_msg_error(fctid,
743 "No value of MINDIT found in extension %" CPL_SIZE_FORMAT,
744 (cpl_size)j);
745 dummy = 1;
746 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
747 return(-1);
748 vircam_chan_free(np,&pp);
749 continue;
750 }
751
752 /* Get a representative from each exposure group and check if it
753 is saturated. If it is, then reject the group from further
754 analysis */
755
756 ngood = 0;
757 exps = cpl_malloc(ps.nddg*sizeof(float));
758 ngood_flats = 0;
759 for (i = 0; i < ps.nddg; i++) {
760 test = casu_fits_load(cpl_frameset_get_position(ps.ddg[i].domes,0),
761 CPL_TYPE_FLOAT,j);
762 med = cpl_image_get_median((const cpl_image*)casu_fits_get_image(test));
763 med *= (1.0 + mindit/ps.ddg[i].exptime);
764 if (med > sat) {
765 ps.ddg[i].flag = SATURATE_FLAG;
766 } else {
767 ngood++;
768 exps[ngood-1] = ps.ddg[i].exptime;
769 ngood_flats += ps.ddg[i].ndomes;
770 ps.ddg[i].flag = OK_FLAG;
771 }
772 casu_fits_delete(test);
773 }
774 exps = cpl_realloc(exps,ngood*sizeof(float));
775
776 /* Are there enough non-saturated exposures for linearity fit? */
777
778 if (ngood < vircam_linearity_analyse_config.norder+1) {
779 cpl_msg_info(fctid,
780 "Too few unsaturated flats for linearity fit for extension %" CPL_SIZE_FORMAT,
781 (cpl_size)j);
782 dummy = 1;
783 cpl_free(exps);
784 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
785 return(-1);
786 vircam_chan_free(np,&pp);
787 continue;
788 }
789
790 /* Sort the exposure array */
791
792 casu_sort(&exps,ngood,1);
793
794 /* Loop for each exposure time. When you have enough flats, then
795 you can quit this loop */
796
797 ps.nuse = min(vircam_linearity_analyse_config.maxbpmfr,ngood_flats);
798 ps.nflatlist = 0;
799 ps.flatlist = cpl_malloc(ps.nuse*sizeof(casu_fits *));
800 for (i = ngood-1; i >= 0; i--) {
801 krem = -1;
802 for (k = 0; k <= ps.nddg; k++) {
803 if (ps.ddg[k].exptime == exps[i]) {
804 krem = k;
805 break;
806 }
807 }
808
809 /* Load the dark frames from this exposure time */
810
811 darks = casu_fits_load_list(ps.ddg[krem].darks,CPL_TYPE_FLOAT,j);
812 ndarks = ps.ddg[krem].ndarks;
813 if (darks == NULL) {
814 cpl_msg_error(fctid,
815 "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
816 (cpl_size)j,ps.ddg[krem].exptime);
817 continue;
818 }
819
820 /* Form a mean dark for this exposure time. If there is only one
821 then don't bother, just load up that one frame. */
822
823 if (ndarks == 1) {
824 outdark = casu_fits_duplicate(darks[0]);
825 } else {
826 status = CASU_OK;
827 (void)casu_imcombine(darks,NULL,ndarks,1,1,1,5.0,"EXPTIME",
828 &outimage,NULL,&rejmask,&rejplus,&drs,
829 &status);
830 freespace(rejmask);
831 freespace(rejplus);
832 freepropertylist(drs);
833 if (status != CASU_OK) {
834 cpl_msg_error(fctid,
835 "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
836 (cpl_size)j,ps.ddg[krem].exptime);
837 freefitslist(darks,ndarks);
838 continue;
839 }
840 outdark = casu_fits_wrap(outimage,darks[0],NULL,NULL);
841 }
842 freefitslist(darks,ndarks);
843
844 /* Load the flats for this group */
845
846 domes = casu_fits_load_list(ps.ddg[krem].domes,CPL_TYPE_FLOAT,j);
847 ndomes = ps.ddg[krem].ndomes;
848 if (domes == NULL) {
849 cpl_msg_error(fctid,
850 "Error loading domes extension %" CPL_SIZE_FORMAT ", exptime %g",
851 (cpl_size)j,ps.ddg[i].exptime);
852 freefits(outdark);
853 continue;
854 }
855
856 /* Now loop for each flat in this group or until you have
857 filled the flats buffer */
858
859 for (kk = 0; kk < ndomes; kk++) {
860 status = CASU_OK;
861 casu_darkcor(domes[kk],outdark,1.0,&status);
862 ps.flatlist[ps.nflatlist] = casu_fits_duplicate(domes[kk]);
863 ps.ddg[krem].proc[kk] = ps.flatlist[ps.nflatlist];
864 ps.nflatlist++;
865 if (ps.nflatlist == ps.nuse)
866 break;
867 }
868
869 /* Tidy up a bit */
870
871 freefitslist(domes,ndomes);
872 freefits(outdark);
873
874 /* Do we have enough yet? */
875
876 if (ps.nflatlist == ps.nuse)
877 break;
878 }
879 freespace(exps);
880 ps.flatlist = cpl_realloc(ps.flatlist,
881 (ps.nflatlist)*sizeof(casu_fits *));
882
883 /* Generate a bad pixel mask now */
884
885 status = CASU_OK;
886 (void)casu_genbpm(ps.flatlist,ps.nflatlist,NULL,
887 vircam_linearity_analyse_config.lthr,
888 vircam_linearity_analyse_config.hthr,"EXPTIME",
889 &(ps.bpm_array),&nbad,&badfrac,&status);
890 bpm = cpl_array_get_data_int(ps.bpm_array);
891
892 /* Store away some useful info */
893
894 vircam_linearity_analyse_config.bad_pixel_num = nbad;
895 vircam_linearity_analyse_config.bad_pixel_stat = badfrac;
896
897 /* Right. Free the pointer for flatlist, but don't delete the
898 casu_fits structures in the list because they are copied into
899 the ddg structure */
900
901 freespace(ps.flatlist);
902 ps.nflatlist = 0;
903
904 /* Get an initial allocation of space to hold the stats */
905
906 nalloc = 16;
907 fdata = cpl_malloc(nalloc*sizeof(double *));
908 dexps = cpl_malloc(nalloc*sizeof(double));
909 mjds = cpl_malloc(nalloc*sizeof(double));
910
911 /* Loop through the ddg structure, missing out any that have
912 overexposed flats. Work out the stats for the images that
913 remain. */
914
915 cpl_msg_info(fctid,
916 "Beginning linearity work on extension %" CPL_SIZE_FORMAT,
917 (cpl_size)j);
918 ps.nfdata = 0;
919 outdark = NULL;
920 for (i = 0; i < ps.nddg; i++) {
921 if (ps.ddg[i].flag == SATURATE_FLAG)
922 continue;
923 for (k = 0; k < ps.ddg[i].ndomes; k++) {
924
925 /* If this particular frame wasn't processed, then you need
926 to form a dark for this group. */
927
928 if (ps.ddg[i].proc[k] == NULL) {
929 if (outdark == NULL) {
930
931 /* Load the dark frames from this exposure time */
932
933 darks = casu_fits_load_list(ps.ddg[i].darks,
934 CPL_TYPE_FLOAT,j);
935 ndarks = ps.ddg[i].ndarks;
936 if (darks == NULL) {
937 cpl_msg_error(fctid,
938 "Error loading darks extension %" CPL_SIZE_FORMAT ", exptime %g",
939 (cpl_size)j,ps.ddg[i].exptime);
940 continue;
941 }
942
943 /* Form a mean dark for this exposure time. If there
944 is only one then don't bother, just load up that
945 one frame. */
946
947 if (ps.ddg[i].ndarks == 1) {
948 outdark = casu_fits_duplicate(darks[0]);
949 } else {
950 status = CASU_OK;
951 (void)casu_imcombine(darks,NULL,ndarks,1,1,1,5.0,
952 "EXPTIME",&outimage,NULL,
953 &rejmask,&rejplus,
954 &drs,&status);
955 freespace(rejmask);
956 freespace(rejplus);
957 freepropertylist(drs);
958 if (status != CASU_OK) {
959 cpl_msg_error(fctid,
960 "Dark combine failure extension %" CPL_SIZE_FORMAT " exposure %g",
961 (cpl_size)j,ps.ddg[i].exptime);
962 freefitslist(darks,ndarks);
963 continue;
964 }
965 outdark = casu_fits_wrap(outimage,darks[0],
966 NULL,NULL);
967 }
968 freefitslist(darks,ndarks);
969 }
970
971 /* Load the flat and dark correct it */
972
973 frame = cpl_frameset_get_position(ps.ddg[i].domes,k);
974 fframe = casu_fits_load(frame,CPL_TYPE_FLOAT,j);
975 casu_darkcor(fframe,outdark,1.0,&status);
976
977 /* If this frame has already been corrected, then use it */
978
979 } else {
980 fframe = ps.ddg[i].proc[k];
981 }
982
983 /* Generate the stats for this frame and store it */
984
985 d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
986 if (ps.nfdata >= nalloc) {
987 nalloc += 16;
988 fdata = cpl_realloc(fdata,nalloc*sizeof(double *));
989 dexps = cpl_realloc(dexps,nalloc*sizeof(double));
990 mjds = cpl_realloc(mjds,nalloc*sizeof(double));
991 }
992 (void)vircam_pfits_get_mjd(casu_fits_get_phu(fframe),&mjd);
993 mjds[ps.nfdata] = mjd;
994 dexps[ps.nfdata] = (double)(ps.ddg[i].exptime);
995 fdata[ps.nfdata] = d;
996
997 /* If doing diagnostic curves, then add the relevant
998 information to the first table now */
999
1000 if (ps.diag1 != NULL) {
1001 cpl_table_set_string(ps.diag1,"filename",
1002 (cpl_size)(ps.nfdata),
1003 casu_fits_get_filename(fframe));
1004 cpl_table_set_double(ps.diag1,"exptime",
1005 (cpl_size)ps.nfdata,
1006 dexps[ps.nfdata]);
1007 cpl_table_set_double(ps.diag1,"mjd",(cpl_size)(ps.nfdata),
1008 mjd);
1009 for (n = 1; n <= np; n++) {
1010 char * colname = cpl_sprintf("rawflux_%02d", n);
1011 // snprintf(colname,16,"rawflux_%02d", n);
1012 cpl_table_set_double(ps.diag1,colname,
1013 (cpl_size)(ps.nfdata),d[n-1]);
1014 cpl_free(colname);
1015 }
1016 }
1017 if (ps.ddg[i].proc[k] != NULL) {
1018 freefits(ps.ddg[i].proc[k]);
1019 } else {
1020 freefits(fframe);
1021 }
1022 ps.nfdata++;
1023 }
1024 freefits(outdark);
1025 }
1026 freefits(outdark);
1027 if (ps.diag1 != NULL)
1028 cpl_table_set_size(ps.diag1,(cpl_size)(ps.nfdata));
1029
1030 /* Now, if we are going to tweak the stats using the monitor exposure
1031 then we should do that now */
1032
1033 if (vircam_linearity_analyse_config.adjust) {
1034
1035 /* Get some workspace for the data array */
1036
1037 cdata = cpl_malloc(ps.ndomecheck*sizeof(double *));
1038
1039 /* Get the exposure time for the monitor set and make sure
1040 that the set isn't saturated */
1041
1042 test = casu_fits_load(cpl_frameset_get_position(ps.domecheck,0),
1043 CPL_TYPE_FLOAT,j);
1045 med = cpl_image_get_median((const cpl_image*)casu_fits_get_image(test));
1046 med *= (1.0 + mindit/expt);
1047 adjust = 1;
1048 if (med > sat) {
1049 cpl_msg_info(fctid,"Monitor exposures saturated. No drift adjustment made");
1050 adjust = 0;
1051 }
1052 casu_fits_delete(test);
1053
1054 /* Ok assuming all that's going well, then form a mean dark */
1055
1056 if (adjust) {
1057
1058 darks = casu_fits_load_list(ps.darkcheck,CPL_TYPE_FLOAT,j);
1059 ndarks = ps.ndarkcheck;
1060 if (darks == NULL) {
1061 cpl_msg_error(fctid,
1062 "Error loading check darks extension %" CPL_SIZE_FORMAT,
1063 (cpl_size)j);
1064 continue;
1065 }
1066
1067 /* Form a mean dark for this exposure time. If there
1068 is only one then don't bother, just load up that
1069 one frame. */
1070
1071 if (ndarks == 1) {
1072 outdark = casu_fits_duplicate(darks[0]);
1073 } else {
1074 status = CASU_OK;
1075 (void)casu_imcombine(darks,NULL,ndarks,1,1,1,5.0,"EXPTIME",
1076 &outimage,NULL,&rejmask,&rejplus,
1077 &drs,&status);
1078 freespace(rejmask);
1079 freespace(rejplus);
1080 freepropertylist(drs);
1081 if (status != CASU_OK) {
1082 cpl_msg_error(fctid,
1083 "Combine failure extension %" CPL_SIZE_FORMAT " monitor",
1084 (cpl_size)j);
1085 freefitslist(darks,ndarks);
1086 continue;
1087 }
1088 outdark = casu_fits_wrap(outimage,darks[0],
1089 NULL,NULL);
1090 }
1091 freefitslist(darks,ndarks);
1092
1093 /* Now, loop through the monitor domes, dark correct and then
1094 do the stats */
1095
1096 mjdcheck = cpl_malloc(ps.ndomecheck*sizeof(double));
1097 for (i = 0; i < ps.ndomecheck; i++) {
1098 frame = cpl_frameset_get_position(ps.domecheck,i);
1099 fframe = casu_fits_load(frame,CPL_TYPE_FLOAT,j);
1100 casu_darkcor(fframe,outdark,1.0,&status);
1101 d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
1102 cdata[i] = d;
1105 mjdcheck[i] = mjd;
1106
1107 /* If doing diagnostics then fill in the table now */
1108
1109 if (ps.diag2 != NULL) {
1110 cpl_table_set_string(ps.diag2,"filename",(cpl_size)i,
1111 casu_fits_get_filename(fframe));
1112 cpl_table_set_double(ps.diag2,"exptime",
1113 (cpl_size)i,(double)expt);
1114 cpl_table_set_double(ps.diag2,"mjd",
1115 (cpl_size)i,mjd);
1116 for (n = 1; n <= np; n++) {
1117 char * colname = cpl_sprintf("rawflux_%02d", n);
1118 // snprintf(colname,16,"rawflux_%02d", n);
1119 cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
1120 d[n-1]);
1121 cpl_free(colname);
1122 colname = cpl_sprintf("linflux_%02d", n);
1123 //snprintf(colname,16,"linflux_%02d", n);
1124 cpl_table_set_double(ps.diag2,colname,(cpl_size)i,
1125 d[n-1]);
1126 cpl_free(colname);
1127 }
1128 }
1129 freefits(fframe);
1130 }
1131 freefits(outdark);
1132
1133 /* Generate the correction factors now */
1134
1135 cf = vircam_linearity_tweakfac(cdata,mjdcheck,ps.ndomecheck,
1136 np,&facrng,&maxdiff);
1137 vircam_linearity_analyse_config.facrng = 100.0*(float)facrng;
1138 vircam_linearity_analyse_config.maxdiff = 100.0*(float)maxdiff;
1139 if (ps.diag2 != NULL) {
1140 for (i = 0; i < ps.ndomecheck; i++)
1141 cpl_table_set_double(ps.diag2,"adjust_fac",
1142 (cpl_size)i,cf[i]);
1143 }
1144
1145 /* Ok, now do the correction for each of the linearity
1146 sequence frames */
1147
1148 for (i = 0; i < ps.nfdata; i++) {
1149 mjd = mjds[i];
1150 krem = -1;
1151 for (k = 0; k < ps.ndomecheck; k++) {
1152 if (mjd < mjdcheck[k]) {
1153 krem = k;
1154 break;
1155 }
1156 }
1157 if (krem == -1) {
1158 fac = cf[ps.ndomecheck-1];
1159 } else if (krem == 0) {
1160 fac = cf[0];
1161 } else {
1162 fac = 0.5*(cf[krem -1] + cf[krem]);
1163 }
1164 for (k = 0; k < np; k++)
1165 fdata[i][k] /= fac;
1166 if (ps.diag1 != NULL)
1167 cpl_table_set_double(ps.diag1,"adjust_fac",
1168 (cpl_size)i,fac);
1169 }
1170
1171 /* Get rid of some stuff now */
1172
1173 freespace2(cdata,ps.ndomecheck);
1174 freespace(cf);
1175 freespace(mjdcheck);
1176 }
1177
1178 cpl_free(cdata);
1179 }
1180
1181 /* Do some intermediate tidying */
1182
1183 freespace(mjds);
1184 vircam_chan_free(np,&pp);
1185
1186 /* Right, there should be no images left in memory now. Do the
1187 linearity analysis now. */
1188
1189 (void)vircam_genlincur(fdata,ps.nfdata,dexps,(double)mindit,ps.chantab,
1190 vircam_linearity_analyse_config.norder,
1191 &(ps.lchantab),&lindata,&status);
1192 if (ps.diag1 != NULL) {
1193 for (i = 0; i < ps.nfdata; i++) {
1194 for (n = 0; n < np; n++) {
1195 char * colname = cpl_sprintf("linflux_%02d", n+1);
1196 // snprintf(colname,16,"linflux_%02d", n+1);
1197 cpl_table_set_double(ps.diag1,colname,
1198 (cpl_size)i,lindata[i*np+n]);
1199 cpl_free(colname);
1200 }
1201 }
1202 }
1203 freespace2(fdata,ps.nfdata);
1204 freespace(dexps);
1205 freespace(lindata);
1206 if (status != CASU_OK) {
1207 cpl_msg_error(fctid,
1208 "Linearity curve fit failed extension %" CPL_SIZE_FORMAT,
1209 (cpl_size)j);
1210 dummy = 1;
1211 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
1212 return(-1);
1213 vircam_linearity_analyse_tidy(1);
1214 continue;
1215 }
1216
1217 /* Get some QC1 parameters by finding the average fit error and
1218 the average percentage non-linearity */
1219
1220 vircam_linearity_analyse_config.linearity =
1221 (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000");
1222 vircam_linearity_analyse_config.linerror =
1223 (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000_err");
1224
1225 /* Save new linearity info */
1226
1227 if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
1228 return(-1);
1229 }
1230
1231 /* Get out of here */
1232
1233 vircam_linearity_analyse_tidy(2);
1234 return(0);
1235}
1236
1237
1238/*---------------------------------------------------------------------------*/
1245/*---------------------------------------------------------------------------*/
1246
1247static int vircam_linearity_analyse_save(cpl_frameset *framelist,
1248 cpl_parameterlist *parlist) {
1249 cpl_propertylist *plist,*elist,*pafprop;
1250 cpl_image *outimg;
1251 const char *outtab = "lchantab.fits";
1252 const char *outbpm = "bpm.fits";
1253 const char *outtabpaf = "lchantab";
1254 const char *outbpmpaf = "bpm";
1255 const char *outdiag1 = "ldiag1.fits";
1256 const char *outdiag2 = "ldiag2.fits";
1257 const char *fctid = "vircam_linearity_analyse_save";
1258 const char *recipeid = "vircam_linearity_analyse";
1259 int nx,ny,nord,*bpm,i;
1260
1261 /* Do some stuff for the first extension to set up the frame */
1262
1263 nord = vircam_linearity_analyse_config.norder;
1264 if (isfirst) {
1265
1266 /* Set up the output frame */
1267
1268 product_frame_chantab = cpl_frame_new();
1269 cpl_frame_set_filename(product_frame_chantab,outtab);
1270 cpl_frame_set_tag(product_frame_chantab,VIRCAM_PRO_CHANTAB);
1271 cpl_frame_set_type(product_frame_chantab,CPL_FRAME_TYPE_TABLE);
1272 cpl_frame_set_group(product_frame_chantab,CPL_FRAME_GROUP_PRODUCT);
1273 cpl_frame_set_level(product_frame_chantab,CPL_FRAME_LEVEL_FINAL);
1274
1275 /* Set up the PRO keywords for primary */
1276
1277 ps.phupaf = vircam_paf_phu_items(ps.plist);
1278 plist = cpl_propertylist_duplicate(ps.plist);
1279 vircam_dfs_set_product_primary_header(plist,product_frame_chantab,
1280 framelist,parlist,
1281 (char *)recipeid,
1282 "PRO-1.15",ps.inherit,0);
1283
1284 /* Now define the table propertylist and give it an extension name and
1285 PRO keywords */
1286
1287 elist = cpl_propertylist_duplicate(ps.elist);
1288 cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
1289 vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
1290 framelist,parlist,(char *)recipeid,
1291 "PRO-1.15",ps.inherit);
1292
1293 /* Add QC1 info */
1294 if (isfinite(vircam_linearity_analyse_config.linearity)){
1295 cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
1296 vircam_linearity_analyse_config.linearity);
1297 } else {
1298 cpl_propertylist_update_float(elist,"ESO QC LINEARITY", 100.);
1299 }
1300 cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
1301 "% non-linearity at 10000 ADU");
1302 cpl_propertylist_update_float(elist,"ESO QC LINERROR",
1303 vircam_linearity_analyse_config.linerror);
1304 cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
1305 "% error non-linearity at 10000 ADU");
1306 cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
1307 vircam_linearity_analyse_config.facrng);
1308 cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
1309 "total % range in screen variation");
1310 cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
1311 vircam_linearity_analyse_config.maxdiff);
1312 cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
1313 "maximum % step in screen variation");
1314 cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
1315
1316 /* Set up a dummy table if necessary */
1317
1318 if (dummy == 1) {
1319 casu_dummy_property(elist);
1320 if (ps.lchantab == NULL)
1321 ps.lchantab = vircam_chantab_new(nord,casu_tfits_get_table(ps.chantab));
1322 }
1323
1324 /* And finally save the table */
1325
1326 if (cpl_table_save(ps.lchantab,plist,elist,outtab,CPL_IO_DEFAULT)
1327 != CPL_ERROR_NONE) {
1328 cpl_msg_error(fctid,"Cannot save product table extension");
1329 freepropertylist(plist);
1330 freepropertylist(elist);
1331 return(-1);
1332 }
1333 cpl_frameset_insert(framelist,product_frame_chantab);
1334
1335 /* Write PAF */
1336
1337 pafprop = vircam_paf_req_items(elist);
1338 casu_merge_propertylists(pafprop,ps.phupaf);
1339 vircam_paf_append(pafprop,plist,"ESO INS FILT1 NAME");
1340 vircam_paf_append(pafprop,elist,"ESO PRO CATG");
1341 vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
1342 if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
1343 "QC file",pafprop) != CASU_OK)
1344 cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
1345 cpl_propertylist_delete(pafprop);
1346
1347 /* Quick tidy */
1348
1349 freepropertylist(plist);
1350 freepropertylist(elist);
1351
1352 /* Set up the output bad pixel mask primary */
1353
1354 product_frame_bpm = cpl_frame_new();
1355 cpl_frame_set_filename(product_frame_bpm,outbpm);
1356 cpl_frame_set_tag(product_frame_bpm,VIRCAM_PRO_BPM);
1357 cpl_frame_set_type(product_frame_bpm,CPL_FRAME_TYPE_IMAGE);
1358 cpl_frame_set_group(product_frame_bpm,CPL_FRAME_GROUP_PRODUCT);
1359 cpl_frame_set_level(product_frame_bpm,CPL_FRAME_LEVEL_FINAL);
1360
1361 /* Set up the PRO keywords for primary header in the bpm */
1362
1363 plist = ps.plist;
1364 vircam_dfs_set_product_primary_header(plist,product_frame_bpm,
1365 framelist,parlist,
1366 (char *)recipeid,"PRO-1.15",
1367 ps.inherit,0);
1368
1369 /* Now save the PHU 'image' */
1370
1371 if (cpl_image_save(NULL,outbpm,CPL_TYPE_UCHAR,plist,
1372 CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
1373 cpl_msg_error(fctid,"Cannot save product PHU");
1374 cpl_frame_delete(product_frame_bpm);
1375 return(-1);
1376 }
1377 cpl_frameset_insert(framelist,product_frame_bpm);
1378
1379 /* Section for saving diagnostic tables. First the linearity
1380 sequence diagnostics */
1381
1382 if (ps.diag1 != NULL) {
1383
1384 /* Set up the output frame */
1385
1386 product_frame_diag1 = cpl_frame_new();
1387 cpl_frame_set_filename(product_frame_diag1,outdiag1);
1388 cpl_frame_set_tag(product_frame_diag1,VIRCAM_PRO_LIN_DIAG1);
1389 cpl_frame_set_type(product_frame_diag1,CPL_FRAME_TYPE_TABLE);
1390 cpl_frame_set_group(product_frame_diag1,CPL_FRAME_GROUP_PRODUCT);
1391 cpl_frame_set_level(product_frame_diag1,CPL_FRAME_LEVEL_FINAL);
1392
1393 /* Set up the PRO keywords for primary */
1394
1395 plist = cpl_propertylist_duplicate(ps.plist);
1396 vircam_dfs_set_product_primary_header(plist,product_frame_diag1,
1397 framelist,parlist,
1398 (char *)recipeid,
1399 "PRO-1.15",ps.inherit,0);
1400
1401 /* Now define the table propertylist and give it an extension name and
1402 PRO keywords */
1403
1404 elist = cpl_propertylist_duplicate(ps.elist);
1405 vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
1406 framelist,parlist,
1407 (char *)recipeid,"PRO-1.15",
1408 ps.inherit);
1409
1410 /* Set up a dummy property if necessary */
1411
1412 if (dummy == 1)
1413 casu_dummy_property(elist);
1414
1415 /* And finally save the table */
1416
1417 if (cpl_table_save(ps.diag1,plist,elist,outdiag1,CPL_IO_DEFAULT)
1418 != CPL_ERROR_NONE) {
1419 cpl_msg_error(fctid,"Cannot save product table extension");
1420 freepropertylist(plist);
1421 freepropertylist(elist);
1422 return(-1);
1423 }
1424 cpl_frameset_insert(framelist,product_frame_diag1);
1425 freepropertylist(plist);
1426 freepropertylist(elist);
1427 }
1428
1429 /* Now the monitor sequence diagnostics */
1430
1431 if (ps.diag2 != NULL) {
1432
1433 /* Set up the output frame */
1434
1435 product_frame_diag2 = cpl_frame_new();
1436 cpl_frame_set_filename(product_frame_diag2,outdiag2);
1437 cpl_frame_set_tag(product_frame_diag2,VIRCAM_PRO_LIN_DIAG2);
1438 cpl_frame_set_type(product_frame_diag2,CPL_FRAME_TYPE_TABLE);
1439 cpl_frame_set_group(product_frame_diag2,CPL_FRAME_GROUP_PRODUCT);
1440 cpl_frame_set_level(product_frame_diag2,CPL_FRAME_LEVEL_FINAL);
1441
1442 /* Set up the PRO keywords for primary */
1443
1444 plist = cpl_propertylist_duplicate(ps.plist);
1445 vircam_dfs_set_product_primary_header(plist,product_frame_diag2,
1446 framelist,parlist,
1447 (char *)recipeid,
1448 "PRO-1.15",ps.inherit,0);
1449
1450 /* Now define the table propertylist and give it an extension name
1451 and PRO keywords */
1452
1453 elist = cpl_propertylist_duplicate(ps.elist);
1454 vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
1455 framelist,parlist,
1456 (char *)recipeid,"PRO-1.15",
1457 ps.inherit);
1458
1459 /* Set up a dummy property if necessary */
1460
1461 if (dummy == 1)
1462 casu_dummy_property(elist);
1463
1464 /* And finally save the table */
1465
1466 if (cpl_table_save(ps.diag2,plist,elist,outdiag2,CPL_IO_DEFAULT)
1467 != CPL_ERROR_NONE) {
1468 cpl_msg_error(fctid,"Cannot save product table extension");
1469 freepropertylist(plist);
1470 freepropertylist(elist);
1471 return(-1);
1472 }
1473 cpl_frameset_insert(framelist,product_frame_diag2);
1474 freepropertylist(plist);
1475 freepropertylist(elist);
1476 }
1477
1478 /* Section for all other extensions */
1479
1480 } else {
1481
1482 /* Do the table extension PRO keywords */
1483
1484 elist = cpl_propertylist_duplicate(ps.elist);
1485 cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
1486 vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
1487 framelist,parlist,
1488 (char *)recipeid,
1489 "PRO-1.15",ps.inherit);
1490
1491 /* Add QC1 info */
1492 if (isnormal(vircam_linearity_analyse_config.linearity)){
1493 cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
1494 vircam_linearity_analyse_config.linearity);
1495 } else {
1496 cpl_propertylist_update_float(elist,"ESO QC LINEARITY", 100.);
1497 }
1498
1499 cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
1500 "% non-linearity at 10000 ADU");
1501 cpl_propertylist_update_float(elist,"ESO QC LINERROR",
1502 vircam_linearity_analyse_config.linerror);
1503 cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
1504 "% error non-linearity at 10000 ADU");
1505 cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
1506 vircam_linearity_analyse_config.facrng);
1507 cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
1508 "total % range in screen variation");
1509 cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
1510 vircam_linearity_analyse_config.maxdiff);
1511 cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
1512 "maximum % step in screen variation");
1513 cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
1514
1515 /* Set up a dummy table if necessary */
1516
1517 if (dummy == 1) {
1518 casu_dummy_property(elist);
1519 if (ps.lchantab == NULL)
1520 ps.lchantab = vircam_chantab_new(nord,casu_tfits_get_table(ps.chantab));
1521 }
1522
1523 /* And finally save the table */
1524
1525 if (cpl_table_save(ps.lchantab,NULL,elist,outtab,CPL_IO_EXTEND)
1526 != CPL_ERROR_NONE) {
1527 cpl_msg_error(fctid,"Cannot save product table extension");
1528 freepropertylist(elist);
1529 return(-1);
1530 }
1531
1532 /* Write PAF */
1533
1534 pafprop = vircam_paf_req_items(elist);
1535 casu_merge_propertylists(pafprop,ps.phupaf);
1536 vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
1537 vircam_paf_append(pafprop,elist,"ESO PRO CATG");
1538 vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
1539 if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
1540 "QC file",pafprop) != CASU_OK)
1541 cpl_msg_warning(fctid,"Unable to save PAF for BPM");
1542 cpl_propertylist_delete(pafprop);
1543
1544 /* Quick tidy */
1545
1546 freepropertylist(elist);
1547
1548 /* Now the diagnostic tables */
1549
1550 if (ps.diag1 != NULL) {
1551 elist = cpl_propertylist_duplicate(ps.elist);
1552 vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
1553 framelist,parlist,
1554 (char *)recipeid,"PRO-1.15",
1555 ps.inherit);
1556
1557 /* Set up a dummy property if necessary */
1558
1559 if (dummy == 1)
1560 casu_dummy_property(elist);
1561
1562 /* And finally save the table */
1563
1564 if (cpl_table_save(ps.diag1,NULL,elist,outdiag1,CPL_IO_EXTEND)
1565 != CPL_ERROR_NONE) {
1566 cpl_msg_error(fctid,"Cannot save product table extension");
1567 freepropertylist(elist);
1568 return(-1);
1569 }
1570 freepropertylist(elist);
1571 }
1572 if (ps.diag2 != NULL) {
1573 elist = cpl_propertylist_duplicate(ps.elist);
1574 vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
1575 framelist,parlist,
1576 (char *)recipeid,"PRO-1.15",
1577 ps.inherit);
1578
1579 /* Set up a dummy property if necessary */
1580
1581 if (dummy == 1)
1582 casu_dummy_property(elist);
1583
1584 /* And finally save the table */
1585
1586 if (cpl_table_save(ps.diag2,NULL,elist,outdiag2,CPL_IO_EXTEND)
1587 != CPL_ERROR_NONE) {
1588 cpl_msg_error(fctid,"Cannot save product table extension");
1589 freepropertylist(elist);
1590 return(-1);
1591 }
1592 freepropertylist(elist);
1593 }
1594
1595 }
1596
1597 /* Save the bpm extension now */
1598
1599 plist = ps.elist;
1600 nx = ps.nx;
1601 ny = ps.ny;
1602 if (dummy && ps.bpm_array == NULL) {
1603 ps.bpm_array = cpl_array_new((cpl_size)(nx*ny),CPL_TYPE_INT);
1604 bpm = cpl_array_get_data_int(ps.bpm_array);
1605 for (i = 0; i < nx*ny; i++)
1606 bpm[i] = 0;
1607 }
1608 bpm = cpl_array_get_data_int(ps.bpm_array);
1609 vircam_dfs_set_product_exten_header(plist,product_frame_bpm,
1610 framelist,parlist,(char *)recipeid,
1611 "PRO-1.15",ps.inherit);
1612 cpl_propertylist_update_float(plist,"ESO QC BAD_PIXEL_STAT",
1613 vircam_linearity_analyse_config.bad_pixel_stat);
1614 cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_STAT",
1615 "Fraction of pixels that are bad");
1616 cpl_propertylist_update_int(plist,"ESO QC BAD_PIXEL_NUM",
1617 vircam_linearity_analyse_config.bad_pixel_num);
1618 cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_NUM",
1619 "Number of pixels that are bad");
1620 cpl_propertylist_update_int(plist,"ESO PRO DATANCOM",ps.nuse);
1621 if (dummy)
1622 casu_dummy_property(plist);
1623 outimg = cpl_image_wrap_int((cpl_size)nx,(cpl_size)ny,bpm);
1624 if (cpl_image_save(outimg,outbpm,CPL_TYPE_UCHAR,plist,
1625 CPL_IO_EXTEND) != CPL_ERROR_NONE) {
1626 cpl_msg_error(fctid,"Cannot save product image extension");
1627 return(-1);
1628 }
1629
1630 /* Write PAF */
1631
1632 pafprop = vircam_paf_req_items(plist);
1633 casu_merge_propertylists(pafprop,ps.phupaf);
1634 vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
1635 vircam_paf_append(pafprop,ps.plist,"ESO PRO CATG");
1636 vircam_paf_append(pafprop,plist,"ESO PRO DATANCOM");
1637 if (vircam_paf_print((char *)outbpmpaf,"VIRCAM/vircam_linearity_analyse",
1638 "QC file",pafprop) != CASU_OK)
1639 cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
1640 cpl_propertylist_delete(pafprop);
1641
1642 /* Quick tidy */
1643
1644 cpl_image_unwrap(outimg);
1645
1646 return(0);
1647}
1648
1649/*---------------------------------------------------------------------------*/
1657/*---------------------------------------------------------------------------*/
1658
1659static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
1660 cpl_parameterlist *parlist) {
1661 int retval;
1662 const char *fctid = "vircam_linearity_analyse_lastbit";
1663
1664 /* Save the new channel table and bad pixel map */
1665
1666 cpl_msg_info(fctid,
1667 "Saving linearity table and bpm for extension %" CPL_SIZE_FORMAT,
1668 (cpl_size)jext);
1669 retval = vircam_linearity_analyse_save(framelist,parlist);
1670 if (retval != 0) {
1671 vircam_linearity_analyse_tidy(2);
1672 return(-1);
1673 }
1674
1675 /* Do some intermediate tidying */
1676
1677 vircam_linearity_analyse_tidy(1);
1678 return(0);
1679}
1680
1681/*---------------------------------------------------------------------------*/
1685/*---------------------------------------------------------------------------*/
1686
1687static int vircam_linearity_analyse_domedark_groups(void) {
1688 int i,j,found;
1689 float texp;
1690 cpl_frame *frame;
1691 cpl_propertylist *plist;
1692 const char *fctid = "vircam_linearity_analyse_domedark_groups";
1693
1694 /* Start by getting the memory for the domedark groups */
1695
1696 ps.ddg = cpl_calloc(ps.ndomes,sizeof(ddgrp));
1697 ps.nddg = 0;
1698
1699 /* Loop for each of the dome frames and get its exposure time. If this
1700 doesn't exist, then signal an error and go on */
1701
1702 for (i = 0; i < ps.ndomes; i++) {
1703 frame = cpl_frameset_get_position(ps.domelist,i);
1704 plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
1705 if (vircam_pfits_get_exptime(plist,&texp) != CASU_OK) {
1706 cpl_msg_warning(fctid,"No exposure time found in %s",
1707 cpl_frame_get_filename(frame));
1708 cpl_propertylist_delete(plist);
1709 continue;
1710 }
1711 cpl_propertylist_delete(plist);
1712
1713 /* Search the domedark groups to see if this exposure time has already
1714 been used. If not, then create a new group. If it has then just add
1715 this frame to the correct group */
1716
1717 found = 0;
1718 for (j = 0; j < ps.nddg; j++) {
1719 if (ps.ddg[j].exptime == texp) {
1720 found = 1;
1721 break;
1722 }
1723 }
1724 if (found) {
1725 cpl_frameset_insert(ps.ddg[j].domes,cpl_frame_duplicate(frame));
1726 ps.ddg[j].ndomes += 1;
1727 } else {
1728 ps.ddg[ps.nddg].exptime = texp;
1729 ps.ddg[ps.nddg].darks = cpl_frameset_new();
1730 ps.ddg[ps.nddg].domes = cpl_frameset_new();
1731 ps.ddg[ps.nddg].ndarks = 0;
1732 ps.ddg[ps.nddg].ndomes = 1;
1733 ps.ddg[ps.nddg].flag = OK_FLAG;
1734 cpl_frameset_insert(ps.ddg[ps.nddg].domes,
1735 cpl_frame_duplicate(frame));
1736 ps.nddg += 1;
1737 }
1738 }
1739
1740 /* Right, now loop through all the darks and get their exposure times */
1741
1742 for (i = 0; i < ps.ndarks; i++) {
1743 frame = cpl_frameset_get_position(ps.darklist,i);
1744 plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
1745 if (vircam_pfits_get_exptime(plist,&texp) != CASU_OK) {
1746 cpl_msg_warning(fctid,"No exposure time found in %s",
1747 cpl_frame_get_filename(frame));
1748 cpl_propertylist_delete(plist);
1749 continue;
1750 }
1751 cpl_propertylist_delete(plist);
1752
1753 /* Search the domedark groups to see if this dark fits into one of
1754 the defined groups. If not, then ignore it. If it does, then
1755 add it into the dark frameset */
1756
1757 found = 0;
1758 for (j = 0; j < ps.nddg; j++) {
1759 if (ps.ddg[j].exptime == texp) {
1760 found = 1;
1761 break;
1762 }
1763 }
1764 if (found) {
1765 cpl_frameset_insert(ps.ddg[j].darks,cpl_frame_duplicate(frame));
1766 ps.ddg[j].ndarks += 1;
1767 }
1768 }
1769
1770 /* Now go through the domedark groups and ditch any that don't have any
1771 dark frames */
1772
1773 i = 0;
1774 while (i < ps.nddg) {
1775 if (ps.ddg[i].ndarks == 0) {
1776 cpl_msg_warning(fctid,
1777 "No dark frames exist for exposure %g\nThrowing these away",
1778 ps.ddg[i].exptime);
1779 freeframeset(ps.ddg[i].darks);
1780 freeframeset(ps.ddg[i].domes);
1781 for (j = i+1; j < ps.nddg; j++)
1782 ps.ddg[j-1] = ps.ddg[j];
1783 ps.nddg -= 1;
1784 } else
1785 i++;
1786 }
1787
1788 /* Allocate some space for casu_fits arrays for processed domes */
1789
1790 for (i = 0; i < ps.nddg; i++) {
1791 ps.ddg[i].proc = cpl_malloc(ps.ddg[i].ndomes*sizeof(casu_fits *));
1792 for (j = 0; j < ps.ddg[i].ndomes; j++)
1793 ps.ddg[i].proc[j] = NULL;
1794 }
1795
1796 /* Resize the output array and return so long as there is anything
1797 left. If there isn't then signal a major error */
1798
1799 if (ps.nddg > 0) {
1800 ps.ddg = cpl_realloc(ps.ddg,ps.nddg*sizeof(ddgrp));
1801 return(0);
1802 } else {
1803 cpl_msg_error(fctid,"There are no darks defined for input domes");
1804 return(-1);
1805 }
1806}
1807
1808/*---------------------------------------------------------------------------*/
1817/*---------------------------------------------------------------------------*/
1818
1819static double *vircam_linearity_analyse_genstat(casu_fits *fframe, int *bpm,
1820 parquet *p, int np) {
1821 int i,ist,ifn,jst,jfn,n,jind2,iind2,jj,nx,ii;
1822 parquet *pp;
1823 double *d;
1824 float *tmp,*data;
1825
1826 /* Get the workspace for the output result */
1827
1828 d = cpl_malloc(np*sizeof(double));
1829
1830 /* Get the input data array */
1831
1832 nx = (int)cpl_image_get_size_x(casu_fits_get_image(fframe));
1833 data = cpl_image_get_data_float(casu_fits_get_image(fframe));
1834
1835 /* Get some workspace for doing the median calculations */
1836
1837 tmp = cpl_malloc(SUBSET*SUBSET*sizeof(float));
1838
1839 /* Loop for each channel in the parquet structure */
1840
1841 for (i = 0; i < np; i++) {
1842 pp = p + i;
1843
1844 /* Take the central part of the channel */
1845
1846 ist = ((pp->delta_i)/2 - SUBSET2);
1847 ifn = ist + SUBSET - 1;
1848 jst = ((pp->delta_j)/2 - SUBSET2);
1849 jfn = jst + SUBSET - 1;
1850
1851 /* Put the data into the workspace and do a median */
1852
1853 n = 0;
1854 for (jj = jst; jj <= jfn; jj++) {
1855 jind2 = (jj + pp->iymin - 1)*nx;
1856 for (ii = ist; ii <= ifn; ii++) {
1857 iind2 = jind2 + ii + pp->ixmin - 1;
1858 if (bpm[iind2] == 0)
1859 tmp[n++] = data[iind2];
1860 }
1861 }
1862 d[i] = (double)casu_med(tmp,NULL,(long)n);
1863 }
1864
1865 /* Tidy and get out of here */
1866
1867 freespace(tmp);
1868 return(d);
1869}
1870
1871/*---------------------------------------------------------------------------*/
1883/*---------------------------------------------------------------------------*/
1884
1885static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
1886 int nchan, double *facrng,
1887 double *maxdiff) {
1888 int i,ist,ifn,j;
1889 double *factors,sum,midval,minfac,maxfac;
1890
1891 /* Get some memory for the output array */
1892
1893 factors = cpl_malloc(nim*sizeof(double));
1894
1895 /* First sort the data into order of mjd */
1896
1897 vircam_mjdsort(fdata,mjd,nim);
1898
1899 /* Which index is the midpoint? */
1900
1901 if (nim % 2 == 0) {
1902 ist = nim/2 - 1;
1903 ifn = ist + 1;
1904 } else {
1905 ist = nim/2;
1906 ifn = ist;
1907 }
1908
1909 /* Loop for each channel */
1910
1911 for (i = 0; i < nchan; i++) {
1912
1913 /* Get midpoint value */
1914
1915 midval = 0.5*(fdata[ist][i] + fdata[ifn][i]);
1916
1917 /* Now normalise all the ith channels by this value */
1918
1919 for (j = 0; j < nim; j++)
1920 fdata[j][i] /= midval;
1921 }
1922
1923 /* Now loop for each image and average the values for all the channels in
1924 in image */
1925
1926 *maxdiff = 0.0;
1927 maxfac = 0.0;
1928 minfac = 0.0;
1929 for (j = 0; j < nim; j++) {
1930 sum = 0.0;
1931 for (i = 0; i < nchan; i++)
1932 sum += fdata[j][i];
1933 factors[j] = sum/(double)nchan;
1934 if (j == 0) {
1935 maxfac = factors[j];
1936 minfac = factors[j];
1937 } else {
1938 minfac = min(minfac,factors[j]);
1939 maxfac = max(maxfac,factors[j]);
1940 *maxdiff = max(*maxdiff,fabs(factors[j]-factors[j-1]));
1941 }
1942 }
1943 *facrng = maxfac - minfac;
1944
1945 /* Get out of here */
1946
1947 return(factors);
1948}
1949
1950/*---------------------------------------------------------------------------*/
1958/*---------------------------------------------------------------------------*/
1959
1960static void vircam_mjdsort(double **fdata, double *mjd, int n) {
1961 int iii,ii,i,ifin,j;
1962 double tmpmjd,*tmpdata;
1963
1964
1965 iii = 2;
1966 while (iii < n)
1967 iii *= 2;
1968 iii = min(n,(3*iii)/4 - 1);
1969
1970 while (iii > 1) {
1971 iii /= 2;
1972 ifin = n - iii;
1973 for (ii = 0; ii < ifin; ii++) {
1974 i = ii;
1975 j = i + iii;
1976 if (mjd[i] > mjd[j]) {
1977 tmpmjd = mjd[j];
1978 tmpdata = fdata[j];
1979 while (1) {
1980 mjd[j] = mjd[i];
1981 fdata[j] = fdata[i];
1982 j = i;
1983 i = i - iii;
1984 if (i < 0 || mjd[0] <= tmpmjd)
1985 break;
1986 }
1987 mjd[j] = tmpmjd;
1988 fdata[j] = tmpdata;
1989 }
1990 }
1991 }
1992}
1993
1994/*---------------------------------------------------------------------------*/
2001/*---------------------------------------------------------------------------*/
2002
2003static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows) {
2004 int i;
2005 cpl_table *t;
2006
2007 /* Create a new table */
2008
2009 t = cpl_table_new(nrows);
2010
2011 /* Add the first few columns */
2012
2013 cpl_table_new_column(t,"filename",CPL_TYPE_STRING);
2014 cpl_table_new_column(t,"exptime",CPL_TYPE_DOUBLE);
2015 cpl_table_set_column_unit(t,"exptime","seconds");
2016 cpl_table_new_column(t,"mjd",CPL_TYPE_DOUBLE);
2017 cpl_table_set_column_unit(t,"mjd","days");
2018
2019 /* Add columns for each of the channels' raw median flux and linearised
2020 median flux */
2021
2022 for (i = 1; i <= np; i++) {
2023 char * colname = cpl_sprintf("rawflux_%02d", i);
2024 // (void)snprintf(colname,16,"rawflux_%02d", i);
2025 cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
2026 cpl_table_set_column_unit(t,colname,"ADU");
2027 cpl_free(colname);
2028 colname = cpl_sprintf("linflux_%02d", i);
2029 //snprintf(colname,16,"linflux_%02d", i);
2030 cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
2031 cpl_table_set_column_unit(t,colname,"ADU");
2032 cpl_free(colname);
2033 }
2034
2035 /* Finally add the correction factors that were used */
2036
2037 cpl_table_new_column(t,"adjust_fac",CPL_TYPE_DOUBLE);
2038
2039 /* Right, get out of here */
2040
2041 return(t);
2042}
2043
2044/*---------------------------------------------------------------------------*/
2048/*---------------------------------------------------------------------------*/
2049
2050static void vircam_linearity_analyse_init(void) {
2051 ps.labels = NULL;
2052 ps.domelist = NULL;
2053 ps.darklist = NULL;
2054 ps.domecheck = NULL;
2055 ps.darkcheck = NULL;
2056 ps.ndomes = 0;
2057 ps.ndarks = 0;
2058 ps.ndomecheck = 0;
2059 ps.ndarkcheck = 0;
2060 ps.chanfrm = NULL;
2061 ps.chantab = NULL;
2062 ps.lchantab = NULL;
2063 ps.flatlist = NULL;
2064 ps.bpm_array = NULL;
2065 ps.ddg = NULL;
2066 ps.plist = NULL;
2067 ps.elist = NULL;
2068 ps.phupaf = NULL;
2069 ps.diag1 = NULL;
2070 ps.diag2 = NULL;
2071 ps.inherit = NULL;
2072}
2073
2074/*---------------------------------------------------------------------------*/
2078/*---------------------------------------------------------------------------*/
2079
2080static void vircam_linearity_analyse_tidy(int level) {
2081 int i;
2082
2083 freetfits(ps.chantab);
2084 freearray(ps.bpm_array);
2085 freefitslist(ps.flatlist,ps.nflatlist);
2086 freetable(ps.lchantab);
2087 freepropertylist(ps.plist);
2088 freepropertylist(ps.elist);
2089 freetable(ps.diag1);
2090 freetable(ps.diag2);
2091 if (level == 1)
2092 return;
2093
2094 freespace(ps.labels);
2095 freeframeset(ps.domelist);
2096 freeframeset(ps.darklist);
2097 freeframeset(ps.domecheck);
2098 freeframeset(ps.darkcheck);
2099 freeframe(ps.chanfrm);
2100 if (ps.ddg != NULL) {
2101 for (i = 0; i < ps.nddg; i++) {
2102 freeframeset(ps.ddg[i].darks);
2103 freeframeset(ps.ddg[i].domes);
2104 freefitslist(ps.ddg[i].proc,ps.ddg[i].ndomes);
2105 }
2106 freespace(ps.ddg);
2107 }
2108 freepropertylist(ps.phupaf);
2109}
2110
2113/*
2114
2115$Log: not supported by cvs2svn $
2116Revision 1.64 2012/01/15 17:40:09 jim
2117Minor modifications to take into accout the changes in cpl API for v6
2118
2119Revision 1.63 2010/12/09 13:20:26 jim
2120Default polynomial order is now 4
2121
2122Revision 1.62 2010/07/02 07:17:35 jim
2123Fixed typo
2124
2125Revision 1.61 2010/06/30 12:42:00 jim
2126A few fixes to stop compiler compaints
2127
2128Revision 1.60 2010/03/09 14:29:55 jim
2129Now modified ESO PRO DATANCOM to reflect the number of images used in
2130the analysis
2131
2132Revision 1.59 2009/09/09 09:50:21 jim
2133Modified to try and get headers right
2134
2135Revision 1.58 2009/07/13 08:17:57 jim
2136Fixed bug that meant saturation level wasn't being propogated
2137
2138Revision 1.57 2009/06/23 05:22:26 jim
2139Adds saturation back into the channel table header
2140
2141Revision 1.56 2008/12/08 06:32:42 jim
2142Changed 'missing dark frame' error to warning
2143
2144Revision 1.55 2008/12/05 13:28:32 jim
2145Fixed save routine so that the correct version of PRO CATG is written to the
2146paf file
2147
2148Revision 1.54 2008/10/01 04:59:13 jim
2149Added call to vircam_frameset_fexists to check input frameset
2150
2151Revision 1.53 2008/09/30 11:33:00 jim
2152Fixed bug where saturation flag wasn't being set to OK.
2153
2154Revision 1.52 2008/09/29 11:23:00 jim
2155Minor fix to docs
2156
2157Revision 1.51 2008/01/22 19:47:56 jim
2158New version to implement new algorithm
2159
2160Revision 1.50 2007/11/26 09:58:49 jim
2161Now fails if given observation files done with NDIT != 1
2162
2163Revision 1.49 2007/11/23 18:34:28 jim
2164fixed memory allocation bug
2165
2166Revision 1.48 2007/11/22 12:36:55 jim
2167Modified to create diagnostic tables
2168
2169Revision 1.47 2007/11/20 09:41:13 jim
2170Added ability to alter dome sequence stats by using the monitoring exposures
2171
2172Revision 1.46 2007/11/14 10:42:25 jim
2173Substantial changes to incorporate new linearity analysis algorithm and to
2174restrict the amount of memory required to do the analysis (especially
2175the BPM work)
2176
2177Revision 1.45 2007/09/07 13:32:12 jim
2178uses a sorted framelist to ensure that the correct information is given
2179to the output product header
2180
2181Revision 1.44 2007/09/06 21:37:53 jim
2182fixed call to vircam_dfs_setup_product_ routines to use the full input
2183frameset
2184
2185Revision 1.43 2007/08/29 09:20:33 jim
2186Primary header is now derived from the same header that forms the PAF rather
2187than starting off empty and allowing CPL to copy everything it thinks you
2188want...
2189
2190Revision 1.42 2007/08/23 09:02:03 jim
2191Modified to check domes for DETLIVE before checking darks
2192
2193Revision 1.41 2007/07/09 13:21:55 jim
2194Modified to use new version of vircam_exten_range
2195
2196Revision 1.40 2007/06/13 08:11:27 jim
2197Modified docs to reflect changes in DFS tags
2198
2199Revision 1.39 2007/04/30 09:40:17 jim
2200Added more stuff to paf files
2201
2202Revision 1.38 2007/04/04 10:36:07 jim
2203Fixed typo preventing output of main PAF. Also modified to use dfs tags
2204
2205Revision 1.37 2007/03/29 12:19:38 jim
2206Little changes to improve documentation
2207
2208Revision 1.36 2007/03/01 12:41:49 jim
2209Modified slightly after code checking
2210
2211Revision 1.35 2007/02/19 21:13:04 jim
2212added bad pixel number QC parameter
2213
2214Revision 1.34 2007/02/15 11:54:09 jim
2215Modified to make a distinction between initial channel table and one that
2216has the proper linearity information
2217
2218Revision 1.33 2007/02/15 06:59:38 jim
2219Added ability to write QC paf files
2220
2221Revision 1.32 2007/02/07 10:12:40 jim
2222Removed calls to vircam_ndit_correct as this is now no longer necessary
2223
2224Revision 1.31 2007/02/06 13:11:12 jim
2225Fixed entry for PRO dictionary in cpl_dfs_set_product_header
2226
2227Revision 1.30 2006/12/13 11:45:36 jim
2228Fixed scaling of sigma error
2229
2230Revision 1.29 2006/12/11 22:47:12 jim
2231Fixed subtle bug in the way that stats were being done.
2232
2233Revision 1.28 2006/11/27 12:15:08 jim
2234changed calls to cpl_propertylist_append to cpl_propertylist_update
2235
2236Revision 1.27 2006/11/10 09:23:46 jim
2237Fixed save routine so to use a new version of vircam_chantab_new
2238
2239Revision 1.26 2006/10/31 10:27:27 jim
2240Fixed a few bugs and modified to make sure than an extension name appear
2241in each fits extension
2242
2243Revision 1.25 2006/09/09 16:49:40 jim
2244Header comment update
2245
2246Revision 1.24 2006/09/08 09:20:22 jim
2247major upgrade to main processing routine: to deal with bad input better; to
2248write out dummy results in the case of failure; to combine raw darks on the
2249fly for use in dark correction, rather than using master darks;
2250
2251Revision 1.23 2006/08/03 13:26:44 jim
2252fixed another typo
2253
2254Revision 1.22 2006/08/03 10:36:32 jim
2255Fixed typo
2256
2257Revision 1.21 2006/06/20 19:06:38 jim
2258Added correction for ndit. Now adjusts the value of norder if the number
2259of frames given is too small for the order of polynomial requested
2260
2261Revision 1.20 2006/06/15 09:58:58 jim
2262Minor changes to docs
2263
2264Revision 1.19 2006/06/06 13:03:42 jim
2265Fixed scaling that was causing funny stats
2266
2267Revision 1.18 2006/05/27 21:40:06 jim
2268Bad pixels are now defined by a number of sigma above or below the mean
2269
2270Revision 1.17 2006/05/09 09:30:47 jim
2271Fixed _save routine so that bad pixel mask is saved with unsigned char
2272data type
2273
2274Revision 1.16 2006/05/08 12:32:12 jim
2275Changed default calling parameters for vircam_imcombine
2276
2277Revision 1.15 2006/05/04 11:53:15 jim
2278Fixed the way the _save routine works to be more consistent with the
2279standard CPL way of doing things
2280
2281Revision 1.14 2006/05/03 12:55:17 jim
2282Fixed some memory leaks
2283
2284Revision 1.13 2006/05/02 13:26:32 jim
2285fixed bug where the wrong amount of memory was being allocated for the dark
2286exposure times
2287
2288Revision 1.12 2006/05/02 11:36:29 jim
2289fixed illegal propertylist_delete calls
2290
2291Revision 1.11 2006/04/27 09:46:01 jim
2292Modified DFS frame types to conform to new dictionary
2293
2294Revision 1.10 2006/04/25 13:45:57 jim
2295Fixed to adhere to new calling sequence for vircam_dfs routines
2296
2297Revision 1.9 2006/04/24 12:12:59 jim
2298Fixed --help documentation and sorted out filename extension problem
2299(.fit -> .fits)
2300
2301Revision 1.8 2006/04/20 11:31:34 jim
2302Added bad pixel masking
2303
2304Revision 1.7 2006/03/23 21:18:45 jim
2305Minor changes mainly to comment headers
2306
2307Revision 1.6 2006/03/22 14:02:51 jim
2308cosmetic changes to keep lint happy
2309
2310Revision 1.5 2006/03/22 12:13:51 jim
2311Modified to use new vircam_mask capability
2312
2313Revision 1.4 2006/03/15 10:43:40 jim
2314Fixed a few things
2315
2316Revision 1.3 2006/03/03 14:29:06 jim
2317Now calls routines with vir_fits.
2318
2319Revision 1.2 2006/02/22 10:01:22 jim
2320Added full documentation
2321
2322Revision 1.1 2006/02/18 11:49:58 jim
2323new file
2324
2325
2326*/
casu_fits * casu_fits_wrap(cpl_image *im, casu_fits *model, cpl_propertylist *phu, cpl_propertylist *ehu)
Definition: casu_fits.c:883
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
casu_fits ** casu_fits_load_list(cpl_frameset *f, cpl_type type, int exten)
Definition: casu_fits.c:318
void casu_fits_delete(casu_fits *p)
Definition: casu_fits.c:364
casu_fits * casu_fits_duplicate(casu_fits *in)
Definition: casu_fits.c:225
cpl_propertylist * casu_fits_get_phu(casu_fits *p)
Definition: casu_fits.c:531
char * casu_fits_get_filename(casu_fits *p)
Definition: casu_fits.c:646
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
casu_fits * casu_fits_load(cpl_frame *frame, cpl_type type, int nexten)
Definition: casu_fits.c:80
int casu_imcombine(casu_fits **fset, casu_fits **fsetv, int nfits, int combtype, int scaletype, int xrej, float thresh, const char *expkey, cpl_image **outimage, cpl_image **outvimage, unsigned char **rejmask, unsigned char **rejplus, cpl_propertylist **drs, int *status)
Stack images into a mean or median image with rejection.
int casu_darkcor(casu_fits *infile, casu_fits *darksrc, float darkscl, int *status)
Correct input data for dark current.
Definition: casu_darkcor.c:86
int casu_genbpm(casu_fits **flatlist, int nflatlist, cpl_image *master, float lthr, float hthr, const char *expkey, cpl_array **bpm_array, int *nbad, float *badfrac, int *status)
Generate a bad pixel mask from a list of dome flats.
Definition: casu_genbpm.c:97
float casu_med(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:89
cpl_propertylist * casu_tfits_get_ehu(casu_tfits *p)
Definition: casu_tfits.c:473
casu_tfits * casu_tfits_load(cpl_frame *table, int nexten)
Definition: casu_tfits.c:78
cpl_table * casu_tfits_get_table(casu_tfits *p)
Definition: casu_tfits.c:364
int casu_compare_tags(const cpl_frame *frame1, const cpl_frame *frame2)
Compare input tags.
Definition: casu_utils.c:96
void casu_merge_propertylists(cpl_propertylist *p1, cpl_propertylist *p2)
Merge two propertylists.
Definition: casu_utils.c:399
void casu_dummy_property(cpl_propertylist *p)
Set dummy property keyword.
Definition: casu_utils.c:445
cpl_frame * casu_frameset_subgroup_1(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Extract a frame of a given label from a frameset.
Definition: casu_utils.c:206
void casu_sort(float **a, int n, int m)
Sort a 2d array by the first column and co-sort the rest.
Definition: casu_utils.c:349
cpl_frameset * casu_frameset_subgroup(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Extract a frameset from another frameset.
Definition: casu_utils.c:149
int vircam_genlincur(double **fdata, int nimages, double *exps, double mindit, casu_tfits *chantab, int norder, cpl_table **lchantab, double **lindata, int *status)
Generate a linearity curve for each readout channel in a list of images.
cpl_table * vircam_chantab_new(int nord, cpl_table *template)
int vircam_chantab_verify(cpl_table *intab)
int vircam_chan_fill(cpl_table *tab, parquet **p, long *np)
void vircam_chan_free(int np, parquet **p)
int vircam_dfs_set_groups(cpl_frameset *set)
Definition: vircam_dfs.c:115
void vircam_dfs_set_product_primary_header(cpl_propertylist *plist, cpl_frame *frame, cpl_frameset *frameset, cpl_parameterlist *parlist, char *recipeid, const char *dict, cpl_frame *inherit, int synch)
Definition: vircam_dfs.c:227
void vircam_dfs_set_product_exten_header(cpl_propertylist *plist, cpl_frame *frame, cpl_frameset *frameset, cpl_parameterlist *parlist, char *recipeid, const char *dict, cpl_frame *inherit)
Definition: vircam_dfs.c:299
int vircam_pfits_get_ndit(const cpl_propertylist *plist, int *ndit)
Get the value of NDIT.
Definition: vircam_pfits.c:583
int vircam_pfits_get_mjd(const cpl_propertylist *plist, double *mjd)
Get the value of the modified Julian date.
Definition: vircam_pfits.c:729
int vircam_pfits_get_detlive(const cpl_propertylist *plist, int *detlive)
Get the value of DET_LIVE.
Definition: vircam_pfits.c:624
int vircam_pfits_get_exptime(const cpl_propertylist *plist, float *exptime)
Get the value of exposure time.
Definition: vircam_pfits.c:245
int vircam_pfits_get_mindit(const cpl_propertylist *plist, float *mindit)
Get the value of mindit time.
Definition: vircam_pfits.c:262
int vircam_pfits_get_saturation(const cpl_propertylist *plist, float *saturation)
Get the saturation level for this detector.
Definition: vircam_pfits.c:696
const char * vircam_get_license(void)
Definition: vircam_utils.c:116
void vircam_exten_range(int inexten, const cpl_frame *fr, int *out1, int *out2)
Definition: vircam_utils.c:165