VISIR Pipeline Reference Manual  4.1.0
visir_util_repack.c
1 /* $Id: visir_util_repack.c,v 1.144 2013-09-24 11:09:55 jtaylor Exp $
2  *
3  * This file is part of the VISIR Pipeline
4  * Copyright (C) 2002,2003,2013,2014 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
19  */
20 
21 #ifdef HAVE_CONFIG_H
22 #include <config.h>
23 #endif
24 
25 /*-----------------------------------------------------------------------------
26  Includes
27  -----------------------------------------------------------------------------*/
28 
29 #include "visir_recipe.h"
30 #include "irplib_framelist.h"
31 #include "irplib_wcs.h"
32 #include <cxlist.h>
33 /* Verify self-sufficiency of CPL header files by including system files last */
34 #include <string.h>
35 /* for strcasecmp */
36 #include <strings.h>
37 #include <math.h> /* isnan */
38 
39 
40 /*-----------------------------------------------------------------------------
41  Defines
42  -----------------------------------------------------------------------------*/
43 
44 #ifndef VISIR_UTIL_REPACK_CONAD
45 #define VISIR_UTIL_REPACK_CONAD 1.1
46 #endif
47 
48 #define RECIPE_STRING "visir_util_repack"
49 
50 #define VISIR_DRS_CONAD "ESO DET OUT1 CONAD"
51 
52 
53 /*-----------------------------------------------------------------------------
54  Private Functions prototypes
55  -----------------------------------------------------------------------------*/
56 typedef enum {
57  VISIR_SUB_NOCORRECT = 0,
58  VISIR_SUB_CHOPCORRECT = 1,
59  VISIR_SUB_CHOPNODCORRECT = 2
60 } visir_sub_type;
61 
62 typedef enum {
63  VISIR_FRAME_A,
64  VISIR_FRAME_B,
65  VISIR_FRAME_STATIC,
66 } visir_frame_type;
67 
68 /* convinience data structure to hold common data needed
69  * to process one frame */
70 typedef struct {
71  int planestart;
72  int planeend;
73  int trimlow;
74  int trimhigh;
75  int bkgcorrect;
76  cpl_boolean normalize;
77  cpl_boolean compress;
78  double conad;
79  int nframes;
80  const irplib_framelist * rawframes;
81  int iframe;
82  const cpl_parameterlist * parlist;
83  const char * tag;
84  const cpl_propertylist * plist;
85  visir_data_type datatype;
86  cpl_type load_type;
87 
88  char * onname;
89  char * offname;
90  const char * remregexp;
91 
92  double pthrow;
93  double posang;
94  double dit;
95  const char * comoffx;
96  const char * comoffy;
97  double offsetx;
98  double offsety;
99  double jitterwidth;
100  double pxspace;
101  double pfov;
102 
103  /* only for burst */
104  int to_off;
105  int halfcycle;
106 
107  double bpmthresh;
108 
109  int naxis3;
110  cpl_boolean is_a;
111  cpl_boolean chop_on;
112  const char * procatgon;
113  const char * procatgoff;
114  cpl_image * mean_on;
115  cpl_size * nmean_on;
116  cpl_image * mean_off;
117  cpl_size * nmean_off;
118 
119  double time_obsstart;
120  double time_filewrite;
121  /* min and max of a two A/B states to write into a chop nod file */
122  double time_min_obsstart;
123  double time_max_filewrite;
125 
126 static cpl_error_code
127 visir_util_repack_two(const repack_framestate * fstatea,
128  const repack_framestate * fstateb,
129  cpl_frameset * products,
130  cpl_image ** pbpm,
131  cpl_image * nonlinear,
132  double, double,
133  const cpl_bivector * lintab);
134 
135 static inline cpl_error_code
136 visir_util_repack_check(const cpl_image *,
137  const cpl_imagelist *,
138  const cpl_imagelist *);
139 
140 #ifdef VISIR_CHAIN
141 #define cpl_plugin_get_info visir_util_repack_get_info
142 #endif
143 cpl_recipe_define(visir_util_repack, VISIR_BINARY_VERSION,
144  "Lars Lundin", PACKAGE_BUGREPORT, "2011",
145  "Conversion of raw CUBE2 or BURST images to on- and off-cubes",
146  "The files listed in the Set Of Frames (sof-file) "
147  "must be tagged:\n"
148  "VISIR-CUBE2-raw-file.fits " VISIR_UTIL_REPACK_RAW
149  "\nor\n"
150  "VISIR-BURST-raw-file.fits " VISIR_IMG_BURST"\n"
151  "VISIR-BURST-bpm-file.fits " VISIR_CALIB_STATIC_MASK
152  "\nFor BURST data it will remove planes where the chopper "
153  "switched from on <-> off based on the "
154  "timestamps in the header."
155  "\nThe product(s) will have a FITS card\n"
156  "'HIERARCH ESO PRO CATG' with a value of one of:\n"
157  VISIR_UTIL_REPACK_A_ON_PROCATG " (NodPos: A, ChopPos: on)\n"
158  VISIR_UTIL_REPACK_A_OFF_PROCATG " (NodPos: A, ChopPos: off)\n"
159  VISIR_UTIL_REPACK_B_ON_PROCATG " (NodPos: B, ChopPos: on)\n"
160  VISIR_UTIL_REPACK_B_OFF_PROCATG " (NodPos: B, ChopPos: off)\n"
161  /* FIXME: reimplement
162  "Additionally, the recipe collapses the on- and off-cubes, "
163  "these product(s) will have a FITS card\n"
164  "'HIERARCH ESO PRO CATG' with a value of one of:\n"
165  VISIR_UTIL_REPACK_MEAN_A_ON_PROCATG
166  " (NodPos: A, ChopPos: on)\n"
167  VISIR_UTIL_REPACK_MEAN_A_OFF_PROCATG
168  " (NodPos: A, ChopPos: off)\n"
169  VISIR_UTIL_REPACK_MEAN_B_ON_PROCATG
170  " (NodPos: B, ChopPos: on)\n"
171  VISIR_UTIL_REPACK_MEAN_B_OFF_PROCATG
172  " (NodPos: B, ChopPos: off)\n" */
173  "For CUBE2, the recipe will produce a static "
174  "bad-pixel map, it will have a FITS card\n"
175  "'HIERARCH ESO PRO CATG' with a value of:\n"
176  VISIR_CALIB_STATIC_MASK"\n"
177  );
178 
179 /*----------------------------------------------------------------------------*/
183 /*----------------------------------------------------------------------------*/
184 
185 /*-----------------------------------------------------------------------------
186  Functions code
187  -----------------------------------------------------------------------------*/
188 
189 
190 /*----------------------------------------------------------------------------*/
197 /*----------------------------------------------------------------------------*/
198 static cpl_image *
199 visir_imagelist_pop(cpl_imagelist * list)
200 {
201  if (cpl_imagelist_get_size(list) == 0)
202  return NULL;
203  return cpl_imagelist_unset(list, cpl_imagelist_get_size(list) - 1);
204 }
205 
206 
207 /*----------------------------------------------------------------------------*/
215 /*----------------------------------------------------------------------------*/
216 static cpl_type get_optimum_save_type(const cpl_image * img)
217 {
218  cpl_type res = CPL_TYPE_UNSPECIFIED;
219  if (cpl_image_get_type(img) == CPL_TYPE_INT) {
220  cpl_stats * stats = cpl_stats_new_from_image(img, CPL_STATS_MIN |
221  CPL_STATS_MAX);
222  if ((int)cpl_stats_get_max(stats) <= CX_MAXSHORT &&
223  (int)cpl_stats_get_min(stats) >= CX_MINSHORT)
224  res = CPL_TYPE_SHORT;
225  cpl_stats_delete(stats);
226  }
227 
228  return res;
229 }
230 
231 
232 /*----------------------------------------------------------------------------*/
240 /*----------------------------------------------------------------------------*/
241 static cpl_error_code
242 visir_util_repack_fill_parameterlist(cpl_parameterlist * self)
243 {
244  const char * context = PACKAGE "." RECIPE_STRING;
245  cpl_error_code err;
246 
247  cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);
248 
249  /* Fill the parameters list */
250 
251  /* --planestart */
252  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "planestart",
253  0, NULL, context, "Plane number to "
254  "start repacking from, earlier planes "
255  "are skipped.");
256  cpl_ensure_code(!err, err);
257 
258  /* --ncycles */
259  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "ncycles",
260  -1, NULL, context, "Number of full "
261  "on-off cycles to repack. <= 0 for all.");
262  cpl_ensure_code(!err, err);
263 
264  /* --trimlow */
265  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "trimlow",
266  0, NULL, context, "Burst data only. "
267  "Number of additional planes to cut "
268  "from before each plane with chopper "
269  "movement.");
270  cpl_ensure_code(!err, err);
271 
272  /* --trimhigh */
273  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "trimhigh",
274  0, NULL, context, "Burst data only. "
275  "Number of additional planes to cut "
276  "from after each plane with chopper "
277  "movement.\n A value of -1 does not "
278  "skip the plane of the movement.");
279  cpl_ensure_code(!err, err);
280 
281  /* --bkgcorrect */
282  err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
283  "bkgcorrect", "none", NULL, context,
284  "Output background corrected planes by "
285  "subtracting chop (on/off) and nod (A/B) "
286  "planes\n Options:\n"
287  " none: no correction\n"
288  " chop: on - off\n"
289  " chopnod: (Aon - Aoff) - (Bon - Boff)");
290  cpl_ensure_code(!err, err);
291 
292 
293  /* --normalize */
294  err =
295  irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING, "normalize",
296  CPL_TRUE, NULL, context,
297  "Normalize planes by DIT");
298  cpl_ensure_code(!err, err);
299 
300 
301  /* --compress */
302  err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
303  "compress", CPL_FALSE, NULL, context,
304  "Apply lossless compression on output"
305  "files. Can only be done for integer "
306  " type results.");
307  cpl_ensure_code(!err, err);
308 
309  return CPL_ERROR_NONE;
310 }
311 
312 
313 /*----------------------------------------------------------------------------*/
324 /*----------------------------------------------------------------------------*/
325 static repack_framestate *
326 repack_framestate_new(irplib_framelist * rawframes, int iframe,
327  const cpl_parameterlist * parlist)
328 {
329  repack_framestate * state = cpl_calloc(1, sizeof(repack_framestate));
330  int ndit, ncycles, planes_per_cycle = 2; /* avoid false uninit warning */
331  cpl_frame * frame = irplib_framelist_get(rawframes, iframe);
332  const char * filename = cpl_frame_get_filename(frame);
333  cpl_size next;
334  state->time_max_filewrite = -1;
335  state->time_min_obsstart = 1e300;
336 
337  state->rawframes = rawframes;
338 
339  state->iframe = iframe;
340 
341  state->tag = cpl_frame_get_tag(frame);
342 
343  error_if(strcmp(state->tag, VISIR_CALIB_STATIC_MASK) == 0,
344  CPL_ERROR_UNSUPPORTED_MODE, " ");
345 
346  /* compression support disabled, can give bad error messages for
347  * INT only acquarius data */
348  /* compressed files have the data in extension 1 */
349  const cpl_size iext = 0;
350 
351  skip_if(irplib_framelist_load_propertylist(rawframes, iframe, iext,
352  ".*", CPL_FALSE));
353  state->parlist = parlist;
354 
355  state->plist = irplib_framelist_get_propertylist_const(rawframes, iframe);
356  skip_if(0);
357 
358  skip_if(visir_get_data_type(frame, state->plist, &state->datatype, &next));
359 
360  if (state->datatype == VISIR_DATA_AQU_HCYCLE) {
361  /* last frame is INT, but is skipped by when loading */
362  state->naxis3 = next;
363  }
364  else if (state->datatype == VISIR_DATA_AQU_INT) {
365  state->naxis3 = 1;
366  }
367  else
368  state->naxis3 = visir_pfits_get_naxis3(state->plist);
369 
370  state->chop_on = visir_pfits_get_chop_stat(state->plist);
371 
372  {
373  const char * bkgc = irplib_parameterlist_get_string(parlist, PACKAGE,
374  RECIPE_STRING,
375  "bkgcorrect");
376  if (visir_str_par_is_empty(bkgc) || !strcasecmp(bkgc, "none"))
377  state->bkgcorrect = VISIR_SUB_NOCORRECT;
378  else if (!strcasecmp(bkgc, "chopnod"))
379  state->bkgcorrect = VISIR_SUB_CHOPNODCORRECT;
380  else if (!strcasecmp(bkgc, "chop"))
381  state->bkgcorrect = VISIR_SUB_CHOPCORRECT;
382  else
383  error_if(1, CPL_ERROR_ILLEGAL_INPUT,
384  "Unknown parameter to --bkgcorrect: %s", bkgc);
385 
386  error_if(!state->chop_on && state->bkgcorrect == VISIR_SUB_CHOPCORRECT,
387  CPL_ERROR_ILLEGAL_INPUT, "Requested chop correction on data "
388  "with "VISIR_PFITS_BOOL_CHOP_STATUS" false");
389  }
390 
391  error_if(state->datatype == VISIR_DATA_AQU_INT &&
392  state->bkgcorrect != VISIR_SUB_CHOPNODCORRECT,
393  CPL_ERROR_INCOMPATIBLE_INPUT,
394  "INT frame only data requires --bkgcorrect to be chopnod");
395 
396  /* cannot extract a and b hcycle from this format */
397  error_if(state->datatype == VISIR_DATA_CUBE1,
398  CPL_ERROR_INCOMPATIBLE_INPUT,
399  "CUBE1 data not supported please use the legacy recipes "
400  "visir_img_combine or visir_spc_obs");
401 
402  state->normalize = irplib_parameterlist_get_bool(parlist, PACKAGE,
403  RECIPE_STRING,
404  "normalize");
405 
406  state->dit = visir_pfits_get_dit(state->plist);
407 
408  error_if(state->normalize && state->dit == 0, CPL_ERROR_ILLEGAL_INPUT,
409  "Cannot normalize by DIT = 0");
410 
411  state->planestart = irplib_parameterlist_get_int(parlist, PACKAGE,
412  RECIPE_STRING,
413  "planestart");
414  error_if(state->planestart >= state->naxis3, CPL_ERROR_ILLEGAL_INPUT,
415  "planestart %d equal or larger than number of planes %d.",
416  state->planestart, state->naxis3);
417 
418  ndit = visir_pfits_get_ndit(state->plist);
419  const double chop_freq = visir_pfits_get_chop_freq(state->plist);
420 
421  IRPLIB_DIAG_PRAGMA_PUSH_ERR(-Wswitch);
422  switch(state->datatype) {
423  case VISIR_DATA_CUBE1:
424  bug_if(1);
425  break;
426  case VISIR_DATA_AQU_HCYCLE:
427  case VISIR_DATA_CUBE2:
428  planes_per_cycle = 2;
429  break;
430  case VISIR_DATA_AQU_INT:
431  /* not actually used */
432  planes_per_cycle = 1;
433  break;
434  case VISIR_DATA_BURST:
435  planes_per_cycle = ndit * 2;
436  break;
437  case VISIR_DATA_AQU_BURST:
438  case VISIR_DATA_AQU_BURST_EXT:
439  /* FIXME: ndit seems to be naxis3 in aqu? */
440  planes_per_cycle = 1. / (state->dit * chop_freq);
441  break;
442  }
443  IRPLIB_DIAG_PRAGMA_POP;
444 
445  ncycles = irplib_parameterlist_get_int(parlist, PACKAGE,
446  RECIPE_STRING, "ncycles");
447  if (ncycles <= 0)
448  ncycles = irplib_pfits_get_int(state->plist,
449  VISIR_PFITS_INT_CHOP_NCYCLES);
450 
451  state->planeend = state->planestart + ncycles * planes_per_cycle;
452  /* ensure we always repack a multiple of a full cycle to get equal number
453  * of on and off planes */
454  if (state->planeend >= state->naxis3 || state->planeend <= 0) {
455  if (state->planestart % planes_per_cycle == 0)
456  state->planeend = state->naxis3;
457  else
458  state->planeend = state->naxis3 -
459  (planes_per_cycle - (state->planestart % planes_per_cycle));
460  }
461 
462  /* we have max 16 bit adc and cpl int is 32 bit so reasonable dits allow
463  * integer normalization without overflow */
464  if (state->normalize && (state->dit < 1e-6 ||
465  (double)((int)(1. / state->dit)) != 1. / state->dit))
466  state->load_type = CPL_TYPE_FLOAT;
467  else
468  state->load_type = CPL_TYPE_UNSPECIFIED;
469 
470  state->trimlow = irplib_parameterlist_get_int(parlist, PACKAGE,
471  RECIPE_STRING,
472  "trimlow");
473  state->trimhigh = irplib_parameterlist_get_int(parlist, PACKAGE,
474  RECIPE_STRING,
475  "trimhigh");
476 
477  state->compress = irplib_parameterlist_get_bool(parlist, PACKAGE,
478  RECIPE_STRING,
479  "compress");
480 
481  if (state->compress && CPL_IO_COMPRESS_RICE == 0) {
482  cpl_msg_warning(cpl_func, "CPL version too old for compression.");
483  state->compress = CPL_FALSE;
484  }
485 
486  state->conad = VISIR_UTIL_REPACK_CONAD;
487 
488  state->nframes = irplib_framelist_get_size(rawframes);
489  state->remregexp = "^(" VISIR_PFITS_DOUBLE_CUMOFFSETX
490  "|" VISIR_PFITS_DOUBLE_CUMOFFSETY ")$";
491 
492  /* Get the chopping throw in pixels */
493  state->pthrow = visir_pfits_get_chop_pthrow(state->plist);
494  state->posang = visir_pfits_get_chop_posang(state->plist);
495 
496  state->pfov = visir_pfits_get_pixscale(state->plist);
497 
498  /* Copy comments from CUMOFFSET[XY] */
499  state->comoffx = cpl_propertylist_get_comment
500  (state->plist, VISIR_PFITS_DOUBLE_CUMOFFSETX);
501  state->comoffy = cpl_propertylist_get_comment
502  (state->plist, VISIR_PFITS_DOUBLE_CUMOFFSETY);
503  state->offsetx = visir_pfits_get_cumoffsetx(state->plist);
504  state->offsety = visir_pfits_get_cumoffsety(state->plist);
505  state->jitterwidth = 0.;
506  if (cpl_propertylist_has(state->plist, "ESO SEQ JITTER WIDTH")) {
507  state->jitterwidth =
508  cpl_propertylist_get_double(state->plist, "ESO SEQ JITTER WIDTH");
509  state->jitterwidth /= state->pfov;
510  }
511 
512  /* we only have one chip, move PIXSPACE from extension to main header */
513  if (next > 0) {
514  cpl_propertylist * extplist = cpl_propertylist_load(filename, 1);
515  skip_if(extplist == NULL);
516  state->pxspace = visir_pfits_get_pixspace(extplist);
517  cpl_propertylist_delete(extplist);
518  skip_if(0);
519  }
520 
521  if (visir_data_is_burst(state->datatype)) {
522  visir_img_burst_find_delta_chop(state->plist,
523  &state->to_off,
524  &state->halfcycle);
525  }
526 
527  state->bpmthresh = VISIR_HCYCLE_BPM_THRESHOLD;
528 
529  if (state->bkgcorrect == VISIR_SUB_NOCORRECT) {
530  const char * nodpos = visir_pfits_get_nodpos(state->plist);
531  state->is_a = nodpos != NULL &&
532  (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
533  state->procatgon = state->is_a ? VISIR_UTIL_REPACK_A_ON_PROCATG
534  : VISIR_UTIL_REPACK_B_ON_PROCATG;
535  state->procatgoff = state->is_a ? VISIR_UTIL_REPACK_A_OFF_PROCATG
536  : VISIR_UTIL_REPACK_B_OFF_PROCATG;
537  state->onname = cpl_sprintf(RECIPE_STRING "_%s_on_%03d"
538  CPL_DFS_FITS, nodpos, 1+iframe);
539  state->offname = cpl_sprintf(RECIPE_STRING "_%s_off_%03d"
540  CPL_DFS_FITS, nodpos, 1+iframe);
541 
542  if (!state->is_a) {
543  /* Verify that the NODPOS is either A or B */
544  skip_if (nodpos == NULL);
545  skip_if (strstr(nodpos, "B") == NULL && strstr(nodpos, "b") == NULL);
546  }
547  } else if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) {
548  const char * nodpos = visir_pfits_get_nodpos(state->plist);
549  state->is_a = nodpos != NULL &&
550  (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
551  state->procatgon = state->is_a ? "BKG_CORRECTED_A" : "BKG_CORRECTED_B";
552  state->onname = cpl_sprintf(RECIPE_STRING "_%s_bkgcor_%03d"
553  CPL_DFS_FITS, nodpos, 1+iframe);
554  } else { /*if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) { */
555  const char * nodpos = visir_pfits_get_nodpos(state->plist);
556  state->is_a = nodpos != NULL &&
557  (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
558  state->procatgon = VISIR_UTIL_CORRECTED;
559  state->onname = cpl_sprintf(RECIPE_STRING "_bkgcor_%03d"
560  CPL_DFS_FITS, 1+iframe);
561  }
562 
563  {
564  long nx, ny;
565  if (state->datatype == VISIR_DATA_AQU_HCYCLE ||
566  state->datatype == VISIR_DATA_AQU_INT ) {
567  cpl_propertylist * plist = cpl_propertylist_load(filename, 1);
568  nx = visir_pfits_get_naxis1(plist);
569  ny = visir_pfits_get_naxis2(plist);
570  cpl_propertylist_delete(plist);
571  }
572  else {
573  nx = visir_pfits_get_naxis1(state->plist);
574  ny = visir_pfits_get_naxis2(state->plist);
575  }
576  skip_if(0);
577 
578  state->mean_on = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
579  state->nmean_on = cpl_calloc(1, sizeof(cpl_size));
580  state->mean_off = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
581  state->nmean_off = cpl_calloc(1, sizeof(cpl_size));
582  }
583 
584 
585  end_skip;
586 
587  return state;
588 }
589 
590 
591 /*----------------------------------------------------------------------------*/
596 /*----------------------------------------------------------------------------*/
597 static void
598 repack_framestate_delete(void * state_)
599 {
600  repack_framestate * state = (repack_framestate*) state_;
601 
602  if (state == NULL)
603  return;
604 
605  cpl_free(state->onname);
606  cpl_free(state->offname);
607  cpl_image_delete(state->mean_on);
608  cpl_image_delete(state->mean_off);
609  cpl_free(state->nmean_on);
610  cpl_free(state->nmean_off);
611  cpl_free(state);
612 }
613 
614 
615 /*----------------------------------------------------------------------------*/
622 /*----------------------------------------------------------------------------*/
623 static cpl_error_code
624 store_means(const visir_sub_type bkgcorrect, const irplib_framelist * rawfr,
625  cpl_frameset * framelist, const cpl_parameterlist * parlist,
626  cpl_image ** mean, cpl_size * nmean, visir_frame_type * frametype)
627 {
628  cpl_frameset * usedframes[] = {cpl_frameset_new(), cpl_frameset_new(),
629  cpl_frameset_new(), cpl_frameset_new()};
630 
631  int normalize = irplib_parameterlist_get_bool(parlist, PACKAGE,
632  RECIPE_STRING,
633  "normalize");
634  cpl_propertylist * plist = cpl_propertylist_new();
635  if (normalize) {
636  cpl_propertylist_append_string(plist, "BUNIT", "adu / s");
637  } else {
638  cpl_propertylist_append_string(plist, "BUNIT", "adu");
639  }
640 
641  if (bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
642  for (int i = 1; i < 4; i++) {
643  cpl_image_add(mean[0], mean[i]);
644  nmean[0] += nmean[i];
645  }
646  skip_if(cpl_image_divide_scalar(mean[0], nmean[0]));
647  for (int i = 0; i < irplib_framelist_get_size(rawfr); i++) {
648  cpl_frame * frm =
649  cpl_frame_duplicate(irplib_framelist_get_const(rawfr, i));
650  cpl_frameset_insert(usedframes[0], frm);
651  }
652  skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[0],
653  mean[0], CPL_TYPE_UNSPECIFIED,
654  RECIPE_STRING, "MEAN",
655  plist, NULL, visir_pipe_id,
656  "visir_util_repack_mean.fits"));
657  }
658  else if (bkgcorrect == VISIR_SUB_CHOPCORRECT) {
659  const char * procatg[] = {"MEAN_A", "MEAN_B"};
660  const char * fn[] ={"visir_util_repack_mean_A.fits",
661  "visir_util_repack_mean_B.fits"};
662  for (int i = 0; i < 2; i++) {
663  int idx = i == 0 ? 0 : 2;
664  cpl_image_add(mean[idx], mean[idx + 1]);
665  nmean[idx] += nmean[idx + 1];
666  skip_if(cpl_image_divide_scalar(mean[idx], nmean[idx]));
667  for (int j = 0; j < irplib_framelist_get_size(rawfr); j++) {
668  if ((i == 0 && frametype[j] == VISIR_FRAME_B) ||
669  (i == 1 && frametype[j] == VISIR_FRAME_A) ||
670  frametype[j] == VISIR_FRAME_STATIC)
671  continue;
672  cpl_frame * frm =
673  cpl_frame_duplicate(irplib_framelist_get_const(rawfr, j));
674  cpl_frameset_insert(usedframes[i], frm);
675  }
676  skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[i],
677  mean[idx], CPL_TYPE_UNSPECIFIED,
678  RECIPE_STRING, procatg[i],
679  plist, NULL, visir_pipe_id, fn[i]));
680  }
681  }
682  else {
683  const char * procatg[] = {"MEAN_A_ON", "MEAN_A_OFF",
684  "MEAN_B_ON", "MEAN_B_OFF"};
685  const char * fn[] ={"visir_util_repack_mean_Aoff.fits",
686  "visir_util_repack_mean_Aon.fits",
687  "visir_util_repack_mean_Boff.fits",
688  "visir_util_repack_mean_Bon.fits"};
689  for (int i = 0; i < 4; i++) {
690  skip_if(cpl_image_divide_scalar(mean[i], nmean[i]));
691  for (int j = 0; j < irplib_framelist_get_size(rawfr); j++) {
692  if ((i < 2 && frametype[j] == VISIR_FRAME_A) ||
693  (i >= 2 && frametype[j] == VISIR_FRAME_B) ||
694  frametype[j] == VISIR_FRAME_STATIC)
695  continue;
696  cpl_frame * frm =
697  cpl_frame_duplicate(irplib_framelist_get_const(rawfr, j));
698  cpl_frameset_insert(usedframes[i], frm);
699  }
700  skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[i],
701  mean[i], CPL_TYPE_UNSPECIFIED,
702  RECIPE_STRING, procatg[i],
703  plist, NULL, visir_pipe_id, fn[i]));
704  }
705  }
706 
707  end_skip;
708 
709  cpl_propertylist_delete(plist);
710  for (size_t i = 0; i < sizeof(usedframes)/sizeof(usedframes[0]); i++)
711  cpl_frameset_delete(usedframes[i]);
712 
713  return cpl_error_get_code();
714 }
715 
716 
717 /*----------------------------------------------------------------------------*/
724 /*----------------------------------------------------------------------------*/
725 static int visir_util_repack(cpl_frameset * framelist,
726  const cpl_parameterlist * parlist)
727 {
728  irplib_framelist * allframes = NULL;
729  irplib_framelist * rawframes = NULL;
730  int n;
731  cx_list * alist = cx_list_new();
732  cx_list * blist = cx_list_new();
733  cpl_image * bpm = NULL;
734  /* nonlinear pixel count */
735  cpl_image * nonlinear = NULL;
736  cpl_bivector * lintable = NULL;
737  cpl_image * mean[] = {NULL, NULL, NULL, NULL};
738  cpl_size nmean[] = {0, 0, 0, 0};
739  visir_sub_type bkgcorrect = VISIR_SUB_NOCORRECT;
740  visir_frame_type * frametype = NULL;
741  double xjitter0, yjitter0;
742 
743  /* Identify the RAW and CALIB frames in the input frameset */
744  skip_if (visir_dfs_set_groups(framelist));
745  cpl_fits_set_mode(CPL_FITS_START_CACHING);
746 
747  /* Objects observation */
748  allframes = irplib_framelist_cast(framelist);
749  skip_if(allframes == NULL);
750  rawframes = irplib_framelist_extract_regexp(allframes, "^("
751  VISIR_SPC_OBS_RAW
752  "|" VISIR_SPC_OBS_ECH_RAW
753  "|" VISIR_SPC_PHOT_RAW
754  "|" VISIR_SPC_PHOT_ECH_RAW
755  "|" VISIR_IMG_PHOT_RAW
756  "|" VISIR_IMG_COMBINE_CN
757  "|" VISIR_IMG_COMBINE_CNJ
758  "|" VISIR_IMG_COMBINE_CJ
759  "|" VISIR_IMG_COMBINE_NJ
760  "|" VISIR_IMG_COMBINE_DJ
761  "|" VISIR_ACQ_CNJ
762  "|" VISIR_IMG_BURST
763  "|" VISIR_UTIL_REPACK_RAW
764  "|" VISIR_UTIL_INPUTS_RAW
765  "|" VISIR_UTIL_DATA
766  ")$",
767  CPL_FALSE);
768  skip_if (rawframes == NULL);
769 
770  n = irplib_framelist_get_size(rawframes);
771  frametype = cpl_malloc(sizeof(visir_frame_type) * n);
772 
773  for (int i = 0; i < n; i++) {
774  repack_framestate * state = NULL;
775 
776  state = repack_framestate_new(rawframes, i, parlist);
777 
778  frametype[i] = state->is_a ? VISIR_FRAME_A : VISIR_FRAME_B;
779  if (state->is_a == CPL_TRUE)
780  cx_list_push_back(alist, state);
781  else
782  cx_list_push_back(blist, state);
783  skip_if(0);
784  }
785 
786  error_if(cx_list_size(alist) == 0, CPL_ERROR_ILLEGAL_INPUT,
787  "No frames with nodpos A");
788  error_if(cx_list_size(blist) == 0, CPL_ERROR_ILLEGAL_INPUT,
789  "No frames with nodpos B");
790  if (cx_list_size(alist) != cx_list_size(blist)) {
791  if (cx_list_size(alist) > cx_list_size(alist)) {
792  cpl_msg_warning(cpl_func,
793  "Expecting even number of files, ignoring the last Nod A frame");
794  repack_framestate_delete(cx_list_pop_back(alist));
795  }
796  if (cx_list_size(blist) > cx_list_size(alist)) {
797  cpl_msg_warning(cpl_func,
798  "Expecting even number of files, ignoring the last Nod B frame");
799  repack_framestate_delete(cx_list_pop_back(blist));
800  }
801  }
802  error_if(cx_list_size(alist) != cx_list_size(blist),
803  CPL_ERROR_ILLEGAL_INPUT,
804  "Unequal number of A and B frames. A: %d, B: %d",
805  (int)cx_list_size(alist), (int)cx_list_size(blist));
806 
807  {
808  cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_STATIC_MASK);
809  repack_framestate * state = ((repack_framestate*)cx_list_front(alist));
810  yjitter0 = ((repack_framestate*)cx_list_front(alist))->offsety;
811  if (frm) {
812  bpm = cpl_image_load(cpl_frame_get_filename(frm),
813  CPL_TYPE_UNSPECIFIED, 0, 0);
814  if (bpm == NULL) {
815  cpl_msg_warning(cpl_func, "Loading of bad pixel map %s failed",
816  cpl_frame_get_filename(frm));
817  skip_if(0);
818  }
819  cpl_msg_info(cpl_func, "Loaded bad pixel mask from %s",
820  cpl_frame_get_filename(frm));
821  if (visir_data_is_aqu(state->datatype)) {
822  cpl_size win_nx = visir_pfits_get_win_nx(state->plist);
823  cpl_size win_ny = visir_pfits_get_win_ny(state->plist);
824  cpl_size start_x = visir_pfits_get_start_x(state->plist);
825  cpl_size start_y = visir_pfits_get_start_y(state->plist);
826  /* cut bpm to detector readout window */
827  cpl_size nx = cpl_image_get_size_x(bpm);
828  cpl_size ny = cpl_image_get_size_y(bpm);
829  cpl_size llx = 1, lly = 1, urx = nx, ury = ny;
830  if (win_nx > 0 && start_x > 0) {
831  llx = start_x;
832  urx = start_x + win_nx - 1;
833  }
834  if (win_ny > 0 && start_y > 0) {
835  lly = start_y;
836  ury = start_y + win_ny - 1;
837  }
838  if (llx != 1 || urx != nx || lly != 1 || ury != ny) {
839  cpl_image * tmp = cpl_image_extract(bpm, llx, lly, urx, ury);
840  cpl_image_delete(bpm);
841  bpm = tmp;
842  }
843  skip_if(0);
844  }
845  nonlinear = cpl_image_new(cpl_image_get_size_x(bpm),
846  cpl_image_get_size_y(bpm), CPL_TYPE_INT);
847  }
848  }
849 
850  xjitter0 = ((repack_framestate*)cx_list_front(alist))->offsetx;
851  yjitter0 = ((repack_framestate*)cx_list_front(alist))->offsety;
852 
853  {
854  repack_framestate * st = cx_list_front(alist);
855  const char * lextname = NULL;
856  if (visir_is_img(st->plist)) {
857  lextname = "IMAGE_LIN";
858  }
859  else if (visir_is_spc(st->plist)) {
860  lextname = "SPEC_LIN";
861  }
862  cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_LIN);
863  if (lextname && frm && visir_data_is_aqu(st->datatype)) {
864  const char * fn = cpl_frame_get_filename(frm);
865  cpl_msg_info(cpl_func, "Correcting linearity using %s extension "
866  "of %s", lextname, fn);
867  cpl_size iext = cpl_fits_find_extension(fn, lextname);
868  error_if(iext < 0, CPL_ERROR_ILLEGAL_INPUT,
869  "Linearity correction extension %s not found in %s",
870  lextname, fn);
871 
872  cpl_table * tab = cpl_table_load(fn, iext, 0);
873  const size_t n = cpl_table_get_nrow(tab);
874  lintable = cpl_bivector_new(n);
875  memcpy(cpl_bivector_get_x_data(lintable),
876  cpl_table_get_data_double(tab, "dc_level"),
877  sizeof(double) * n);
878  memcpy(cpl_bivector_get_y_data(lintable),
879  cpl_table_get_data_double(tab, "conv_gain"),
880  sizeof(double) * n);
881  cpl_table_delete(tab);
882 
883  /* normalize by average */
884  cpl_vector_divide_scalar(cpl_bivector_get_y(lintable),
885  cpl_vector_get_mean(cpl_bivector_get_y(lintable)));
886  }
887  }
888 
889  while (1) {
890  repack_framestate * fstatea = NULL, * fstateb = NULL;
891  if (!cx_list_empty(alist))
892  fstatea = (repack_framestate*)cx_list_pop_front(alist);
893  if (!cx_list_empty(blist))
894  fstateb = (repack_framestate*)cx_list_pop_front(blist);
895 
896  if (fstatea == NULL || fstateb == NULL)
897  break;
898 
899  if (lintable) {
900  fstatea->load_type = CPL_TYPE_FLOAT;
901  fstateb->load_type = CPL_TYPE_FLOAT;
902  }
903 
904 
905  if (cpl_propertylist_has(fstatea->plist, "DATE-OBS") &&
906  cpl_propertylist_has(fstatea->plist, "DATE") &&
907  cpl_propertylist_has(fstateb->plist, "DATE-OBS") &&
908  cpl_propertylist_has(fstateb->plist, "DATE")) {
909  const char * str =
910  cpl_propertylist_get_string(fstatea->plist, "DATE-OBS");
911  irplib_wcs_mjd_from_string(&(fstatea->time_obsstart), str);
912  str = cpl_propertylist_get_string(fstatea->plist, "DATE");
913  irplib_wcs_mjd_from_string(&(fstatea->time_filewrite), str);
914 
915  str = cpl_propertylist_get_string(fstateb->plist, "DATE-OBS");
916  irplib_wcs_mjd_from_string(&(fstateb->time_obsstart), str);
917  str = cpl_propertylist_get_string(fstateb->plist, "DATE");
918  irplib_wcs_mjd_from_string(&(fstateb->time_filewrite), str);
919  }
920 
921  fstatea->time_min_obsstart = CX_MIN(fstatea->time_obsstart,
922  fstateb->time_obsstart);
923  fstateb->time_min_obsstart = CX_MIN(fstatea->time_obsstart,
924  fstateb->time_obsstart);
925  fstatea->time_max_filewrite = CX_MAX(fstatea->time_filewrite,
926  fstateb->time_filewrite);
927  fstateb->time_max_filewrite = CX_MAX(fstatea->time_filewrite,
928  fstateb->time_filewrite);
929 
930  /* run single threaded but allow async tasks */
931  OMP_PRAGMA(omp parallel num_threads(2))
932  OMP_PRAGMA(omp single)
933  {
934  visir_util_repack_two(fstatea, fstateb, framelist, &bpm, nonlinear,
935  xjitter0, yjitter0, lintable);
936  }
937  skip_if(0);
938 
939  bkgcorrect = fstatea->bkgcorrect;
940  repack_framestate * st[4] = {fstatea, fstatea, fstateb, fstateb};
941  for (int i = 0; i < 4; i++) {
942  cpl_image * stimg = i % 2 == 0 ? st[i]->mean_on : st[i]->mean_off;
943  cpl_size nst = i % 2 == 0 ? *st[i]->nmean_on : *st[i]->nmean_off;
944  if (mean[i] == NULL)
945  mean[i] = cpl_image_duplicate(stimg);
946  else
947  cpl_image_add(mean[i], stimg);
948  nmean[i] += nst;
949  }
950 
951  repack_framestate_delete(fstatea);
952  repack_framestate_delete(fstateb);
953  cpl_fits_set_mode(CPL_FITS_RESTART_CACHING);
954  }
955 
956  skip_if(store_means(bkgcorrect, rawframes, framelist, parlist,
957  mean, nmean, frametype));
958 
959  /* store bpm, skip if wrong size, there is no keyword one can
960  * use in the OCA rules to associate DRS/AQU properly */
961  if (bpm &&
962  cpl_image_get_size_x(bpm) == cpl_image_get_size_x(mean[0]) &&
963  cpl_image_get_size_y(bpm) == cpl_image_get_size_y(mean[0])) {
964  /* apply all-nonlinear bpm */
965  if (lintable) {
966  cpl_size total = nmean[0] + nmean[1] + nmean[2] + nmean[3];
967  cpl_mask * mnonlinear =
968  cpl_mask_threshold_image_create(nonlinear, total * 0.9, total * 2);
969  cpl_msg_info(cpl_func, "%lld (%.3g%%) pixels non-linear in 90%% "
970  "of %lld frames", cpl_mask_count(mnonlinear),
971  (100. * cpl_mask_count(mnonlinear)) /
972  (cpl_mask_get_size_x(mnonlinear) *
973  cpl_mask_get_size_y(mnonlinear)), total);
974  /* disabled due to problems in HRS with so many bad pixels */
975  //cpl_image_reject_from_mask(bpm, mnonlinear);
976  //cpl_image_fill_rejected(bpm, 1);
977  cpl_mask_delete(mnonlinear);
978  }
979 
980  cpl_frameset * usedframes = cpl_frameset_new();
981  cpl_frameset_insert(usedframes, cpl_frame_duplicate
982  (irplib_framelist_get_const(rawframes, 0)));
983  irplib_dfs_save_image(framelist, parlist, usedframes, bpm,
984  CPL_BPP_8_UNSIGNED, RECIPE_STRING,
985  VISIR_CALIB_STATIC_MASK, NULL, NULL,
986  visir_pipe_id, RECIPE_STRING "_bpm"
987  CPL_DFS_FITS);
988  cpl_frameset_delete(usedframes);
989  }
990  else if (bpm) {
991  cpl_msg_warning(cpl_func, "Provided bad pixel mask does not have "
992  "correct size, skipping");
993  }
994 
995  end_skip;
996 
997  irplib_framelist_delete(allframes);
998  irplib_framelist_delete(rawframes);
999 
1000  cx_list_destroy(alist, repack_framestate_delete);
1001  cx_list_destroy(blist, repack_framestate_delete);
1002  cpl_image_delete(bpm);
1003  cpl_image_delete(nonlinear);
1004  cpl_bivector_delete(lintable);
1005  cpl_free(frametype);
1006 
1007  for (size_t i = 0; i < sizeof(mean)/sizeof(mean[0]); i++)
1008  cpl_image_delete(mean[i]);
1009 
1010  return cpl_error_get_code();
1011 }
1012 
1013 
1014 /*----------------------------------------------------------------------------*/
1026 /*----------------------------------------------------------------------------*/
1027 static cpl_error_code
1028 save_subtract_images(const repack_framestate * state,
1029  const cpl_imagelist * subtracted,
1030  const cpl_imagelist * jit_cor,
1031  const cpl_propertylist * plist)
1032 {
1033  const cpl_size nsub = cpl_imagelist_get_size(subtracted);
1034  cpl_ensure_code(jit_cor == NULL || cpl_imagelist_get_size(jit_cor) == nsub,
1035  CPL_ERROR_ILLEGAL_INPUT);
1036 
1037 
1038  for (cpl_size j = 0; j < nsub; j++) {
1039  const cpl_image * img = cpl_imagelist_get_const(subtracted, j);
1040  cpl_io_type compress = 0;
1041  cpl_type save_type;
1042 
1043  if (jit_cor)
1044  cpl_image_add(state->mean_on, cpl_imagelist_get_const(jit_cor, j));
1045  else
1046  cpl_image_add(state->mean_on, img);
1047  (*state->nmean_on)++;
1048 
1049  if (state->compress && cpl_image_get_type(img) == CPL_TYPE_INT) {
1050  /* saves space even if it could be stored in a short */
1051  save_type = CPL_TYPE_INT;
1052  compress = CPL_IO_COMPRESS_RICE;
1053  }
1054  else
1055  save_type = get_optimum_save_type(img);
1056 
1057  skip_if(cpl_image_save(img, state->onname, save_type,
1058  plist, CPL_IO_EXTEND | compress));
1059  }
1060 
1061  end_skip;
1062 
1063  return cpl_error_get_code();
1064 }
1065 
1066 
1067 /*----------------------------------------------------------------------------*/
1080 /*----------------------------------------------------------------------------*/
1081 static
1082 cpl_error_code append_images(const repack_framestate * state,
1083  cpl_imagelist * images,
1084  const cpl_propertylist * plist,
1085  const cpl_boolean on)
1086 {
1087  const char * name = on ? state->onname : state->offname;
1088  const cpl_size n = cpl_imagelist_get_size(images);
1089  cpl_image * mimg = on ? state->mean_on : state->mean_off;
1090  cpl_size * nmean = on ? state->nmean_on : state->nmean_off;
1091 
1092  for (cpl_size j = 0; j < n; j++) {
1093  cpl_image * img = cpl_imagelist_get(images, j);
1094  cpl_io_type compress = 0;
1095  cpl_type save_type;
1096  /* drs burst data is not preprocessed so it needs flipping */
1097  if (state->datatype == VISIR_DATA_BURST) {
1098  cpl_image_multiply_scalar(img, -1);
1099  cpl_image_add_scalar(img, VISIR_HCYCLE_OFFSET);
1100  }
1101 
1102  cpl_image_add(mimg, img);
1103  (*nmean)++;
1104 
1105  if (state->compress && cpl_image_get_type(img) == CPL_TYPE_INT) {
1106  /* saves space even if it could be stored in a short */
1107  save_type = CPL_TYPE_INT;
1108  compress = CPL_IO_COMPRESS_RICE;
1109  }
1110  else
1111  save_type = get_optimum_save_type(img);
1112 
1113  skip_if(cpl_image_save(img, name, save_type,
1114  plist, CPL_IO_EXTEND | compress));
1115  }
1116 
1117  end_skip;
1118 
1119  return cpl_error_get_code();
1120 }
1121 
1122 
1123 /*----------------------------------------------------------------------------*/
1131 /*----------------------------------------------------------------------------*/
1132 static cpl_error_code
1133 cast_list(cpl_imagelist * l, const cpl_type type)
1134 {
1135  const cpl_size n = cpl_imagelist_get_size(l);
1136  cpl_imagelist * cl;
1137  if (type == CPL_TYPE_UNSPECIFIED)
1138  return CPL_ERROR_NONE;
1139 
1140  cl = cpl_imagelist_new();
1141 
1142  for (cpl_size i = 0; i < n; i++) {
1143  cpl_image * img = cpl_imagelist_get(l, i);
1144  cpl_image * cast;
1145  if (cpl_image_get_type(img) == type) {
1146  cpl_imagelist_delete(cl);
1147  return cpl_error_get_code();
1148  }
1149  cast = cpl_image_cast(img, type);
1150  cpl_imagelist_set(cl, cast, i);
1151  }
1152 
1153  cpl_imagelist_empty(l);
1154  for (cpl_size i = 0; i < n; i++)
1155  cpl_imagelist_set(l, cpl_imagelist_get(cl, i), i);
1156 
1157  visir_imagelist_unwrap(cl);
1158 
1159  return cpl_error_get_code();
1160 }
1161 
1162 
1163 /*----------------------------------------------------------------------------*/
1174 /*----------------------------------------------------------------------------*/
1175 static
1176 cpl_error_code load_chunk(cpl_imagelist * on, cpl_imagelist * off,
1177  const repack_framestate * state,
1178  const int pstart, const int pend)
1179 {
1180  /* sort a chunk of data */
1181  if (state->datatype == VISIR_DATA_AQU_HCYCLE) {
1182  const cpl_frame * frame = irplib_framelist_get_const(state->rawframes,
1183  state->iframe);
1184  const char * filename = cpl_frame_get_filename(frame);
1185  cpl_errorstate prestate = cpl_errorstate_get();
1186  skip_if(0);
1187 
1188  for (cpl_size iext = pstart + 1; iext < pend + 1; iext++) {
1189  cpl_image * img = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, iext);
1190  cpl_propertylist * plist;
1191  const char * ftype;
1192 
1193  if (img == NULL) {
1194  cpl_msg_debug(cpl_func, "No image-data in extension %d",
1195  (int)iext);
1196  cpl_errorstate_set(prestate);
1197  continue;
1198  }
1199  plist = cpl_propertylist_load_regexp(filename, iext,
1200  VISIR_PFITS_STRING_FRAME_TYPE,
1201  CPL_FALSE);
1202  skip_if(plist == NULL);
1203  ftype = visir_pfits_get_frame_type(plist);
1204  error_if(ftype == NULL, CPL_ERROR_DATA_NOT_FOUND, "ESO DET FRAM "
1205  "TYPE keyword missing in extension %d of file %s",
1206  (int)iext, filename);
1207  if (strcmp(ftype, "HCYCLE1") == 0)
1208  cpl_imagelist_set(on, img, cpl_imagelist_get_size(on));
1209  else if (strcmp(ftype, "HCYCLE2") == 0)
1210  cpl_imagelist_set(off, img, cpl_imagelist_get_size(off));
1211  else {
1212  cpl_msg_info(cpl_func, "Skipping \"%s\" frame type", ftype);
1213  }
1214  cpl_propertylist_delete(plist);
1215  skip_if(0);
1216  }
1217  }
1218  else if (state->datatype == VISIR_DATA_CUBE2) {
1219  skip_if(visir_load_cube2_split(on, off, state->rawframes,
1220  state->iframe, pstart, pend));
1221 
1222  }
1223  else if (visir_data_is_burst(state->datatype))
1224  skip_if(visir_load_burst(on, off,
1225  irplib_framelist_get_const(state->rawframes,
1226  state->iframe),
1227  state->plist, state->to_off,
1228  state->halfcycle, pstart, pend,
1229  state->trimlow, state->trimhigh));
1230  else
1231  error_if(1, CPL_ERROR_ILLEGAL_INPUT, "invalid data tag");
1232 
1233  /* FIXME: pass load type to the loading functions instead */
1234  cast_list(on, state->load_type);
1235  cast_list(off, state->load_type);
1236 
1237  end_skip;
1238 
1239  return cpl_error_get_code();
1240 }
1241 
1242 
1243 /*----------------------------------------------------------------------------*/
1253 /*----------------------------------------------------------------------------*/
1254 static cpl_size
1255 equalize_lists(cpl_imagelist * a, cpl_imagelist * b)
1256 {
1257  const cpl_size na = cpl_imagelist_get_size(a);
1258  const cpl_size nb = cpl_imagelist_get_size(b);
1259 
1260  if (na != nb)
1261  cpl_msg_warning(cpl_func, "Unequal number of planes in on "
1262  "and off list: #on %d, #off %d. Skipping %ld"
1263  " planes.", (int)na, (int)nb, labs(na - nb));
1264 
1265  if (na > nb) {
1266  for (cpl_size i = 0; i < na - nb; i++)
1267  cpl_image_delete(visir_imagelist_pop(a));
1268  }
1269  else if (na < nb) {
1270  for (cpl_size i = 0; i < nb - na; i++)
1271  cpl_image_delete(visir_imagelist_pop(b));
1272  }
1273 
1274  if (cpl_error_get_code() != CPL_ERROR_NONE)
1275  return -1;
1276  return cpl_imagelist_get_size(a);
1277 }
1278 
1279 static cpl_error_code
1280 check_and_fix_cd_wcs(cpl_propertylist * plist, const cpl_propertylist * iplist)
1281 {
1282  if (!cpl_propertylist_has(plist, "CD1_1") ||
1283  cpl_propertylist_get_double(plist, "CD1_1") == 0.) {
1284  double pfov = visir_pfits_get_pixscale(iplist);
1285  cpl_msg_warning(cpl_func, "CD1_1 WCS key missing or zero, fixing to "
1286  "%g (pfov) / 3600", pfov);
1287  cpl_propertylist_update_double(plist, "CD1_1", pfov / 3600.);
1288  }
1289  if (!cpl_propertylist_has(plist, "CD2_2") ||
1290  cpl_propertylist_get_double(plist, "CD2_2") == 0.) {
1291  double pfov = visir_pfits_get_pixscale(iplist);
1292  cpl_msg_warning(cpl_func, "CD2_2 WCS key missing or zero, fixing to "
1293  "%g (-pfov) / 3600", - pfov);
1294  cpl_propertylist_update_double(plist, "CD2_2", -pfov / 3600.);
1295  }
1296 
1297  return cpl_error_get_code();
1298 }
1299 
1300 
1301 /*----------------------------------------------------------------------------*/
1314 /*----------------------------------------------------------------------------*/
1315 static cpl_error_code
1316 prepare_output(const repack_framestate * state, cpl_frameset * products,
1317  double xjittera, double yjittera,
1318  double xjitterb, double yjitterb,
1319  double bg_mean)
1320 {
1321  cpl_frameset * usedframes = cpl_frameset_new();
1322  cpl_propertylist * onlist = cpl_propertylist_new();
1323  cpl_propertylist * offlist = cpl_propertylist_new();
1324 
1325  /* Need a copy of the WCS cards for subsequent beam stacking */
1326  bug_if(cpl_propertylist_copy_property_regexp(onlist, state->plist, "^("
1327  IRPLIB_PFITS_WCS_REGEXP ")$",
1328  CPL_FALSE));
1329 
1330  bug_if(cpl_propertylist_append_double(onlist, VISIR_DRS_CONAD, state->conad));
1331  bug_if(cpl_propertylist_set_comment(onlist, VISIR_DRS_CONAD, "Default "
1332  "single frame value: " IRPLIB_STRINGIFY
1333  (VISIR_UTIL_REPACK_CONAD)));
1334 
1335  bug_if(cpl_propertylist_append_double(onlist, VISIR_PFITS_DOUBLE_PIXSPACE,
1336  state->pxspace));
1337 
1338  cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETXA, xjittera);
1339  cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETYA, yjittera);
1340  cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETXA,
1341  state->comoffx);
1342  cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETYA,
1343  state->comoffy);
1344 
1345  cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETXB, xjitterb);
1346  cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETYB, yjitterb);
1347  cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETXB,
1348  state->comoffx);
1349  cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETYB,
1350  state->comoffy);
1351 
1352  bug_if(cpl_propertylist_append(offlist, onlist));
1353 
1354  cpl_propertylist_append_int(onlist, "ESO DRS DTYPE", state->datatype);
1355  cpl_propertylist_append_int(offlist, "ESO DRS DTYPE", state->datatype);
1356  bug_if(0);
1357 
1358  if (state->time_max_filewrite > 0) {
1359  cpl_propertylist_append_double(onlist, "ESO DRS DATE",
1360  state->time_max_filewrite);
1361  cpl_propertylist_append_double(offlist, "ESO DRS DATE",
1362  state->time_max_filewrite);
1363  cpl_propertylist_append_double(onlist, "ESO DRS DATE-OBS",
1364  state->time_min_obsstart);
1365  cpl_propertylist_append_double(offlist, "ESO DRS DATE-OBS",
1366  state->time_min_obsstart);
1367  }
1368  else {
1369  /* always write so following recipes don't need to check */
1370  cpl_propertylist_append_double(onlist, "ESO DRS DATE", -1.);
1371  cpl_propertylist_append_double(offlist, "ESO DRS DATE", -1.);
1372  cpl_propertylist_append_double(onlist, "ESO DRS DATE-OBS", -1.);
1373  cpl_propertylist_append_double(offlist, "ESO DRS DATE-OBS", -1.);
1374  }
1375  if (state->normalize) {
1376  cpl_propertylist_append_string(onlist, "BUNIT", "adu / s");
1377  cpl_propertylist_append_string(offlist, "BUNIT", "adu / s");
1378  }
1379  else {
1380  cpl_propertylist_append_string(onlist, "BUNIT", "adu");
1381  cpl_propertylist_append_string(offlist, "BUNIT", "adu");
1382  }
1383 
1384  /* record all inputs as used, even though not all are actually used for
1385  * each output but not recording them here would then require merging input
1386  * headers in subsequent recipes which do join the output files into a
1387  * single product */
1388  for (int i = 0; i < state->nframes; i++) {
1389  const cpl_frame * frm =
1390  irplib_framelist_get_const(state->rawframes, i);
1391  bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm)));
1392  }
1393 
1394  skip_if(check_and_fix_cd_wcs(onlist, state->plist));
1395  skip_if(check_and_fix_cd_wcs(offlist, state->plist));
1396 
1397  if (products == NULL)
1398  products = cpl_frameset_new();
1399 
1400 
1401  if (state->datatype == VISIR_DATA_CUBE2) {
1402  skip_if(visir_qc_append_background(onlist, state->rawframes, 0, 0));
1403  skip_if(visir_qc_append_background(offlist, state->rawframes, 0, 0));
1404  }
1405  else if (!isnan(bg_mean)) {
1406  bug_if (cpl_propertylist_append_double(onlist, "ESO QC BACKGD MEAN",
1407  bg_mean));
1408  bug_if (cpl_propertylist_append_double(offlist, "ESO QC BACKGD MEAN",
1409  bg_mean));
1410  }
1411 
1412  if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
1413  state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
1414  skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1415  RECIPE_STRING, state->procatgon,
1416  onlist, state->remregexp,
1417  visir_pipe_id,
1418  state->onname));
1419  } else {
1420  /* Save each image in a separate extension */
1421  /* Do not save data in the primary data unit (DFS10475) */
1422  skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1423  RECIPE_STRING, state->procatgon,
1424  onlist, state->remregexp,
1425  visir_pipe_id, state->onname));
1426  if (state->chop_on)
1427  skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1428  RECIPE_STRING, state->procatgoff,
1429  offlist, state->remregexp,
1430  visir_pipe_id, state->offname));
1431  }
1432 
1433  end_skip;
1434  cpl_frameset_delete(usedframes);
1435  cpl_propertylist_delete(onlist);
1436  cpl_propertylist_delete(offlist);
1437 
1438  return cpl_error_get_code();
1439 }
1440 
1441 
1442 static cpl_imagelist *
1443 correct_jitter(const cpl_imagelist * imgs, double dx, double dy)
1444 {
1445  cpl_imagelist * jit_cor = cpl_imagelist_new();
1446  cpl_msg_info(cpl_func, "Correcting jitter, x: %g y: %g", dx, dy);
1447  for (cpl_size i = 0, n = cpl_imagelist_get_size(imgs); i < n; i++) {
1448  cpl_image * img = cpl_image_duplicate(cpl_imagelist_get_const(imgs, i));
1449  /* only used for mean which in turn is only used for
1450  * initial beam guess, pixel precision sufficient */
1451  skip_if(cpl_image_shift(img,
1452  -visir_round_to_int(dx),
1453  -visir_round_to_int(dy)));
1454  cpl_imagelist_set(jit_cor, img, i);
1455  }
1456 
1457  end_skip;
1458 
1459  return jit_cor;
1460 }
1461 
1462 
1463 /* ---------------------------------------------------------------------------*/
1479 /* ---------------------------------------------------------------------------*/
1480 static cpl_imagelist *
1481 chop_correct(const repack_framestate * state,
1482  cpl_imagelist * on,
1483  cpl_imagelist * off,
1484  const cpl_boolean last_chunk)
1485 {
1486  cpl_imagelist * sub;
1487  /* burst data is not preprocessed and needs sign flipping */
1488  cpl_imagelist * subt;
1489 
1490  subt = state->datatype == VISIR_DATA_BURST ? on : off;
1491  sub = state->datatype == VISIR_DATA_BURST ? off : on;
1492 
1493  if (last_chunk)
1494  equalize_lists(on, off);
1495 
1496  cpl_imagelist_subtract(sub, subt);
1497  if (state->normalize) {
1498  cpl_imagelist_multiply_scalar(sub, 1. / (state->dit * 2));
1499  }
1500  /* empty now to reduce memory usage */
1501  cpl_imagelist_empty(subt);
1502 
1503  return sub;
1504 }
1505 
1506 /* ---------------------------------------------------------------------------*/
1513 /* ---------------------------------------------------------------------------*/
1514 static double
1515 compute_qc_median(const cpl_imagelist * list, cpl_size * nimages)
1516 {
1517  double sum = 0.;
1518  for (cpl_size i = 0; i < cpl_imagelist_get_size(list); i++) {
1519  const cpl_image * img = cpl_imagelist_get_const(list, i);
1520  sum += cpl_image_get_median(img);
1521  }
1522 
1523  *nimages += cpl_imagelist_get_size(list);
1524 
1525  return sum;
1526 }
1527 
1528 /* ---------------------------------------------------------------------------*/
1536 /* ---------------------------------------------------------------------------*/
1537 static cpl_error_code
1538 correct_linearity(const repack_framestate * state,
1539  cpl_imagelist * images,
1540  const cpl_bivector * lintable, cpl_image * nonlinear)
1541 {
1542  if (lintable == NULL || !visir_data_is_aqu(state->datatype)) {
1543  return CPL_ERROR_NONE;
1544  }
1545  for (size_t i = 0; i < (size_t)cpl_imagelist_get_size(images); i++) {
1546  cpl_image * img = cpl_imagelist_get(images, i);
1547  cpl_ensure_code(cpl_image_get_type(img) == CPL_TYPE_FLOAT ||
1548  cpl_image_get_type(img) == CPL_TYPE_DOUBLE,
1549  CPL_ERROR_ILLEGAL_INPUT);
1550  cpl_image * dvals = cpl_image_cast(img, CPL_TYPE_DOUBLE);
1551  cpl_image * corvals = visir_linintp_values(dvals, lintable);
1552  if (nonlinear) {
1553  cpl_image * bad =
1554  cpl_image_new_from_mask(cpl_image_get_bpm(corvals));
1555  cpl_image_add(nonlinear, bad);
1556  cpl_image_delete(bad);
1557  }
1558  cpl_image_divide(img, corvals);
1559  cpl_image_delete(dvals);
1560  cpl_image_delete(corvals);
1561  }
1562 
1563  return cpl_error_get_code();
1564 }
1565 
1566 /* ---------------------------------------------------------------------------*/
1582 /* ---------------------------------------------------------------------------*/
1583 static cpl_error_code
1584 save_corrected(const repack_framestate * fstatea,
1585  const repack_framestate * fstateb,
1586  cpl_imagelist * aon,
1587  cpl_imagelist * aoff,
1588  cpl_imagelist * bon,
1589  cpl_imagelist * boff,
1590  double dxa, double dya,
1591  double dxb, double dyb,
1592  const cpl_boolean last_chunk)
1593 {
1594  cpl_imagelist * chop_corrected_a, * chop_corrected_b;
1595  const repack_framestate * state = fstatea;
1596 
1597  /* chop correct if chopper is on */
1598  if (state->chop_on) {
1599  chop_corrected_a = chop_correct(fstatea, aon, aoff, last_chunk);
1600  chop_corrected_b = chop_correct(fstateb, bon, boff, last_chunk);
1601  }
1602  else {
1603  chop_corrected_a = aon;
1604  chop_corrected_b = bon;
1605  }
1606 
1607  if (cpl_error_get_code())
1608  return cpl_error_get_code();
1609 
1610  if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) {
1611  /* only chop correction requested */
1612  save_subtract_images(fstatea, chop_corrected_a, NULL, NULL);
1613  save_subtract_images(fstateb, chop_corrected_b, NULL, NULL);
1614  }
1615  else {
1616  /* perform nod and jitter correction
1617  * jitter is only used for the mean used for object detection pivoting
1618  * correction is done in later recipes */
1619  cpl_imagelist * jca = NULL;
1620  cpl_imagelist * jcb = NULL;
1621 
1622  if (last_chunk)
1623  equalize_lists(chop_corrected_a, chop_corrected_b);
1624 
1625  if (state->normalize) {
1626  cpl_imagelist_divide_scalar(chop_corrected_a, 2);
1627  cpl_imagelist_divide_scalar(chop_corrected_b, 2);
1628  }
1629 
1630  if (visir_round_to_int(dxa) != 0 ||
1631  visir_round_to_int(dya) != 0 ||
1632  visir_round_to_int(dxb) != 0 ||
1633  visir_round_to_int(dyb) != 0) {
1634  jca = correct_jitter(chop_corrected_a, dxa, dya);
1635  jcb = correct_jitter(chop_corrected_b, dxb, dyb);
1636  cpl_imagelist_subtract(jca, jcb);
1637  }
1638 
1639  cpl_imagelist_subtract(chop_corrected_a, chop_corrected_b);
1640 
1641  save_subtract_images(state, chop_corrected_a, jca, NULL);
1642  cpl_imagelist_delete(jcb);
1643  cpl_imagelist_delete(jca);
1644  }
1645 
1646  return cpl_error_get_code();
1647 }
1648 
1649 static cpl_error_code
1650 save_uncorrected(const repack_framestate * fstatea,
1651  const repack_framestate * fstateb,
1652  cpl_imagelist * aon,
1653  cpl_imagelist * aoff,
1654  cpl_imagelist * bon,
1655  cpl_imagelist * boff)
1656 {
1657  const repack_framestate * state = fstatea;
1658  if (state->normalize) {
1659  cpl_imagelist_multiply_scalar(aon, 1. / fstatea->dit);
1660  cpl_imagelist_multiply_scalar(aoff, 1. / fstatea->dit);
1661  cpl_imagelist_multiply_scalar(bon, 1. / fstateb->dit);
1662  cpl_imagelist_multiply_scalar(boff, 1. / fstateb->dit);
1663  }
1664  append_images(fstatea, aon, NULL, CPL_TRUE);
1665  append_images(fstatea, aoff, NULL, CPL_FALSE);
1666  append_images(fstateb, bon, NULL, CPL_TRUE);
1667  append_images(fstateb, boff, NULL, CPL_FALSE);
1668  cpl_imagelist_empty(aon);
1669  cpl_imagelist_empty(aoff);
1670  cpl_imagelist_empty(bon);
1671  cpl_imagelist_empty(boff);
1672 
1673  return cpl_error_get_code();
1674 }
1675 
1676 #define VISIR_SWAP(a, b)\
1677  do {\
1678  cpl_imagelist * tmp = a; \
1679  a = b; \
1680  b = tmp; \
1681  } while (0)
1682 
1683 /*----------------------------------------------------------------------------*/
1696 /*----------------------------------------------------------------------------*/
1697 static cpl_error_code
1698 visir_util_repack_two(const repack_framestate * fstatea,
1699  const repack_framestate * fstateb,
1700  cpl_frameset * products,
1701  cpl_image ** pbpm,
1702  cpl_image * nonlinear,
1703  double xjitter0, double yjitter0,
1704  const cpl_bivector * lintable)
1705 {
1706  /* process the data */
1707  cpl_errorstate cleanstate = cpl_errorstate_get();
1708  const repack_framestate * state = fstatea;
1709  int chunksize = 200;
1710  cpl_imagelist * aon = NULL;
1711  cpl_imagelist * aoff = NULL;
1712  cpl_imagelist * bon = NULL;
1713  cpl_imagelist * boff = NULL;
1714  /* preload lists */
1715  cpl_imagelist * aon_next = NULL;
1716  cpl_imagelist * aoff_next = NULL;
1717  cpl_imagelist * bon_next = NULL;
1718  cpl_imagelist * boff_next = NULL;
1719  const char * chopnod_dir = visir_pfits_get_chopnod_dir(state->plist);
1720 
1721  double dxa = fstatea->offsetx - xjitter0;
1722  double dya = fstatea->offsety - yjitter0;
1723  double dxb = fstateb->offsetx - xjitter0;
1724  double dyb = fstateb->offsety - yjitter0;
1725 
1726  double qc_bkg_sum = 0.;
1727  cpl_size qc_bkg_count = 0;
1728 
1729  /* chunksize must be a multiple of the period to avoid having a
1730  * #on-#off difference larger than chunksize */
1731 
1732  if (visir_data_is_burst(state->datatype)) {
1733  chunksize = 2 * state->halfcycle;
1734  /* parameter sanity check, ensures equal length A and B lists */
1735  error_if(state->planeend - state->planestart < 2 * state->halfcycle,
1736  CPL_ERROR_ILLEGAL_INPUT,
1737  "Number of planes to be repacked must be larger than "
1738  "a full cycle of %d planes.", state->halfcycle * 2);
1739  }
1740 
1741  /* can be missing for old spec data */
1742  if (chopnod_dir == NULL) {
1743  chopnod_dir = "PARALLEL";
1744  cpl_errorstate_set(cleanstate);
1745  }
1746 
1747  /* try to correct the cumoffset based on throw
1748  * POSANG is not a reliable mean to determine which offset has the throw
1749  * added to the jitter
1750  * apply heuristic that the jitter is smaller than the throw and correct
1751  * those offsets that are larger than throw with a 10 pixel margin */
1752  cpl_msg_info(cpl_func, "POSANG: %g; Offset: (%g, %g); Throw: %g; "
1753  "Jitterwidth %g", fstateb->posang * CPL_MATH_RAD_DEG, dxb,
1754  dyb, fstateb->pthrow, fstateb->jitterwidth);
1755  if ((int)rint(fstateb->posang * CPL_MATH_RAD_DEG) % 90 == 0) {
1756  if (dxb >= fstateb->pthrow - fstateb->jitterwidth - 10.) {
1757  dxb -= fstateb->pthrow;
1758  }
1759  else if (dxb <= -(fstateb->pthrow - fstateb->jitterwidth - 10.)) {
1760  dxb += fstateb->pthrow;
1761  }
1762  if (dyb >= fstateb->pthrow - fstateb->jitterwidth - 10.) {
1763  dyb -= fstateb->pthrow;
1764  }
1765  else if (dyb <= -(fstateb->pthrow - fstateb->jitterwidth - 10.)) {
1766  dyb += fstateb->pthrow;
1767  }
1768  /* remove rounding errors on no jitter */
1769  dxb = fabs(dxb) < 0.001 ? 0. : dxb;
1770  dyb = fabs(dyb) < 0.001 ? 0. : dyb;
1771  }
1772  else {
1773  /* offset keywords don't seem to respect angle. assume no jitter */
1774  if (fstateb->jitterwidth != 0.) {
1775  cpl_msg_warning(cpl_func,
1776  "Can't do jitter correction on non normal POSANG");
1777  }
1778  dxb = 0.;
1779  dyb = 0.;
1780  }
1781 
1782  aon = cpl_imagelist_new();
1783  aoff = cpl_imagelist_new();
1784  bon = cpl_imagelist_new();
1785  boff = cpl_imagelist_new();
1786 
1787 
1788  /* load the first chunk of data to fill QC parameters*/
1789  if (state->datatype != VISIR_DATA_AQU_INT) {
1790  const int pstart = state->planestart;
1791  const int pend = CX_MIN(pstart + chunksize, state->planeend);
1792  load_chunk(aon, aoff, fstatea, pstart, pend);
1793  load_chunk(bon, boff, fstateb, pstart, pend);
1794  }
1795  if ((visir_data_is_aqu(state->datatype) || visir_data_is_burst(state->datatype)) &&
1796  state->datatype != VISIR_DATA_AQU_INT) {
1797  /* cube2 is computed in prepare_output */
1798  qc_bkg_sum += compute_qc_median(aon, &qc_bkg_count);
1799  qc_bkg_sum += compute_qc_median(aoff, &qc_bkg_count);
1800  qc_bkg_sum += compute_qc_median(bon, &qc_bkg_count);
1801  qc_bkg_sum += compute_qc_median(boff, &qc_bkg_count);
1802  }
1803 
1804 
1805  if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
1806  state->bkgcorrect == VISIR_SUB_NOCORRECT) {
1807  prepare_output(fstatea, products, dxa, dya, dxb, dyb, qc_bkg_sum / qc_bkg_count);
1808  prepare_output(fstateb, products, dxa, dya, dxb, dyb, qc_bkg_sum / qc_bkg_count);
1809  }
1810  else if (state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
1811  if (state->chop_on == CPL_FALSE)
1812  cpl_msg_info(cpl_func, VISIR_PFITS_BOOL_CHOP_STATUS
1813  " false, only doing nod correction.");
1814  prepare_output(state, products, dxa, dya, dxb, dyb, qc_bkg_sum / qc_bkg_count);
1815  }
1816  skip_if(0);
1817 
1818  if (state->datatype == VISIR_DATA_AQU_INT) {
1819  cpl_imagelist * a = cpl_imagelist_new();
1820  cpl_imagelist * b = cpl_imagelist_new();
1821  cpl_imagelist * jca = NULL;
1822  cpl_imagelist * jcb = NULL;
1823  const cpl_frame * frame;
1824  const char * filename;
1825  cpl_image * img;
1826 
1827  frame = irplib_framelist_get_const(fstatea->rawframes,
1828  fstatea->iframe);
1829  filename = cpl_frame_get_filename(frame);
1830  img = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 1);
1831  cpl_imagelist_set(a, img, cpl_imagelist_get_size(a));
1832 
1833  frame = irplib_framelist_get_const(fstateb->rawframes,
1834  fstateb->iframe);
1835  filename = cpl_frame_get_filename(frame);
1836  img = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 1);
1837  cpl_imagelist_set(b, img, cpl_imagelist_get_size(b));
1838 
1839  if (fstatea->normalize) {
1840  cpl_imagelist_multiply_scalar(a, 1. / (state->dit * 4));
1841  cpl_imagelist_multiply_scalar(b, 1. / (state->dit * 4));
1842  }
1843 
1844  if (visir_round_to_int(dxa) != 0 ||
1845  visir_round_to_int(dya) != 0 ||
1846  visir_round_to_int(dxb) != 0 ||
1847  visir_round_to_int(dyb) != 0) {
1848  jca = correct_jitter(a, dxa, dya);
1849  jcb = correct_jitter(b, dxb, dyb);
1850  cpl_imagelist_subtract(jca, jcb);
1851  cpl_imagelist_subtract(a, b);
1852  save_subtract_images(fstatea, a, jca, NULL);
1853  }
1854  else {
1855  cpl_imagelist_subtract(a, b);
1856  save_subtract_images(fstatea, a, jca, NULL);
1857  }
1858 
1859  cpl_imagelist_delete(a);
1860  cpl_imagelist_delete(b);
1861  cpl_imagelist_delete(jca);
1862  cpl_imagelist_delete(jcb);
1863  }
1864 
1865  aon_next = cpl_imagelist_new();
1866  aoff_next = cpl_imagelist_new();
1867  bon_next = cpl_imagelist_new();
1868  boff_next = cpl_imagelist_new();
1869 
1870  for (int pstart = state->planestart;
1871  pstart < state->planeend + chunksize - 1 &&
1872  state->datatype != VISIR_DATA_AQU_INT;
1873  pstart += chunksize) {
1874  const int pend = CX_MIN(pstart + chunksize, state->planeend);
1875 
1876  if (pstart >= pend)
1877  break;
1878 
1879 
1880  /* the process->save part has a throughput of ~200-400MB/s
1881  * SSDs can match this so by performing one preload in parallel with
1882  * processing we archive ideal performance also on these systems
1883  * first chunk already loaded synchronously */
1884  if (pstart != state->planestart) {
1885  /* empty the currently worked on lists
1886  * may have already been emptied by save_ functions */
1887  cpl_imagelist_empty(aon);
1888  cpl_imagelist_empty(aoff);
1889  cpl_imagelist_empty(bon);
1890  cpl_imagelist_empty(boff);
1891 
1892  /* wait for preload of last iteration to finish */
1893  OMP3_PRAGMA(omp taskwait);
1894  /* swap empty current and full preload lists */
1895  VISIR_SWAP(aon, aon_next);
1896  VISIR_SWAP(aoff, aoff_next);
1897  VISIR_SWAP(bon, bon_next);
1898  VISIR_SWAP(boff, boff_next);
1899  }
1900 
1901  /* preload the next chunk asynchronous */
1902  if (pend < state->planeend) {
1903  int pend_next = CX_MIN(pend + chunksize, state->planeend);
1904  OMP3_PRAGMA(omp task) {
1905  load_chunk(aon_next, aoff_next, fstatea, pend, pend_next);
1906  load_chunk(bon_next, boff_next, fstateb, pend, pend_next);
1907  }
1908  }
1909 
1910  if (*pbpm == NULL && state->datatype == VISIR_DATA_CUBE2) {
1911 
1912  *pbpm = cpl_image_duplicate(cpl_imagelist_get(aon, 0));
1913  cpl_image_threshold(*pbpm, state->bpmthresh,
1914  state->bpmthresh, 0.0, 1.0);
1915  skip_if(0);
1916  }
1917 
1918  correct_linearity(fstatea, aon, lintable, nonlinear);
1919  correct_linearity(fstatea, aoff, lintable, nonlinear);
1920  correct_linearity(fstateb, bon, lintable, nonlinear);
1921  correct_linearity(fstateb, boff, lintable, nonlinear);
1922 
1923  if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
1924  state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
1925  cpl_boolean last_chunk = pstart + chunksize >= pend;
1926  skip_if(save_corrected(fstatea, fstateb,
1927  aon, aoff, bon, boff,
1928  dxa, dya, dxb, dyb, last_chunk));
1929  }
1930  else {
1931  skip_if(save_uncorrected(fstatea, fstateb, aon, aoff, bon, boff));
1932  }
1933  }
1934 
1935  end_skip;
1936  cpl_imagelist_delete(aon);
1937  cpl_imagelist_delete(aoff);
1938  cpl_imagelist_delete(bon);
1939  cpl_imagelist_delete(boff);
1940  cpl_imagelist_delete(aon_next);
1941  cpl_imagelist_delete(aoff_next);
1942  cpl_imagelist_delete(bon_next);
1943  cpl_imagelist_delete(boff_next);
1944 
1945  return cpl_error_get_code();
1946 }
1947 
1948 
1949 /*----------------------------------------------------------------------------*/
1958 /*----------------------------------------------------------------------------*/
1959 static inline cpl_error_code visir_util_repack_check(const cpl_image * self,
1960  const cpl_imagelist * on,
1961  const cpl_imagelist * off)
1962 {
1963  /* UNUSED due to chunking */
1964  cpl_image * meanon = cpl_imagelist_collapse_create(on);
1965  cpl_image * meanoff = cpl_imagelist_collapse_create(off);
1966  const cpl_error_code err1 = cpl_image_subtract(meanon, meanoff);
1967  const cpl_error_code err2 = cpl_image_subtract(meanon, self);
1968  const unsigned bitmask = CPL_STATS_MIN | CPL_STATS_MAX | CPL_STATS_MEAN
1969  | CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV | CPL_STATS_STDEV;
1970 
1971  cpl_stats * stats = cpl_stats_new_from_image(meanon, bitmask);
1972 
1973  bug_if(err1 + err2);
1974 
1975  bug_if(cpl_stats_dump(stats, bitmask, stderr));
1976 
1977  end_skip;
1978 
1979  cpl_image_delete(meanon);
1980  cpl_image_delete(meanoff);
1981  cpl_stats_delete(stats);
1982 
1983  return cpl_error_get_code();
1984 
1985 }
double visir_pfits_get_cumoffsetx(const cpl_propertylist *self)
The cumulative offset in X.
Definition: visir_pfits.c:241
int visir_pfits_get_naxis2(const cpl_propertylist *self)
The NAXIS2 key.
Definition: visir_pfits.c:505
int visir_pfits_get_win_ny(const cpl_propertylist *self)
The WIN NY key.
Definition: visir_pfits.c:550
double visir_pfits_get_chop_pthrow(const cpl_propertylist *self)
The chopping throw in pixels.
Definition: visir_pfits.c:214
cpl_error_code visir_load_cube2_split(cpl_imagelist *alist, cpl_imagelist *blist, const irplib_framelist *rawframes, int pos, const int planestart, const int planeend)
Load and split the data in a CUBE2 file.
Definition: visir_inputs.c:238
int visir_pfits_get_naxis3(const cpl_propertylist *self)
The NAXIS3 key.
Definition: visir_pfits.c:520
int visir_pfits_get_start_x(const cpl_propertylist *self)
The WIN STRX key.
Definition: visir_pfits.c:565
int visir_pfits_get_naxis1(const cpl_propertylist *self)
The NAXIS1 key.
Definition: visir_pfits.c:490
cpl_error_code irplib_dfs_save_propertylist(cpl_frameset *allframes, const cpl_parameterlist *parlist, const cpl_frameset *usedframes, const char *recipe, const char *procat, const cpl_propertylist *applist, const char *remregexp, const char *pipe_id, const char *filename)
Save a propertylist as a DFS-compliant pipeline product.
Definition: irplib_utils.c:240
irplib_framelist * irplib_framelist_extract_regexp(const irplib_framelist *self, const char *regexp, cpl_boolean invert)
Extract the frames with the given tag from a framelist.
const char * visir_pfits_get_frame_type(const cpl_propertylist *self)
The frame type.
Definition: visir_pfits.c:395
const cpl_propertylist * irplib_framelist_get_propertylist_const(const irplib_framelist *self, int pos)
Get the propertylist of the specified frame in the framelist.
double visir_pfits_get_chop_posang(const cpl_propertylist *self)
The chopping position angle in rad.
Definition: visir_pfits.c:197
cpl_error_code irplib_dfs_save_image(cpl_frameset *allframes, const cpl_parameterlist *parlist, const cpl_frameset *usedframes, const cpl_image *image, cpl_type_bpp bpp, const char *recipe, const char *procat, const cpl_propertylist *applist, const char *remregexp, const char *pipe_id, const char *filename)
Save an image as a DFS-compliant pipeline product.
Definition: irplib_utils.c:192
int visir_pfits_get_win_nx(const cpl_propertylist *self)
The WIN NX key.
Definition: visir_pfits.c:535
int visir_pfits_get_ndit(const cpl_propertylist *self)
The NDIT keyword.
Definition: visir_pfits.c:479
double visir_pfits_get_cumoffsety(const cpl_propertylist *self)
The cumulative offset in Y.
Definition: visir_pfits.c:253
cpl_frame * irplib_framelist_get(irplib_framelist *self, int pos)
Get the specified frame from the framelist.
double visir_pfits_get_chop_freq(const cpl_propertylist *self)
The chopping frequency.
Definition: visir_pfits.c:162
double visir_pfits_get_pixspace(const cpl_propertylist *self)
The pixel spacing.
Definition: visir_pfits.c:647
const char * visir_pfits_get_nodpos(const cpl_propertylist *self)
The nodding position.
Definition: visir_pfits.c:619
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.
Definition: visir_dfs.c:72
double visir_pfits_get_dit(const cpl_propertylist *self)
The DIT.
Definition: visir_pfits.c:301
const cpl_frame * irplib_framelist_get_const(const irplib_framelist *self, int pos)
Get the specified frame from the framelist.
double visir_pfits_get_pixscale(const cpl_propertylist *self)
The pixel scale.
Definition: visir_pfits.c:631
double visir_pfits_get_chop_stat(const cpl_propertylist *self)
The chopping status.
Definition: visir_pfits.c:179
void irplib_framelist_delete(irplib_framelist *self)
Deallocate an irplib_framelist with its frames and properties.
const char * visir_pfits_get_chopnod_dir(const cpl_propertylist *self)
The chopping direction.
Definition: visir_pfits.c:131
int visir_pfits_get_start_y(const cpl_propertylist *self)
The WIN STRY key.
Definition: visir_pfits.c:580
cpl_error_code irplib_framelist_load_propertylist(irplib_framelist *self, int pos, int ind, const char *regexp, cpl_boolean invert)
Load the propertylist of the specified frame in the framelist.
irplib_framelist * irplib_framelist_cast(const cpl_frameset *frameset)
Create an irplib_framelist from a cpl_framelist.
int irplib_framelist_get_size(const irplib_framelist *self)
Get the size of a framelist.