VISIR Pipeline Reference Manual  4.1.7
visir_util_repack.c
1 /* $Id: visir_util_repack.c,v 1.144 2013-09-24 11:09:55 jtaylor Exp $
2  *
3  * This file is part of the VISIR Pipeline
4  * Copyright (C) 2002,2003,2013,2014 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
19  */
20 
21 #ifdef HAVE_CONFIG_H
22 #include <config.h>
23 #endif
24 
25 /*-----------------------------------------------------------------------------
26  Includes
27  -----------------------------------------------------------------------------*/
28 
29 #include "visir_recipe.h"
30 #include "irplib_framelist.h"
31 #include "irplib_wcs.h"
32 #include <cxlist.h>
33 /* Verify self-sufficiency of CPL header files by including system files last */
34 #include <string.h>
35 /* for strcasecmp */
36 #include <strings.h>
37 #include <math.h> /* isnan */
38 
39 
40 /*-----------------------------------------------------------------------------
41  Defines
42  -----------------------------------------------------------------------------*/
43 
44 #ifndef VISIR_UTIL_REPACK_CONAD
45 #define VISIR_UTIL_REPACK_CONAD 1.1
46 #endif
47 
48 #define RECIPE_STRING "visir_util_repack"
49 
50 #define VISIR_DRS_CONAD "ESO DET OUT1 CONAD"
51 
52 
53 /*-----------------------------------------------------------------------------
54  Private Functions prototypes
55  -----------------------------------------------------------------------------*/
56 typedef enum {
57  VISIR_SUB_NOCORRECT = 0,
58  VISIR_SUB_CHOPCORRECT = 1,
59  VISIR_SUB_CHOPNODCORRECT = 2
60 } visir_sub_type;
61 
62 typedef enum {
63  VISIR_FRAME_A,
64  VISIR_FRAME_B,
65  VISIR_FRAME_STATIC,
66 } visir_frame_type;
67 
68 /* convinience data structure to hold common data needed
69  * to process one frame */
70 typedef struct {
71  int planestart;
72  int planeend;
73  int trimlow;
74  int trimhigh;
75  int bkgcorrect;
76  cpl_boolean normalize;
77  cpl_boolean compress;
78  double conad;
79  int nframes;
80  const irplib_framelist * rawframes;
81  int iframe;
82  const cpl_parameterlist * parlist;
83  const char * tag;
84  const cpl_propertylist * plist;
85  visir_data_type datatype;
86  cpl_type load_type;
87 
88  char * onname;
89  char * offname;
90  const char * remregexp;
91 
92  double pthrow;
93  double posang;
94  double dit;
95  const char * comoffx;
96  const char * comoffy;
97  double offsetx;
98  double offsety;
99  double jitterwidth;
100  double pxspace;
101  double pfov;
102  cpl_wcs * wcs;
103 
104  /* only for burst */
105  int to_off;
106  int halfcycle;
107 
108  double bpmthresh;
109 
110  int naxis3;
111  cpl_boolean is_a;
112  const char * procatgon;
113  const char * procatgoff;
114  cpl_image * mean_on;
115  cpl_size * nmean_on;
116  cpl_image * mean_off;
117  cpl_size * nmean_off;
118 
119  double time_obsstart;
120  double time_filewrite;
121  /* min and max of a two A/B states to write into a chop nod file */
122  double time_min_obsstart;
123  double time_max_filewrite;
125 
126 static cpl_error_code
127 visir_util_repack_two(const repack_framestate * fstatea,
128  const repack_framestate * fstateb,
129  cpl_frameset * products,
130  cpl_image ** pbpm,
131  cpl_image * nonlinear,
132  double, double,
133  const cpl_bivector * lintab);
134 
135 static inline cpl_error_code
136 visir_util_repack_check(const cpl_image *,
137  const cpl_imagelist *,
138  const cpl_imagelist *);
139 
140 #ifdef VISIR_CHAIN
141 #define cpl_plugin_get_info visir_util_repack_get_info
142 #endif
143 cpl_recipe_define(visir_util_repack, VISIR_BINARY_VERSION,
144  "Lars Lundin", PACKAGE_BUGREPORT, "2011",
145  "Conversion of raw CUBE2 or BURST images to on- and off-cubes",
146  "The files listed in the Set Of Frames (sof-file) "
147  "must be tagged:\n"
148  "VISIR-CUBE2-raw-file.fits " VISIR_UTIL_REPACK_RAW
149  "\nor\n"
150  "VISIR-BURST-raw-file.fits " VISIR_IMG_BURST"\n"
151  "VISIR-BURST-bpm-file.fits " VISIR_CALIB_STATIC_MASK
152  "\nFor BURST data it will remove planes where the chopper "
153  "switched from on <-> off based on the "
154  "timestamps in the header."
155  "\nThe product(s) will have a FITS card\n"
156  "'HIERARCH ESO PRO CATG' with a value of one of:\n"
157  VISIR_UTIL_REPACK_A_ON_PROCATG " (NodPos: A, ChopPos: on)\n"
158  VISIR_UTIL_REPACK_A_OFF_PROCATG " (NodPos: A, ChopPos: off)\n"
159  VISIR_UTIL_REPACK_B_ON_PROCATG " (NodPos: B, ChopPos: on)\n"
160  VISIR_UTIL_REPACK_B_OFF_PROCATG " (NodPos: B, ChopPos: off)\n"
161  /* FIXME: reimplement
162  "Additionally, the recipe collapses the on- and off-cubes, "
163  "these product(s) will have a FITS card\n"
164  "'HIERARCH ESO PRO CATG' with a value of one of:\n"
165  VISIR_UTIL_REPACK_MEAN_A_ON_PROCATG
166  " (NodPos: A, ChopPos: on)\n"
167  VISIR_UTIL_REPACK_MEAN_A_OFF_PROCATG
168  " (NodPos: A, ChopPos: off)\n"
169  VISIR_UTIL_REPACK_MEAN_B_ON_PROCATG
170  " (NodPos: B, ChopPos: on)\n"
171  VISIR_UTIL_REPACK_MEAN_B_OFF_PROCATG
172  " (NodPos: B, ChopPos: off)\n" */
173  "For CUBE2, the recipe will produce a static "
174  "bad-pixel map, it will have a FITS card\n"
175  "'HIERARCH ESO PRO CATG' with a value of:\n"
176  VISIR_CALIB_STATIC_MASK"\n"
177  );
178 
179 /*----------------------------------------------------------------------------*/
183 /*----------------------------------------------------------------------------*/
184 
185 /*-----------------------------------------------------------------------------
186  Functions code
187  -----------------------------------------------------------------------------*/
188 
189 
190 /*----------------------------------------------------------------------------*/
197 /*----------------------------------------------------------------------------*/
198 static cpl_image *
199 visir_imagelist_pop(cpl_imagelist * list)
200 {
201  if (cpl_imagelist_get_size(list) == 0)
202  return NULL;
203  return cpl_imagelist_unset(list, cpl_imagelist_get_size(list) - 1);
204 }
205 
206 
207 /*----------------------------------------------------------------------------*/
215 /*----------------------------------------------------------------------------*/
216 static cpl_type get_optimum_save_type(const cpl_image * img)
217 {
218  cpl_type res = CPL_TYPE_UNSPECIFIED;
219  if (cpl_image_get_type(img) == CPL_TYPE_INT) {
220  cpl_stats * stats = cpl_stats_new_from_image(img, CPL_STATS_MIN |
221  CPL_STATS_MAX);
222  if ((int)cpl_stats_get_max(stats) <= CX_MAXSHORT &&
223  (int)cpl_stats_get_min(stats) >= CX_MINSHORT)
224  res = CPL_TYPE_SHORT;
225  cpl_stats_delete(stats);
226  }
227 
228  return res;
229 }
230 
231 
232 /*----------------------------------------------------------------------------*/
240 /*----------------------------------------------------------------------------*/
241 static cpl_error_code
242 visir_util_repack_fill_parameterlist(cpl_parameterlist * self)
243 {
244  const char * context = PACKAGE "." RECIPE_STRING;
245  cpl_error_code err;
246 
247  cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);
248 
249  /* Fill the parameters list */
250 
251  /* --planestart */
252  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "planestart",
253  0, NULL, context, "Plane number to "
254  "start repacking from, earlier planes "
255  "are skipped.");
256  cpl_ensure_code(!err, err);
257 
258  /* --ncycles */
259  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "ncycles",
260  -1, NULL, context, "Number of full "
261  "on-off cycles to repack. <= 0 for all.");
262  cpl_ensure_code(!err, err);
263 
264  /* --trimlow */
265  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "trimlow",
266  0, NULL, context, "Burst data only. "
267  "Number of additional planes to cut "
268  "from before each plane with chopper "
269  "movement.");
270  cpl_ensure_code(!err, err);
271 
272  /* --trimhigh */
273  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "trimhigh",
274  0, NULL, context, "Burst data only. "
275  "Number of additional planes to cut "
276  "from after each plane with chopper "
277  "movement.\n A value of -1 does not "
278  "skip the plane of the movement.");
279  cpl_ensure_code(!err, err);
280 
281  /* --bkgcorrect */
282  err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
283  "bkgcorrect", "none", NULL, context,
284  "Output background corrected planes by "
285  "subtracting chop (on/off) and nod (A/B) "
286  "planes\n Options:\n"
287  " none: no correction\n"
288  " chop: on - off\n"
289  " chopnod: (Aon - Aoff) - (Bon - Boff)");
290  cpl_ensure_code(!err, err);
291 
292 
293  /* --normalize */
294  err =
295  irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING, "normalize",
296  CPL_TRUE, NULL, context,
297  "Normalize planes by DIT");
298  cpl_ensure_code(!err, err);
299 
300 
301  /* --compress */
302  err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
303  "compress", CPL_FALSE, NULL, context,
304  "Apply lossless compression on output"
305  "files. Can only be done for integer "
306  " type results.");
307  cpl_ensure_code(!err, err);
308 
309  return CPL_ERROR_NONE;
310 }
311 
312 
313 /*----------------------------------------------------------------------------*/
324 /*----------------------------------------------------------------------------*/
325 static repack_framestate *
326 repack_framestate_new(irplib_framelist * rawframes, int iframe,
327  const cpl_parameterlist * parlist)
328 {
329  repack_framestate * state = cpl_calloc(1, sizeof(repack_framestate));
330  int ndit, navrg, ncycles, planes_per_cycle = 2;
331  cpl_frame * frame = irplib_framelist_get(rawframes, iframe);
332  const char * filename = cpl_frame_get_filename(frame);
333  cpl_size next;
334  state->time_max_filewrite = -1;
335  state->time_min_obsstart = 1e300;
336 
337  state->rawframes = rawframes;
338 
339  state->iframe = iframe;
340 
341  state->tag = cpl_frame_get_tag(frame);
342 
343  error_if(strcmp(state->tag, VISIR_CALIB_STATIC_MASK) == 0,
344  CPL_ERROR_UNSUPPORTED_MODE, " ");
345 
346  /* compression support disabled, can give bad error messages for
347  * INT only acquarius data */
348  /* compressed files have the data in extension 1 */
349  const cpl_size iext = 0;
350 
351  skip_if(irplib_framelist_load_propertylist(rawframes, iframe, iext,
352  ".*", CPL_FALSE));
353  state->parlist = parlist;
354 
355  state->plist = irplib_framelist_get_propertylist_const(rawframes, iframe);
356  skip_if(0);
357 
358  skip_if(visir_get_data_type(frame, state->plist, &state->datatype, &next));
359 
360  if (state->datatype == VISIR_DATA_AQU_HCYCLE) {
361  /* last frame is INT, but is skipped by when loading */
362  state->naxis3 = next;
363  }
364  else if (state->datatype == VISIR_DATA_AQU_INT) {
365  state->naxis3 = 1;
366  }
367  else if (state->datatype == VISIR_DATA_AQU_BURST_EXT) {
368  cpl_propertylist * dplist = cpl_propertylist_load(filename, 1);
369  state->naxis3 = visir_pfits_get_naxis3(dplist);
370  cpl_propertylist_delete(dplist);
371  }
372  else
373  state->naxis3 = visir_pfits_get_naxis3(state->plist);
374 
375  {
376  const char * bkgc = irplib_parameterlist_get_string(parlist, PACKAGE,
377  RECIPE_STRING,
378  "bkgcorrect");
379  if (visir_str_par_is_empty(bkgc) || !strcasecmp(bkgc, "none"))
380  state->bkgcorrect = VISIR_SUB_NOCORRECT;
381  else if (!strcasecmp(bkgc, "chopnod"))
382  state->bkgcorrect = VISIR_SUB_CHOPNODCORRECT;
383  else if (!strcasecmp(bkgc, "chop"))
384  state->bkgcorrect = VISIR_SUB_CHOPCORRECT;
385  else
386  error_if(1, CPL_ERROR_ILLEGAL_INPUT,
387  "Unknown parameter to --bkgcorrect: %s", bkgc);
388  }
389 
390  error_if(state->datatype == VISIR_DATA_AQU_INT &&
391  state->bkgcorrect != VISIR_SUB_CHOPNODCORRECT,
392  CPL_ERROR_INCOMPATIBLE_INPUT,
393  "INT frame only data requires --bkgcorrect to be chopnod");
394 
395  /* cannot extract a and b hcycle from this format */
396  error_if(state->datatype == VISIR_DATA_CUBE1,
397  CPL_ERROR_INCOMPATIBLE_INPUT,
398  "CUBE1 data not supported please use the legacy recipes "
399  "visir_img_combine or visir_spc_obs");
400 
401  state->normalize = irplib_parameterlist_get_bool(parlist, PACKAGE,
402  RECIPE_STRING,
403  "normalize");
404 
405  state->dit = visir_pfits_get_dit(state->plist);
406 
407  error_if(state->normalize && state->dit == 0, CPL_ERROR_ILLEGAL_INPUT,
408  "Cannot normalize by DIT = 0");
409 
410  state->planestart = irplib_parameterlist_get_int(parlist, PACKAGE,
411  RECIPE_STRING,
412  "planestart");
413  error_if(state->planestart >= state->naxis3, CPL_ERROR_ILLEGAL_INPUT,
414  "planestart %d equal or larger than number of planes %d.",
415  state->planestart, state->naxis3);
416 
417  ndit = visir_pfits_get_ndit(state->plist);
418  const double chop_freq = visir_pfits_get_chop_freq(state->plist);
419  navrg = visir_pfits_get_navrg(state->plist);
420 
421  IRPLIB_DIAG_PRAGMA_PUSH_ERR(-Wswitch);
422  switch(state->datatype) {
423  case VISIR_DATA_CUBE1:
424  bug_if(1);
425  break;
426  case VISIR_DATA_AQU_HCYCLE:
427  case VISIR_DATA_CUBE2:
428  planes_per_cycle = 2;
429  break;
430  case VISIR_DATA_AQU_INT:
431  /* not actually used */
432  planes_per_cycle = 1;
433  break;
434  case VISIR_DATA_BURST:
435  case VISIR_DATA_AQU_BURST:
436  case VISIR_DATA_AQU_BURST_EXT:
437  planes_per_cycle = ndit * 2 / navrg;
438  break;
439  }
440  IRPLIB_DIAG_PRAGMA_POP;
441 
442  ncycles = irplib_parameterlist_get_int(parlist, PACKAGE,
443  RECIPE_STRING, "ncycles");
444  if (ncycles <= 0)
445  ncycles = irplib_pfits_get_int(state->plist,
446  VISIR_PFITS_INT_CHOP_NCYCLES);
447 
448  state->planeend = state->planestart + ncycles * planes_per_cycle;
449  /* ensure we always repack a multiple of a full cycle to get equal number
450  * of on and off planes */
451  if (state->planeend >= state->naxis3 || state->planeend <= 0) {
452  if (state->planestart % planes_per_cycle == 0)
453  state->planeend = state->naxis3;
454  else
455  state->planeend = state->naxis3 -
456  (planes_per_cycle - (state->planestart % planes_per_cycle));
457  }
458 
459  /* we have max 16 bit adc and cpl int is 32 bit so reasonable dits allow
460  * integer normalization without overflow */
461  if (state->normalize && (state->dit < 1e-6 ||
462  (double)((int)(1. / state->dit)) != 1. / state->dit))
463  state->load_type = CPL_TYPE_FLOAT;
464  else
465  state->load_type = CPL_TYPE_UNSPECIFIED;
466 
467  state->trimlow = irplib_parameterlist_get_int(parlist, PACKAGE,
468  RECIPE_STRING,
469  "trimlow");
470  state->trimhigh = irplib_parameterlist_get_int(parlist, PACKAGE,
471  RECIPE_STRING,
472  "trimhigh");
473 
474  state->compress = irplib_parameterlist_get_bool(parlist, PACKAGE,
475  RECIPE_STRING,
476  "compress");
477 
478  if (state->compress && CPL_IO_COMPRESS_RICE == 0) {
479  cpl_msg_warning(cpl_func, "CPL version too old for compression.");
480  state->compress = CPL_FALSE;
481  }
482 
483  state->conad = VISIR_UTIL_REPACK_CONAD;
484 
485  state->nframes = irplib_framelist_get_size(rawframes);
486  state->remregexp = "^(" VISIR_PFITS_DOUBLE_CUMOFFSETX
487  "|" VISIR_PFITS_DOUBLE_CUMOFFSETY ")$";
488 
489  /* Get the chopping throw in pixels */
490  state->pthrow = visir_pfits_get_chop_pthrow(state->plist);
491  state->posang = visir_pfits_get_chop_posang(state->plist);
492 
493  state->pfov = visir_pfits_get_pixscale(state->plist);
494 
495  /* Copy comments from CUMOFFSET[XY] */
496  state->comoffx = cpl_propertylist_get_comment
497  (state->plist, VISIR_PFITS_DOUBLE_CUMOFFSETX);
498  state->comoffy = cpl_propertylist_get_comment
499  (state->plist, VISIR_PFITS_DOUBLE_CUMOFFSETY);
500  state->offsetx = visir_pfits_get_cumoffsetx(state->plist);
501  state->offsety = visir_pfits_get_cumoffsety(state->plist);
502  state->jitterwidth = 0.;
503  if (cpl_propertylist_has(state->plist, "ESO SEQ JITTER WIDTH")) {
504  state->jitterwidth =
505  cpl_propertylist_get_double(state->plist, "ESO SEQ JITTER WIDTH");
506  state->jitterwidth /= state->pfov;
507  }
508 
509  /* we only have one chip, move PIXSPACE from extension to main header */
510  /* TODO still needed? */
511  if (next > 0 &&
512  !cpl_propertylist_has(state->plist, "ESO DET CHIP1 PXSPACE") &&
513  !cpl_propertylist_has(state->plist, "ESO DET CHIP PXSPACE")) {
514  cpl_propertylist * extplist = cpl_propertylist_load(filename, 1);
515  skip_if(extplist == NULL);
516  state->pxspace = visir_pfits_get_pixspace(extplist);
517  cpl_propertylist_delete(extplist);
518  skip_if(0);
519  }
520 
521  if (visir_data_is_burst(state->datatype) &&
522  visir_data_is_drs(state->datatype)) {
523  visir_img_burst_find_delta_chop(state->plist,
524  &state->to_off,
525  &state->halfcycle);
526  }
527  else if (visir_data_is_burst(state->datatype)) {
528  error_if(ndit % navrg != 0, CPL_ERROR_ILLEGAL_INPUT,
529  "ndit (%d) not divisible by navrg %d, not supported",
530  ndit, navrg);
531  state->halfcycle = ndit / navrg;
532  }
533 
534  state->bpmthresh = VISIR_HCYCLE_BPM_THRESHOLD;
535 
536  if (state->bkgcorrect == VISIR_SUB_NOCORRECT) {
537  const char * nodpos = visir_pfits_get_nodpos(state->plist);
538  state->is_a = nodpos != NULL &&
539  (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
540  state->procatgon = state->is_a ? VISIR_UTIL_REPACK_A_ON_PROCATG
541  : VISIR_UTIL_REPACK_B_ON_PROCATG;
542  state->procatgoff = state->is_a ? VISIR_UTIL_REPACK_A_OFF_PROCATG
543  : VISIR_UTIL_REPACK_B_OFF_PROCATG;
544  state->onname = cpl_sprintf(RECIPE_STRING "_%s_on_%03d"
545  CPL_DFS_FITS, nodpos, 1+iframe);
546  state->offname = cpl_sprintf(RECIPE_STRING "_%s_off_%03d"
547  CPL_DFS_FITS, nodpos, 1+iframe);
548 
549  if (!state->is_a) {
550  /* Verify that the NODPOS is either A or B */
551  skip_if (nodpos == NULL);
552  skip_if (strstr(nodpos, "B") == NULL && strstr(nodpos, "b") == NULL);
553  }
554  } else if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) {
555  const char * nodpos = visir_pfits_get_nodpos(state->plist);
556  state->is_a = nodpos != NULL &&
557  (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
558  state->procatgon = state->is_a ? "BKG_CORRECTED_A" : "BKG_CORRECTED_B";
559  state->onname = cpl_sprintf(RECIPE_STRING "_%s_bkgcor_%03d"
560  CPL_DFS_FITS, nodpos, 1+iframe);
561  } else { /*if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) { */
562  const char * nodpos = visir_pfits_get_nodpos(state->plist);
563  state->is_a = nodpos != NULL &&
564  (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
565  state->procatgon = VISIR_UTIL_CORRECTED;
566  state->onname = cpl_sprintf(RECIPE_STRING "_bkgcor_%03d"
567  CPL_DFS_FITS, 1+iframe);
568  }
569 
570  {
571  long nx, ny;
572  if (state->datatype == VISIR_DATA_AQU_HCYCLE ||
573  state->datatype == VISIR_DATA_AQU_INT ||
574  state->datatype == VISIR_DATA_AQU_BURST_EXT) {
575  cpl_errorstate cleanstate = cpl_errorstate_get();
576  cpl_propertylist * plist = cpl_propertylist_load(filename, 1);
577  nx = visir_pfits_get_naxis1(plist);
578  ny = visir_pfits_get_naxis2(plist);
579  state->wcs = cpl_wcs_new_from_propertylist(plist);
580  if (state->wcs == NULL) {
581  cpl_errorstate_set(cleanstate);
582  }
583  cpl_propertylist_delete(plist);
584  }
585  else {
586  nx = visir_pfits_get_naxis1(state->plist);
587  ny = visir_pfits_get_naxis2(state->plist);
588  }
589  skip_if(0);
590 
591  state->mean_on = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
592  state->nmean_on = cpl_calloc(1, sizeof(cpl_size));
593  state->mean_off = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
594  state->nmean_off = cpl_calloc(1, sizeof(cpl_size));
595  }
596 
597 
598  end_skip;
599 
600  return state;
601 }
602 
603 
604 /*----------------------------------------------------------------------------*/
609 /*----------------------------------------------------------------------------*/
610 static void
611 repack_framestate_delete(void * state_)
612 {
613  repack_framestate * state = (repack_framestate*) state_;
614 
615  if (state == NULL)
616  return;
617 
618  cpl_wcs_delete(state->wcs);
619  cpl_free(state->onname);
620  cpl_free(state->offname);
621  cpl_image_delete(state->mean_on);
622  cpl_image_delete(state->mean_off);
623  cpl_free(state->nmean_on);
624  cpl_free(state->nmean_off);
625  cpl_free(state);
626 }
627 
628 
629 /*----------------------------------------------------------------------------*/
636 /*----------------------------------------------------------------------------*/
637 static cpl_error_code
638 store_means(const visir_sub_type bkgcorrect, const irplib_framelist * rawfr,
639  cpl_frameset * framelist, const cpl_parameterlist * parlist,
640  cpl_image ** mean, cpl_size * nmean, visir_frame_type * frametype)
641 {
642  cpl_frameset * usedframes[] = {cpl_frameset_new(), cpl_frameset_new(),
643  cpl_frameset_new(), cpl_frameset_new()};
644 
645  int normalize = irplib_parameterlist_get_bool(parlist, PACKAGE,
646  RECIPE_STRING,
647  "normalize");
648  cpl_propertylist * plist = cpl_propertylist_new();
649  if (normalize) {
650  cpl_propertylist_append_string(plist, "BUNIT", "adu / s");
651  } else {
652  cpl_propertylist_append_string(plist, "BUNIT", "adu");
653  }
654 
655  if (bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
656  for (int i = 1; i < 4; i++) {
657  cpl_image_add(mean[0], mean[i]);
658  nmean[0] += nmean[i];
659  }
660  skip_if(cpl_image_divide_scalar(mean[0], nmean[0]));
661  for (int i = 0; i < irplib_framelist_get_size(rawfr); i++) {
662  cpl_frame * frm =
663  cpl_frame_duplicate(irplib_framelist_get_const(rawfr, i));
664  cpl_frameset_insert(usedframes[0], frm);
665  }
666  skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[0],
667  mean[0], CPL_TYPE_UNSPECIFIED,
668  RECIPE_STRING, "MEAN",
669  plist, NULL, visir_pipe_id,
670  "visir_util_repack_mean.fits"));
671  }
672  else if (bkgcorrect == VISIR_SUB_CHOPCORRECT) {
673  const char * procatg[] = {"MEAN_A", "MEAN_B"};
674  const char * fn[] ={"visir_util_repack_mean_A.fits",
675  "visir_util_repack_mean_B.fits"};
676  for (int i = 0; i < 2; i++) {
677  int idx = i == 0 ? 0 : 2;
678  cpl_image_add(mean[idx], mean[idx + 1]);
679  nmean[idx] += nmean[idx + 1];
680  skip_if(cpl_image_divide_scalar(mean[idx], nmean[idx]));
681  for (int j = 0; j < irplib_framelist_get_size(rawfr); j++) {
682  if ((i == 0 && frametype[j] == VISIR_FRAME_B) ||
683  (i == 1 && frametype[j] == VISIR_FRAME_A) ||
684  frametype[j] == VISIR_FRAME_STATIC)
685  continue;
686  cpl_frame * frm =
687  cpl_frame_duplicate(irplib_framelist_get_const(rawfr, j));
688  cpl_frameset_insert(usedframes[i], frm);
689  }
690  skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[i],
691  mean[idx], CPL_TYPE_UNSPECIFIED,
692  RECIPE_STRING, procatg[i],
693  plist, NULL, visir_pipe_id, fn[i]));
694  }
695  }
696  else {
697  const char * procatg[] = {"MEAN_A_ON", "MEAN_A_OFF",
698  "MEAN_B_ON", "MEAN_B_OFF"};
699  const char * fn[] ={"visir_util_repack_mean_Aoff.fits",
700  "visir_util_repack_mean_Aon.fits",
701  "visir_util_repack_mean_Boff.fits",
702  "visir_util_repack_mean_Bon.fits"};
703  for (int i = 0; i < 4; i++) {
704  skip_if(cpl_image_divide_scalar(mean[i], nmean[i]));
705  for (int j = 0; j < irplib_framelist_get_size(rawfr); j++) {
706  if ((i < 2 && frametype[j] == VISIR_FRAME_A) ||
707  (i >= 2 && frametype[j] == VISIR_FRAME_B) ||
708  frametype[j] == VISIR_FRAME_STATIC)
709  continue;
710  cpl_frame * frm =
711  cpl_frame_duplicate(irplib_framelist_get_const(rawfr, j));
712  cpl_frameset_insert(usedframes[i], frm);
713  }
714  skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[i],
715  mean[i], CPL_TYPE_UNSPECIFIED,
716  RECIPE_STRING, procatg[i],
717  plist, NULL, visir_pipe_id, fn[i]));
718  }
719  }
720 
721  end_skip;
722 
723  cpl_propertylist_delete(plist);
724  for (size_t i = 0; i < sizeof(usedframes)/sizeof(usedframes[0]); i++)
725  cpl_frameset_delete(usedframes[i]);
726 
727  return cpl_error_get_code();
728 }
729 
730 
731 /*----------------------------------------------------------------------------*/
738 /*----------------------------------------------------------------------------*/
739 static int visir_util_repack(cpl_frameset * framelist,
740  const cpl_parameterlist * parlist)
741 {
742  irplib_framelist * allframes = NULL;
743  irplib_framelist * rawframes = NULL;
744  int n;
745  cx_list * alist = cx_list_new();
746  cx_list * blist = cx_list_new();
747  cpl_image * bpm = NULL;
748  /* nonlinear pixel count */
749  cpl_image * nonlinear = NULL;
750  cpl_bivector * lintable = NULL;
751  cpl_image * mean[] = {NULL, NULL, NULL, NULL};
752  cpl_size nmean[] = {0, 0, 0, 0};
753  visir_sub_type bkgcorrect = VISIR_SUB_NOCORRECT;
754  visir_frame_type * frametype = NULL;
755  double xjitter0, yjitter0;
756 
757  /* Identify the RAW and CALIB frames in the input frameset */
758  skip_if (visir_dfs_set_groups(framelist));
759  cpl_fits_set_mode(CPL_FITS_START_CACHING);
760 
761  /* Objects observation */
762  allframes = irplib_framelist_cast(framelist);
763  skip_if(allframes == NULL);
764  rawframes = irplib_framelist_extract_regexp(allframes, "^("
765  VISIR_SPC_OBS_RAW
766  "|" VISIR_SPC_OBS_ECH_RAW
767  "|" VISIR_SPC_PHOT_RAW
768  "|" VISIR_SPC_PHOT_ECH_RAW
769  "|" VISIR_IMG_PHOT_RAW
770  "|" VISIR_IMG_COMBINE_CN
771  "|" VISIR_IMG_COMBINE_CN_BURST
772  "|" VISIR_IMG_COMBINE_CNJ
773  "|" VISIR_IMG_COMBINE_CNJ_BURST
774  "|" VISIR_IMG_COMBINE_CJ
775  "|" VISIR_IMG_COMBINE_NJ
776  "|" VISIR_IMG_COMBINE_NJ
777  "|" VISIR_IMG_CAL_PHOT
778  "|" VISIR_IMG_CAL_PHOT_BURST
779  "|" VISIR_IMG_CAL_OBJ
780  "|" VISIR_IMG_CAL_OBJ_BURST
781  "|" VISIR_ACQ_CNJ
782  "|" VISIR_SAM_CAL_N_RAW
783  "|" VISIR_SAM_CAL_CN_RAW
784  "|" VISIR_SAM_CAL_NJ_RAW
785  "|" VISIR_SAM_CAL_CNJ_RAW
786  "|" VISIR_CORO_CAL_CNJ_RAW
787  "|" VISIR_SAM_OBS_N_RAW
788  "|" VISIR_SAM_OBS_CN_RAW
789  "|" VISIR_SAM_OBS_NJ_RAW
790  "|" VISIR_SAM_OBS_CNJ_RAW
791  "|" VISIR_CORO_OBS_CNJ_RAW
792  "|" VISIR_IMG_BURST
793  "|" VISIR_UTIL_REPACK_RAW
794  "|" VISIR_UTIL_INPUTS_RAW
795  "|" VISIR_UTIL_DATA
796  ")$",
797  CPL_FALSE);
798  skip_if (rawframes == NULL);
799 
800  n = irplib_framelist_get_size(rawframes);
801  frametype = cpl_malloc(sizeof(visir_frame_type) * n);
802 
803  for (int i = 0; i < n; i++) {
804  repack_framestate * state = NULL;
805 
806  state = repack_framestate_new(rawframes, i, parlist);
807 
808  frametype[i] = state->is_a ? VISIR_FRAME_A : VISIR_FRAME_B;
809  if (state->is_a == CPL_TRUE)
810  cx_list_push_back(alist, state);
811  else
812  cx_list_push_back(blist, state);
813  skip_if(0);
814  }
815 
816  error_if(cx_list_size(alist) == 0, CPL_ERROR_ILLEGAL_INPUT,
817  "No frames with nodpos A");
818  error_if(cx_list_size(blist) == 0, CPL_ERROR_ILLEGAL_INPUT,
819  "No frames with nodpos B");
820  if (cx_list_size(alist) != cx_list_size(blist)) {
821  if (cx_list_size(alist) > cx_list_size(alist)) {
822  cpl_msg_warning(cpl_func,
823  "Expecting even number of files, ignoring the last Nod A frame");
824  repack_framestate_delete(cx_list_pop_back(alist));
825  }
826  if (cx_list_size(blist) > cx_list_size(alist)) {
827  cpl_msg_warning(cpl_func,
828  "Expecting even number of files, ignoring the last Nod B frame");
829  repack_framestate_delete(cx_list_pop_back(blist));
830  }
831  }
832  error_if(cx_list_size(alist) != cx_list_size(blist),
833  CPL_ERROR_ILLEGAL_INPUT,
834  "Unequal number of A and B frames. A: %d, B: %d",
835  (int)cx_list_size(alist), (int)cx_list_size(blist));
836 
837  {
838  cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_STATIC_MASK);
839  if (!frm) {
840  frm = cpl_frameset_find(framelist, VISIR_CALIB_STATIC_MASK_IMG);
841  }
842  if (!frm) {
843  frm = cpl_frameset_find(framelist, VISIR_CALIB_STATIC_MASK_SPC);
844  }
845  repack_framestate * state = ((repack_framestate*)cx_list_front(alist));
846  yjitter0 = ((repack_framestate*)cx_list_front(alist))->offsety;
847  if (frm) {
848  bpm = cpl_image_load(cpl_frame_get_filename(frm),
849  CPL_TYPE_UNSPECIFIED, 0, 0);
850  if (bpm == NULL) {
851  cpl_msg_warning(cpl_func, "Loading of bad pixel map %s failed",
852  cpl_frame_get_filename(frm));
853  skip_if(0);
854  }
855  cpl_msg_info(cpl_func, "Loaded bad pixel mask from %s",
856  cpl_frame_get_filename(frm));
857  if (visir_data_is_aqu(state->datatype)) {
858  cpl_size win_nx = visir_pfits_get_win_nx(state->plist);
859  cpl_size win_ny = visir_pfits_get_win_ny(state->plist);
860  cpl_size start_x = visir_pfits_get_start_x(state->plist);
861  cpl_size start_y = visir_pfits_get_start_y(state->plist);
862  /* cut bpm to detector readout window */
863  cpl_size nx = cpl_image_get_size_x(bpm);
864  cpl_size ny = cpl_image_get_size_y(bpm);
865  cpl_size llx = 1, lly = 1, urx = nx, ury = ny;
866  if (win_nx > 0 && start_x > 0) {
867  llx = start_x;
868  urx = start_x + win_nx - 1;
869  }
870  if (win_ny > 0 && start_y > 0) {
871  lly = start_y;
872  ury = start_y + win_ny - 1;
873  }
874  if (llx != 1 || urx != nx || lly != 1 || ury != ny) {
875  cpl_image * tmp = cpl_image_extract(bpm, llx, lly, urx, ury);
876  cpl_image_delete(bpm);
877  bpm = tmp;
878  }
879  skip_if(0);
880  }
881  nonlinear = cpl_image_new(cpl_image_get_size_x(bpm),
882  cpl_image_get_size_y(bpm), CPL_TYPE_INT);
883  }
884  }
885 
886  xjitter0 = ((repack_framestate*)cx_list_front(alist))->offsetx;
887  yjitter0 = ((repack_framestate*)cx_list_front(alist))->offsety;
888 
889  {
890  repack_framestate * st = cx_list_front(alist);
891  const char * lextname = NULL;
892  if (visir_is_img(st->plist)) {
893  lextname = "IMAGE_LIN";
894  }
895  else if (visir_is_spc(st->plist)) {
896  lextname = "SPEC_LIN";
897  }
898  cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_LIN);
899  if (lextname && frm && visir_data_is_aqu(st->datatype)) {
900  const char * fn = cpl_frame_get_filename(frm);
901  cpl_msg_info(cpl_func, "Correcting linearity using %s extension "
902  "of %s", lextname, fn);
903  cpl_size iext = cpl_fits_find_extension(fn, lextname);
904  error_if(iext < 0, CPL_ERROR_ILLEGAL_INPUT,
905  "Linearity correction extension %s not found in %s",
906  lextname, fn);
907 
908  cpl_table * tab = cpl_table_load(fn, iext, 0);
909  const size_t n = cpl_table_get_nrow(tab);
910  lintable = cpl_bivector_new(n);
911  memcpy(cpl_bivector_get_x_data(lintable),
912  cpl_table_get_data_double(tab, "dc_level"),
913  sizeof(double) * n);
914  memcpy(cpl_bivector_get_y_data(lintable),
915  cpl_table_get_data_double(tab, "conv_gain"),
916  sizeof(double) * n);
917  cpl_table_delete(tab);
918 
919  /* normalize by average */
920  cpl_vector_divide_scalar(cpl_bivector_get_y(lintable),
921  cpl_vector_get_mean(cpl_bivector_get_y(lintable)));
922  }
923  }
924 
925  while (1) {
926  repack_framestate * fstatea = NULL, * fstateb = NULL;
927  if (!cx_list_empty(alist))
928  fstatea = (repack_framestate*)cx_list_pop_front(alist);
929  if (!cx_list_empty(blist))
930  fstateb = (repack_framestate*)cx_list_pop_front(blist);
931 
932  if (fstatea == NULL || fstateb == NULL)
933  break;
934 
935  if (lintable) {
936  fstatea->load_type = CPL_TYPE_FLOAT;
937  fstateb->load_type = CPL_TYPE_FLOAT;
938  }
939 
940 
941  if (cpl_propertylist_has(fstatea->plist, "DATE-OBS") &&
942  cpl_propertylist_has(fstatea->plist, "DATE") &&
943  cpl_propertylist_has(fstateb->plist, "DATE-OBS") &&
944  cpl_propertylist_has(fstateb->plist, "DATE")) {
945  const char * str =
946  cpl_propertylist_get_string(fstatea->plist, "DATE-OBS");
947  irplib_wcs_mjd_from_string(&(fstatea->time_obsstart), str);
948  str = cpl_propertylist_get_string(fstatea->plist, "DATE");
949  irplib_wcs_mjd_from_string(&(fstatea->time_filewrite), str);
950 
951  str = cpl_propertylist_get_string(fstateb->plist, "DATE-OBS");
952  irplib_wcs_mjd_from_string(&(fstateb->time_obsstart), str);
953  str = cpl_propertylist_get_string(fstateb->plist, "DATE");
954  irplib_wcs_mjd_from_string(&(fstateb->time_filewrite), str);
955  }
956 
957  fstatea->time_min_obsstart = CX_MIN(fstatea->time_obsstart,
958  fstateb->time_obsstart);
959  fstateb->time_min_obsstart = CX_MIN(fstatea->time_obsstart,
960  fstateb->time_obsstart);
961  fstatea->time_max_filewrite = CX_MAX(fstatea->time_filewrite,
962  fstateb->time_filewrite);
963  fstateb->time_max_filewrite = CX_MAX(fstatea->time_filewrite,
964  fstateb->time_filewrite);
965 
966  /* run single threaded but allow async tasks */
967  OMP_PRAGMA(omp parallel num_threads(2))
968  OMP_PRAGMA(omp single)
969  {
970  visir_util_repack_two(fstatea, fstateb, framelist, &bpm, nonlinear,
971  xjitter0, yjitter0, lintable);
972  }
973  skip_if(0);
974 
975  bkgcorrect = fstatea->bkgcorrect;
976  repack_framestate * st[4] = {fstatea, fstatea, fstateb, fstateb};
977  for (int i = 0; i < 4; i++) {
978  cpl_image * stimg = i % 2 == 0 ? st[i]->mean_on : st[i]->mean_off;
979  cpl_size nst = i % 2 == 0 ? *st[i]->nmean_on : *st[i]->nmean_off;
980  if (mean[i] == NULL)
981  mean[i] = cpl_image_duplicate(stimg);
982  else
983  cpl_image_add(mean[i], stimg);
984  nmean[i] += nst;
985  }
986 
987  repack_framestate_delete(fstatea);
988  repack_framestate_delete(fstateb);
989  cpl_fits_set_mode(CPL_FITS_RESTART_CACHING);
990  }
991 
992  skip_if(store_means(bkgcorrect, rawframes, framelist, parlist,
993  mean, nmean, frametype));
994 
995  /* store bpm, skip if wrong size, there is no keyword one can
996  * use in the OCA rules to associate DRS/AQU properly */
997  if (bpm &&
998  cpl_image_get_size_x(bpm) == cpl_image_get_size_x(mean[0]) &&
999  cpl_image_get_size_y(bpm) == cpl_image_get_size_y(mean[0])) {
1000  /* apply all-nonlinear bpm */
1001  if (lintable) {
1002  cpl_size total = nmean[0] + nmean[1] + nmean[2] + nmean[3];
1003  cpl_mask * mnonlinear =
1004  cpl_mask_threshold_image_create(nonlinear, total * 0.9, total * 2);
1005  cpl_msg_info(cpl_func, "%lld (%.3g%%) pixels non-linear in 90%% "
1006  "of %lld frames", cpl_mask_count(mnonlinear),
1007  (100. * cpl_mask_count(mnonlinear)) /
1008  (cpl_mask_get_size_x(mnonlinear) *
1009  cpl_mask_get_size_y(mnonlinear)), total);
1010  /* disabled due to problems in HRS with so many bad pixels */
1011  //cpl_image_reject_from_mask(bpm, mnonlinear);
1012  //cpl_image_fill_rejected(bpm, 1);
1013  cpl_mask_delete(mnonlinear);
1014  }
1015 
1016  cpl_frameset * usedframes = cpl_frameset_new();
1017  cpl_frameset_insert(usedframes, cpl_frame_duplicate
1018  (irplib_framelist_get_const(rawframes, 0)));
1019  irplib_dfs_save_image(framelist, parlist, usedframes, bpm,
1020  CPL_BPP_8_UNSIGNED, RECIPE_STRING,
1021  VISIR_CALIB_STATIC_MASK, NULL, NULL,
1022  visir_pipe_id, RECIPE_STRING "_bpm"
1023  CPL_DFS_FITS);
1024  cpl_frameset_delete(usedframes);
1025  }
1026  else if (bpm) {
1027  cpl_msg_warning(cpl_func, "Provided bad pixel mask does not have "
1028  "correct size, skipping");
1029  }
1030 
1031  end_skip;
1032 
1033  irplib_framelist_delete(allframes);
1034  irplib_framelist_delete(rawframes);
1035 
1036  cx_list_destroy(alist, repack_framestate_delete);
1037  cx_list_destroy(blist, repack_framestate_delete);
1038  cpl_image_delete(bpm);
1039  cpl_image_delete(nonlinear);
1040  cpl_bivector_delete(lintable);
1041  cpl_free(frametype);
1042 
1043  for (size_t i = 0; i < sizeof(mean)/sizeof(mean[0]); i++)
1044  cpl_image_delete(mean[i]);
1045 
1046  return cpl_error_get_code();
1047 }
1048 
1049 
1050 /*----------------------------------------------------------------------------*/
1062 /*----------------------------------------------------------------------------*/
1063 static cpl_error_code
1064 save_subtract_images(const repack_framestate * state,
1065  const cpl_imagelist * subtracted,
1066  const cpl_imagelist * jit_cor,
1067  const cpl_propertylist * plist)
1068 {
1069  const cpl_size nsub = cpl_imagelist_get_size(subtracted);
1070  cpl_ensure_code(jit_cor == NULL || cpl_imagelist_get_size(jit_cor) == nsub,
1071  CPL_ERROR_ILLEGAL_INPUT);
1072 
1073 
1074  for (cpl_size j = 0; j < nsub; j++) {
1075  const cpl_image * img = cpl_imagelist_get_const(subtracted, j);
1076  cpl_io_type compress = 0;
1077  cpl_type save_type;
1078 
1079  if (jit_cor)
1080  cpl_image_add(state->mean_on, cpl_imagelist_get_const(jit_cor, j));
1081  else
1082  cpl_image_add(state->mean_on, img);
1083  (*state->nmean_on)++;
1084 
1085  if (state->compress && cpl_image_get_type(img) == CPL_TYPE_INT) {
1086  /* saves space even if it could be stored in a short */
1087  save_type = CPL_TYPE_INT;
1088  compress = CPL_IO_COMPRESS_RICE;
1089  }
1090  else
1091  save_type = get_optimum_save_type(img);
1092 
1093  skip_if(cpl_image_save(img, state->onname, save_type,
1094  plist, CPL_IO_EXTEND | compress));
1095  }
1096 
1097  end_skip;
1098 
1099  return cpl_error_get_code();
1100 }
1101 
1102 
1103 /*----------------------------------------------------------------------------*/
1116 /*----------------------------------------------------------------------------*/
1117 static
1118 cpl_error_code append_images(const repack_framestate * state,
1119  cpl_imagelist * images,
1120  const cpl_propertylist * plist,
1121  const cpl_boolean on)
1122 {
1123  const char * name = on ? state->onname : state->offname;
1124  const cpl_size n = cpl_imagelist_get_size(images);
1125  cpl_image * mimg = on ? state->mean_on : state->mean_off;
1126  cpl_size * nmean = on ? state->nmean_on : state->nmean_off;
1127 
1128  for (cpl_size j = 0; j < n; j++) {
1129  cpl_image * img = cpl_imagelist_get(images, j);
1130  cpl_io_type compress = 0;
1131  cpl_type save_type;
1132  /* drs burst data is not preprocessed so it needs flipping */
1133  if (state->datatype == VISIR_DATA_BURST) {
1134  cpl_image_multiply_scalar(img, -1);
1135  cpl_image_add_scalar(img, VISIR_HCYCLE_OFFSET);
1136  }
1137 
1138  cpl_image_add(mimg, img);
1139  (*nmean)++;
1140 
1141  if (state->compress && cpl_image_get_type(img) == CPL_TYPE_INT) {
1142  /* saves space even if it could be stored in a short */
1143  save_type = CPL_TYPE_INT;
1144  compress = CPL_IO_COMPRESS_RICE;
1145  }
1146  else
1147  save_type = get_optimum_save_type(img);
1148 
1149  skip_if(cpl_image_save(img, name, save_type,
1150  plist, CPL_IO_EXTEND | compress));
1151  }
1152 
1153  end_skip;
1154 
1155  return cpl_error_get_code();
1156 }
1157 
1158 
1159 /*----------------------------------------------------------------------------*/
1167 /*----------------------------------------------------------------------------*/
1168 static cpl_error_code
1169 cast_list(cpl_imagelist * l, const cpl_type type)
1170 {
1171  const cpl_size n = cpl_imagelist_get_size(l);
1172  cpl_imagelist * cl;
1173  if (type == CPL_TYPE_UNSPECIFIED)
1174  return CPL_ERROR_NONE;
1175 
1176  cl = cpl_imagelist_new();
1177 
1178  for (cpl_size i = 0; i < n; i++) {
1179  cpl_image * img = cpl_imagelist_get(l, i);
1180  cpl_image * cast;
1181  if (cpl_image_get_type(img) == type) {
1182  cpl_imagelist_delete(cl);
1183  return cpl_error_get_code();
1184  }
1185  cast = cpl_image_cast(img, type);
1186  cpl_imagelist_set(cl, cast, i);
1187  }
1188 
1189  cpl_imagelist_empty(l);
1190  for (cpl_size i = 0; i < n; i++)
1191  cpl_imagelist_set(l, cpl_imagelist_get(cl, i), i);
1192 
1193  visir_imagelist_unwrap(cl);
1194 
1195  return cpl_error_get_code();
1196 }
1197 
1198 
1199 /*----------------------------------------------------------------------------*/
1210 /*----------------------------------------------------------------------------*/
1211 static
1212 cpl_error_code load_chunk(cpl_imagelist * on, cpl_imagelist * off,
1213  const repack_framestate * state,
1214  const int pstart, const int pend)
1215 {
1216  /* sort a chunk of data */
1217  if (state->datatype == VISIR_DATA_AQU_HCYCLE) {
1218  const cpl_frame * frame = irplib_framelist_get_const(state->rawframes,
1219  state->iframe);
1220  const char * fn = cpl_frame_get_filename(frame);
1221  skip_if(0);
1222 
1223  for (cpl_size iext = pstart + 1; iext < pend + 1; iext++) {
1224  const char * ftype;
1225  cpl_propertylist * plist =
1226  cpl_propertylist_load_regexp(fn, iext,
1227  VISIR_PFITS_STRING_FRAME_TYPE,
1228  CPL_FALSE);
1229  skip_if(plist == NULL);
1230  ftype = visir_pfits_get_frame_type(plist);
1231  error_if(ftype == NULL, CPL_ERROR_DATA_NOT_FOUND, "ESO DET FRAM "
1232  "TYPE keyword missing in extension %d of file %s",
1233  (int)iext, fn);
1234  if (strcmp(ftype, "HCYCLE1") == 0)
1235  cpl_imagelist_set(on,
1236  cpl_image_load(fn, CPL_TYPE_FLOAT, 0, iext),
1237  cpl_imagelist_get_size(on));
1238  else if (strcmp(ftype, "HCYCLE2") == 0)
1239  cpl_imagelist_set(off,
1240  cpl_image_load(fn, CPL_TYPE_FLOAT, 0, iext),
1241  cpl_imagelist_get_size(off));
1242  else {
1243  cpl_msg_debug(cpl_func, "Skipping \"%s\" frame type", ftype);
1244  }
1245  cpl_propertylist_delete(plist);
1246  skip_if(0);
1247  }
1248  }
1249  else if (state->datatype == VISIR_DATA_CUBE2) {
1250  skip_if(visir_load_cube2_split(on, off, state->rawframes,
1251  state->iframe, pstart, pend));
1252 
1253  }
1254  else if (visir_data_is_burst(state->datatype)) {
1255  const cpl_frame * frame = irplib_framelist_get_const(state->rawframes,
1256  state->iframe);
1257  /* TODO rework loading from first extension */
1258  if (state->datatype == VISIR_DATA_AQU_BURST_EXT) {
1259  cpl_propertylist * plist =
1260  cpl_propertylist_load(cpl_frame_get_filename(frame), 1);
1261  /* hack to get it to load from 1. extension */
1262  cpl_propertylist_update_int(plist, "ZNAXIS3", visir_pfits_get_naxis3(plist));
1263  skip_if(visir_load_burst_aqu(on, off, frame, plist,
1264  state->halfcycle , pstart, pend));
1265  cpl_propertylist_delete(plist);
1266  }
1267  else {
1268  skip_if(visir_load_burst(on, off, frame,
1269  state->plist, state->to_off,
1270  state->halfcycle, pstart, pend,
1271  state->trimlow, state->trimhigh));
1272  }
1273  }
1274  else
1275  error_if(1, CPL_ERROR_ILLEGAL_INPUT, "invalid data tag");
1276 
1277  /* FIXME: pass load type to the loading functions instead */
1278  cast_list(on, state->load_type);
1279  cast_list(off, state->load_type);
1280 
1281  end_skip;
1282 
1283  return cpl_error_get_code();
1284 }
1285 
1286 
1287 /*----------------------------------------------------------------------------*/
1297 /*----------------------------------------------------------------------------*/
1298 static cpl_size
1299 equalize_lists(cpl_imagelist * a, cpl_imagelist * b)
1300 {
1301  const cpl_size na = cpl_imagelist_get_size(a);
1302  const cpl_size nb = cpl_imagelist_get_size(b);
1303 
1304  if (na != nb)
1305  cpl_msg_warning(cpl_func, "Unequal number of planes in on "
1306  "and off list: #on %d, #off %d. Skipping %ld"
1307  " planes.", (int)na, (int)nb, labs(na - nb));
1308 
1309  if (na > nb) {
1310  for (cpl_size i = 0; i < na - nb; i++)
1311  cpl_image_delete(visir_imagelist_pop(a));
1312  }
1313  else if (na < nb) {
1314  for (cpl_size i = 0; i < nb - na; i++)
1315  cpl_image_delete(visir_imagelist_pop(b));
1316  }
1317 
1318  if (cpl_error_get_code() != CPL_ERROR_NONE)
1319  return -1;
1320  return cpl_imagelist_get_size(a);
1321 }
1322 
1323 static cpl_error_code
1324 check_and_fix_cd_wcs(const repack_framestate * state, cpl_propertylist * plist)
1325 {
1326  /* move wcs from extension to primary as needed by detect_shift
1327  * but only if primary does not already have wcs as then the ext is wrong
1328  * (aqu data < 2016.02) */
1329  if (state->wcs && !cpl_propertylist_has(plist, "CRVAL1")) {
1330  const cpl_matrix * m;
1331  const cpl_array * v = cpl_wcs_get_crval(state->wcs);
1332  cpl_propertylist_update_double(plist, "CRVAL1",
1333  cpl_array_get_double(v, 0, NULL));
1334  cpl_propertylist_update_double(plist, "CRVAL2",
1335  cpl_array_get_double(v, 1, NULL));
1336  v = cpl_wcs_get_ctype(state->wcs);
1337  cpl_propertylist_update_string(plist, "CTYPE1",
1338  cpl_array_get_string(v, 0));
1339  cpl_propertylist_update_string(plist, "CTYPE2",
1340  cpl_array_get_string(v, 1));
1341  v = cpl_wcs_get_crpix(state->wcs);
1342  cpl_propertylist_update_double(plist, "CRPIX1",
1343  cpl_array_get_double(v, 0, NULL));
1344  cpl_propertylist_update_double(plist, "CRPIX2",
1345  cpl_array_get_double(v, 1, NULL));
1346  m = cpl_wcs_get_cd(state->wcs);
1347  cpl_propertylist_update_double(plist, "CD1_1",
1348  cpl_matrix_get(m, 0, 0));
1349  cpl_propertylist_append_double(plist, "CD1_2",
1350  cpl_matrix_get(m, 0, 1));
1351  cpl_propertylist_update_double(plist, "CD2_1",
1352  cpl_matrix_get(m, 1, 0));
1353  cpl_propertylist_update_double(plist, "CD2_2",
1354  cpl_matrix_get(m, 1, 1));
1355  v = cpl_wcs_get_cunit(state->wcs);
1356  cpl_propertylist_update_string(plist, "CUNIT1",
1357  cpl_array_get_string(v, 0));
1358  cpl_propertylist_update_string(plist, "CUNIT2",
1359  cpl_array_get_string(v, 1));
1360  }
1361  if (!cpl_propertylist_has(plist, "CD1_1") ||
1362  cpl_propertylist_get_double(plist, "CD1_1") == 0.) {
1363  double pfov = visir_pfits_get_pixscale(state->plist);
1364  cpl_msg_warning(cpl_func, "CD1_1 WCS key missing or zero, fixing to "
1365  "%g (pfov) / 3600", -pfov);
1366  cpl_propertylist_update_double(plist, "CD1_1", -pfov / 3600.);
1367  }
1368  if (!cpl_propertylist_has(plist, "CD2_2") ||
1369  cpl_propertylist_get_double(plist, "CD2_2") == 0.) {
1370  double pfov = visir_pfits_get_pixscale(state->plist);
1371  cpl_msg_warning(cpl_func, "CD2_2 WCS key missing or zero, fixing to "
1372  "%g (-pfov) / 3600", pfov);
1373  cpl_propertylist_update_double(plist, "CD2_2", pfov / 3600.);
1374  }
1375 
1376  return cpl_error_get_code();
1377 }
1378 
1379 
1380 /*----------------------------------------------------------------------------*/
1393 /*----------------------------------------------------------------------------*/
1394 static cpl_error_code
1395 prepare_output(const repack_framestate * state, cpl_frameset * products,
1396  double xjittera, double yjittera,
1397  double xjitterb, double yjitterb,
1398  double bg_mean)
1399 {
1400  cpl_frameset * usedframes = cpl_frameset_new();
1401  cpl_propertylist * onlist = cpl_propertylist_new();
1402  cpl_propertylist * offlist = cpl_propertylist_new();
1403 
1404  /* Need a copy of the WCS cards for subsequent beam stacking */
1405  bug_if(cpl_propertylist_copy_property_regexp(onlist, state->plist, "^("
1406  IRPLIB_PFITS_WCS_REGEXP ")$",
1407  CPL_FALSE));
1408 
1409  bug_if(cpl_propertylist_append_double(onlist, VISIR_DRS_CONAD, state->conad));
1410  bug_if(cpl_propertylist_set_comment(onlist, VISIR_DRS_CONAD, "Default "
1411  "single frame value: " IRPLIB_STRINGIFY
1412  (VISIR_UTIL_REPACK_CONAD)));
1413 
1414  bug_if(cpl_propertylist_append_double(onlist, VISIR_PFITS_DOUBLE_PIXSPACE,
1415  state->pxspace));
1416 
1417  cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETXA, xjittera);
1418  cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETYA, yjittera);
1419  cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETXA,
1420  state->comoffx);
1421  cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETYA,
1422  state->comoffy);
1423 
1424  cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETXB, xjitterb);
1425  cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETYB, yjitterb);
1426  cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETXB,
1427  state->comoffx);
1428  cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETYB,
1429  state->comoffy);
1430 
1431  bug_if(cpl_propertylist_append(offlist, onlist));
1432 
1433  cpl_propertylist_append_int(onlist, "ESO DRS DTYPE", state->datatype);
1434  cpl_propertylist_append_int(offlist, "ESO DRS DTYPE", state->datatype);
1435  cpl_propertylist_append_string(offlist, "ESO DRS CATG", state->tag);
1436  cpl_propertylist_append_string(onlist, "ESO DRS CATG", state->tag);
1437  bug_if(0);
1438 
1439  if (state->time_max_filewrite > 0) {
1440  cpl_propertylist_append_double(onlist, "ESO DRS DATE",
1441  state->time_max_filewrite);
1442  cpl_propertylist_append_double(offlist, "ESO DRS DATE",
1443  state->time_max_filewrite);
1444  cpl_propertylist_append_double(onlist, "ESO DRS DATE-OBS",
1445  state->time_min_obsstart);
1446  cpl_propertylist_append_double(offlist, "ESO DRS DATE-OBS",
1447  state->time_min_obsstart);
1448  }
1449  else {
1450  /* always write so following recipes don't need to check */
1451  cpl_propertylist_append_double(onlist, "ESO DRS DATE", -1.);
1452  cpl_propertylist_append_double(offlist, "ESO DRS DATE", -1.);
1453  cpl_propertylist_append_double(onlist, "ESO DRS DATE-OBS", -1.);
1454  cpl_propertylist_append_double(offlist, "ESO DRS DATE-OBS", -1.);
1455  }
1456  if (state->normalize) {
1457  cpl_propertylist_append_string(onlist, "BUNIT", "adu / s");
1458  cpl_propertylist_append_string(offlist, "BUNIT", "adu / s");
1459  }
1460  else {
1461  cpl_propertylist_append_string(onlist, "BUNIT", "adu");
1462  cpl_propertylist_append_string(offlist, "BUNIT", "adu");
1463  }
1464 
1465  /* record all inputs as used, even though not all are actually used for
1466  * each output but not recording them here would then require merging input
1467  * headers in subsequent recipes which do join the output files into a
1468  * single product */
1469  for (int i = 0; i < state->nframes; i++) {
1470  const cpl_frame * frm =
1471  irplib_framelist_get_const(state->rawframes, i);
1472  bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm)));
1473  }
1474 
1475  skip_if(check_and_fix_cd_wcs(state, onlist));
1476  skip_if(check_and_fix_cd_wcs(state, offlist));
1477 
1478  if (products == NULL)
1479  products = cpl_frameset_new();
1480 
1481 
1482  if (state->datatype == VISIR_DATA_CUBE2) {
1483  skip_if(visir_qc_append_background(onlist, state->rawframes, 0, 0));
1484  skip_if(visir_qc_append_background(offlist, state->rawframes, 0, 0));
1485  }
1486  else if (!isnan(bg_mean)) {
1487  bug_if (cpl_propertylist_append_double(onlist, "ESO QC BACKGD MEAN",
1488  bg_mean));
1489  bug_if (cpl_propertylist_append_double(offlist, "ESO QC BACKGD MEAN",
1490  bg_mean));
1491  }
1492 
1493  if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
1494  state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
1495  skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1496  RECIPE_STRING, state->procatgon,
1497  onlist, state->remregexp,
1498  visir_pipe_id,
1499  state->onname));
1500  } else {
1501  /* Save each image in a separate extension */
1502  /* Do not save data in the primary data unit (DFS10475) */
1503  skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1504  RECIPE_STRING, state->procatgon,
1505  onlist, state->remregexp,
1506  visir_pipe_id, state->onname));
1507  skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1508  RECIPE_STRING, state->procatgoff,
1509  offlist, state->remregexp,
1510  visir_pipe_id, state->offname));
1511  }
1512 
1513  end_skip;
1514  cpl_frameset_delete(usedframes);
1515  cpl_propertylist_delete(onlist);
1516  cpl_propertylist_delete(offlist);
1517 
1518  return cpl_error_get_code();
1519 }
1520 
1521 
1522 static cpl_imagelist *
1523 correct_jitter(const cpl_imagelist * imgs, double dx, double dy)
1524 {
1525  cpl_imagelist * jit_cor = cpl_imagelist_new();
1526  cpl_msg_info(cpl_func, "Correcting jitter, x: %g y: %g", dx, dy);
1527  for (cpl_size i = 0, n = cpl_imagelist_get_size(imgs); i < n; i++) {
1528  cpl_image * img = cpl_image_duplicate(cpl_imagelist_get_const(imgs, i));
1529  /* only used for mean which in turn is only used for
1530  * initial beam guess, pixel precision sufficient */
1531  skip_if(cpl_image_shift(img,
1532  -visir_round_to_int(dx),
1533  -visir_round_to_int(dy)));
1534  cpl_imagelist_set(jit_cor, img, i);
1535  }
1536 
1537  end_skip;
1538 
1539  return jit_cor;
1540 }
1541 
1542 
1543 /* ---------------------------------------------------------------------------*/
1559 /* ---------------------------------------------------------------------------*/
1560 static cpl_imagelist *
1561 chop_correct(const repack_framestate * state,
1562  cpl_imagelist * on,
1563  cpl_imagelist * off,
1564  const cpl_boolean last_chunk)
1565 {
1566  cpl_imagelist * sub;
1567  /* burst data is not preprocessed and needs sign flipping */
1568  cpl_imagelist * subt;
1569 
1570  subt = state->datatype == VISIR_DATA_BURST ? on : off;
1571  sub = state->datatype == VISIR_DATA_BURST ? off : on;
1572 
1573  if (last_chunk)
1574  equalize_lists(on, off);
1575 
1576  cpl_imagelist_subtract(sub, subt);
1577  if (state->normalize) {
1578  cpl_imagelist_multiply_scalar(sub, 1. / (state->dit * 2));
1579  }
1580  /* empty now to reduce memory usage */
1581  cpl_imagelist_empty(subt);
1582 
1583  return sub;
1584 }
1585 
1586 /* ---------------------------------------------------------------------------*/
1593 /* ---------------------------------------------------------------------------*/
1594 static double
1595 compute_qc_median(const cpl_imagelist * list, cpl_size * nimages)
1596 {
1597  double sum = 0.;
1598  for (cpl_size i = 0; i < cpl_imagelist_get_size(list); i++) {
1599  const cpl_image * img = cpl_imagelist_get_const(list, i);
1600  sum += cpl_image_get_median(img);
1601  }
1602 
1603  *nimages += cpl_imagelist_get_size(list);
1604 
1605  return sum;
1606 }
1607 
1608 /* ---------------------------------------------------------------------------*/
1616 /* ---------------------------------------------------------------------------*/
1617 static cpl_error_code
1618 correct_linearity(const repack_framestate * state,
1619  cpl_imagelist * images,
1620  const cpl_bivector * lintable, cpl_image * nonlinear)
1621 {
1622  if (lintable == NULL || !visir_data_is_aqu(state->datatype)) {
1623  return CPL_ERROR_NONE;
1624  }
1625  for (size_t i = 0; i < (size_t)cpl_imagelist_get_size(images); i++) {
1626  cpl_image * img = cpl_imagelist_get(images, i);
1627  cpl_ensure_code(cpl_image_get_type(img) == CPL_TYPE_FLOAT ||
1628  cpl_image_get_type(img) == CPL_TYPE_DOUBLE,
1629  CPL_ERROR_ILLEGAL_INPUT);
1630  cpl_image * dvals = cpl_image_cast(img, CPL_TYPE_DOUBLE);
1631  cpl_image * corvals = visir_linintp_values(dvals, lintable);
1632  if (nonlinear) {
1633  cpl_image * bad =
1634  cpl_image_new_from_mask(cpl_image_get_bpm(corvals));
1635  cpl_image_add(nonlinear, bad);
1636  cpl_image_delete(bad);
1637  }
1638  cpl_image_divide(img, corvals);
1639  cpl_image_delete(dvals);
1640  cpl_image_delete(corvals);
1641  }
1642 
1643  return cpl_error_get_code();
1644 }
1645 
1646 /* ---------------------------------------------------------------------------*/
1662 /* ---------------------------------------------------------------------------*/
1663 static cpl_error_code
1664 save_corrected(const repack_framestate * fstatea,
1665  const repack_framestate * fstateb,
1666  cpl_imagelist * aon,
1667  cpl_imagelist * aoff,
1668  cpl_imagelist * bon,
1669  cpl_imagelist * boff,
1670  double dxa, double dya,
1671  double dxb, double dyb,
1672  const cpl_boolean last_chunk)
1673 {
1674  cpl_imagelist * chop_corrected_a, * chop_corrected_b;
1675  const repack_framestate * state = fstatea;
1676 
1677  chop_corrected_a = chop_correct(fstatea, aon, aoff, last_chunk);
1678  chop_corrected_b = chop_correct(fstateb, bon, boff, last_chunk);
1679 
1680  if (cpl_error_get_code())
1681  return cpl_error_get_code();
1682 
1683  if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) {
1684  /* only chop correction requested */
1685  save_subtract_images(fstatea, chop_corrected_a, NULL, NULL);
1686  save_subtract_images(fstateb, chop_corrected_b, NULL, NULL);
1687  }
1688  else {
1689  /* perform nod and jitter correction
1690  * jitter is only used for the mean used for object detection pivoting
1691  * correction is done in later recipes */
1692  cpl_imagelist * jca = NULL;
1693  cpl_imagelist * jcb = NULL;
1694 
1695  if (last_chunk)
1696  equalize_lists(chop_corrected_a, chop_corrected_b);
1697 
1698  if (state->normalize) {
1699  cpl_imagelist_divide_scalar(chop_corrected_a, 2);
1700  cpl_imagelist_divide_scalar(chop_corrected_b, 2);
1701  }
1702 
1703  if (visir_round_to_int(dxa) != 0 ||
1704  visir_round_to_int(dya) != 0 ||
1705  visir_round_to_int(dxb) != 0 ||
1706  visir_round_to_int(dyb) != 0) {
1707  jca = correct_jitter(chop_corrected_a, dxa, dya);
1708  jcb = correct_jitter(chop_corrected_b, dxb, dyb);
1709  cpl_imagelist_subtract(jca, jcb);
1710  }
1711 
1712  cpl_imagelist_subtract(chop_corrected_a, chop_corrected_b);
1713 
1714  save_subtract_images(state, chop_corrected_a, jca, NULL);
1715  cpl_imagelist_delete(jcb);
1716  cpl_imagelist_delete(jca);
1717  }
1718 
1719  return cpl_error_get_code();
1720 }
1721 
1722 static cpl_error_code
1723 save_uncorrected(const repack_framestate * fstatea,
1724  const repack_framestate * fstateb,
1725  cpl_imagelist * aon,
1726  cpl_imagelist * aoff,
1727  cpl_imagelist * bon,
1728  cpl_imagelist * boff)
1729 {
1730  const repack_framestate * state = fstatea;
1731  if (state->normalize) {
1732  cpl_imagelist_multiply_scalar(aon, 1. / fstatea->dit);
1733  cpl_imagelist_multiply_scalar(aoff, 1. / fstatea->dit);
1734  cpl_imagelist_multiply_scalar(bon, 1. / fstateb->dit);
1735  cpl_imagelist_multiply_scalar(boff, 1. / fstateb->dit);
1736  }
1737  append_images(fstatea, aon, NULL, CPL_TRUE);
1738  append_images(fstatea, aoff, NULL, CPL_FALSE);
1739  append_images(fstateb, bon, NULL, CPL_TRUE);
1740  append_images(fstateb, boff, NULL, CPL_FALSE);
1741  cpl_imagelist_empty(aon);
1742  cpl_imagelist_empty(aoff);
1743  cpl_imagelist_empty(bon);
1744  cpl_imagelist_empty(boff);
1745 
1746  return cpl_error_get_code();
1747 }
1748 
1749 #define VISIR_SWAP(a, b)\
1750  do {\
1751  cpl_imagelist * tmp = a; \
1752  a = b; \
1753  b = tmp; \
1754  } while (0)
1755 
1756 /*----------------------------------------------------------------------------*/
1769 /*----------------------------------------------------------------------------*/
1770 static cpl_error_code
1771 visir_util_repack_two(const repack_framestate * fstatea,
1772  const repack_framestate * fstateb,
1773  cpl_frameset * products,
1774  cpl_image ** pbpm,
1775  cpl_image * nonlinear,
1776  double xjitter0, double yjitter0,
1777  const cpl_bivector * lintable)
1778 {
1779  /* process the data */
1780  cpl_errorstate cleanstate = cpl_errorstate_get();
1781  const repack_framestate * state = fstatea;
1782  int chunksize = 200;
1783  cpl_imagelist * aon = NULL;
1784  cpl_imagelist * aoff = NULL;
1785  cpl_imagelist * bon = NULL;
1786  cpl_imagelist * boff = NULL;
1787  /* preload lists */
1788  cpl_imagelist * aon_next = NULL;
1789  cpl_imagelist * aoff_next = NULL;
1790  cpl_imagelist * bon_next = NULL;
1791  cpl_imagelist * boff_next = NULL;
1792  const char * chopnod_dir = visir_pfits_get_chopnod_dir(state->plist);
1793 
1794  double dxa = fstatea->offsetx - xjitter0;
1795  double dya = fstatea->offsety - yjitter0;
1796  double dxb = fstateb->offsetx - xjitter0;
1797  double dyb = fstateb->offsety - yjitter0;
1798 
1799  double qc_bkg_sum = 0.;
1800  cpl_size qc_bkg_count = 0;
1801 
1802  /* chunksize must be a multiple of the period to avoid having a
1803  * #on-#off difference larger than chunksize */
1804 
1805  if (visir_data_is_burst(state->datatype)) {
1806  chunksize = 2 * state->halfcycle *
1807  CX_MAX(1, chunksize / (2 * state->halfcycle));
1808  /* parameter sanity check, ensures equal length A and B lists */
1809  error_if(state->planeend - state->planestart < 2 * state->halfcycle,
1810  CPL_ERROR_ILLEGAL_INPUT,
1811  "Number of planes to be repacked must be larger than "
1812  "a full cycle of %d planes.", state->halfcycle * 2);
1813  }
1814 
1815  /* can be missing for old spec data */
1816  if (chopnod_dir == NULL) {
1817  chopnod_dir = "PARALLEL";
1818  cpl_errorstate_set(cleanstate);
1819  }
1820 
1821  /* try to correct the cumoffset based on throw
1822  * POSANG is not a reliable mean to determine which offset has the throw
1823  * added to the jitter
1824  * apply heuristic that the jitter is smaller than the throw and correct
1825  * those offsets that are larger than throw with a 10 pixel margin */
1826  cpl_msg_info(cpl_func, "POSANG: %g; Offset: (%g, %g); Throw: %g; "
1827  "Jitterwidth %g", fstateb->posang * CPL_MATH_RAD_DEG, dxb,
1828  dyb, fstateb->pthrow, fstateb->jitterwidth);
1829  if ((int)rint(fstateb->posang * CPL_MATH_RAD_DEG) % 90 == 0) {
1830  if (dxb >= fstateb->pthrow - fstateb->jitterwidth - 10.) {
1831  dxb -= fstateb->pthrow;
1832  }
1833  else if (dxb <= -(fstateb->pthrow - fstateb->jitterwidth - 10.)) {
1834  dxb += fstateb->pthrow;
1835  }
1836  if (dyb >= fstateb->pthrow - fstateb->jitterwidth - 10.) {
1837  dyb -= fstateb->pthrow;
1838  }
1839  else if (dyb <= -(fstateb->pthrow - fstateb->jitterwidth - 10.)) {
1840  dyb += fstateb->pthrow;
1841  }
1842  /* remove rounding errors on no jitter */
1843  dxb = fabs(dxb) < 0.001 ? 0. : dxb;
1844  dyb = fabs(dyb) < 0.001 ? 0. : dyb;
1845  }
1846  else {
1847  /* offset keywords don't seem to respect angle. assume no jitter */
1848  if (fstateb->jitterwidth != 0.) {
1849  cpl_msg_warning(cpl_func,
1850  "Can't do jitter correction on non normal POSANG");
1851  }
1852  dxb = 0.;
1853  dyb = 0.;
1854  }
1855 
1856  aon = cpl_imagelist_new();
1857  aoff = cpl_imagelist_new();
1858  bon = cpl_imagelist_new();
1859  boff = cpl_imagelist_new();
1860 
1861 
1862  /* load the first chunk of data to fill QC parameters*/
1863  if (state->datatype != VISIR_DATA_AQU_INT) {
1864  const int pstart = state->planestart;
1865  const int pend = CX_MIN(pstart + chunksize, state->planeend);
1866  load_chunk(aon, aoff, fstatea, pstart, pend);
1867  load_chunk(bon, boff, fstateb, pstart, pend);
1868  }
1869  if ((visir_data_is_aqu(state->datatype) || visir_data_is_burst(state->datatype)) &&
1870  state->datatype != VISIR_DATA_AQU_INT) {
1871  /* cube2 is computed in prepare_output */
1872  qc_bkg_sum += compute_qc_median(aon, &qc_bkg_count);
1873  qc_bkg_sum += compute_qc_median(aoff, &qc_bkg_count);
1874  qc_bkg_sum += compute_qc_median(bon, &qc_bkg_count);
1875  qc_bkg_sum += compute_qc_median(boff, &qc_bkg_count);
1876  }
1877 
1878 
1879  if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
1880  state->bkgcorrect == VISIR_SUB_NOCORRECT) {
1881  prepare_output(fstatea, products, dxa, dya, dxb, dyb, qc_bkg_sum / qc_bkg_count);
1882  prepare_output(fstateb, products, dxa, dya, dxb, dyb, qc_bkg_sum / qc_bkg_count);
1883  }
1884  else if (state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
1885  prepare_output(state, products, dxa, dya, dxb, dyb, qc_bkg_sum / qc_bkg_count);
1886  }
1887  skip_if(0);
1888 
1889  if (state->datatype == VISIR_DATA_AQU_INT) {
1890  cpl_imagelist * a = cpl_imagelist_new();
1891  cpl_imagelist * b = cpl_imagelist_new();
1892  cpl_imagelist * jca = NULL;
1893  cpl_imagelist * jcb = NULL;
1894  const cpl_frame * frame;
1895  const char * filename;
1896  cpl_image * img;
1897 
1898  frame = irplib_framelist_get_const(fstatea->rawframes,
1899  fstatea->iframe);
1900  filename = cpl_frame_get_filename(frame);
1901  img = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 1);
1902  cpl_imagelist_set(a, img, cpl_imagelist_get_size(a));
1903 
1904  frame = irplib_framelist_get_const(fstateb->rawframes,
1905  fstateb->iframe);
1906  filename = cpl_frame_get_filename(frame);
1907  img = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 1);
1908  cpl_imagelist_set(b, img, cpl_imagelist_get_size(b));
1909 
1910  if (fstatea->normalize) {
1911  cpl_imagelist_multiply_scalar(a, 1. / (state->dit * 4));
1912  cpl_imagelist_multiply_scalar(b, 1. / (state->dit * 4));
1913  }
1914 
1915  if (visir_round_to_int(dxa) != 0 ||
1916  visir_round_to_int(dya) != 0 ||
1917  visir_round_to_int(dxb) != 0 ||
1918  visir_round_to_int(dyb) != 0) {
1919  jca = correct_jitter(a, dxa, dya);
1920  jcb = correct_jitter(b, dxb, dyb);
1921  cpl_imagelist_subtract(jca, jcb);
1922  cpl_imagelist_subtract(a, b);
1923  save_subtract_images(fstatea, a, jca, NULL);
1924  }
1925  else {
1926  cpl_imagelist_subtract(a, b);
1927  save_subtract_images(fstatea, a, jca, NULL);
1928  }
1929 
1930  cpl_imagelist_delete(a);
1931  cpl_imagelist_delete(b);
1932  cpl_imagelist_delete(jca);
1933  cpl_imagelist_delete(jcb);
1934  }
1935 
1936  aon_next = cpl_imagelist_new();
1937  aoff_next = cpl_imagelist_new();
1938  bon_next = cpl_imagelist_new();
1939  boff_next = cpl_imagelist_new();
1940 
1941  for (int pstart = state->planestart;
1942  pstart < state->planeend + chunksize - 1 &&
1943  state->datatype != VISIR_DATA_AQU_INT;
1944  pstart += chunksize) {
1945  const int pend = CX_MIN(pstart + chunksize, state->planeend);
1946 
1947  if (pstart >= pend)
1948  break;
1949 
1950 
1951  /* the process->save part has a throughput of ~200-400MB/s
1952  * SSDs can match this so by performing one preload in parallel with
1953  * processing we archive ideal performance also on these systems
1954  * first chunk already loaded synchronously */
1955  if (pstart != state->planestart) {
1956  /* empty the currently worked on lists
1957  * may have already been emptied by save_ functions */
1958  cpl_imagelist_empty(aon);
1959  cpl_imagelist_empty(aoff);
1960  cpl_imagelist_empty(bon);
1961  cpl_imagelist_empty(boff);
1962 
1963  /* wait for preload of last iteration to finish */
1964  OMP3_PRAGMA(omp taskwait);
1965  /* swap empty current and full preload lists */
1966  VISIR_SWAP(aon, aon_next);
1967  VISIR_SWAP(aoff, aoff_next);
1968  VISIR_SWAP(bon, bon_next);
1969  VISIR_SWAP(boff, boff_next);
1970  }
1971 
1972  /* preload the next chunk asynchronous */
1973  if (pend < state->planeend) {
1974  int pend_next = CX_MIN(pend + chunksize, state->planeend);
1975  OMP3_PRAGMA(omp task) {
1976  load_chunk(aon_next, aoff_next, fstatea, pend, pend_next);
1977  load_chunk(bon_next, boff_next, fstateb, pend, pend_next);
1978  }
1979  }
1980 
1981  if (*pbpm == NULL && state->datatype == VISIR_DATA_CUBE2) {
1982 
1983  *pbpm = cpl_image_duplicate(cpl_imagelist_get(aon, 0));
1984  cpl_image_threshold(*pbpm, state->bpmthresh,
1985  state->bpmthresh, 0.0, 1.0);
1986  skip_if(0);
1987  }
1988 
1989  correct_linearity(fstatea, aon, lintable, nonlinear);
1990  correct_linearity(fstatea, aoff, lintable, nonlinear);
1991  correct_linearity(fstateb, bon, lintable, nonlinear);
1992  correct_linearity(fstateb, boff, lintable, nonlinear);
1993 
1994  if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
1995  state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
1996  cpl_boolean last_chunk = pstart + chunksize >= pend;
1997  skip_if(save_corrected(fstatea, fstateb,
1998  aon, aoff, bon, boff,
1999  dxa, dya, dxb, dyb, last_chunk));
2000  }
2001  else {
2002  skip_if(save_uncorrected(fstatea, fstateb, aon, aoff, bon, boff));
2003  }
2004  }
2005 
2006  end_skip;
2007  cpl_imagelist_delete(aon);
2008  cpl_imagelist_delete(aoff);
2009  cpl_imagelist_delete(bon);
2010  cpl_imagelist_delete(boff);
2011  cpl_imagelist_delete(aon_next);
2012  cpl_imagelist_delete(aoff_next);
2013  cpl_imagelist_delete(bon_next);
2014  cpl_imagelist_delete(boff_next);
2015 
2016  return cpl_error_get_code();
2017 }
2018 
2019 
2020 /*----------------------------------------------------------------------------*/
2029 /*----------------------------------------------------------------------------*/
2030 static inline cpl_error_code visir_util_repack_check(const cpl_image * self,
2031  const cpl_imagelist * on,
2032  const cpl_imagelist * off)
2033 {
2034  /* UNUSED due to chunking */
2035  cpl_image * meanon = cpl_imagelist_collapse_create(on);
2036  cpl_image * meanoff = cpl_imagelist_collapse_create(off);
2037  const cpl_error_code err1 = cpl_image_subtract(meanon, meanoff);
2038  const cpl_error_code err2 = cpl_image_subtract(meanon, self);
2039  const unsigned bitmask = CPL_STATS_MIN | CPL_STATS_MAX | CPL_STATS_MEAN
2040  | CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV | CPL_STATS_STDEV;
2041 
2042  cpl_stats * stats = cpl_stats_new_from_image(meanon, bitmask);
2043 
2044  bug_if(err1 + err2);
2045 
2046  bug_if(cpl_stats_dump(stats, bitmask, stderr));
2047 
2048  end_skip;
2049 
2050  cpl_image_delete(meanon);
2051  cpl_image_delete(meanoff);
2052  cpl_stats_delete(stats);
2053 
2054  return cpl_error_get_code();
2055 
2056 }
double visir_pfits_get_cumoffsetx(const cpl_propertylist *self)
The cumulative offset in X.
Definition: visir_pfits.c:249
int visir_pfits_get_naxis2(const cpl_propertylist *self)
The NAXIS2 key.
Definition: visir_pfits.c:528
int visir_pfits_get_win_ny(const cpl_propertylist *self)
The WIN NY key.
Definition: visir_pfits.c:579
double visir_pfits_get_chop_pthrow(const cpl_propertylist *self)
The chopping throw in pixels.
Definition: visir_pfits.c:222
cpl_error_code visir_load_cube2_split(cpl_imagelist *alist, cpl_imagelist *blist, const irplib_framelist *rawframes, int pos, const int planestart, const int planeend)
Load and split the data in a CUBE2 file.
Definition: visir_inputs.c:238
int visir_pfits_get_naxis3(const cpl_propertylist *self)
The NAXIS3 key.
Definition: visir_pfits.c:543
int visir_pfits_get_navrg(const cpl_propertylist *self)
The NAVRG.
Definition: visir_pfits.c:323
int visir_pfits_get_start_x(const cpl_propertylist *self)
The WIN STRX key.
Definition: visir_pfits.c:600
int visir_pfits_get_naxis1(const cpl_propertylist *self)
The NAXIS1 key.
Definition: visir_pfits.c:513
cpl_error_code irplib_dfs_save_propertylist(cpl_frameset *allframes, const cpl_parameterlist *parlist, const cpl_frameset *usedframes, const char *recipe, const char *procat, const cpl_propertylist *applist, const char *remregexp, const char *pipe_id, const char *filename)
Save a propertylist as a DFS-compliant pipeline product.
Definition: irplib_utils.c:240
irplib_framelist * irplib_framelist_extract_regexp(const irplib_framelist *self, const char *regexp, cpl_boolean invert)
Extract the frames with the given tag from a framelist.
const char * visir_pfits_get_frame_type(const cpl_propertylist *self)
The frame type.
Definition: visir_pfits.c:418
const cpl_propertylist * irplib_framelist_get_propertylist_const(const irplib_framelist *self, int pos)
Get the propertylist of the specified frame in the framelist.
double visir_pfits_get_chop_posang(const cpl_propertylist *self)
The chopping position angle in rad.
Definition: visir_pfits.c:205
cpl_error_code irplib_dfs_save_image(cpl_frameset *allframes, const cpl_parameterlist *parlist, const cpl_frameset *usedframes, const cpl_image *image, cpl_type_bpp bpp, const char *recipe, const char *procat, const cpl_propertylist *applist, const char *remregexp, const char *pipe_id, const char *filename)
Save an image as a DFS-compliant pipeline product.
Definition: irplib_utils.c:192
int visir_pfits_get_win_nx(const cpl_propertylist *self)
The WIN NX key.
Definition: visir_pfits.c:558
int visir_pfits_get_ndit(const cpl_propertylist *self)
The NDIT keyword.
Definition: visir_pfits.c:502
double visir_pfits_get_cumoffsety(const cpl_propertylist *self)
The cumulative offset in Y.
Definition: visir_pfits.c:261
cpl_frame * irplib_framelist_get(irplib_framelist *self, int pos)
Get the specified frame from the framelist.
double visir_pfits_get_chop_freq(const cpl_propertylist *self)
The chopping frequency.
Definition: visir_pfits.c:170
double visir_pfits_get_pixspace(const cpl_propertylist *self)
The pixel spacing.
Definition: visir_pfits.c:694
const char * visir_pfits_get_nodpos(const cpl_propertylist *self)
The nodding position.
Definition: visir_pfits.c:666
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.
Definition: visir_dfs.c:72
double visir_pfits_get_dit(const cpl_propertylist *self)
The DIT.
Definition: visir_pfits.c:309
const cpl_frame * irplib_framelist_get_const(const irplib_framelist *self, int pos)
Get the specified frame from the framelist.
double visir_pfits_get_pixscale(const cpl_propertylist *self)
The pixel scale.
Definition: visir_pfits.c:678
void irplib_framelist_delete(irplib_framelist *self)
Deallocate an irplib_framelist with its frames and properties.
const char * visir_pfits_get_chopnod_dir(const cpl_propertylist *self)
The chopping direction.
Definition: visir_pfits.c:139
int visir_pfits_get_start_y(const cpl_propertylist *self)
The WIN STRY key.
Definition: visir_pfits.c:621
cpl_error_code irplib_framelist_load_propertylist(irplib_framelist *self, int pos, int ind, const char *regexp, cpl_boolean invert)
Load the propertylist of the specified frame in the framelist.
irplib_framelist * irplib_framelist_cast(const cpl_frameset *frameset)
Create an irplib_framelist from a cpl_framelist.
int irplib_framelist_get_size(const irplib_framelist *self)
Get the size of a framelist.