IIINSTRUMENT Pipeline Reference Manual 4.5.1
visir_util_repack.c
1/* $Id: visir_util_repack.c,v 1.144 2013-09-24 11:09:55 jtaylor Exp $
2 *
3 * This file is part of the VISIR Pipeline
4 * Copyright (C) 2002,2003,2013,2014 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
19 */
20
21#ifdef HAVE_CONFIG_H
22#include <config.h>
23#endif
24
25/*-----------------------------------------------------------------------------
26 Includes
27 -----------------------------------------------------------------------------*/
28
29#include "visir_recipe.h"
30#include "irplib_framelist.h"
31#include "visir_spc_optmod.h"
32#include "visir_spectro.h"
33#include "irplib_wcs.h"
34#include <cxlist.h>
35/* Verify self-sufficiency of CPL header files by including system files last */
36#include <string.h>
37/* for strcasecmp */
38#include <strings.h>
39#include <math.h> /* isnan */
40
41
42/*-----------------------------------------------------------------------------
43 Defines
44 -----------------------------------------------------------------------------*/
45
46#ifndef VISIR_UTIL_REPACK_CONAD
47#define VISIR_UTIL_REPACK_CONAD 1.1
48#endif
49
50#define RECIPE_STRING "visir_util_repack"
51
52#define VISIR_DRS_CONAD "ESO DET OUT1 CONAD"
53enum nodtype_e {
54 ANOD,
55 BNOD,
56 ABNOD
57};
58
59
60/*-----------------------------------------------------------------------------
61 Private Functions prototypes
62 -----------------------------------------------------------------------------*/
63typedef enum {
64 VISIR_SUB_NOCORRECT = 0,
65 VISIR_SUB_CHOPCORRECT = 1,
66 VISIR_SUB_CHOPNODCORRECT = 2
67} visir_sub_type;
68
69typedef enum {
70 VISIR_FRAME_A,
71 VISIR_FRAME_B,
72 VISIR_FRAME_STATIC,
73} visir_frame_type;
74
75/* convinience data structure to hold common data needed
76 * to process one frame */
77typedef struct {
78 int planestart;
79 int planeend;
80 int trimlow;
81 int trimhigh;
82 int bkgcorrect;
83 cpl_boolean normalize;
84 cpl_boolean compress;
85 double conad;
86 int nframes;
87 const irplib_framelist * rawframes;
88 int iframe;
89 const cpl_parameterlist * parlist;
90 const char * tag;
91 const cpl_propertylist * plist;
92 visir_data_type datatype;
93 cpl_type load_type;
94
95 char * onname;
96 char * offname;
97 const char * remregexp;
98
99 double pthrow;
100 double posang;
101 double dit;
102 int ncycles;
103 int nplanes_per_cycle;
104 const char * comoffx;
105 const char * comoffy;
106 double offsetx;
107 double offsety;
108 double jitterwidth;
109 double pxspace;
110 double pfov;
111 cpl_wcs * wcs;
112 visir_spc_resol resol;
113
114 /* only for burst */
115 int to_off;
116 int halfcycle;
117
118 double bpmthresh;
119
120 int naxis3;
121 cpl_boolean is_a;
122 const char * procatgon;
123 const char * procatgoff;
124 cpl_size nx, ny;
125 /* initialized from framelist global storage */
126 cpl_image * mean_on;
127 cpl_size * nmean_on;
128 cpl_image * mean_off;
129 cpl_size * nmean_off;
130
131 double time_obsstart;
132 double time_filewrite;
133 /* min and max of a two A/B states to write into a chop nod file */
134 double time_min_obsstart;
135 double time_max_filewrite;
136} repack_framestate;
137
138static cpl_error_code
139visir_util_repack_two(const repack_framestate * fstatea,
140 const repack_framestate * fstateb,
141 cpl_frameset * products,
142 cpl_image ** pbpm,
143 cpl_image * nonlinear,
144 const cpl_bivector * lintab);
145
146static inline cpl_error_code
147visir_util_repack_check(const cpl_image *,
148 const cpl_imagelist *,
149 const cpl_imagelist *);
150
151#ifdef VISIR_CHAIN
152#define cpl_plugin_get_info visir_util_repack_get_info
153#endif
154cpl_recipe_define(visir_util_repack, VISIR_BINARY_VERSION,
155 "Lars Lundin", PACKAGE_BUGREPORT, "2011",
156 "Conversion of raw CUBE2 or BURST images to on- and off-cubes",
157 "The files listed in the Set Of Frames (sof-file) "
158 "must be tagged:\n"
159 "VISIR-CUBE2-raw-file.fits " VISIR_UTIL_REPACK_RAW
160 "\nor\n"
161 "VISIR-BURST-raw-file.fits " VISIR_IMG_BURST"\n"
162 "VISIR-BURST-bpm-file.fits " VISIR_CALIB_STATIC_MASK
163 "\nFor BURST data it will remove planes where the chopper "
164 "switched from on <-> off based on the "
165 "timestamps in the header."
166 "\nThe product(s) will have a FITS card\n"
167 "'HIERARCH ESO PRO CATG' with a value of one of:\n"
168 VISIR_UTIL_REPACK_A_ON_PROCATG " (NodPos: A, ChopPos: on)\n"
169 VISIR_UTIL_REPACK_A_OFF_PROCATG " (NodPos: A, ChopPos: off)\n"
170 VISIR_UTIL_REPACK_B_ON_PROCATG " (NodPos: B, ChopPos: on)\n"
171 VISIR_UTIL_REPACK_B_OFF_PROCATG " (NodPos: B, ChopPos: off)\n"
172 /* FIXME: reimplement
173 "Additionally, the recipe collapses the on- and off-cubes, "
174 "these product(s) will have a FITS card\n"
175 "'HIERARCH ESO PRO CATG' with a value of one of:\n"
176 VISIR_UTIL_REPACK_MEAN_A_ON_PROCATG
177 " (NodPos: A, ChopPos: on)\n"
178 VISIR_UTIL_REPACK_MEAN_A_OFF_PROCATG
179 " (NodPos: A, ChopPos: off)\n"
180 VISIR_UTIL_REPACK_MEAN_B_ON_PROCATG
181 " (NodPos: B, ChopPos: on)\n"
182 VISIR_UTIL_REPACK_MEAN_B_OFF_PROCATG
183 " (NodPos: B, ChopPos: off)\n" */
184 "For CUBE2, the recipe will produce a static "
185 "bad-pixel map, it will have a FITS card\n"
186 "'HIERARCH ESO PRO CATG' with a value of:\n"
187 VISIR_CALIB_STATIC_MASK"\n"
188 );
189
190/*----------------------------------------------------------------------------*/
194/*----------------------------------------------------------------------------*/
195
196/*-----------------------------------------------------------------------------
197 Functions code
198 -----------------------------------------------------------------------------*/
199
200
201/*----------------------------------------------------------------------------*/
208/*----------------------------------------------------------------------------*/
209static cpl_image *
210visir_imagelist_pop(cpl_imagelist * list)
211{
212 if (cpl_imagelist_get_size(list) == 0)
213 return NULL;
214 return cpl_imagelist_unset(list, cpl_imagelist_get_size(list) - 1);
215}
216
217
218/*----------------------------------------------------------------------------*/
226/*----------------------------------------------------------------------------*/
227static cpl_type get_optimum_save_type(const cpl_image * img)
228{
229 cpl_type res = CPL_TYPE_UNSPECIFIED;
230 if (cpl_image_get_type(img) == CPL_TYPE_INT) {
231 cpl_stats * stats = cpl_stats_new_from_image(img, CPL_STATS_MIN |
232 CPL_STATS_MAX);
233 if ((int)cpl_stats_get_max(stats) <= CX_MAXSHORT &&
234 (int)cpl_stats_get_min(stats) >= CX_MINSHORT)
235 res = CPL_TYPE_SHORT;
236 cpl_stats_delete(stats);
237 }
238
239 return res;
240}
241
242
243/*----------------------------------------------------------------------------*/
251/*----------------------------------------------------------------------------*/
252static cpl_error_code
253visir_util_repack_fill_parameterlist(cpl_parameterlist * self)
254{
255 const char * context = PACKAGE "." RECIPE_STRING;
256 cpl_error_code err;
257
258 cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);
259
260 /* Fill the parameters list */
261
262 /* --planestart */
263 err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "planestart",
264 0, NULL, context, "Plane number to "
265 "start repacking from in each nod cycle,"
266 "earlier planes are skipped.");
267 cpl_ensure_code(!err, err);
268
269 /* --planelimit */
270 err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "planelimit",
271 -1, NULL, context, "Limit number of processed input"
272 "planes. It will repack until at least this number "
273 "of input images have been processed or the full "
274 "dataset has been repacked. Always full chop cycles "
275 "need to be repacked so the number is adjusted "
276 "upward to the next multiple of images per chop cycle."
277 " <= 0 for no limit.");
278 cpl_ensure_code(!err, err);
279
280 /* --ncycles */
281 err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "ncycles",
282 -1, NULL, context, "Number of full "
283 "on-off cycles to repack per nod cycles."
284 " <= 0 for all.");
285 cpl_ensure_code(!err, err);
286
287
288
289 /* --trimlow */
290 err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "trimlow",
291 0, NULL, context, "Burst data only. "
292 "Number of additional planes to cut "
293 "from before each plane with chopper "
294 "movement.");
295 cpl_ensure_code(!err, err);
296
297 /* --trimhigh */
298 err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "trimhigh",
299 0, NULL, context, "Burst data only. "
300 "Number of additional planes to cut "
301 "from after each plane with chopper "
302 "movement.\n A value of -1 does not "
303 "skip the plane of the movement.");
304 cpl_ensure_code(!err, err);
305
306 /* --bkgcorrect */
307 err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
308 "bkgcorrect", "none", NULL, context,
309 "Output background corrected planes by "
310 "subtracting chop (on/off) and nod (A/B) "
311 "planes\n Options:\n"
312 " none: no correction\n"
313 " chop: on - off\n"
314 " chopnod: (Aon - Aoff) - (Bon - Boff)");
315 cpl_ensure_code(!err, err);
316
317
318 /* --normalize */
319 err =
320 irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING, "normalize",
321 CPL_TRUE, NULL, context,
322 "Normalize planes by DIT");
323 cpl_ensure_code(!err, err);
324
325
326 /* --compress */
327 err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
328 "compress", CPL_FALSE, NULL, context,
329 "Apply lossless compression on output"
330 " files. Can only be done for integer"
331 " type results.");
332 cpl_ensure_code(!err, err);
333
334 /* --lincorrect */
335 err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
336 "lincorrect", CPL_FALSE, NULL, context,
337 "Apply linearity correction. Should "
338 "only be enabled on high flux observations"
339 ", may degrade results otherwise.");
340 cpl_ensure_code(!err, err);
341
342 return CPL_ERROR_NONE;
343}
344
345
346/*----------------------------------------------------------------------------*/
357/*----------------------------------------------------------------------------*/
358static repack_framestate *
359repack_framestate_new(irplib_framelist * rawframes, int iframe,
360 const cpl_parameterlist * parlist)
361{
362 repack_framestate * state = cpl_calloc(1, sizeof(repack_framestate));
363 int ndit, navrg;
364 cpl_frame * frame = irplib_framelist_get(rawframes, iframe);
365 const char * filename = cpl_frame_get_filename(frame);
366 cpl_size next;
367 state->time_max_filewrite = -1;
368 state->time_min_obsstart = 1e300;
369 cpl_errorstate cleanstate = cpl_errorstate_get();
370
371 state->rawframes = rawframes;
372
373 state->iframe = iframe;
374
375 state->tag = cpl_frame_get_tag(frame);
376
377 error_if(strcmp(state->tag, VISIR_CALIB_STATIC_MASK) == 0,
378 CPL_ERROR_UNSUPPORTED_MODE, " ");
379
380 /* compression support disabled, can give bad error messages for
381 * INT only acquarius data */
382 /* compressed files have the data in extension 1 */
383 const cpl_size iext = 0;
384
385 skip_if(irplib_framelist_load_propertylist(rawframes, iframe, iext,
386 ".*", CPL_FALSE));
387 state->parlist = parlist;
388
389 state->plist = irplib_framelist_get_propertylist_const(rawframes, iframe);
390 skip_if(0);
391
392 skip_if(visir_get_data_type(frame, state->plist, &state->datatype, &next));
393
394 if (state->datatype == VISIR_DATA_AQU_HCYCLE) {
395 /* last frame is INT, but is skipped by when loading */
396 state->naxis3 = next;
397 }
398 else if (state->datatype == VISIR_DATA_AQU_INT) {
399 state->naxis3 = 1;
400 }
401 else if (state->datatype == VISIR_DATA_AQU_BURST_EXT) {
402 cpl_propertylist * dplist = cpl_propertylist_load(filename, 1);
403 state->naxis3 = visir_pfits_get_naxis3(dplist);
404 cpl_propertylist_delete(dplist);
405 }
406 else
407 state->naxis3 = visir_pfits_get_naxis3(state->plist);
408
409 {
410 const char * bkgc = irplib_parameterlist_get_string(parlist, PACKAGE,
411 RECIPE_STRING,
412 "bkgcorrect");
413 if (visir_str_par_is_empty(bkgc) || !strcasecmp(bkgc, "none"))
414 state->bkgcorrect = VISIR_SUB_NOCORRECT;
415 else if (!strcasecmp(bkgc, "chopnod"))
416 state->bkgcorrect = VISIR_SUB_CHOPNODCORRECT;
417 else if (!strcasecmp(bkgc, "chop"))
418 state->bkgcorrect = VISIR_SUB_CHOPCORRECT;
419 else
420 error_if(1, CPL_ERROR_ILLEGAL_INPUT,
421 "Unknown parameter to --bkgcorrect: %s", bkgc);
422 }
423
424 error_if(state->datatype == VISIR_DATA_AQU_INT &&
425 state->bkgcorrect != VISIR_SUB_CHOPNODCORRECT,
426 CPL_ERROR_INCOMPATIBLE_INPUT,
427 "INT frame only data requires --bkgcorrect to be chopnod");
428
429 /* cannot extract a and b hcycle from this format */
430 error_if(state->datatype == VISIR_DATA_CUBE1,
431 CPL_ERROR_INCOMPATIBLE_INPUT,
432 "CUBE1 data not supported please use the legacy recipes "
433 "visir_img_combine or visir_spc_obs");
434
435 state->normalize = irplib_parameterlist_get_bool(parlist, PACKAGE,
436 RECIPE_STRING,
437 "normalize");
438
439 state->dit = visir_pfits_get_dit(state->plist);
440
441 error_if(state->normalize && state->dit == 0, CPL_ERROR_ILLEGAL_INPUT,
442 "Cannot normalize by DIT = 0");
443
444 state->planestart = irplib_parameterlist_get_int(parlist, PACKAGE,
445 RECIPE_STRING,
446 "planestart");
447 error_if(state->planestart >= state->naxis3, CPL_ERROR_ILLEGAL_INPUT,
448 "planestart %d equal or larger than number of planes %d.",
449 state->planestart, state->naxis3);
450
451 ndit = visir_pfits_get_ndit(state->plist);
452 navrg = visir_pfits_get_navrg(state->plist);
453
454 IRPLIB_DIAG_PRAGMA_PUSH_ERR(-Wswitch);
455 switch(state->datatype) {
456 case VISIR_DATA_CUBE1:
457 bug_if(1);
458 break;
459 case VISIR_DATA_AQU_HCYCLE:
460 case VISIR_DATA_CUBE2:
461 state->nplanes_per_cycle = 2;
462 break;
463 case VISIR_DATA_AQU_INT:
464 /* not actually used */
465 state->nplanes_per_cycle = 1;
466 break;
467 case VISIR_DATA_BURST:
468 case VISIR_DATA_AQU_BURST:
469 case VISIR_DATA_AQU_BURST_EXT:
470 state->nplanes_per_cycle = ndit * 2 / navrg;
471 break;
472 }
473 IRPLIB_DIAG_PRAGMA_POP;
474
475 state->ncycles = irplib_parameterlist_get_int(parlist, PACKAGE,
476 RECIPE_STRING, "ncycles");
477 if (state->ncycles <= 0)
478 state->ncycles = irplib_pfits_get_int(state->plist,
479 VISIR_PFITS_INT_CHOP_NCYCLES);
480
481 state->planeend = state->planestart + state->ncycles *
482 state->nplanes_per_cycle;
483 /* ensure we always repack a multiple of a full cycle to get equal number
484 * of on and off planes */
485 if (state->planeend >= state->naxis3 || state->planeend <= 0) {
486 if (state->planestart % state->nplanes_per_cycle == 0)
487 state->planeend = state->naxis3;
488 else
489 state->planeend = state->naxis3 -
490 (state->nplanes_per_cycle -
491 (state->planestart % state->nplanes_per_cycle));
492 }
493
494 /* we have max 16 bit adc and cpl int is 32 bit so reasonable dits allow
495 * integer normalization without overflow */
496 if (state->normalize && (state->dit < 1e-6 ||
497 (double)((int)(1. / state->dit)) != 1. / state->dit))
498 state->load_type = CPL_TYPE_FLOAT;
499 else
500 state->load_type = CPL_TYPE_UNSPECIFIED;
501
502 state->trimlow = irplib_parameterlist_get_int(parlist, PACKAGE,
503 RECIPE_STRING,
504 "trimlow");
505 state->trimhigh = irplib_parameterlist_get_int(parlist, PACKAGE,
506 RECIPE_STRING,
507 "trimhigh");
508
509 state->compress = irplib_parameterlist_get_bool(parlist, PACKAGE,
510 RECIPE_STRING,
511 "compress");
512
513 if (state->compress && CPL_IO_COMPRESS_RICE == 0) {
514 cpl_msg_warning(cpl_func, "CPL version too old for compression.");
515 state->compress = CPL_FALSE;
516 }
517
518 state->conad = VISIR_UTIL_REPACK_CONAD;
519
520 state->nframes = irplib_framelist_get_size(rawframes);
521 state->remregexp = "^(" VISIR_PFITS_DOUBLE_CUMOFFSETX
522 "|" VISIR_PFITS_DOUBLE_CUMOFFSETY ")$";
523
524
525 state->pfov = visir_pfits_get_pixscale(state->plist);
526
527 /* Copy comments from CUMOFFSET[XY] */
528 state->comoffx = cpl_propertylist_get_comment
529 (state->plist, VISIR_PFITS_DOUBLE_CUMOFFSETX);
530 state->comoffy = cpl_propertylist_get_comment
531 (state->plist, VISIR_PFITS_DOUBLE_CUMOFFSETY);
532 state->offsetx = visir_pfits_get_cumoffsetx(state->plist);
533 state->offsety = visir_pfits_get_cumoffsety(state->plist);
534 state->jitterwidth = 0.;
535 if (cpl_propertylist_has(state->plist, "ESO SEQ JITTER WIDTH")) {
536 state->jitterwidth =
537 cpl_propertylist_get_double(state->plist, "ESO SEQ JITTER WIDTH");
538 state->jitterwidth /= state->pfov;
539 }
540
541 /* we only have one chip, move PIXSPACE from extension to main header */
542 /* TODO still needed? */
543 if (next > 0 &&
544 !cpl_propertylist_has(state->plist, "ESO DET CHIP1 PXSPACE") &&
545 !cpl_propertylist_has(state->plist, "ESO DET CHIP PXSPACE")) {
546 cpl_propertylist * extplist = cpl_propertylist_load(filename, 1);
547 skip_if(extplist == NULL);
548 state->pxspace = visir_pfits_get_pixspace(extplist);
549 cpl_propertylist_delete(extplist);
550 skip_if(0);
551 }
552
553 if (visir_data_is_burst(state->datatype) &&
554 visir_data_is_drs(state->datatype)) {
555 visir_img_burst_find_delta_chop(state->plist,
556 &state->to_off,
557 &state->halfcycle);
558 }
559 else if (visir_data_is_burst(state->datatype)) {
560 error_if(ndit % navrg != 0, CPL_ERROR_ILLEGAL_INPUT,
561 "ndit (%d) not divisible by navrg %d, not supported",
562 ndit, navrg);
563 state->halfcycle = ndit / navrg;
564 }
565
566 state->bpmthresh = VISIR_HCYCLE_BPM_THRESHOLD;
567
568 if (state->bkgcorrect == VISIR_SUB_NOCORRECT) {
569 const char * nodpos = visir_pfits_get_nodpos(state->plist);
570 state->is_a = nodpos != NULL &&
571 (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
572 state->procatgon = state->is_a ? VISIR_UTIL_REPACK_A_ON_PROCATG
573 : VISIR_UTIL_REPACK_B_ON_PROCATG;
574 state->procatgoff = state->is_a ? VISIR_UTIL_REPACK_A_OFF_PROCATG
575 : VISIR_UTIL_REPACK_B_OFF_PROCATG;
576 state->onname = cpl_sprintf(RECIPE_STRING "_%s_on_%03d"
577 CPL_DFS_FITS, nodpos, 1+iframe);
578 state->offname = cpl_sprintf(RECIPE_STRING "_%s_off_%03d"
579 CPL_DFS_FITS, nodpos, 1+iframe);
580
581 if (!state->is_a) {
582 /* Verify that the NODPOS is either A or B */
583 skip_if (nodpos == NULL);
584 skip_if (strstr(nodpos, "B") == NULL && strstr(nodpos, "b") == NULL);
585 }
586 } else if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) {
587 const char * nodpos = visir_pfits_get_nodpos(state->plist);
588 state->is_a = nodpos != NULL &&
589 (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
590 state->procatgon = state->is_a ? "BKG_CORRECTED_A" : "BKG_CORRECTED_B";
591 state->onname = cpl_sprintf(RECIPE_STRING "_%s_bkgcor_%03d"
592 CPL_DFS_FITS, nodpos, 1+iframe);
593 } else { /*if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) { */
594 const char * nodpos = visir_pfits_get_nodpos(state->plist);
595 state->is_a = nodpos != NULL &&
596 (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
597 state->procatgon = VISIR_UTIL_CORRECTED;
598 state->onname = cpl_sprintf(RECIPE_STRING "_bkgcor_%03d"
599 CPL_DFS_FITS, 1+iframe);
600 }
601
602 /* Get the chopping throw in pixels */
603 state->pthrow = visir_pfits_get_chop_pthrow(state->plist);
604 state->posang = visir_pfits_get_chop_posang(state->plist) -
605 visir_pfits_get_ada_posang(state->plist);
606
607 const char * chopnod_dir = visir_pfits_get_chopnod_dir(state->plist);
608 /* can be missing for old spec data */
609 if (chopnod_dir == NULL) {
610 chopnod_dir = "PARALLEL";
611 cpl_errorstate_set(cleanstate);
612 }
613
614 /* compute jitter offset */
615 if (strcmp(visir_pfits_get_insmode(state->plist), "IMG") == 0) {
616 if (strcmp(chopnod_dir, "PARALLEL") == 0) {
617 state->posang -= 90 * CPL_MATH_RAD_DEG;
618 }
619 }
620 else if (strncmp(visir_pfits_get_insmode(state->plist), "SPC", 3) == 0) {
621 /* spc detector is always parallel and rotated another 90 degree */
622 state->posang -= 180 * CPL_MATH_RAD_DEG;
623 }
624 else {
625 cpl_errorstate_set(cleanstate);
626 cpl_msg_warning(cpl_func, "Unknown INS MODE %s",
627 visir_pfits_get_insmode(state->plist));
628 }
629 if (!state->is_a) {
630 state->offsetx -= cos(state->posang) * state->pthrow;// - sin(state->posang) * state->pthrow;
631 state->offsety += sin(state->posang) * state->pthrow;// + cos(state->posang) * state->pthrow;
632 }
633
634 /* spectrograph rotation also swaps -x and y */
635 if (visir_data_is_aqu(state->datatype) &&
636 strncmp(visir_pfits_get_insmode(state->plist), "SPC", 3) == 0) {
637 double tmp = state->offsety;
638 state->offsety = -state->offsetx;
639 state->offsetx = tmp;
640 }
641
642
643 /* remove rounding errors on no jitter */
644 state->offsetx = fabs(state->offsetx) < 0.001 ? 0. : state->offsetx;
645 state->offsety = fabs(state->offsety) < 0.001 ? 0. : state->offsety;
646 cpl_msg_info(cpl_func, "POSANG: %g; Offset: (%g, %g); Throw: %g; "
647 "Jitterwidth %g", state->posang * CPL_MATH_DEG_RAD, state->offsetx,
648 state->offsety, state->pthrow, state->jitterwidth);
649
650 const char * sresol = visir_pfits_get_resol(state->plist);
651 if (sresol && strcmp(sresol, "LRP") == 0) {
652 state->resol = VISIR_SPC_R_LRP;
653 }
654 else if (sresol && strcmp(sresol, "LR") == 0) {
655 state->resol = VISIR_SPC_R_LR;
656 }
657 else if (sresol && strcmp(sresol, "MR") == 0) {
658 state->resol = VISIR_SPC_R_MR;
659 }
660 else if (sresol && strcmp(sresol, "HRS") == 0) {
661 state->resol = VISIR_SPC_R_HR;
662 }
663 else if (sresol && strcmp(sresol, "HRG") == 0) {
664 state->resol = VISIR_SPC_R_GHR;
665 }
666 else {
667 state->resol = VISIR_SPC_R_ERR;
668 }
669
670 {
671 long nx, ny;
672 if (state->datatype == VISIR_DATA_AQU_HCYCLE ||
673 state->datatype == VISIR_DATA_AQU_INT ||
674 state->datatype == VISIR_DATA_AQU_BURST_EXT) {
675 cpl_propertylist * plist = cpl_propertylist_load(filename, 1);
676 nx = visir_pfits_get_naxis1(plist);
677 ny = visir_pfits_get_naxis2(plist);
678 state->wcs = cpl_wcs_new_from_propertylist(plist);
679 if (state->wcs == NULL) {
680 cpl_errorstate_set(cleanstate);
681 }
682 cpl_propertylist_delete(plist);
683 }
684 else {
685 nx = visir_pfits_get_naxis1(state->plist);
686 ny = visir_pfits_get_naxis2(state->plist);
687 }
688 skip_if(0);
689
690 state->nx = nx;
691 state->ny = ny;
692 }
693
694
695 end_skip;
696
697 return state;
698}
699
700/* ---------------------------------------------------------------------------*/
707/* ---------------------------------------------------------------------------*/
708static void
709repack_framestate_adjust_planeend(repack_framestate * state, int maxplanes)
710{
711 if (maxplanes <= 0) {
712 return;
713 }
714 int nplanes = state->planeend - state->planestart;
715 maxplanes = CX_MAX(maxplanes, state->nplanes_per_cycle);
716 if (nplanes - state->nplanes_per_cycle > maxplanes) {
717 state->planeend = state->planestart + maxplanes;
718 if (maxplanes % state->nplanes_per_cycle != 0) {
719 state->planeend += state->nplanes_per_cycle -
720 (maxplanes % state->nplanes_per_cycle);
721 }
722 }
723}
724
725/*----------------------------------------------------------------------------*/
730/*----------------------------------------------------------------------------*/
731static void
732repack_framestate_delete(void * state_)
733{
734 repack_framestate * state = (repack_framestate*) state_;
735
736 if (state == NULL)
737 return;
738
739 cpl_wcs_delete(state->wcs);
740 cpl_free(state->onname);
741 cpl_free(state->offname);
742 cpl_free(state);
743}
744
745
746/*----------------------------------------------------------------------------*/
753/*----------------------------------------------------------------------------*/
754static cpl_error_code
755store_means(const visir_sub_type bkgcorrect, const irplib_framelist * rawfr,
756 cpl_frameset * framelist, const cpl_parameterlist * parlist,
757 cpl_image ** mean, cpl_size * nmean, visir_frame_type * frametype)
758{
759 cpl_frameset * usedframes[] = {cpl_frameset_new(), cpl_frameset_new(),
760 cpl_frameset_new(), cpl_frameset_new()};
761
762 int normalize = irplib_parameterlist_get_bool(parlist, PACKAGE,
763 RECIPE_STRING,
764 "normalize");
765 cpl_propertylist * plist = cpl_propertylist_new();
766 if (normalize) {
767 cpl_propertylist_append_string(plist, "BUNIT", "adu / s");
768 } else {
769 cpl_propertylist_append_string(plist, "BUNIT", "adu");
770 }
771
772 if (bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
773 for (int i = 1; i < 4; i++) {
774 cpl_image_add(mean[0], mean[i]);
775 nmean[0] += nmean[i];
776 }
777 skip_if(cpl_image_divide_scalar(mean[0], nmean[0]));
778 for (int i = 0; i < irplib_framelist_get_size(rawfr); i++) {
779 cpl_frame * frm =
780 cpl_frame_duplicate(irplib_framelist_get_const(rawfr, i));
781 cpl_frameset_insert(usedframes[0], frm);
782 }
783 skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[0],
784 mean[0], CPL_TYPE_UNSPECIFIED,
785 RECIPE_STRING, VISIR_UTIL_REPACK_MEAN_PROCATG,
786 plist, NULL, visir_pipe_id,
787 "visir_util_repack_mean.fits"));
788 }
789 else if (bkgcorrect == VISIR_SUB_CHOPCORRECT) {
790 const char * procatg[] = {
791 VISIR_UTIL_REPACK_MEAN_A_PROCATG,
792 VISIR_UTIL_REPACK_MEAN_B_PROCATG};
793 const char * fn[] ={"visir_util_repack_mean_A.fits",
794 "visir_util_repack_mean_B.fits"};
795 for (int i = 0; i < 2; i++) {
796 int idx = i == 0 ? 0 : 2;
797 cpl_image_add(mean[idx], mean[idx + 1]);
798 nmean[idx] += nmean[idx + 1];
799 skip_if(cpl_image_divide_scalar(mean[idx], nmean[idx]));
800 for (int j = 0; j < irplib_framelist_get_size(rawfr); j++) {
801 if ((i == 0 && frametype[j] == VISIR_FRAME_B) ||
802 (i == 1 && frametype[j] == VISIR_FRAME_A) ||
803 frametype[j] == VISIR_FRAME_STATIC)
804 continue;
805 cpl_frame * frm =
806 cpl_frame_duplicate(irplib_framelist_get_const(rawfr, j));
807 cpl_frameset_insert(usedframes[i], frm);
808 }
809 skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[i],
810 mean[idx], CPL_TYPE_UNSPECIFIED,
811 RECIPE_STRING, procatg[i],
812 plist, NULL, visir_pipe_id, fn[i]));
813 }
814 }
815 else {
816 const char * procatg[] = {
817 VISIR_UTIL_REPACK_MEAN_A_ON_PROCATG,
818 VISIR_UTIL_REPACK_MEAN_A_OFF_PROCATG,
819 VISIR_UTIL_REPACK_MEAN_B_ON_PROCATG,
820 VISIR_UTIL_REPACK_MEAN_B_OFF_PROCATG,
821 };
822 const char * fn[] ={"visir_util_repack_mean_Aoff.fits",
823 "visir_util_repack_mean_Aon.fits",
824 "visir_util_repack_mean_Boff.fits",
825 "visir_util_repack_mean_Bon.fits"};
826 for (int i = 0; i < 4; i++) {
827 skip_if(cpl_image_divide_scalar(mean[i], nmean[i]));
828 for (int j = 0; j < irplib_framelist_get_size(rawfr); j++) {
829 if ((i < 2 && frametype[j] == VISIR_FRAME_A) ||
830 (i >= 2 && frametype[j] == VISIR_FRAME_B) ||
831 frametype[j] == VISIR_FRAME_STATIC)
832 continue;
833 cpl_frame * frm =
834 cpl_frame_duplicate(irplib_framelist_get_const(rawfr, j));
835 cpl_frameset_insert(usedframes[i], frm);
836 }
837 skip_if(irplib_dfs_save_image(framelist, parlist, usedframes[i],
838 mean[i], CPL_TYPE_UNSPECIFIED,
839 RECIPE_STRING, procatg[i],
840 plist, NULL, visir_pipe_id, fn[i]));
841 }
842 }
843
844 end_skip;
845
846 cpl_propertylist_delete(plist);
847 for (size_t i = 0; i < sizeof(usedframes)/sizeof(usedframes[0]); i++)
848 cpl_frameset_delete(usedframes[i]);
849
850 return cpl_error_get_code();
851}
852
853
854/*----------------------------------------------------------------------------*/
861/*----------------------------------------------------------------------------*/
862static int visir_util_repack(cpl_frameset * framelist,
863 const cpl_parameterlist * parlist)
864{
865 irplib_framelist * allframes = NULL;
866 irplib_framelist * rawframes = NULL;
867 int n;
868 cx_list * alist = cx_list_new();
869 cx_list * blist = cx_list_new();
870 cpl_image * bpm = NULL;
871 /* nonlinear pixel count */
872 cpl_image * nonlinear = NULL;
873 cpl_bivector * lintable = NULL;
874 cpl_image * mean[] = {NULL, NULL, NULL, NULL};
875 cpl_size nmean[] = {0, 0, 0, 0};
876 visir_sub_type bkgcorrect = VISIR_SUB_NOCORRECT;
877 visir_frame_type * frametype = NULL;
878 int planelimit = 0, nreduced = 0;
879
880 /* Identify the RAW and CALIB frames in the input frameset */
881 skip_if (visir_dfs_set_groups(framelist));
882 cpl_fits_set_mode(CPL_FITS_START_CACHING);
883
884 /* Objects observation */
885 allframes = irplib_framelist_cast(framelist);
886 skip_if(allframes == NULL);
887 rawframes = irplib_framelist_extract_regexp(allframes, "^("
888 VISIR_SPC_OBS_RAW
889 "|" VISIR_SPC_OBS_ECH_RAW
890 "|" VISIR_SPC_PHOT_RAW
891 "|" VISIR_SPC_PHOT_ECH_RAW
892 "|" VISIR_IMG_PHOT_RAW
893 "|" VISIR_IMG_COMBINE_CN
894 "|" VISIR_IMG_COMBINE_CN_BURST
895 "|" VISIR_IMG_COMBINE_CNJ
896 "|" VISIR_IMG_COMBINE_CNJ_BURST
897 "|" VISIR_IMG_COMBINE_CJ
898 "|" VISIR_IMG_COMBINE_NJ
899 "|" VISIR_IMG_COMBINE_NJ
900 "|" VISIR_IMG_CAL_PHOT
901 "|" VISIR_IMG_CAL_PHOT_BURST
902 "|" VISIR_IMG_CAL_OBJ
903 "|" VISIR_IMG_CAL_OBJ_BURST
904 "|" VISIR_ACQ_CNJ
905 "|" VISIR_SAM_CAL_N_RAW
906 "|" VISIR_SAM_CAL_CN_RAW
907 "|" VISIR_SAM_CAL_NJ_RAW
908 "|" VISIR_SAM_CAL_CNJ_RAW
909 "|" VISIR_CORO_CAL_CNJ_RAW
910 "|" VISIR_SAM_OBS_N_RAW
911 "|" VISIR_SAM_OBS_CN_RAW
912 "|" VISIR_SAM_OBS_NJ_RAW
913 "|" VISIR_SAM_OBS_CNJ_RAW
914 "|" VISIR_CORO_OBS_CNJ_RAW
915 "|" VISIR_IMG_BURST
916 "|" VISIR_UTIL_REPACK_RAW
917 "|" VISIR_UTIL_INPUTS_RAW
918 "|" VISIR_UTIL_DATA
919 ")$",
920 CPL_FALSE);
921 skip_if (rawframes == NULL);
922
923 n = irplib_framelist_get_size(rawframes);
924 planelimit = irplib_parameterlist_get_int(parlist, PACKAGE,
925 RECIPE_STRING, "planelimit");
926 frametype = cpl_malloc(sizeof(visir_frame_type) * n);
927
928 for (int i = 0; i < n; i++) {
929 repack_framestate * state = NULL;
930
931 state = repack_framestate_new(rawframes, i, parlist);
932
933 frametype[i] = state->is_a ? VISIR_FRAME_A : VISIR_FRAME_B;
934 if (state->is_a == CPL_TRUE)
935 cx_list_push_back(alist, state);
936 else
937 cx_list_push_back(blist, state);
938 skip_if(0);
939 }
940
941 error_if(cx_list_size(alist) == 0, CPL_ERROR_ILLEGAL_INPUT,
942 "No frames with nodpos A");
943 error_if(cx_list_size(blist) == 0, CPL_ERROR_ILLEGAL_INPUT,
944 "No frames with nodpos B");
945 if (cx_list_size(alist) != cx_list_size(blist)) {
946 if (cx_list_size(alist) > cx_list_size(alist)) {
947 cpl_msg_warning(cpl_func,
948 "Expecting even number of files, ignoring the last Nod A frame");
949 repack_framestate_delete(cx_list_pop_back(alist));
950 }
951 if (cx_list_size(blist) > cx_list_size(alist)) {
952 cpl_msg_warning(cpl_func,
953 "Expecting even number of files, ignoring the last Nod B frame");
954 repack_framestate_delete(cx_list_pop_back(blist));
955 }
956 }
957 error_if(cx_list_size(alist) != cx_list_size(blist),
958 CPL_ERROR_ILLEGAL_INPUT,
959 "Unequal number of A and B frames. A: %d, B: %d",
960 (int)cx_list_size(alist), (int)cx_list_size(blist));
961
962 {
963 cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_STATIC_MASK);
964 repack_framestate * state = ((repack_framestate*)cx_list_front(alist));
965 if (frm) {
966 bpm = visir_load_bpm(frm, state->datatype,
967 strncmp(visir_pfits_get_insmode(state->plist),
968 "SPC", 3) == 0);
969 if (bpm == NULL) {
970 cpl_msg_warning(cpl_func, "Loading of bad pixel map %s failed",
971 cpl_frame_get_filename(frm));
972 skip_if(0);
973 }
974 if (visir_data_is_aqu(state->datatype)) {
975 cpl_size win_nx = visir_pfits_get_win_nx(state->plist);
976 cpl_size win_ny = visir_pfits_get_win_ny(state->plist);
977 cpl_size start_x = visir_pfits_get_start_x(state->plist);
978 cpl_size start_y = visir_pfits_get_start_y(state->plist);
979 /* cut bpm to detector readout window */
980 cpl_size nx = cpl_image_get_size_x(bpm);
981 cpl_size ny = cpl_image_get_size_y(bpm);
982 cpl_size llx = 1, lly = 1, urx = nx, ury = ny;
983 if (win_nx > 0 && start_x > 0) {
984 llx = start_x;
985 urx = start_x + win_nx - 1;
986 }
987 if (win_ny > 0 && start_y > 0) {
988 lly = start_y;
989 ury = start_y + win_ny - 1;
990 }
991 if (llx != 1 || urx != nx || lly != 1 || ury != ny) {
992 cpl_image * tmp = cpl_image_extract(bpm, llx, lly, urx, ury);
993 cpl_image_delete(bpm);
994 bpm = tmp;
995 }
996 skip_if(0);
997 }
998 nonlinear = cpl_image_new(cpl_image_get_size_x(bpm),
999 cpl_image_get_size_y(bpm), CPL_TYPE_INT);
1000 }
1001 }
1002
1003 /* off by default, not accurate enough to work on low count signal data */
1004 if (irplib_parameterlist_get_bool(parlist, PACKAGE, RECIPE_STRING,
1005 "lincorrect")) {
1006 repack_framestate * st = cx_list_front(alist);
1007 cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_LIN);
1008 error_if(frm == NULL, CPL_ERROR_INCOMPATIBLE_INPUT,
1009 "Linearity correction requested by no input file tagged "
1010 "with LINEARITY_TABLE");
1011 if (visir_data_is_aqu(st->datatype)) {
1012 if (visir_is_img(st->plist)) {
1013 lintable = visir_load_lintable(frm, CPL_FALSE);
1014 }
1015 else if (visir_is_spc(st->plist)) {
1016 lintable = visir_load_lintable(frm, CPL_TRUE);
1017 }
1018 }
1019 }
1020
1021 while (1) {
1022 repack_framestate * fstatea = NULL, * fstateb = NULL;
1023 if (!cx_list_empty(alist))
1024 fstatea = (repack_framestate*)cx_list_pop_front(alist);
1025 if (!cx_list_empty(blist))
1026 fstateb = (repack_framestate*)cx_list_pop_front(blist);
1027
1028 if (fstatea == NULL || fstateb == NULL)
1029 break;
1030
1031 if (lintable) {
1032 fstatea->load_type = CPL_TYPE_FLOAT;
1033 fstateb->load_type = CPL_TYPE_FLOAT;
1034 }
1035
1036
1037 if (cpl_propertylist_has(fstatea->plist, "DATE-OBS") &&
1038 cpl_propertylist_has(fstatea->plist, "DATE") &&
1039 cpl_propertylist_has(fstateb->plist, "DATE-OBS") &&
1040 cpl_propertylist_has(fstateb->plist, "DATE")) {
1041 const char * str =
1042 cpl_propertylist_get_string(fstatea->plist, "DATE-OBS");
1043 irplib_wcs_mjd_from_string(&(fstatea->time_obsstart), str);
1044 str = cpl_propertylist_get_string(fstatea->plist, "DATE");
1045 irplib_wcs_mjd_from_string(&(fstatea->time_filewrite), str);
1046
1047 str = cpl_propertylist_get_string(fstateb->plist, "DATE-OBS");
1048 irplib_wcs_mjd_from_string(&(fstateb->time_obsstart), str);
1049 str = cpl_propertylist_get_string(fstateb->plist, "DATE");
1050 irplib_wcs_mjd_from_string(&(fstateb->time_filewrite), str);
1051 }
1052
1053 fstatea->time_min_obsstart = CX_MIN(fstatea->time_obsstart,
1054 fstateb->time_obsstart);
1055 fstateb->time_min_obsstart = CX_MIN(fstatea->time_obsstart,
1056 fstateb->time_obsstart);
1057 fstatea->time_max_filewrite = CX_MAX(fstatea->time_filewrite,
1058 fstateb->time_filewrite);
1059 fstateb->time_max_filewrite = CX_MAX(fstatea->time_filewrite,
1060 fstateb->time_filewrite);
1061
1062 for (int i = 0; i < 4; i++) {
1063 if (mean[i] == NULL) {
1064 mean[i] = cpl_image_new(fstatea->nx, fstatea->ny, CPL_TYPE_DOUBLE);
1065 }
1066 }
1067 skip_if(0);
1068 /* order is important, see store_means
1069 * updates are currently single threaded, if not add locks or allocate
1070 * on demand as there can be many nod pairs */
1071 fstatea->mean_on = mean[0];
1072 fstatea->mean_off = mean[1];
1073 fstateb->mean_on = mean[2];
1074 fstateb->mean_off = mean[3];
1075 fstatea->nmean_on = &nmean[0];
1076 fstatea->nmean_off = &nmean[1];
1077 fstateb->nmean_on = &nmean[2];
1078 fstateb->nmean_off = &nmean[3];
1079
1080 repack_framestate_adjust_planeend(fstatea, planelimit - nreduced);
1081 repack_framestate_adjust_planeend(fstateb, planelimit - nreduced);
1082
1083 /* run single threaded but allow two async reader tasks */
1084 OMP_PRAGMA(omp parallel num_threads(3))
1085 OMP_PRAGMA(omp single)
1086 {
1087 visir_util_repack_two(fstatea, fstateb, framelist, &bpm, nonlinear,
1088 lintable);
1089 }
1090 skip_if(0);
1091
1092 bkgcorrect = fstatea->bkgcorrect;
1093
1094 nreduced += fstatea->planeend - fstatea->planestart;
1095
1096 repack_framestate_delete(fstatea);
1097 repack_framestate_delete(fstateb);
1098 cpl_fits_set_mode(CPL_FITS_RESTART_CACHING);
1099 if (planelimit > 0 && nreduced >= planelimit)
1100 break;
1101 }
1102
1103 skip_if(store_means(bkgcorrect, rawframes, framelist, parlist,
1104 mean, nmean, frametype));
1105
1106 /* store bpm, skip if wrong size, there is no keyword one can
1107 * use in the OCA rules to associate DRS/AQU properly */
1108 if (bpm &&
1109 cpl_image_get_size_x(bpm) == cpl_image_get_size_x(mean[0]) &&
1110 cpl_image_get_size_y(bpm) == cpl_image_get_size_y(mean[0])) {
1111 /* apply all-nonlinear bpm */
1112 if (lintable) {
1113 cpl_size total = nmean[0] + nmean[1] + nmean[2] + nmean[3];
1114 cpl_mask * mnonlinear =
1115 cpl_mask_threshold_image_create(nonlinear, total * 0.9, total * 2);
1116 cpl_msg_info(cpl_func, "%lld (%.3g%%) pixels non-linear in 90%% "
1117 "of %lld frames", cpl_mask_count(mnonlinear),
1118 (100. * cpl_mask_count(mnonlinear)) /
1119 (cpl_mask_get_size_x(mnonlinear) *
1120 cpl_mask_get_size_y(mnonlinear)), total);
1121 /* disabled due to problems in HRS with so many bad pixels */
1122 //cpl_image_reject_from_mask(bpm, mnonlinear);
1123 //cpl_image_fill_rejected(bpm, 1);
1124 cpl_mask_delete(mnonlinear);
1125 }
1126
1127 cpl_frameset * usedframes = cpl_frameset_new();
1128 cpl_frameset_insert(usedframes, cpl_frame_duplicate
1129 (irplib_framelist_get_const(rawframes, 0)));
1130 irplib_dfs_save_image(framelist, parlist, usedframes, bpm,
1131 CPL_BPP_8_UNSIGNED, RECIPE_STRING,
1132 VISIR_CALIB_STATIC_MASK, NULL, NULL,
1133 visir_pipe_id, RECIPE_STRING "_bpm"
1134 CPL_DFS_FITS);
1135 cpl_frameset_delete(usedframes);
1136 }
1137 else if (bpm) {
1138 cpl_msg_warning(cpl_func, "Provided bad pixel mask does not have "
1139 "correct size, skipping");
1140 }
1141
1142 end_skip;
1143
1144 irplib_framelist_delete(allframes);
1145 irplib_framelist_delete(rawframes);
1146
1147 cx_list_destroy(alist, repack_framestate_delete);
1148 cx_list_destroy(blist, repack_framestate_delete);
1149 cpl_image_delete(bpm);
1150 cpl_image_delete(nonlinear);
1151 cpl_bivector_delete(lintable);
1152 cpl_free(frametype);
1153
1154 for (size_t i = 0; i < sizeof(mean)/sizeof(mean[0]); i++)
1155 cpl_image_delete(mean[i]);
1156
1157 return cpl_error_get_code();
1158}
1159
1160
1161/*----------------------------------------------------------------------------*/
1173/*----------------------------------------------------------------------------*/
1174static cpl_error_code
1175save_subtract_images(const repack_framestate * state,
1176 const cpl_imagelist * subtracted,
1177 const cpl_imagelist * jit_cor,
1178 const cpl_propertylist * plist)
1179{
1180 const cpl_size nsub = cpl_imagelist_get_size(subtracted);
1181 cpl_ensure_code(jit_cor == NULL || cpl_imagelist_get_size(jit_cor) == nsub,
1182 CPL_ERROR_ILLEGAL_INPUT);
1183
1184
1185 for (cpl_size j = 0; j < nsub; j++) {
1186 const cpl_image * img = cpl_imagelist_get_const(subtracted, j);
1187 cpl_io_type compress = 0;
1188 cpl_type save_type;
1189
1190 if (jit_cor)
1191 cpl_image_add(state->mean_on, cpl_imagelist_get_const(jit_cor, j));
1192 else
1193 cpl_image_add(state->mean_on, img);
1194 (*state->nmean_on)++;
1195
1196 if (state->compress && cpl_image_get_type(img) == CPL_TYPE_INT) {
1197 /* saves space even if it could be stored in a short */
1198 save_type = CPL_TYPE_INT;
1199 compress = CPL_IO_COMPRESS_RICE;
1200 }
1201 else
1202 save_type = get_optimum_save_type(img);
1203
1204 skip_if(cpl_image_save(img, state->onname, save_type,
1205 plist, CPL_IO_EXTEND | compress));
1206 }
1207
1208 end_skip;
1209
1210 return cpl_error_get_code();
1211}
1212
1213
1214/*----------------------------------------------------------------------------*/
1227/*----------------------------------------------------------------------------*/
1228static
1229cpl_error_code append_images(const repack_framestate * state,
1230 cpl_imagelist * images,
1231 const cpl_propertylist * plist,
1232 const cpl_boolean on)
1233{
1234 const char * name = on ? state->onname : state->offname;
1235 const cpl_size n = cpl_imagelist_get_size(images);
1236 cpl_image * mimg = on ? state->mean_on : state->mean_off;
1237 cpl_size * nmean = on ? state->nmean_on : state->nmean_off;
1238
1239 for (cpl_size j = 0; j < n; j++) {
1240 cpl_image * img = cpl_imagelist_get(images, j);
1241 cpl_io_type compress = 0;
1242 cpl_type save_type;
1243 /* drs burst data is not preprocessed so it needs flipping */
1244 if (state->datatype == VISIR_DATA_BURST) {
1245 cpl_image_multiply_scalar(img, -1);
1246 cpl_image_add_scalar(img, VISIR_HCYCLE_OFFSET);
1247 }
1248
1249 cpl_image_add(mimg, img);
1250 (*nmean)++;
1251
1252 if (state->compress && cpl_image_get_type(img) == CPL_TYPE_INT) {
1253 /* saves space even if it could be stored in a short */
1254 save_type = CPL_TYPE_INT;
1255 compress = CPL_IO_COMPRESS_RICE;
1256 }
1257 else
1258 save_type = get_optimum_save_type(img);
1259
1260 skip_if(cpl_image_save(img, name, save_type,
1261 plist, CPL_IO_EXTEND | compress));
1262 }
1263
1264 end_skip;
1265
1266 return cpl_error_get_code();
1267}
1268
1269
1270/*----------------------------------------------------------------------------*/
1278/*----------------------------------------------------------------------------*/
1279static cpl_error_code
1280cast_list(cpl_imagelist * l, const cpl_type type)
1281{
1282 const cpl_size n = cpl_imagelist_get_size(l);
1283 cpl_imagelist * cl;
1284 if (type == CPL_TYPE_UNSPECIFIED)
1285 return CPL_ERROR_NONE;
1286
1287 cl = cpl_imagelist_new();
1288
1289 for (cpl_size i = 0; i < n; i++) {
1290 cpl_image * img = cpl_imagelist_get(l, i);
1291 cpl_image * cast;
1292 if (cpl_image_get_type(img) == type) {
1293 cpl_imagelist_delete(cl);
1294 return cpl_error_get_code();
1295 }
1296 cast = cpl_image_cast(img, type);
1297 cpl_imagelist_set(cl, cast, i);
1298 }
1299
1300 cpl_imagelist_empty(l);
1301 for (cpl_size i = 0; i < n; i++)
1302 cpl_imagelist_set(l, cpl_imagelist_get(cl, i), i);
1303
1304 visir_imagelist_unwrap(cl);
1305
1306 return cpl_error_get_code();
1307}
1308
1309
1310/*----------------------------------------------------------------------------*/
1321/*----------------------------------------------------------------------------*/
1322static
1323cpl_error_code load_chunk(cpl_imagelist * on, cpl_imagelist * off,
1324 const repack_framestate * state,
1325 const int pstart, const int pend)
1326{
1327 /* sort a chunk of data */
1328 if (state->datatype == VISIR_DATA_AQU_HCYCLE) {
1329 const cpl_frame * frame = irplib_framelist_get_const(state->rawframes,
1330 state->iframe);
1331 const char * fn = cpl_frame_get_filename(frame);
1332 skip_if(0);
1333
1334 for (cpl_size iext = pstart + 1; iext < pend + 1; iext++) {
1335 const char * ftype;
1336 cpl_propertylist * plist =
1337 cpl_propertylist_load_regexp(fn, iext,
1338 VISIR_PFITS_STRING_FRAME_TYPE,
1339 CPL_FALSE);
1340 skip_if(plist == NULL);
1341 ftype = visir_pfits_get_frame_type(plist);
1342 error_if(ftype == NULL, CPL_ERROR_DATA_NOT_FOUND, "ESO DET FRAM "
1343 "TYPE keyword missing in extension %d of file %s",
1344 (int)iext, fn);
1345 if (strcmp(ftype, "HCYCLE1") == 0)
1346 cpl_imagelist_set(on,
1347 cpl_image_load(fn, CPL_TYPE_FLOAT, 0, iext),
1348 cpl_imagelist_get_size(on));
1349 else if (strcmp(ftype, "HCYCLE2") == 0)
1350 cpl_imagelist_set(off,
1351 cpl_image_load(fn, CPL_TYPE_FLOAT, 0, iext),
1352 cpl_imagelist_get_size(off));
1353 else {
1354 cpl_msg_debug(cpl_func, "Skipping \"%s\" frame type", ftype);
1355 }
1356 cpl_propertylist_delete(plist);
1357 skip_if(0);
1358 }
1359 }
1360 else if (state->datatype == VISIR_DATA_CUBE2) {
1361 skip_if(visir_load_cube2_split(on, off, state->rawframes,
1362 state->iframe, pstart, pend));
1363
1364 }
1365 else if (visir_data_is_burst(state->datatype)) {
1366 const cpl_frame * frame = irplib_framelist_get_const(state->rawframes,
1367 state->iframe);
1368 /* TODO rework loading from first extension */
1369 if (state->datatype == VISIR_DATA_AQU_BURST_EXT) {
1370 cpl_propertylist * plist =
1371 cpl_propertylist_load(cpl_frame_get_filename(frame), 1);
1372 /* hack to get it to load from 1. extension */
1373 cpl_propertylist_update_int(plist, "ZNAXIS3", visir_pfits_get_naxis3(plist));
1374 skip_if(visir_load_burst_aqu(on, off, frame, plist,
1375 state->halfcycle , pstart, pend));
1376 cpl_propertylist_delete(plist);
1377 }
1378 else {
1379 skip_if(visir_load_burst(on, off, frame,
1380 state->plist, state->to_off,
1381 state->halfcycle, pstart, pend,
1382 state->trimlow, state->trimhigh));
1383 }
1384 }
1385 else
1386 error_if(1, CPL_ERROR_ILLEGAL_INPUT, "invalid data tag");
1387
1388 /* FIXME: pass load type to the loading functions instead */
1389 cast_list(on, state->load_type);
1390 cast_list(off, state->load_type);
1391
1392 end_skip;
1393
1394 return cpl_error_get_code();
1395}
1396
1397
1398/*----------------------------------------------------------------------------*/
1408/*----------------------------------------------------------------------------*/
1409static cpl_size
1410equalize_lists(cpl_imagelist * a, cpl_imagelist * b)
1411{
1412 const cpl_size na = cpl_imagelist_get_size(a);
1413 const cpl_size nb = cpl_imagelist_get_size(b);
1414
1415 if (na != nb)
1416 cpl_msg_warning(cpl_func, "Unequal number of planes in on "
1417 "and off list: #on %d, #off %d. Skipping %ld"
1418 " planes.", (int)na, (int)nb, labs(na - nb));
1419
1420 if (na > nb) {
1421 for (cpl_size i = 0; i < na - nb; i++)
1422 cpl_image_delete(visir_imagelist_pop(a));
1423 }
1424 else if (na < nb) {
1425 for (cpl_size i = 0; i < nb - na; i++)
1426 cpl_image_delete(visir_imagelist_pop(b));
1427 }
1428
1429 if (cpl_error_get_code() != CPL_ERROR_NONE)
1430 return -1;
1431 return cpl_imagelist_get_size(a);
1432}
1433
1434static cpl_error_code
1435check_and_fix_cd_wcs(const repack_framestate * state, cpl_propertylist * plist)
1436{
1437 cpl_errorstate cleanstate = cpl_errorstate_get();
1438 /* move wcs from extension to primary as needed by detect_shift
1439 * but only if primary does not already have wcs as then the ext is wrong
1440 * (aqu data < 2016.02) */
1441 if (state->wcs && !cpl_propertylist_has(plist, "CRVAL1")) {
1442 const cpl_matrix * m;
1443 const cpl_array * v = cpl_wcs_get_crval(state->wcs);
1444 cpl_propertylist_update_double(plist, "CRVAL1",
1445 cpl_array_get_double(v, 0, NULL));
1446 cpl_propertylist_update_double(plist, "CRVAL2",
1447 cpl_array_get_double(v, 1, NULL));
1448 v = cpl_wcs_get_ctype(state->wcs);
1449 cpl_propertylist_update_string(plist, "CTYPE1",
1450 cpl_array_get_string(v, 0));
1451 cpl_propertylist_update_string(plist, "CTYPE2",
1452 cpl_array_get_string(v, 1));
1453 v = cpl_wcs_get_crpix(state->wcs);
1454 cpl_propertylist_update_double(plist, "CRPIX1",
1455 cpl_array_get_double(v, 0, NULL));
1456 cpl_propertylist_update_double(plist, "CRPIX2",
1457 cpl_array_get_double(v, 1, NULL));
1458 m = cpl_wcs_get_cd(state->wcs);
1459 cpl_propertylist_update_double(plist, "CD1_1",
1460 cpl_matrix_get(m, 0, 0));
1461 cpl_propertylist_append_double(plist, "CD1_2",
1462 cpl_matrix_get(m, 0, 1));
1463 cpl_propertylist_update_double(plist, "CD2_1",
1464 cpl_matrix_get(m, 1, 0));
1465 cpl_propertylist_update_double(plist, "CD2_2",
1466 cpl_matrix_get(m, 1, 1));
1467 v = cpl_wcs_get_cunit(state->wcs);
1468 cpl_propertylist_update_string(plist, "CUNIT1",
1469 cpl_array_get_string(v, 0));
1470 cpl_propertylist_update_string(plist, "CUNIT2",
1471 cpl_array_get_string(v, 1));
1472 }
1473 cpl_boolean invalid =
1474 (cpl_propertylist_get_double(plist, "CD1_1") == 0. &&
1475 cpl_propertylist_get_double(plist, "CD1_2") == 0.) ||
1476 (cpl_propertylist_get_double(plist, "CD2_1") == 0. &&
1477 cpl_propertylist_get_double(plist, "CD2_2") == 0.);
1478 invalid = invalid || (cpl_error_get_code() != CPL_ERROR_NONE);
1479 cpl_errorstate_set(cleanstate);
1480
1481 if (invalid) {
1482 double pfov = visir_pfits_get_pixscale(state->plist);
1483 cpl_msg_warning(cpl_func, "CDX_Y WCS key missing or zero, fixing to "
1484 "%g (pfov) / 3600", pfov);
1485 cpl_propertylist_update_double(plist, "CD1_1", -pfov / 3600.);
1486 cpl_propertylist_update_double(plist, "CD1_2", 0.);
1487 cpl_propertylist_update_double(plist, "CD2_1", 0.);
1488 cpl_propertylist_update_double(plist, "CD2_2", pfov / 3600.);
1489 }
1490
1491 return cpl_error_get_code();
1492}
1493
1494
1495/*----------------------------------------------------------------------------*/
1509/*----------------------------------------------------------------------------*/
1510static cpl_error_code
1511prepare_output(const repack_framestate * state, cpl_frameset * products,
1512 double bg_mean,
1513 enum nodtype_e nodtype)
1514{
1515 cpl_frameset * usedframes = cpl_frameset_new();
1516 cpl_propertylist * onlist = cpl_propertylist_new();
1517 cpl_propertylist * offlist = cpl_propertylist_new();
1518
1519 /* Need a copy of the WCS cards for subsequent beam stacking */
1520 bug_if(cpl_propertylist_copy_property_regexp(onlist, state->plist, "^("
1521 IRPLIB_PFITS_WCS_REGEXP ")$",
1522 CPL_FALSE));
1523
1524 bug_if(cpl_propertylist_append_double(onlist, VISIR_DRS_CONAD, state->conad));
1525 bug_if(cpl_propertylist_set_comment(onlist, VISIR_DRS_CONAD, "Default "
1526 "single frame value: " IRPLIB_STRINGIFY
1527 (VISIR_UTIL_REPACK_CONAD)));
1528
1529 bug_if(cpl_propertylist_append_double(onlist, VISIR_PFITS_DOUBLE_PIXSPACE,
1530 state->pxspace));
1531
1532 if (nodtype == ANOD || nodtype == ABNOD) {
1533 const char * keyx = nodtype == ABNOD ?
1534 VISIR_DRS_CUMOFFSETXA : VISIR_DRS_CUMOFFSETX;
1535 const char * keyy = nodtype == ABNOD ?
1536 VISIR_DRS_CUMOFFSETYA : VISIR_DRS_CUMOFFSETY;
1537 cpl_propertylist_append_double(onlist, keyx, state->offsetx);
1538 cpl_propertylist_append_double(onlist, keyy, state->offsety);
1539 cpl_propertylist_set_comment(onlist, keyx, state->comoffx);
1540 cpl_propertylist_set_comment(onlist, keyy, state->comoffy);
1541 }
1542
1543 if (nodtype == BNOD || nodtype == ABNOD) {
1544 const char * keyx = nodtype == ABNOD ?
1545 VISIR_DRS_CUMOFFSETXB : VISIR_DRS_CUMOFFSETX;
1546 const char * keyy = nodtype == ABNOD ?
1547 VISIR_DRS_CUMOFFSETYB : VISIR_DRS_CUMOFFSETY;
1548 cpl_propertylist_append_double(onlist, keyx, state->offsetx);
1549 cpl_propertylist_append_double(onlist, keyy, state->offsety);
1550 cpl_propertylist_set_comment(onlist, keyx, state->comoffx);
1551 cpl_propertylist_set_comment(onlist, keyy, state->comoffy);
1552 }
1553
1554 bug_if(cpl_propertylist_append(offlist, onlist));
1555
1556 cpl_propertylist_append_int(onlist, "ESO DRS DTYPE", state->datatype);
1557 cpl_propertylist_append_int(offlist, "ESO DRS DTYPE", state->datatype);
1558 cpl_propertylist_append_string(offlist, "ESO DRS CATG", state->tag);
1559 cpl_propertylist_append_string(onlist, "ESO DRS CATG", state->tag);
1560 bug_if(0);
1561
1562 if (state->time_max_filewrite > 0) {
1563 cpl_propertylist_append_double(onlist, "ESO DRS DATE",
1564 state->time_max_filewrite);
1565 cpl_propertylist_append_double(offlist, "ESO DRS DATE",
1566 state->time_max_filewrite);
1567 cpl_propertylist_append_double(onlist, "ESO DRS DATE-OBS",
1568 state->time_min_obsstart);
1569 cpl_propertylist_append_double(offlist, "ESO DRS DATE-OBS",
1570 state->time_min_obsstart);
1571 }
1572 else {
1573 /* always write so following recipes don't need to check */
1574 cpl_propertylist_append_double(onlist, "ESO DRS DATE", -1.);
1575 cpl_propertylist_append_double(offlist, "ESO DRS DATE", -1.);
1576 cpl_propertylist_append_double(onlist, "ESO DRS DATE-OBS", -1.);
1577 cpl_propertylist_append_double(offlist, "ESO DRS DATE-OBS", -1.);
1578 }
1579 if (state->normalize) {
1580 cpl_propertylist_append_string(onlist, "BUNIT", "adu / s");
1581 cpl_propertylist_append_string(offlist, "BUNIT", "adu / s");
1582 }
1583 else {
1584 cpl_propertylist_append_string(onlist, "BUNIT", "adu");
1585 cpl_propertylist_append_string(offlist, "BUNIT", "adu");
1586 }
1587
1588 /* record all inputs as used, even though not all are actually used for
1589 * each output but not recording them here would then require merging input
1590 * headers in subsequent recipes which do join the output files into a
1591 * single product */
1592 for (int i = 0; i < state->nframes; i++) {
1593 const cpl_frame * frm =
1594 irplib_framelist_get_const(state->rawframes, i);
1595 bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm)));
1596 }
1597
1598 skip_if(check_and_fix_cd_wcs(state, onlist));
1599 skip_if(check_and_fix_cd_wcs(state, offlist));
1600
1601 if (products == NULL)
1602 products = cpl_frameset_new();
1603
1604
1605 if (state->datatype == VISIR_DATA_CUBE2) {
1606 skip_if(visir_qc_append_background(onlist, state->rawframes, 0, 0));
1607 skip_if(visir_qc_append_background(offlist, state->rawframes, 0, 0));
1608 }
1609 else if (!isnan(bg_mean)) {
1610 bug_if (cpl_propertylist_append_double(onlist, "ESO QC BACKGD MEAN",
1611 bg_mean));
1612 bug_if (cpl_propertylist_append_double(offlist, "ESO QC BACKGD MEAN",
1613 bg_mean));
1614 }
1615
1616 if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
1617 state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
1618 skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1619 RECIPE_STRING, state->procatgon,
1620 onlist, state->remregexp,
1621 visir_pipe_id,
1622 state->onname));
1623 } else {
1624 /* Save each image in a separate extension */
1625 /* Do not save data in the primary data unit (DFS10475) */
1626 skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1627 RECIPE_STRING, state->procatgon,
1628 onlist, state->remregexp,
1629 visir_pipe_id, state->onname));
1630 skip_if(irplib_dfs_save_propertylist(products, state->parlist, usedframes,
1631 RECIPE_STRING, state->procatgoff,
1632 offlist, state->remregexp,
1633 visir_pipe_id, state->offname));
1634 }
1635
1636 end_skip;
1637 cpl_frameset_delete(usedframes);
1638 cpl_propertylist_delete(onlist);
1639 cpl_propertylist_delete(offlist);
1640
1641 return cpl_error_get_code();
1642}
1643
1644
1645static cpl_imagelist *
1646correct_jitter(const cpl_imagelist * imgs, double dx, double dy)
1647{
1648 cpl_imagelist * jit_cor = cpl_imagelist_new();
1649 cpl_msg_info(cpl_func, "Correcting jitter, x: %g y: %g", dx, dy);
1650 for (cpl_size i = 0, n = cpl_imagelist_get_size(imgs); i < n; i++) {
1651 cpl_image * img = cpl_image_duplicate(cpl_imagelist_get_const(imgs, i));
1652 /* only used for mean which in turn is only used for
1653 * initial beam guess, pixel precision sufficient */
1654 skip_if(cpl_image_shift(img,
1655 -visir_round_to_int(dx),
1656 -visir_round_to_int(dy)));
1657 cpl_imagelist_set(jit_cor, img, i);
1658 }
1659
1660 end_skip;
1661
1662 return jit_cor;
1663}
1664
1665
1666/* ---------------------------------------------------------------------------*/
1682/* ---------------------------------------------------------------------------*/
1683static cpl_imagelist *
1684chop_correct(const repack_framestate * state,
1685 cpl_imagelist * on,
1686 cpl_imagelist * off,
1687 const cpl_boolean last_chunk)
1688{
1689 cpl_imagelist * sub;
1690 /* burst data is not preprocessed and needs sign flipping */
1691 cpl_imagelist * subt;
1692
1693 subt = state->datatype == VISIR_DATA_BURST ? on : off;
1694 sub = state->datatype == VISIR_DATA_BURST ? off : on;
1695
1696 if (last_chunk)
1697 equalize_lists(on, off);
1698
1699 cpl_imagelist_subtract(sub, subt);
1700 if (state->normalize) {
1701 cpl_imagelist_multiply_scalar(sub, 1. / (state->dit * 2));
1702 }
1703 /* empty now to reduce memory usage */
1704 cpl_imagelist_empty(subt);
1705
1706 return sub;
1707}
1708
1709/* ---------------------------------------------------------------------------*/
1720/* ---------------------------------------------------------------------------*/
1721static double
1722compute_qc_median(const cpl_imagelist * list, cpl_size * nimages,
1723 cpl_size llx, cpl_size lly, cpl_size urx, cpl_size ury)
1724{
1725 double sum = 0.;
1726 for (cpl_size i = 0; i < cpl_imagelist_get_size(list); i++) {
1727 const cpl_image * img = cpl_imagelist_get_const(list, i);
1728 llx = CPL_MIN(cpl_image_get_size_x(img), llx);
1729 lly = CPL_MIN(cpl_image_get_size_y(img), lly);
1730 urx = CPL_MIN(cpl_image_get_size_x(img), urx);
1731 ury = CPL_MIN(cpl_image_get_size_y(img), ury);
1732 sum += cpl_image_get_median_window(img, llx, lly, urx, ury);
1733 }
1734
1735 *nimages += cpl_imagelist_get_size(list);
1736
1737 return sum;
1738}
1739
1740/* ---------------------------------------------------------------------------*/
1748/* ---------------------------------------------------------------------------*/
1749static cpl_error_code
1750correct_linearity(const repack_framestate * state,
1751 cpl_imagelist * images,
1752 const cpl_bivector * lintable, cpl_image * nonlinear)
1753{
1754 if (lintable == NULL || !visir_data_is_aqu(state->datatype)) {
1755 return CPL_ERROR_NONE;
1756 }
1757 for (size_t i = 0; i < (size_t)cpl_imagelist_get_size(images); i++) {
1758 cpl_image * img = cpl_imagelist_get(images, i);
1759 cpl_ensure_code(cpl_image_get_type(img) == CPL_TYPE_FLOAT ||
1760 cpl_image_get_type(img) == CPL_TYPE_DOUBLE,
1761 CPL_ERROR_ILLEGAL_INPUT);
1762 cpl_image * dvals = cpl_image_cast(img, CPL_TYPE_DOUBLE);
1763 cpl_image * corvals = visir_linintp_values(dvals, lintable);
1764 if (nonlinear) {
1765 cpl_image * bad =
1766 cpl_image_new_from_mask(cpl_image_get_bpm(corvals));
1767 cpl_image_add(nonlinear, bad);
1768 cpl_image_delete(bad);
1769 }
1770 cpl_image_divide(img, corvals);
1771 cpl_image_delete(dvals);
1772 cpl_image_delete(corvals);
1773 }
1774
1775 return cpl_error_get_code();
1776}
1777
1778/* ---------------------------------------------------------------------------*/
1790/* ---------------------------------------------------------------------------*/
1791static cpl_error_code
1792save_corrected(const repack_framestate * fstatea,
1793 const repack_framestate * fstateb,
1794 cpl_imagelist * aon,
1795 cpl_imagelist * aoff,
1796 cpl_imagelist * bon,
1797 cpl_imagelist * boff,
1798 const cpl_boolean last_chunk)
1799{
1800 cpl_imagelist * chop_corrected_a, * chop_corrected_b;
1801 const repack_framestate * state = fstatea;
1802
1803 chop_corrected_a = chop_correct(fstatea, aon, aoff, last_chunk);
1804 chop_corrected_b = chop_correct(fstateb, bon, boff, last_chunk);
1805
1806 if (cpl_error_get_code())
1807 return cpl_error_get_code();
1808
1809 if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT) {
1810 /* only chop correction requested */
1811 save_subtract_images(fstatea, chop_corrected_a, NULL, NULL);
1812 save_subtract_images(fstateb, chop_corrected_b, NULL, NULL);
1813 }
1814 else {
1815 /* perform nod and jitter correction
1816 * jitter is only used for the mean used for object detection pivoting
1817 * correction is done in later recipes */
1818 cpl_imagelist * jca = NULL;
1819 cpl_imagelist * jcb = NULL;
1820
1821 if (last_chunk)
1822 equalize_lists(chop_corrected_a, chop_corrected_b);
1823
1824 if (state->normalize) {
1825 cpl_imagelist_divide_scalar(chop_corrected_a, 2);
1826 cpl_imagelist_divide_scalar(chop_corrected_b, 2);
1827 }
1828
1829 if (visir_round_to_int(fstatea->offsetx) != 0 ||
1830 visir_round_to_int(fstatea->offsety) != 0 ||
1831 visir_round_to_int(fstateb->offsetx) != 0 ||
1832 visir_round_to_int(fstateb->offsety) != 0) {
1833 jca = correct_jitter(chop_corrected_a, fstatea->offsetx, fstatea->offsety);
1834 jcb = correct_jitter(chop_corrected_b, fstateb->offsetx, fstateb->offsety);
1835 cpl_imagelist_subtract(jca, jcb);
1836 }
1837
1838 cpl_imagelist_subtract(chop_corrected_a, chop_corrected_b);
1839
1840 save_subtract_images(state, chop_corrected_a, jca, NULL);
1841 cpl_imagelist_delete(jcb);
1842 cpl_imagelist_delete(jca);
1843 }
1844
1845 return cpl_error_get_code();
1846}
1847
1848static cpl_error_code
1849save_uncorrected(const repack_framestate * fstatea,
1850 const repack_framestate * fstateb,
1851 cpl_imagelist * aon,
1852 cpl_imagelist * aoff,
1853 cpl_imagelist * bon,
1854 cpl_imagelist * boff)
1855{
1856 const repack_framestate * state = fstatea;
1857 if (state->normalize) {
1858 cpl_imagelist_multiply_scalar(aon, 1. / fstatea->dit);
1859 cpl_imagelist_multiply_scalar(aoff, 1. / fstatea->dit);
1860 cpl_imagelist_multiply_scalar(bon, 1. / fstateb->dit);
1861 cpl_imagelist_multiply_scalar(boff, 1. / fstateb->dit);
1862 }
1863 append_images(fstatea, aon, NULL, CPL_TRUE);
1864 append_images(fstatea, aoff, NULL, CPL_FALSE);
1865 append_images(fstateb, bon, NULL, CPL_TRUE);
1866 append_images(fstateb, boff, NULL, CPL_FALSE);
1867 cpl_imagelist_empty(aon);
1868 cpl_imagelist_empty(aoff);
1869 cpl_imagelist_empty(bon);
1870 cpl_imagelist_empty(boff);
1871
1872 return cpl_error_get_code();
1873}
1874
1875#define VISIR_SWAP(a, b)\
1876 do {\
1877 cpl_imagelist * tmp = a; \
1878 a = b; \
1879 b = tmp; \
1880 } while (0)
1881
1882/*----------------------------------------------------------------------------*/
1893/*----------------------------------------------------------------------------*/
1894static cpl_error_code
1895visir_util_repack_two(const repack_framestate * fstatea,
1896 const repack_framestate * fstateb,
1897 cpl_frameset * products,
1898 cpl_image ** pbpm,
1899 cpl_image * nonlinear,
1900 const cpl_bivector * lintable)
1901{
1902 /* process the data */
1903 const repack_framestate * state = fstatea;
1904 int chunksize = 200;
1905 cpl_imagelist * aon = NULL;
1906 cpl_imagelist * aoff = NULL;
1907 cpl_imagelist * bon = NULL;
1908 cpl_imagelist * boff = NULL;
1909 /* preload lists */
1910 cpl_imagelist * aon_next = NULL;
1911 cpl_imagelist * aoff_next = NULL;
1912 cpl_imagelist * bon_next = NULL;
1913 cpl_imagelist * boff_next = NULL;
1914
1915 double qc_bkg_sum = 0.;
1916 cpl_size qc_bkg_count = 0;
1917
1918 /* chunksize must be a multiple of the period to avoid having a
1919 * #on-#off difference larger than chunksize */
1920
1921 if (visir_data_is_burst(state->datatype)) {
1922 chunksize = 2 * state->halfcycle *
1923 CX_MAX(1, chunksize / (2 * state->halfcycle));
1924 /* parameter sanity check, ensures equal length A and B lists */
1925 error_if(state->planeend - state->planestart < 2 * state->halfcycle,
1926 CPL_ERROR_ILLEGAL_INPUT,
1927 "Number of planes to be repacked must be larger than "
1928 "a full cycle of %d planes.", state->halfcycle * 2);
1929 }
1930
1931
1932 aon = cpl_imagelist_new();
1933 aoff = cpl_imagelist_new();
1934 bon = cpl_imagelist_new();
1935 boff = cpl_imagelist_new();
1936
1937
1938 /* load the first chunk of data to fill QC parameters*/
1939 if (state->datatype != VISIR_DATA_AQU_INT) {
1940 const int pstart = state->planestart;
1941 const int pend = CX_MIN(pstart + chunksize, state->planeend);
1942 OMP3_PRAGMA(omp task) {
1943 load_chunk(aon, aoff, fstatea, pstart, pend);
1944 }
1945 OMP3_PRAGMA(omp task) {
1946 load_chunk(bon, boff, fstateb, pstart, pend);
1947 }
1948 OMP3_PRAGMA(omp taskwait);
1949 }
1950 if ((visir_data_is_aqu(state->datatype) ||
1951 visir_data_is_burst(state->datatype)) &&
1952 state->datatype != VISIR_DATA_AQU_INT) {
1953 /* cube2 is computed in prepare_output */
1954 cpl_size llx = 1, lly = 1, urx = 1000000, ury = 1000000;
1955 /* limit wavelength (PIPE-6744) aqu LR wavelength decreasing in x */
1956 if (state->resol == VISIR_SPC_R_LRP) {
1957 llx = 1024 - VISIR_AQU_APPROX_WLEN13;
1958 urx = 1024 - VISIR_AQU_APPROX_WLEN8;
1959 }
1960 qc_bkg_sum += compute_qc_median(aon, &qc_bkg_count,
1961 llx, lly, urx, ury);
1962 qc_bkg_sum += compute_qc_median(aoff, &qc_bkg_count,
1963 llx, lly, urx, ury);
1964 qc_bkg_sum += compute_qc_median(bon, &qc_bkg_count,
1965 llx, lly, urx, ury);
1966 qc_bkg_sum += compute_qc_median(boff, &qc_bkg_count,
1967 llx, lly, urx, ury);
1968 }
1969
1970
1971 if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
1972 state->bkgcorrect == VISIR_SUB_NOCORRECT) {
1973 prepare_output(fstatea, products,
1974 qc_bkg_sum / qc_bkg_count, ANOD);
1975 prepare_output(fstateb, products,
1976 qc_bkg_sum / qc_bkg_count, BNOD);
1977 }
1978 else if (state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
1979 prepare_output(state, products,
1980 qc_bkg_sum / qc_bkg_count, ABNOD);
1981 }
1982 skip_if(0);
1983
1984 if (state->datatype == VISIR_DATA_AQU_INT) {
1985 cpl_imagelist * a = cpl_imagelist_new();
1986 cpl_imagelist * b = cpl_imagelist_new();
1987 cpl_imagelist * jca = NULL;
1988 cpl_imagelist * jcb = NULL;
1989 const cpl_frame * frame;
1990 const char * filename;
1991 cpl_image * img;
1992
1993 frame = irplib_framelist_get_const(fstatea->rawframes,
1994 fstatea->iframe);
1995 filename = cpl_frame_get_filename(frame);
1996 img = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 1);
1997 cpl_imagelist_set(a, img, cpl_imagelist_get_size(a));
1998
1999 frame = irplib_framelist_get_const(fstateb->rawframes,
2000 fstateb->iframe);
2001 filename = cpl_frame_get_filename(frame);
2002 img = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 1);
2003 cpl_imagelist_set(b, img, cpl_imagelist_get_size(b));
2004
2005 if (fstatea->normalize) {
2006 cpl_imagelist_multiply_scalar(a, 1. / (state->dit * 4));
2007 cpl_imagelist_multiply_scalar(b, 1. / (state->dit * 4));
2008 }
2009
2010 if (visir_round_to_int(fstatea->offsetx) != 0 ||
2011 visir_round_to_int(fstatea->offsety) != 0 ||
2012 visir_round_to_int(fstateb->offsetx) != 0 ||
2013 visir_round_to_int(fstateb->offsety) != 0) {
2014 jca = correct_jitter(a, fstatea->offsetx, fstatea->offsety);
2015 jcb = correct_jitter(b, fstateb->offsetx, fstateb->offsety);
2016 cpl_imagelist_subtract(jca, jcb);
2017 cpl_imagelist_subtract(a, b);
2018 save_subtract_images(fstatea, a, jca, NULL);
2019 }
2020 else {
2021 cpl_imagelist_subtract(a, b);
2022 save_subtract_images(fstatea, a, jca, NULL);
2023 }
2024
2025 cpl_imagelist_delete(a);
2026 cpl_imagelist_delete(b);
2027 cpl_imagelist_delete(jca);
2028 cpl_imagelist_delete(jcb);
2029 }
2030
2031 aon_next = cpl_imagelist_new();
2032 aoff_next = cpl_imagelist_new();
2033 bon_next = cpl_imagelist_new();
2034 boff_next = cpl_imagelist_new();
2035
2036 size_t nbytes = 0;
2037 double t1 = 0;
2038 double t_loading = 0;
2039 double t_block_in = 0;
2040
2041 for (int pstart = state->planestart;
2042 pstart < state->planeend + chunksize - 1 &&
2043 state->datatype != VISIR_DATA_AQU_INT;
2044 pstart += chunksize) {
2045 const int pend = CX_MIN(pstart + chunksize, state->planeend);
2046
2047 if (pstart >= pend)
2048 break;
2049
2050
2051 /* the process->save part has a throughput of ~200-400MB/s
2052 * SSDs can match this so by performing one preload in parallel with
2053 * processing we archive ideal performance also on these systems
2054 * first chunk already loaded synchronously */
2055 if (pstart != state->planestart) {
2056 /* empty the currently worked on lists
2057 * may have already been emptied by save_ functions */
2058 cpl_imagelist_empty(aon);
2059 cpl_imagelist_empty(aoff);
2060 cpl_imagelist_empty(bon);
2061 cpl_imagelist_empty(boff);
2062
2063 /* wait for preload of last iteration to finish */
2064 double t2 = cpl_test_get_walltime();
2065 OMP3_PRAGMA(omp taskwait);
2066 t_block_in += cpl_test_get_walltime() - t2;
2067 /* swap empty current and full preload lists */
2068 VISIR_SWAP(aon, aon_next);
2069 VISIR_SWAP(aoff, aoff_next);
2070 VISIR_SWAP(bon, bon_next);
2071 VISIR_SWAP(boff, boff_next);
2072 t_loading += cpl_test_get_walltime() - t1;
2073 if (cpl_imagelist_get_size(aon) > 0) {
2074 cpl_image * img = cpl_imagelist_get(aon, 0);
2075 nbytes += 4 * cpl_imagelist_get_size(aon) *
2076 cpl_image_get_size_x(img) * cpl_image_get_size_y(img) *
2077 cpl_type_get_sizeof(cpl_image_get_type(img));
2078 }
2079 }
2080
2081 /* preload the next chunk asynchronous */
2082 if (pend < state->planeend) {
2083 int pend_next = CX_MIN(pend + chunksize, state->planeend);
2084 t1 = cpl_test_get_walltime();
2085 OMP3_PRAGMA(omp task) {
2086 load_chunk(aon_next, aoff_next, fstatea, pend, pend_next);
2087 }
2088 OMP3_PRAGMA(omp task) {
2089 load_chunk(bon_next, boff_next, fstateb, pend, pend_next);
2090 }
2091 }
2092
2093 if (*pbpm == NULL && state->datatype == VISIR_DATA_CUBE2) {
2094
2095 *pbpm = cpl_image_duplicate(cpl_imagelist_get(aon, 0));
2096 cpl_image_threshold(*pbpm, state->bpmthresh,
2097 state->bpmthresh, 0.0, 1.0);
2098 skip_if(0);
2099 }
2100
2101 correct_linearity(fstatea, aon, lintable, nonlinear);
2102 correct_linearity(fstatea, aoff, lintable, nonlinear);
2103 correct_linearity(fstateb, bon, lintable, nonlinear);
2104 correct_linearity(fstateb, boff, lintable, nonlinear);
2105
2106 if (state->bkgcorrect == VISIR_SUB_CHOPCORRECT ||
2107 state->bkgcorrect == VISIR_SUB_CHOPNODCORRECT) {
2108 cpl_boolean last_chunk = pstart + chunksize >= pend;
2109 skip_if(save_corrected(fstatea, fstateb,
2110 aon, aoff, bon, boff, last_chunk));
2111 }
2112 else {
2113 skip_if(save_uncorrected(fstatea, fstateb, aon, aoff, bon, boff));
2114 }
2115 }
2116 if (t_loading > 0) {
2117 cpl_msg_info(cpl_func, "Loading data with, %.3g MB/s.",
2118 (nbytes / 1024. / 1024.) / t_loading);
2119 cpl_msg_info(cpl_func, "Time spent waiting on input: %3.3gs",
2120 t_block_in);
2121 }
2122
2123 end_skip;
2124 cpl_imagelist_delete(aon);
2125 cpl_imagelist_delete(aoff);
2126 cpl_imagelist_delete(bon);
2127 cpl_imagelist_delete(boff);
2128 cpl_imagelist_delete(aon_next);
2129 cpl_imagelist_delete(aoff_next);
2130 cpl_imagelist_delete(bon_next);
2131 cpl_imagelist_delete(boff_next);
2132
2133 return cpl_error_get_code();
2134}
2135
2136
2137/*----------------------------------------------------------------------------*/
2146/*----------------------------------------------------------------------------*/
2147static cpl_error_code VISIR_ATTR_UNUSED
2148visir_util_repack_check(const cpl_image * self,
2149 const cpl_imagelist * on,
2150 const cpl_imagelist * off)
2151{
2152 /* UNUSED due to chunking */
2153 cpl_image * meanon = cpl_imagelist_collapse_create(on);
2154 cpl_image * meanoff = cpl_imagelist_collapse_create(off);
2155 const cpl_error_code err1 = cpl_image_subtract(meanon, meanoff);
2156 const cpl_error_code err2 = cpl_image_subtract(meanon, self);
2157 const unsigned bitmask = CPL_STATS_MIN | CPL_STATS_MAX | CPL_STATS_MEAN
2158 | CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV | CPL_STATS_STDEV;
2159
2160 cpl_stats * stats = cpl_stats_new_from_image(meanon, bitmask);
2161
2162 bug_if(err1 + err2);
2163
2164 bug_if(cpl_stats_dump(stats, bitmask, stderr));
2165
2166 end_skip;
2167
2168 cpl_image_delete(meanon);
2169 cpl_image_delete(meanoff);
2170 cpl_stats_delete(stats);
2171
2172 return cpl_error_get_code();
2173
2174}
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.
Definition: visir_dfs.c:72
cpl_error_code visir_load_cube2_split(cpl_imagelist *alist, cpl_imagelist *blist, const irplib_framelist *rawframes, int pos, const int planestart, const int planeend)
Load and split the data in a CUBE2 file.
Definition: visir_inputs.c:242
cpl_bivector * visir_load_lintable(cpl_frame *linframe, cpl_boolean is_spec)
load and normalize linearity table into a bivector
Definition: visir_inputs.c:536
int visir_pfits_get_navrg(const cpl_propertylist *self)
The NAVRG.
Definition: visir_pfits.c:339
double visir_pfits_get_dit(const cpl_propertylist *self)
The DIT.
Definition: visir_pfits.c:325
int visir_pfits_get_win_nx(const cpl_propertylist *self)
The WIN NX key.
Definition: visir_pfits.c:591
int visir_pfits_get_win_ny(const cpl_propertylist *self)
The WIN NY key.
Definition: visir_pfits.c:612
double visir_pfits_get_pixscale(const cpl_propertylist *self)
The pixel scale.
Definition: visir_pfits.c:711
double visir_pfits_get_ada_posang(const cpl_propertylist *self)
The adapter position angle in rad.
Definition: visir_pfits.c:249
double visir_pfits_get_chop_posang(const cpl_propertylist *self)
The chopping position angle in rad.
Definition: visir_pfits.c:205
const char * visir_pfits_get_insmode(const cpl_propertylist *self)
The mode.
Definition: visir_pfits.c:474
double visir_pfits_get_cumoffsety(const cpl_propertylist *self)
The cumulative offset in Y.
Definition: visir_pfits.c:277
double visir_pfits_get_pixspace(const cpl_propertylist *self)
The pixel spacing.
Definition: visir_pfits.c:727
const char * visir_pfits_get_chopnod_dir(const cpl_propertylist *self)
The chopping direction.
Definition: visir_pfits.c:139
int visir_pfits_get_ndit(const cpl_propertylist *self)
The NDIT keyword.
Definition: visir_pfits.c:535
int visir_pfits_get_naxis3(const cpl_propertylist *self)
The NAXIS3 key.
Definition: visir_pfits.c:576
double visir_pfits_get_chop_pthrow(const cpl_propertylist *self)
The chopping throw in pixels.
Definition: visir_pfits.c:222
int visir_pfits_get_naxis1(const cpl_propertylist *self)
The NAXIS1 key.
Definition: visir_pfits.c:546
int visir_pfits_get_naxis2(const cpl_propertylist *self)
The NAXIS2 key.
Definition: visir_pfits.c:561
int visir_pfits_get_start_y(const cpl_propertylist *self)
The WIN STRY key.
Definition: visir_pfits.c:654
const char * visir_pfits_get_frame_type(const cpl_propertylist *self)
The frame type.
Definition: visir_pfits.c:434
double visir_pfits_get_cumoffsetx(const cpl_propertylist *self)
The cumulative offset in X.
Definition: visir_pfits.c:265
int visir_pfits_get_start_x(const cpl_propertylist *self)
The WIN STRX key.
Definition: visir_pfits.c:633
const char * visir_pfits_get_nodpos(const cpl_propertylist *self)
The nodding position.
Definition: visir_pfits.c:699
const char * visir_pfits_get_resol(const cpl_propertylist *self)
The spectral resolution.
Definition: visir_pfits.c:820