IIINSTRUMENT Pipeline Reference Manual 4.6.2
visir_util_detect_shift.c
1/* $Id: visir_util_detect_shift.c,v 1.97 2013-05-14 12:59:23 jtaylor Exp $
2 *
3 * This file is part of the VISIR Pipeline
4 * Copyright (C) 2012,2013,2014 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
19 */
20
21#ifdef HAVE_CONFIG_H
22#include <config.h>
23#endif
24
25/*-----------------------------------------------------------------------------
26 Includes
27 -----------------------------------------------------------------------------*/
28
29#include "visir_recipe.h"
30
31#include "visir_spc_distortion.h"
32#include "visir_utils.h"
33#include "irplib_framelist.h"
34#include <cxlist.h>
35#include <string.h>
36#include <assert.h>
37#include <string.h>
38#include <math.h>
39#include <time.h>
40#ifdef _OPENMP
41#include <omp.h>
42#endif
43/*-----------------------------------------------------------------------------
44 Defines
45 -----------------------------------------------------------------------------*/
46
47#define RECIPE_STRING "visir_util_detect_shift"
48
49#define MAX_BEAMS 4
50
51#define SQR(a) ((a)*(a))
52
53/*-----------------------------------------------------------------------------
54 Private Functions prototypes
55 -----------------------------------------------------------------------------*/
56
57typedef enum {
58 VISIR_SHIFT_XCORRELATE,
59 VISIR_SHIFT_BRIGHTEST
60} visir_shift_method;
61
62typedef enum {
63 VISIR_PLANE_AOFF = 0,
64 VISIR_PLANE_BON = 1,
65 VISIR_PLANE_AON = 2,
66 VISIR_PLANE_BOFF = 3,
67 VISIR_PLANE_UNDEFINED = 4,
68} visir_plane_type;
69
70static const char * type2string[] = {"Aoff", "Bon", "Aon", "Boff", "full"};
71
72typedef struct {
73 cpl_size seqnb;
74 cpl_image * img;
75 cpl_propertylist * plist;
76 visir_plane_type type;
77 /* original detected or set beam position */
78 double x;
79 double y;
80 /* detected shift on original position */
81 double xshift;
82 double yshift;
83 double correlation;
84 double median;
85 double mad;
86 double stdev;
87 double fwhmx;
88 double fwhmy;
89 double min;
90 double max;
91 double crval1;
92 double crval2;
93} visir_plane;
94
95
96typedef struct {
97 double min_cor;
98 double max_shift;
99 double max_mad;
100} stats_limits;
101
102static
103cpl_error_code visir_util_detect_shift_one(cpl_frameset *,
104 irplib_framelist * , int,
105 const cpl_parameterlist *,
106 cpl_mask *,
107 visir_plane **);
108
109#ifdef VISIR_OMP3
110static double t_loading = 0.;
111static double t_loading_blocked = 0.;
112static double t_handling_blocked_in = 0.;
113static double t_handling = 0.;
114static double t_handling_blocked_out = 0.;
115static double t_writing = 0.;
116static double t_writing_blocked = 0.;
117#endif
118static size_t nbytes_load = 0;
119static size_t nbytes_load_eff = 0;
120static size_t nbytes_save = 0;
121
122static int is_aqu_data = -1;
123
124#ifdef VISIR_CHAIN
125#define cpl_plugin_get_info visir_util_detect_shift_get_info
126#endif
127cpl_recipe_define(visir_util_detect_shift, VISIR_BINARY_VERSION,
128 "Find beam positions and reject bad images",
129 "Julian Tayor", PACKAGE_BUGREPORT, "2012",
130 "Detects beams in a background subtracted multi-beam image,"
131 " determines the shifts between different images and their "
132 "correlation. Images can be rejected if their correlation "
133 "is bad.\n"
134 "The files listed in the Set Of Frames (sof-file) "
135 "must be tagged: \n"
136 "VISIR-chopnod-corrected-file.fits "
137 VISIR_UTIL_CORRECTED
138 "\nbad-pixel-mask.fits " VISIR_CALIB_STATIC_MASK
139 "\nThe output consists of single beam images of the size "
140 "of the throw. The shifts will be written int CRPIX[12].");
141
142/*----------------------------------------------------------------------------*/
146/*----------------------------------------------------------------------------*/
147
148/*-----------------------------------------------------------------------------
149 Functions code
150 -----------------------------------------------------------------------------*/
151
152static visir_plane * visir_plane_new(void)
153{
154 return cpl_calloc(1, sizeof(visir_plane));
155}
156
157static void visir_plane_delete(visir_plane * pl)
158{
159 if (pl) {
160 cpl_image_delete(pl->img);
161 cpl_propertylist_delete(pl->plist);
162 cpl_free(pl);
163 }
164}
165
166#ifdef _OPENMP
167static inline int
168pl_compare_seqnb(const visir_plane * a, const visir_plane * b)
169{
170 if (a->seqnb < b->seqnb)
171 return -1;
172 if (a->seqnb > b->seqnb)
173 return 1;
174 return 0;
175}
176#endif
177
178static void planelist_delete(cx_list * l)
179{
180 if (l)
181 cx_list_destroy(l, (cx_free_func)&visir_plane_delete);
182}
183
184static inline cpl_image *
185window_extract(const cpl_image * img,
186 const cpl_size x, const cpl_size y, const cpl_size window)
187{
188 const cpl_size nx = cpl_image_get_size_x(img);
189 const cpl_size ny = cpl_image_get_size_y(img);
190 if (x > nx || y > ny || window <= 0)
191 return NULL;
192 return cpl_image_extract(img,
193 CX_MAX(1, x - window / 2),
194 CX_MAX(1, y - window / 2),
195 CX_MIN(nx, x + window / 2),
196 CX_MIN(ny, y + window / 2));
197}
198
199
200static inline void
201window_extract_p(visir_plane * pl, const cpl_image * img,
202 const double x, const double y, const cpl_size window)
203{
204 pl->img = window_extract(img, x, y, window);
205 pl->x = x - CX_MAX(1, (cpl_size)x - window / 2) + 1;
206 pl->y = y - CX_MAX(1, (cpl_size)y - window / 2) + 1;
207}
208
209/* ---------------------------------------------------------------------------*/
215/* ---------------------------------------------------------------------------*/
216static void cut_aqu_illuminated(cpl_image ** img)
217{
218 assert(is_aqu_data >= 0);
219 if (!is_aqu_data) {
220 return;
221 }
222 /* only cut full sized aqu images, skip e.g. smaller aquisition images */
223 cpl_size llx = cpl_image_get_size_x(*img) == 1024 ? 22 : 1;
224 cpl_size lly = cpl_image_get_size_y(*img) == 1024 ? 92 : 1;
225 cpl_size urx = cpl_image_get_size_x(*img) == 1024 ?
226 878 : cpl_image_get_size_x(*img);
227 cpl_size ury = cpl_image_get_size_y(*img) == 1024 ?
228 948 : cpl_image_get_size_y(*img);
229
230 cpl_image * tmp = cpl_image_extract(*img, llx, lly, urx, ury);
231 cpl_image_delete(*img);
232 *img = tmp;
233}
234
235static inline cpl_boolean
236is_image_bad(const visir_plane * pl, const stats_limits * limits)
237{
238 return pl->correlation < limits->min_cor ||
239 hypot(pl->xshift, pl->yshift) > limits->max_shift ||
240 (limits->max_mad > 0 && pl->mad > limits->max_mad);
241}
242
243
244/*----------------------------------------------------------------------------*/
252/*----------------------------------------------------------------------------*/
253static cpl_error_code
254visir_util_detect_shift_fill_parameterlist(cpl_parameterlist * self)
255{
256 const char * context = PACKAGE "." RECIPE_STRING;
257 cpl_error_code err;
258
259 cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);
260
261 /* Fill the parameters list */
262
263 /* --template */
264 err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
265 "template", VISIR_STR_PAR_EMPTY, NULL,
266 context, "Correlation template used "
267 "to detect shifts and reject bad "
268 "images.\n If none given an "
269 "averaged image is used");
270 cpl_ensure_code(!err, err);
271
272 /* --min-correlation */
273 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
274 "min-correlation", 0.5, NULL, context,
275 "Minimal correlation to the template "
276 "required to be accepted");
277 cpl_ensure_code(!err, err);
278
279 /* --max-shift */
280 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
281 "max-shift", 10., NULL, context,
282 "Maximal allowed object shift for an "
283 "image to be accepted");
284 cpl_ensure_code(!err, err);
285
286 /* --max-mad */
287 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
288 "max-mad", 0., NULL, context,
289 "Maximal allowed median absolute "
290 "deviation for an image to be "
291 "accepted. <= 0 equals no limit");
292 cpl_ensure_code(!err, err);
293
294 /* --naverage */
295 err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
296 "naverage", 1, NULL, context,
297 "Number of planes to average before "
298 "attempting to detect the shifts.");
299 cpl_ensure_code(!err, err);
300
301 /* --method */
302 err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
303 "method", "xcorrelate", NULL, context,
304 "Method to determine beam shifts.\n"
305 " xcorrelate: use the cross "
306 "correlation shift.\n"
307 " brightest: use the position of "
308 "the brightest pixel");
309 cpl_ensure_code(!err, err);
310
311 /* --no-shift */
312 err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
313 "no-shift", CPL_FALSE, NULL, context,
314 "Sets whether to apply the "
315 "determined shifts in following "
316 "recipes. Set to TRUE of shifts "
317 "appear too large.");
318 cpl_ensure_code(!err, err);
319
320 /* --no-reject */
321 err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
322 "no-reject", CPL_TRUE, NULL, context,
323 "If true images with bad "
324 "statistics will not be rejected "
325 "and no shift correction is done");
326 cpl_ensure_code(!err, err);
327
328 /* --beampos */
329 err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
330 "beampos", VISIR_STR_PAR_EMPTY,
331 NULL, context,
332 "Define the positions of the beams. "
333 "These positions are cut by the "
334 "window to extract the single "
335 "beam images.\n Format: "
336 "sign:x,y,window;sign:x,y,window;... "
337 "\n where sign is \"pos\" or \"neg\" "
338 "depending on whether the beam is "
339 "positive or negative.\n "
340 "The window is optional and defines "
341 "the size of the cut image around "
342 "the beam. The default window is "
343 "the chop throw.\n E.g.: "
344 "pos:50,60;neg:50,160;\n By default "
345 "autodetection is attempted.\n");
346 cpl_ensure_code(!err, err);
347
348 err =
349 irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING, "eccmax",
350 3.0, NULL, context,
351 "The maximum eccentricity allowed in the combination "
352 "of the three (in parallel nod/chopping) or four (in "
353 "perpendicular nod/chopping) beams. In parallel mode, "
354 "three perfectly aligned points spaced with the "
355 "chopnod throw will have eccentricity 0, while in "
356 "perpedicular mode a square with the chopnod throw as "
357 "the side length will have eccentricity 0"
358 );
359 cpl_ensure_code(!err, err);
360
361 return CPL_ERROR_NONE;
362}
363
364
365/*----------------------------------------------------------------------------*/
372/*----------------------------------------------------------------------------*/
373static int visir_util_detect_shift(cpl_frameset * framelist,
374 const cpl_parameterlist * parlist)
375{
376 irplib_framelist * allframes = NULL;
377 irplib_framelist * rawframes = NULL;
378 int n;
379 cpl_mask * bpm = NULL;
380 visir_plane * template = NULL;
381
382#ifdef _OPENMP
383 omp_set_num_threads(visir_get_num_threads(CPL_FALSE));
384#endif
385
386 /* Identify the RAW and CALIB frames in the input frameset */
387 skip_if (visir_dfs_set_groups(framelist));
388 cpl_fits_set_mode(CPL_FITS_START_CACHING);
389
390 /* Objects observation */
391 allframes = irplib_framelist_cast(framelist);
392 skip_if(allframes == NULL);
393 rawframes = irplib_framelist_extract_regexp(allframes, "^("
394 VISIR_UTIL_CORRECTED
395 ")$", CPL_FALSE);
396 skip_if (rawframes == NULL);
397 n = irplib_framelist_get_size(rawframes);
398
399
400 if (cpl_frameset_find_const(framelist, VISIR_CALIB_STATIC_MASK)) {
401 irplib_framelist * bpmframes =
402 irplib_framelist_extract(allframes, VISIR_CALIB_STATIC_MASK);
403 if (irplib_framelist_get_size(bpmframes) == 1) {
404 const cpl_frame * bpmframe =
405 irplib_framelist_get_const(bpmframes, 0);
406 const char * bpmname = cpl_frame_get_filename(bpmframe);
407 cpl_image * bpmimg =
408 cpl_image_load(bpmname, CPL_TYPE_UNSPECIFIED, 0, 0);
409 cpl_error_code errori;
410
411 bpm = cpl_mask_threshold_image_create(bpmimg, 0.1, DBL_MAX);
412 cpl_image_delete(bpmimg);
413
414 errori = cpl_error_set_where(cpl_func);
415 error_if(errori, errori, "Could not load bad pixel map");
416 cpl_msg_info(cpl_func, "Loaded bad pixel map");
417 }
418 else
419 error_if(1, CPL_ERROR_ILLEGAL_INPUT,
420 "Unexpected number of bad pixel maps");
421 irplib_framelist_delete(bpmframes);
422 }
423
424 for (cpl_size i = 0; i < n; i++) {
425 skip_if(visir_util_detect_shift_one(framelist, rawframes, i,
426 parlist, bpm, & template));
427 cpl_fits_set_mode(CPL_FITS_RESTART_CACHING);
428 }
429
430 end_skip;
431
432 irplib_framelist_delete(allframes);
433 irplib_framelist_delete(rawframes);
434 cpl_mask_delete(bpm);
435 visir_plane_delete(template);
436
437 return cpl_error_get_code();
438}
439
440/* ---------------------------------------------------------------------------*/
454/* ---------------------------------------------------------------------------*/
455static void
456parallel_load(const char * filename, cpl_size iext, cpl_size lim,
457 cpl_image * imgbuf[], cpl_size nimgbuf)
458{
459 assert(nimgbuf % 2 == 0);
460 for (cpl_size i = 0; i < nimgbuf; i+=2) {
461 OMP3_PRAGMA(omp task firstprivate(i) if (iext + i < lim))
462 {
463 cpl_errorstate prestate = cpl_errorstate_get();
464 if (iext + i < lim) {
465 imgbuf[i] = cpl_image_load(filename, CPL_TYPE_FLOAT, 0,
466 iext + i);
467 }
468 if (iext + i + 1 < lim) {
469 imgbuf[i + 1] = cpl_image_load(filename, CPL_TYPE_FLOAT, 0,
470 iext + i + 1);
471 }
472 cpl_errorstate_set(prestate);
473 }
474 }
475 OMP3_PRAGMA(omp taskwait)
476}
477
478
479static visir_imglist *
480load_range(const cpl_frame *frame, cpl_size * start, cpl_size nplanes,
481 const cpl_size naverage, cpl_mask * bpm)
482{
483 const cpl_size next = cpl_frame_get_nextensions(frame);
484 visir_imglist * range = visir_imglist_new(nplanes, NULL);
485 const char * filename = cpl_frame_get_filename(frame);
486 cpl_size iext;
487 cpl_propertylist * plist = NULL;
488 size_t *pbpm = NULL;
489 size_t ndata;
490 /* image fifo for parallel loading */
491 cpl_image * imgbuf[20] = {NULL};
492 size_t nimgbuf = sizeof(imgbuf) / sizeof(imgbuf[0]);
493
494 nplanes = CX_MAX(nplanes, naverage);
495
496 cpl_ensure(naverage > 0, CPL_ERROR_ILLEGAL_INPUT, NULL);
497 cpl_imagelist * av = cpl_imagelist_new();
498
499 for (iext = *start; iext < 1 + next; iext++) {
500 cpl_errorstate prestate = cpl_errorstate_get();
501 size_t nbytes;
502
503 cpl_propertylist_delete(plist);
504
505 plist = cpl_propertylist_load(filename, iext);
506 if (plist == NULL) {
507 cpl_msg_info(cpl_func, "No propertylist in extension %d",
508 (int)iext);
509 cpl_errorstate_set(prestate);
510 continue;
511 }
512
513
514 /* load more images if our fifo is empty
515 * as the input can be compressed and is often still in the cache from
516 * repack we profit from loading with more then one thread */
517 if (imgbuf[0] == NULL) {
518 parallel_load(filename, iext, next + 1, imgbuf, nimgbuf);
519 }
520
521 /* pop one image of the buffer */
522 cpl_image * img = imgbuf[0];
523 memmove(imgbuf, imgbuf + 1, sizeof(imgbuf) - sizeof(imgbuf[0]));
524 imgbuf[nimgbuf - 1] = NULL;
525
526 if (img == NULL) {
527 cpl_msg_debug(cpl_func, "No image-data in extension %d",
528 (int)iext);
529 cpl_propertylist_delete(plist);
530 plist = NULL;
531 cpl_errorstate_set(prestate);
532 continue;
533 }
534
535 nbytes = visir_get_nbytes_plist(plist);
536
537 /* readahead the file for optimal disk IO
538 * start + 1 as first extension is normally empty */
539 if (iext == *start + 1) {
540 visir_readahead(filename, iext * nbytes, nplanes * nbytes);
541 }
542
543 nbytes_load += nbytes;
544 /* uncompressed data loaded assuming bitpix = float */
545 nbytes_load_eff += cpl_image_get_size_x(img) *
546 cpl_image_get_size_y(img) * 4;
547 cpl_imagelist_set(av, img, cpl_imagelist_get_size(av));
548
549 if (cpl_imagelist_get_size(av) == naverage) {
550 cpl_image * avimg = naverage > 1 ? cpl_imagelist_collapse_create(av)
551 : cpl_imagelist_unset(av, 0);
552 if (bpm) {
553 cpl_image_reject_from_mask(avimg, bpm);
554 /* cpl_detector_interpolate_rejected(avimg); too slow */
555 visir_interpolate_rejected(avimg, &pbpm, &ndata);
556 }
557
558 cut_aqu_illuminated(&avimg);
559 visir_imglist_append(range, avimg, plist);
560
561 plist = NULL;
562 cpl_imagelist_delete(av);
563 av = cpl_imagelist_new();
564 }
565
566 if (visir_imglist_get_size(range) == nplanes/naverage)
567 break;
568 }
569
570 if (cpl_imagelist_get_size(av) != 0) {
571 cpl_image * avimg = cpl_imagelist_collapse_create(av);
572 if (bpm) {
573 cpl_image_reject_from_mask(avimg, bpm);
574 /* cpl_detector_interpolate_rejected(avimg); */
575 visir_interpolate_rejected(avimg, &pbpm, &ndata);
576 }
577 cut_aqu_illuminated(&avimg);
578 visir_imglist_append(range, avimg, plist);
579 }
580
581 cpl_free(pbpm);
582 cpl_imagelist_delete(av);
583 *start = iext + 1;
584
585 /* we don't need the input in the page cache anymore
586 * TODO: we might not want to do this when called from reflex */
587 if (visir_imglist_get_size(range) > 1) {
588 /* approximation assuming non padded header and all extensions have
589 * similar compression, take first extension as primary has no data */
590 size_t nbytes = visir_get_nbytes_plist(visir_imglist_get_data(range, 1));
591 visir_drop_cache(filename, 0, nbytes * (*start));
592 }
593
594 return range;
595}
596
597typedef struct {
598 double x;
599 double y;
600 cpl_size window;
601 int sign;
602 /* weight of beam image compared to the original input beams (4 or 2)
603 * in parallel mode one beam is the sum of two input beams -> weight = 2
604 * parallel mode images in util_qc */
605 float weight;
606 int type;
607} beam_info;
608
609
610static cx_list *
611parse_beampos_cmd(const char * bpstr)
612{
613 char *str, *token, *istr = cpl_strdup(bpstr);
614 cx_list * bpl = cx_list_new();
615
616 for (str = istr; ; str = NULL) {
617 beam_info * bi;
618 double w = 0;
619 int ret;
620
621 token = strtok(str, ";");
622 if (token == NULL)
623 break;
624
625 bi = cpl_calloc(1, sizeof(beam_info));
626 bi->type = cx_list_size(bpl);
627 cx_list_push_back(bpl, bi);
628
629 bi->weight = 1;
630 if (strncmp(token, "pos", 3) == 0)
631 bi->sign=1;
632 else if (strncmp(token, "neg", 3) == 0)
633 bi->sign=-1;
634 else
635 error_if(1, CPL_ERROR_ILLEGAL_INPUT,
636 "Invalid beam sign: %s", bpstr);
637
638 ret = sscanf(token+4, "%lf,%lf,%lf", &bi->x, &bi->y, &w);
639 if ((ret != 2 && ret != 3) || bi->x < 0 || bi->y < 0 || w < 0)
640 error_if(1, CPL_ERROR_ILLEGAL_INPUT,
641 "Invalid beam position: %s", bpstr);
642
643 if (ret == 3)
644 bi->window = ceil(w);
645 else
646 bi->window = 0;
647 }
648
649 error_if(cx_list_size(bpl) < 1, CPL_ERROR_ILLEGAL_INPUT,
650 "Invalid beam position: %s", bpstr);
651 {
652 beam_info * pos = NULL;
653 int npos = 0, nneg = 0;
654 FOR_EACH_T(beam_info * bi, bpl) {
655 if (bi->sign > 0) {
656 pos = bi;
657 npos++;
658 }
659 else
660 nneg--;
661 }
662 /* give positive beam in parallel case double weight */
663 if (nneg == 2 && npos == 1)
664 pos->weight *= 2.;
665 }
666 end_skip;
667
668 cpl_free(istr);
669
670 return bpl;
671}
672
673static cx_list *
674get_beam_positions(cpl_image * average, const cpl_parameterlist * parlist,
675 cpl_propertylist * plist)
676{
677 cpl_errorstate cleanstate = cpl_errorstate_get();
678 cpl_propertylist * qclist = cpl_propertylist_new();
679 cpl_image * inverse = cpl_image_duplicate(average);
680 double x[4], y[4];
681 const cpl_boolean chop_on =
682 cpl_propertylist_get_bool(plist, VISIR_PFITS_BOOL_CHOP_STATUS);
683 cx_list * beamposl = cx_list_new();
684 int n;
685
686 skip_if(0);
687
688 cpl_image_multiply_scalar(inverse, -1);
689
690 visir_chopnod_mode mode =
691 visir_img_find_beam(qclist, average, inverse,
692 plist, parlist, RECIPE_STRING,
693 x, y);
694 if (cpl_error_get_code() != CPL_ERROR_NONE) {
695 irplib_error_recover(cleanstate, "Could not detect objects,"
696 " using full image. Photometry "
697 "will not work without object positions.");
698 beam_info * bi = cpl_malloc(sizeof(beam_info));
699 const cpl_size nx = cpl_image_get_size_x(average);
700 const cpl_size ny = cpl_image_get_size_y(average);
701 bi->x = nx / 2;
702 bi->y = ny / 2;
703 bi->window = CX_MAX(nx, ny);
704 bi->sign = 1;
705 bi->weight = 1;
706 bi->type = VISIR_PLANE_UNDEFINED;
707 cx_list_push_back(beamposl, bi);
708 }
709 else {
710
711 if (mode == VISIR_CHOPNOD_PERPENDICULAR)
712 n = 4;
713 else if (mode == VISIR_CHOPNOD_PARALLEL)
714 n = 3;
715 else
716 error_if(1, CPL_ERROR_UNSUPPORTED_MODE, "Unsupported chop mode");
717
718 if (chop_on == CPL_FALSE)
719 n = 2;
720
721 for (int i = 0; i < n; i++) {
722 beam_info * bi = cpl_calloc(1, sizeof(beam_info));
723 bi->x = x[i];
724 bi->y = y[i];
725 /* the first are always the positive apertures */
726 if (n == 4 || n == 2) {
727 bi->sign = i < n / 2 ? 1 : -1;
728 bi->weight = 1.;
729 }
730 else {
731 bug_if(n != 3);
732 /* give positive beam in parallel case double weight */
733 bi->sign = i < 1 ? 1 : -1;
734 bi->weight = i < 1 ? 2. : 1.;
735 }
736 bi->type = i;
737 cx_list_push_back(beamposl, bi);
738 }
739 }
740
741 end_skip;
742 cpl_image_delete(inverse);
743 cpl_propertylist_delete(qclist);
744
745 return beamposl;
746}
747
748static cx_list *
749cut_image(const cpl_image * img, const cpl_propertylist * plist,
750 const cx_list * beampos, cpl_size window)
751{
752 cx_list * l = cx_list_new();
753
754 FOR_EACH_T(const beam_info * bi, beampos) {
755 const cpl_size win = bi->window > 0 ? bi->window : window;
756
757 visir_plane * pl = visir_plane_new();
758 bug_if(0);
759
760 pl->plist = plist ?
761 cpl_propertylist_duplicate(plist) : cpl_propertylist_new();
762
763 window_extract_p(pl, img, bi->x, bi->y, win);
764 skip_if(pl->img == NULL);
765
766 pl->type = bi->type;
767
768 if (bi->sign != 1)
769 cpl_image_multiply_scalar(pl->img, bi->sign);
770
771 cx_list_push_back(l, pl);
772 }
773
774 end_skip;
775
776 return l;
777}
778
779
780static void
781generate_statistics(visir_plane * pl)
782{
783 cpl_size mx, my;
784 cpl_errorstate prestate = cpl_errorstate_get();
785 cpl_stats * stat =
786 cpl_stats_new_from_image(pl->img, CPL_STATS_MAD |
787 CPL_STATS_MEDIAN | CPL_STATS_MAXPOS |
788 CPL_STATS_MAX | CPL_STATS_MIN |
789 CPL_STATS_STDEV);
790 pl->stdev = cpl_stats_get_stdev(stat);
791 pl->median = cpl_stats_get_median(stat);
792 pl->mad = cpl_stats_get_mad(stat);
793 pl->max = cpl_stats_get_max(stat);
794 pl->min = cpl_stats_get_min(stat);
795 mx = cpl_stats_get_max_x(stat);
796 my = cpl_stats_get_max_y(stat);
797 if (cpl_image_get_fwhm(pl->img, mx, my,
798 &pl->fwhmx, &pl->fwhmy) != CPL_ERROR_NONE) {
799 pl->fwhmx = 0;
800 pl->fwhmy = 0;
801 cpl_msg_warning(cpl_func, "Could not determine fwhm.");
802 cpl_errorstate_set(prestate);
803 }
804
805 cpl_stats_delete(stat);
806}
807
808
809static cx_list *
810handle_chunk(visir_imglist * chunk, const visir_plane * template,
811 const cx_list * beampos,
812 const visir_shift_method method,
813 const stats_limits * limits)
814{
815 const cpl_size n = visir_imglist_get_size(chunk);
816 cpl_propertylist * mplist = visir_imglist_get_mplist(chunk);
817 double pthrow = visir_pfits_get_chop_pthrow(mplist);
818 cx_list * res = cx_list_new();
819
820 /* prepare the fft transformed template, one per beam */
821 visir_fftx_cache * fft_cache[MAX_BEAMS];
822 for (size_t i = 0; i < MAX_BEAMS; i++) {
823 fft_cache[i] = visir_new_fftx_cache();
824 }
825 if (n > 0) {
826 cx_list * beams = cut_image(visir_imglist_get_img(chunk, 0),
827 NULL, beampos, pthrow);
828 skip_if(0);
829 for (cpl_size j = 0, m = cx_list_size(beams); j < m; j++) {
830 visir_plane * pl = cx_list_pop_front(beams);
831 visir_fftxcorrelate(template->img, pl->img, CPL_TRUE,
832 NULL, NULL, NULL, fft_cache[j]);
833 visir_plane_delete(pl);
834 }
835 cx_list_delete(beams);
836 }
837
838 skip_if(0);
839
840 /* tasks are used instead of a parallel for to yield resources to the
841 * reader thread if necessary (e.g. to decompress data)
842 * with 1k x 1k burst data the handling is already starved for data
843 * with ~8 threads */
844 for (cpl_size i = 0; i < n; i++)
845OMP3_PRAGMA(omp task firstprivate(i))
846 {
847 cx_list * beams = cut_image(visir_imglist_get_img(chunk, i),
848 visir_imglist_get_data(chunk, i),
849 beampos, pthrow);
850
851 for (cpl_size j = 0, m = cx_list_size(beams); j < m; j++) {
852 double xshift = 0, yshift = 0, max_cor = 1;
853 visir_plane * pl = cx_list_pop_front(beams);
854
855 if (limits->min_cor > 0 || method == VISIR_SHIFT_XCORRELATE) {
856 /* FIXME: cropping around beampos might improve robustness */
857 visir_fftxcorrelate(template->img, pl->img, CPL_TRUE,
858 &xshift, &yshift, &max_cor, fft_cache[j]);
859 xshift = template->x + xshift;
860 yshift = template->y + yshift;
861 }
862
863 if (method == VISIR_SHIFT_BRIGHTEST) {
864 cpl_size ixshift, iyshift;
865 double xsub, ysub;
866 cpl_image_get_maxpos(pl->img, &ixshift, &iyshift);
867 visir_get_subpixel_maxpos(pl->img, ixshift, iyshift,
868 &xsub, &ysub);
869 xshift = ixshift + xsub;
870 yshift = iyshift + ysub;
871 }
872
873 cpl_msg_debug(cpl_func, "%4s: correlation %5.3f shift %5.2f %5.2f",
874 type2string[pl->type], max_cor, xshift, yshift);
875
876 pl->seqnb = i;
877 pl->xshift = xshift - pl->x;
878 pl->yshift = yshift - pl->y;
879 pl->correlation = max_cor;
880 /* with shift the wcs are not same as the template */
881 pl->crval1 = template->crval1;
882 pl->crval2 = template->crval2;
883
884 generate_statistics(pl);
885OMP_PRAGMA(omp critical(chunk_lock))
886 cx_list_push_back(res, pl);
887 }
888
889 cx_list_delete(beams);
890 }
891OMP3_PRAGMA(omp taskwait)
892
893#ifdef _OPENMP
894 /* a priority queue or per thread lists would be more efficient
895 * but time spent here is negligible */
896 cx_list_sort(res, (cx_compare_func)pl_compare_seqnb);
897#endif
898
899 end_skip;
900
901 for (size_t i = 0; i < sizeof(fft_cache)/sizeof(fft_cache[0]); i++)
902 visir_delete_fftx_cache(fft_cache[i]);
903
904 return res;
905}
906
907static visir_plane *
908improve_template(const cx_list *imgs, const stats_limits * limits)
909{
910 cpl_imagelist * l = cpl_imagelist_new();
911 visir_plane * template = visir_plane_new();
912 cx_list_iterator it = cx_list_begin(imgs);
913 const cx_list_iterator end = cx_list_end(imgs);
914 cpl_size nx = 0;
915 cpl_size ny = 0;
916
917 skip_if(cx_list_empty(imgs));
918
919 while (it != end) {
920 visir_plane * pl = cx_list_get(imgs, it);
921 cpl_image * img = cpl_image_duplicate(pl->img);
922 nx = nx == 0 ? cpl_image_get_size_x(img) : nx;
923 ny = ny == 0 ? cpl_image_get_size_y(img) : ny;
924
925 /* sizes of different beams can diverge if they are close to the
926 * detector edge */
927 if (cpl_image_get_size_x(img) != nx ||
928 cpl_image_get_size_y(img) != ny) {
929 it = cx_list_next(imgs, it);
930 continue;
931 }
932
933 // FIXME: use cpl_geom_img_offset_saa for subpixel shifting
934 if (is_image_bad(pl, limits) == CPL_FALSE)
935 cpl_image_shift(img, -visir_round_to_int(pl->xshift),
936 -visir_round_to_int(pl->yshift));
937
938 cpl_imagelist_set(l, img, cpl_imagelist_get_size(l));
939
940 it = cx_list_next(imgs, it);
941 skip_if(0);
942 }
943
944 {
945 /* average and shrink it a bit around the beam for better correlation*/
946 cpl_image * tmp = visir_parallel_median_collapse(l);
947 visir_plane * firstpl = cx_list_get(imgs, cx_list_begin(imgs));
948 template->x = firstpl->x;
949 template->y = firstpl->y;
950 window_extract_p(template, tmp, firstpl->x, firstpl->y,
951 cpl_image_get_size_x(tmp) / 2);
952 cpl_image_delete(tmp);
953 }
954
955 end_skip;
956 cpl_imagelist_delete(l);
957
958 return template;
959}
960
961static cpl_error_code
962save_images(cx_list * imgs, const cpl_propertylist * mplist,
963 const cpl_parameterlist * parlist, const int iframe,
964 const stats_limits * limits)
965{
966 const cpl_boolean noshift =
967 irplib_parameterlist_get_bool(parlist, PACKAGE,
968 RECIPE_STRING, "no-shift");
969 const cpl_boolean noreject =
970 irplib_parameterlist_get_bool(parlist, PACKAGE,
971 RECIPE_STRING, "no-reject");
972 const int naverage =
973 irplib_parameterlist_get_int(parlist, PACKAGE,
974 RECIPE_STRING, "naverage");
975
976 for (cpl_size j = 0, n = cx_list_size(imgs); j < n; j++) {
977 char buffer[128];
978 visir_plane * pl = cx_list_pop_front(imgs);
979
980 cpl_boolean bad = is_image_bad(pl, limits);
981 cpl_type save_type = CPL_TYPE_FLOAT;
982
983 if (noreject == CPL_FALSE && bad) {
984 visir_plane_delete(pl);
985 continue;
986 }
987
988 if (pl->max <= CX_MAXSHORT && pl->min >= CX_MINSHORT &&
989 naverage == 1 && cpl_propertylist_get_int(pl->plist, "BITPIX") > 0)
990 save_type = CPL_TYPE_SHORT;
991
992 cpl_propertylist_copy_property_regexp(pl->plist, mplist, "^("
993 IRPLIB_PFITS_WCS_REGEXP ")$",
994 CPL_FALSE);
995
996 /* don't correct for shift if the stats are bad in no-reject mode */
997 if (noshift || (noreject && bad)) {
998 cpl_propertylist_append_double(pl->plist, "CRPIX1", pl->x);
999 cpl_propertylist_append_double(pl->plist, "CRPIX2", pl->y);
1000 } else {
1001 cpl_propertylist_append_double(pl->plist, "CRPIX1",
1002 pl->x + pl->xshift);
1003 cpl_propertylist_append_double(pl->plist, "CRPIX2",
1004 pl->y + pl->yshift);
1005 }
1006 cpl_propertylist_append_double(pl->plist, "CRVAL1", pl->crval1);
1007 cpl_propertylist_append_double(pl->plist, "CRVAL2", pl->crval2);
1008
1009 cpl_propertylist_append_double(pl->plist, "ESO QC CORR",
1010 pl->correlation);
1011
1012 cpl_propertylist_append_double(pl->plist, "ESO QC MEDIAN",
1013 pl->median);
1014 cpl_propertylist_append_double(pl->plist, "ESO QC MAD",
1015 pl->mad);
1016 cpl_propertylist_append_double(pl->plist, "ESO QC STDEV",
1017 pl->stdev);
1018 cpl_propertylist_append_double(pl->plist, "ESO QC FWHMX",
1019 pl->fwhmx);
1020 cpl_propertylist_append_double(pl->plist, "ESO QC FWHMY",
1021 pl->fwhmy);
1022
1023 sprintf(buffer, RECIPE_STRING"_%03d_%s"CPL_DFS_FITS,
1024 iframe+1, type2string[pl->type]);
1025
1026 /* Update the headers before saving */
1027 visir_dfs_update_header(pl->plist);
1028 cpl_image_save(pl->img, buffer, save_type,
1029 pl->plist, CPL_IO_EXTEND);
1030 nbytes_save += cpl_image_get_size_x(pl->img) *
1031 cpl_image_get_size_y(pl->img) * cpl_type_get_sizeof(save_type);
1032
1033 visir_plane_delete(pl);
1034 skip_if(0);
1035 }
1036
1037 end_skip;
1038
1039 return cpl_error_get_code();
1040}
1041
1042
1043#ifdef VISIR_OMP3
1044static cpl_error_code
1045read_data(visir_queue * readq, cpl_propertylist * mplist,
1046 cpl_frame * frame, const cpl_size chunksize,
1047 const cpl_size naverage, cpl_mask * bpm, cpl_size * cur)
1048{
1049 visir_imglist * range = NULL;
1050 do {
1051 double t = cpl_test_get_walltime(), t2;
1052 range = load_range(frame, cur, chunksize,
1053 naverage, bpm);
1054 skip_if(0);
1055
1056 if (visir_imglist_get_size(range) == 0) {
1057 visir_imglist_delete(range, (visir_free)cpl_propertylist_delete);
1058 break;
1059 }
1060
1061 visir_imglist_set_mplist(range, mplist);
1062 t2 = cpl_test_get_walltime();
1063 t_loading += t2 - t;
1064 skip_if(visir_queue_put(readq, range));
1065 t_loading_blocked += cpl_test_get_walltime() - t2;
1066 } while (visir_imglist_get_size(range) != 0);
1067
1068 cpl_msg_info(cpl_func,
1069 "Loading thread done, %.3g MB/s (%.3g MB/s uncompressed)",
1070 (nbytes_load / 1024. / 1024.) / t_loading,
1071 (nbytes_load_eff / 1024. / 1024.) / t_loading);
1072
1073 end_skip;
1074 /* signal end of data */
1075 visir_queue_put(readq, NULL);
1076
1077 cpl_error_code err = cpl_error_get_code();
1078 if (err != CPL_ERROR_NONE)
1079 visir_queue_set_error(readq, err);
1080
1081 return err;
1082}
1083
1084static cpl_error_code
1085handle_data(visir_queue * readq, visir_queue * writeq,
1086 const visir_plane * template,
1087 cx_list * beampos, const visir_shift_method method,
1088 const stats_limits * limits)
1089{
1090 visir_imglist * range = NULL;
1091 do {
1092 double t = cpl_test_get_walltime(), t2;
1093 range = visir_queue_pop(readq);
1094 t2 = cpl_test_get_walltime();
1095 t_handling_blocked_in += t2 - t;
1096 if (range == NULL)
1097 break;
1098 cx_list * res = handle_chunk(range, template, beampos,
1099 method, limits);
1100
1101 visir_imglist_delete(range, (visir_free)cpl_propertylist_delete);
1102
1103 t = cpl_test_get_walltime();
1104 t_handling += t - t2;
1105 skip_if(visir_queue_put(writeq, res));
1106 t_handling_blocked_out += cpl_test_get_walltime() - t;
1107 } while(range != NULL);
1108
1109 end_skip;
1110 /* signal end of data */
1111 visir_queue_put(writeq, NULL);
1112
1113 cpl_error_code err = cpl_error_get_code();
1114 if (err != CPL_ERROR_NONE) {
1115 visir_queue_set_error(writeq, err);
1116 visir_queue_set_error(readq, err);
1117 }
1118
1119 return err;
1120}
1121
1122
1123static cpl_error_code
1124write_data(visir_queue * writeq, cpl_propertylist * mplist,
1125 const cpl_parameterlist * parlist,
1126 const int iframe, const stats_limits * limits,
1127 const cpl_size chunksize, const cpl_size next)
1128{
1129 cx_list * res = NULL;
1130 cpl_size count = 1;
1131 do {
1132 double t = cpl_test_get_walltime(), t2;
1133 res = visir_queue_pop(writeq);
1134 t2 = cpl_test_get_walltime();
1135 t_writing_blocked += t2 - t;
1136 if (res == NULL)
1137 break;
1138
1139 save_images(res, mplist, parlist, iframe, limits);
1140 planelist_delete(res);
1141 t_writing += cpl_test_get_walltime() - t2;
1142 skip_if(0);
1143 cpl_msg_info(cpl_func, "done: %d/%d",
1144 (int)CX_MIN((count * chunksize), next + 1),
1145 (int)next + 1);
1146 count++;
1147 } while(res != NULL);
1148
1149 cpl_msg_info(cpl_func, "Writing thread done, %.3g MB /s",
1150 (nbytes_save / 1024. / 1024.) / t_writing);
1151
1152 end_skip;
1153
1154 cpl_error_code err = cpl_error_get_code();
1155 if (err != CPL_ERROR_NONE)
1156 visir_queue_set_error(writeq, err);
1157
1158 return err;
1159}
1160#endif
1161
1162/*----------------------------------------------------------------------------*/
1171/*----------------------------------------------------------------------------*/
1172static
1173cpl_error_code visir_util_detect_shift_one(cpl_frameset * framelist,
1174 irplib_framelist * rawframes,
1175 int iframe,
1176 const cpl_parameterlist * parlist,
1177 cpl_mask * bpm,
1178 visir_plane ** ptemplate)
1179{
1180 cpl_frameset * products = cpl_frameset_new();
1181 visir_plane * template = NULL;
1182 cpl_image * av = NULL;
1183 cpl_boolean bfirst = CPL_FALSE;
1184 cx_list * res = NULL;
1185 cpl_frame * frame = irplib_framelist_get(rawframes, iframe);
1186 const char * filename = cpl_frame_get_filename(frame);
1187 const char * fn_template =
1188 irplib_parameterlist_get_string(parlist, PACKAGE,
1189 RECIPE_STRING, "template");
1190 const int naverage =
1191 irplib_parameterlist_get_int(parlist, PACKAGE,
1192 RECIPE_STRING, "naverage");
1193 cpl_size cur = 0;
1194 const cpl_size chunksize = 200 * naverage;
1195 cpl_frame_set_group(frame, CPL_FRAME_GROUP_RAW);
1196
1197 /* load the first chunk for the template */
1198 cpl_size next;
1199 cpl_propertylist * mplist = cpl_propertylist_load(filename, 0);
1200 {
1201 visir_data_type datatype;
1202 visir_get_data_type(frame, mplist, &datatype, &next);
1203 is_aqu_data = visir_data_is_aqu(datatype);
1204 }
1205 visir_imglist * range = load_range(frame, &cur, chunksize,
1206 naverage, bpm);
1207 visir_imglist_set_mplist(range, mplist);
1208 double pthrow = visir_pfits_get_chop_pthrow(mplist);
1209 const char * sbeampos =
1210 irplib_parameterlist_get_string(parlist, PACKAGE,
1211 RECIPE_STRING, "beampos");
1212 cx_list * beampos = NULL;
1213
1214 skip_if(range == NULL);
1215
1216 if (cpl_frameset_find(framelist, VISIR_UTIL_REPACK_MEAN_PROCATG)) {
1217 cpl_frame * fn = cpl_frameset_find(framelist, VISIR_UTIL_REPACK_MEAN_PROCATG);
1218 av = cpl_image_load(cpl_frame_get_filename(fn),
1219 CPL_TYPE_FLOAT, 0,0);
1220 cut_aqu_illuminated(&av);
1221 skip_if(0);
1222
1223 if (bpm) {
1224 cpl_image * ibpm = cpl_image_new_from_mask(bpm);
1225 cut_aqu_illuminated(&ibpm);
1226 cpl_mask * nbpm = cpl_mask_threshold_image_create(ibpm, 0.1, DBL_MAX);
1227
1228 cpl_image_reject_from_mask(av, nbpm);
1229
1230 cpl_image_delete(ibpm);
1231 cpl_mask_delete(nbpm);
1232
1233 skip_if(visir_interpolate_rejected(av, NULL, NULL));
1234 }
1235 }
1236 else {
1237 const cpl_imagelist * tmpl = visir_imglist_get_imglist(range);
1238 av = visir_parallel_median_collapse(tmpl);
1239 skip_if(0);
1240 }
1241
1242 if (!visir_str_par_is_empty(sbeampos)) {
1243 beampos = parse_beampos_cmd(sbeampos);
1244 /* adapt input beampos to unilluminated cut on aqu data */
1245 if (is_aqu_data) {
1246 FOR_EACH_T(beam_info * bi, beampos) {
1247 bi->x -= cpl_image_get_size_x(av) == 1024 - 22 - (1024 - 878 - 1) ? 22 : 0;
1248 bi->y -= cpl_image_get_size_y(av) == 1024 - 92 - (1024 - 948 - 1) ? 92 : 0;
1249 }
1250 }
1251 }
1252 else
1253 beampos = get_beam_positions(av, parlist, mplist);
1254
1255 skip_if(0);
1256
1257 error_if(cx_list_size(beampos) > MAX_BEAMS, CPL_ERROR_ILLEGAL_INPUT,
1258 "Too many beams, maximum supported: %d", MAX_BEAMS);
1259 error_if(ptemplate == NULL, CPL_ERROR_NULL_INPUT, "Bug: ptemplate == NULL");
1260
1261 /* when beam positions is determined once for all datasets we need to adapt
1262 * for the jitter */
1263 if ((!visir_str_par_is_empty(sbeampos) ||
1264 cpl_frameset_find(framelist, VISIR_UTIL_REPACK_MEAN_PROCATG)) &&
1265 cpl_propertylist_has(mplist, VISIR_DRS_CUMOFFSETXA)) {
1266 double dxa = cpl_propertylist_get_double(mplist, VISIR_DRS_CUMOFFSETXA);
1267 double dya = cpl_propertylist_get_double(mplist, VISIR_DRS_CUMOFFSETYA);
1268 double dxb = cpl_propertylist_get_double(mplist, VISIR_DRS_CUMOFFSETXB);
1269 double dyb = cpl_propertylist_get_double(mplist, VISIR_DRS_CUMOFFSETYB);
1270 FOR_EACH_T(beam_info * bi, beampos) {
1271 if (bi->type == VISIR_PLANE_AOFF || bi->type == VISIR_PLANE_AON ||
1272 bi->type == VISIR_PLANE_UNDEFINED) {
1273 bi->x += dxa;
1274 bi->y += dya;
1275 }
1276 else if (bi->type == VISIR_PLANE_BOFF || bi->type == VISIR_PLANE_BON) {
1277 bi->x += dxb;
1278 bi->y += dyb;
1279 }
1280 }
1281 }
1282
1283 FOR_EACH_T(const beam_info * bi, beampos) {
1284 double window = bi->window > 0 ? bi->window : pthrow;
1285 cpl_msg_info(cpl_func, "Beam position: x %5.1f, y %5.1f, window %4.0f,"
1286 " sign %2d", bi->x, bi->y, window, bi->sign);
1287 }
1288
1289 stats_limits limits;
1290 limits.min_cor =
1291 irplib_parameterlist_get_double(parlist, PACKAGE,
1292 RECIPE_STRING, "min-correlation");
1293 limits.max_shift =
1294 irplib_parameterlist_get_double(parlist, PACKAGE,
1295 RECIPE_STRING, "max-shift");
1296 limits.max_mad =
1297 irplib_parameterlist_get_double(parlist, PACKAGE,
1298 RECIPE_STRING, "max-mad");
1299
1300 const char * smethod =
1301 irplib_parameterlist_get_string(parlist, PACKAGE,
1302 RECIPE_STRING, "method");
1303 visir_shift_method method = VISIR_SHIFT_XCORRELATE;
1304
1305 if (strncmp(smethod, "xcorrelate", sizeof("xcorrelate")) == 0)
1306 method = VISIR_SHIFT_XCORRELATE;
1307 else if (strncmp(smethod, "brightest", sizeof("brightest")) == 0)
1308 method = VISIR_SHIFT_BRIGHTEST;
1309 else
1310 cpl_msg_warning(cpl_func, "Unknown method %s. Using xcorrelate",
1311 smethod);
1312
1313 /* FIXME: add option for template per file but make sure
1314 * center is the same in all versions */
1315 if (*ptemplate != NULL)
1316 template = *ptemplate;
1317 else if (!visir_str_par_is_empty(fn_template)) {
1318 cpl_msg_info(cpl_func, "Loading %s as template", fn_template);
1319 template = visir_plane_new();
1320 template->plist = cpl_propertylist_load(fn_template, 0);
1321 template->img = cpl_image_load(fn_template, CPL_TYPE_UNSPECIFIED, 0, 0);
1322 skip_if(template->img == NULL || template->plist == NULL);
1323 if (cpl_propertylist_has(template->plist, "CRPIX1") &&
1324 cpl_propertylist_has(template->plist, "CRPIX2") &&
1325 cpl_propertylist_has(template->plist, "CRVAL1") &&
1326 cpl_propertylist_has(template->plist, "CRVAL2")) {
1327 template->x = cpl_propertylist_get_double(template->plist, "CRPIX1");
1328 template->y = cpl_propertylist_get_double(template->plist, "CRPIX2");
1329 template->crval1 = cpl_propertylist_get_double(template->plist, "CRVAL1");
1330 template->crval2 = cpl_propertylist_get_double(template->plist, "CRVAL2");
1331 }
1332 else {
1333 error_if(1, CPL_ERROR_ILLEGAL_INPUT,
1334 "Input template must have WCS coordinates");
1335
1336 }
1337 *ptemplate = template;
1338 bfirst = CPL_TRUE;
1339 }
1340 else {
1341 template = visir_plane_new();
1342 cpl_msg_info(cpl_func, "Using average as template");
1343 const cpl_boolean noshift =
1344 irplib_parameterlist_get_bool(parlist, PACKAGE,
1345 RECIPE_STRING, "no-shift");
1346 beam_info * bi = cx_list_get(beampos, cx_list_begin(beampos));
1347 /* crops a little extra to get less background in the template */
1348 const cpl_size window = bi->window > 0 ?
1349 bi->window * 2 / 3. : pthrow * 2 / 3.;
1350 window_extract_p(template, av, bi->x, bi->y, window);
1351 cpl_image_multiply_scalar(template->img, bi->sign);
1352
1353 if (!noshift &&
1354 (limits.min_cor > 0 || method == VISIR_SHIFT_XCORRELATE)) {
1355 /* improve the template by doing a coarse shift correction */
1356 stats_limits lim;
1357 lim.min_cor = 0;
1358 lim.max_shift = 5000;
1359 lim.max_mad = 0;
1360OMP_PRAGMA(omp parallel)
1361OMP_PRAGMA(omp single)
1362 res = handle_chunk(range, template, beampos, method, &lim);
1363 visir_plane_delete(template);
1364 template = improve_template(res, &limits);
1365
1366 planelist_delete(res);
1367 res = NULL;
1368 }
1369 template->plist = cpl_propertylist_new();
1370 cpl_propertylist_append_double(template->plist, "CRPIX1", template->x);
1371 cpl_propertylist_append_double(template->plist, "CRPIX2", template->y);
1372 *ptemplate = template;
1373
1374 /* get world coordinate of template center */
1375 cpl_wcs * wcs = cpl_wcs_new_from_propertylist(mplist);
1376 cpl_matrix * from = cpl_matrix_new(1, 2);
1377 cpl_matrix * to = NULL;
1378 cpl_matrix_set(from, 0, 0, bi->x);
1379 cpl_matrix_set(from, 0, 1, bi->y);
1380 cpl_array * status = NULL;
1381 skip_if(cpl_wcs_convert(wcs, from, &to, &status, CPL_WCS_PHYS2WORLD));
1382 template->crval1 = cpl_matrix_get(to, 0, 0);
1383 template->crval2 = cpl_matrix_get(to, 0, 1);
1384 cpl_wcs_delete(wcs);
1385 cpl_matrix_delete(from);
1386 cpl_matrix_delete(to);
1387 cpl_array_delete(status);
1388
1389 bfirst = CPL_TRUE;
1390 }
1391 skip_if(0);
1392
1393 if (bpm) {
1394 int i = 0;
1395 cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_STATIC_MASK);
1396 cpl_image * ibpm = cpl_image_new_from_mask(bpm);
1397 cut_aqu_illuminated(&ibpm);
1398 cx_list * cbpml = cut_image(ibpm, NULL, beampos, pthrow);
1399 cpl_frameset * usedframes = cpl_frameset_new();
1400 cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
1401
1402 FOR_EACH(it, cbpml) {
1403 char buffer[128];
1404 cpl_image * img = ((visir_plane*)cx_list_get(cbpml, it))->img;
1405 cpl_image_abs(img);
1406 sprintf(buffer, RECIPE_STRING"_bpm_%03d_%s"CPL_DFS_FITS,
1407 iframe+1, type2string[i++]);
1408 irplib_dfs_save_image(products, parlist,
1409 usedframes, img, CPL_BPP_8_UNSIGNED,
1410 RECIPE_STRING,
1411 VISIR_CALIB_BPM,
1412 NULL, NULL, visir_pipe_id,
1413 buffer);
1414 }
1415 planelist_delete(cbpml);
1416 cpl_image_delete(ibpm);
1417 cpl_frameset_delete(usedframes);
1418 }
1419
1420 skip_if(0);
1421
1422 if (bfirst) {
1423 char buffer[128];
1424 cpl_frameset * usedframes = cpl_frameset_new();
1425 sprintf(buffer, RECIPE_STRING"_template_%03d"CPL_DFS_FITS,
1426 iframe+1);
1427
1428 cpl_frameset_insert(usedframes, cpl_frame_duplicate(frame));
1429 skip_if(irplib_dfs_save_image(products, parlist,
1430 usedframes, template->img, CPL_TYPE_FLOAT,
1431 RECIPE_STRING,
1432 VISIR_UTIL_DETECT_SHIFT_TEMPLATE_PROCATG,
1433 template->plist, NULL, visir_pipe_id,
1434 buffer));
1435 cpl_frameset_delete(usedframes);
1436 }
1437
1438 FOR_EACH_T(const beam_info * bi, beampos) {
1439 char buffer[128];
1440 cpl_frameset * usedframes = cpl_frameset_new();
1441 cpl_propertylist * _mplist = cpl_propertylist_new();
1442 cpl_propertylist_copy_property_regexp(_mplist, mplist, ".*", 0);
1443
1444 if (bi->type != VISIR_PLANE_UNDEFINED)
1445 sprintf(buffer, "%01d", bi->type);
1446 else
1447 sprintf(buffer, "UNDEFINED");
1448 cpl_propertylist_append_string(_mplist, "ESO QC BEAMID", buffer);
1449 cpl_propertylist_append_float(_mplist, "ESO DRS IMGWGT", bi->weight);
1450 cpl_propertylist_append_double(_mplist, "ESO QC BEAMX", bi->x);
1451 cpl_propertylist_append_double(_mplist, "ESO QC BEAMY", bi->y);
1452
1453 sprintf(buffer, RECIPE_STRING"_%03d_%s"CPL_DFS_FITS,
1454 iframe+1, type2string[bi->type]);
1455
1456 cpl_frameset_insert(usedframes, cpl_frame_duplicate(frame));
1457 irplib_dfs_save_propertylist(products, parlist,
1458 usedframes, RECIPE_STRING,
1459 VISIR_UTIL_DETECT_SHIFT_PROCATG,
1460 _mplist, NULL, visir_pipe_id, buffer);
1461 cpl_frameset_delete(usedframes);
1462 cpl_propertylist_delete(_mplist);
1463 skip_if(0);
1464 }
1465
1466 cpl_msg_info(cpl_func, "Working on frame %d", iframe);
1467 /* first range already loaded
1468 * process it and load next range in parallel*/
1469#ifdef VISIR_OMP3
1470 cpl_error_code serr, werr, lerr;
1471 serr = werr = lerr = CPL_ERROR_NONE;
1472 visir_queue * readq = visir_queue_init(3);
1473 visir_queue * writeq = visir_queue_init(3);
1474 visir_queue_put(readq, range);
1475 range = NULL;
1476 /* Start a thread to read data, a thread to handle the data and a thread to
1477 * write it. Interleaving of reads and write not a big issue as modern
1478 * file systems will delay the actual disk write for a very long time.
1479 * The reader and worker parallize by openmp tasks so no nested
1480 * parallization is required.
1481 * While the scheduling is more expensive than the simple static of the
1482 * nested case, this has the advantage that when the worker thread is
1483 * stalled for data the reader thread can take all resources it needs to
1484 * load new data as fast as possible. This is very relevant when reading
1485 * cached or compressed data which can be CPU bound on fast disks */
1486
1487 /* oversubscribe by two so the queues can't deadlock */
1488 int nthreads = CX_MAX(visir_get_num_threads(CPL_FALSE) + 2, 3);
1489#pragma omp parallel num_threads(nthreads)
1490 {
1491#pragma omp single nowait
1492 lerr = read_data(readq, mplist, frame, chunksize, naverage, bpm, &cur);
1493#pragma omp single nowait
1494 serr = handle_data(readq, writeq, template, beampos, method, &limits);
1495#pragma omp single nowait
1496 werr = write_data(writeq, mplist, parlist, iframe, &limits,
1497 chunksize, next);
1498 }
1499 visir_queue_delete(readq);
1500 visir_queue_delete(writeq);
1501
1502 error_if(serr, serr, "Failed to save data");
1503 error_if(werr, werr, "Failed to handle data");
1504 error_if(lerr, lerr, "Failed to load data");
1505
1506 cpl_msg_info(cpl_func, "Time spent loading data: %3.3gs",
1507 t_loading);
1508 cpl_msg_info(cpl_func, "Time loading thread was blocked on output: %3.3gs",
1509 t_loading_blocked < 1e-3 ? 0. : t_loading_blocked);
1510 cpl_msg_info(cpl_func, "Time working thread was blocked on input: %3.3gs",
1511 t_handling_blocked_in < 1e-3 ? 0. : t_handling_blocked_in);
1512 cpl_msg_info(cpl_func, "Time working on data: %3.3gs",
1513 t_handling);
1514 cpl_msg_info(cpl_func, "Time working thread was blocked on output: %3.3gs",
1515 t_handling_blocked_out < 1e-3 ? 0. : t_handling_blocked_out);
1516 cpl_msg_info(cpl_func, "Time saving thread was blocked on input: %3.3gs",
1517 t_writing_blocked < 1e-3 ? 0. : t_writing_blocked);
1518 cpl_msg_info(cpl_func, "Time spent saving data: %3.3gs",
1519 t_writing);
1520#else
1521 /* without omp 3.0 just do it synchrounous */
1522 do {
1523 res = handle_chunk(range, template, beampos,
1524 method, &limits);
1525
1526 skip_if(0);
1527 cpl_msg_info(cpl_func, "done: %d/%d",
1528 (int)cur - 1, (int)next + 1);
1529 visir_imglist_delete(range,
1530 (visir_free)cpl_propertylist_delete);
1531 skip_if(0);
1532
1533 range = load_range(frame, &cur, chunksize,
1534 naverage, bpm);
1535 skip_if(0);
1536 visir_imglist_set_mplist(range, mplist);
1537
1538 save_images(res, mplist, parlist, iframe, &limits);
1539 planelist_delete(res);
1540 res = NULL;
1541 skip_if(0);
1542 } while (visir_imglist_get_size(range) != 0);
1543#endif
1544
1545 /* only output products that have accepted images */
1546 FOR_EACH_FRAMESET_C(frm, products) {
1547 if (cpl_frame_get_nextensions(frm) > 0 ||
1548 strcmp(cpl_frame_get_tag(frm),
1549 VISIR_UTIL_DETECT_SHIFT_TEMPLATE_PROCATG) == 0 ||
1550 strcmp(cpl_frame_get_tag(frm),
1551 VISIR_CALIB_BPM) == 0 ||
1552 cpl_frame_get_group(frm) != CPL_FRAME_GROUP_PRODUCT) {
1553 cpl_frameset_insert(framelist, cpl_frame_duplicate(frm));
1554 }
1555 }
1556
1557 end_skip;
1558 planelist_delete(res);
1559 cpl_image_delete(av);
1560 cx_list_destroy(beampos, (visir_free)cpl_free);
1561 visir_imglist_delete(range, (visir_free)cpl_propertylist_delete);
1562 cpl_propertylist_delete(mplist);
1563 cpl_frameset_delete(products);
1564
1565 return cpl_error_get_code();
1566}
1567
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.
Definition: visir_dfs.c:72
double visir_pfits_get_chop_pthrow(const cpl_propertylist *self)
The chopping throw in pixels.
Definition: visir_pfits.c:222