VISIR Pipeline Reference Manual  4.1.0
visir_util_detect_shift.c
1 /* $Id: visir_util_detect_shift.c,v 1.97 2013-05-14 12:59:23 jtaylor Exp $
2  *
3  * This file is part of the VISIR Pipeline
4  * Copyright (C) 2012,2013,2014 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
19  */
20 
21 #ifdef HAVE_CONFIG_H
22 #include <config.h>
23 #endif
24 
25 /*-----------------------------------------------------------------------------
26  Includes
27  -----------------------------------------------------------------------------*/
28 
29 #include "visir_recipe.h"
30 
31 #include "visir_spc_distortion.h"
32 #include "visir_utils.h"
33 #include "irplib_framelist.h"
34 #include <cxlist.h>
35 #include <string.h>
36 #include <assert.h>
37 #include <string.h>
38 #include <math.h>
39 #include <time.h>
40 #ifdef _OPENMP
41 #include <omp.h>
42 #endif
43 /*-----------------------------------------------------------------------------
44  Defines
45  -----------------------------------------------------------------------------*/
46 
47 #define RECIPE_STRING "visir_util_detect_shift"
48 // FIXME:
49 #define VISIR_UTIL_DETECT_SHIFT_PROCATG "BEAM_DETECTED"
50 #define VISIR_UTIL_DETECT_SHIFT_TEMPLATE_PROCATG "CORRELATION_TEMPLATE"
51 
52 #define MAX_BEAMS 4
53 
54 #define SQR(a) ((a)*(a))
55 
56 /*-----------------------------------------------------------------------------
57  Private Functions prototypes
58  -----------------------------------------------------------------------------*/
59 
60 typedef enum {
61  VISIR_SHIFT_XCORRELATE,
62  VISIR_SHIFT_BRIGHTEST
63 } visir_shift_method;
64 
65 typedef enum {
66  VISIR_PLANE_AOFF = 0,
67  VISIR_PLANE_BON = 1,
68  VISIR_PLANE_AON = 2,
69  VISIR_PLANE_BOFF = 3,
70  VISIR_PLANE_UNDEFINED = 4,
71 } visir_plane_type;
72 
73 static const char * type2string[] = {"Aoff", "Bon", "Aon", "Boff", "full"};
74 
75 typedef struct {
76  cpl_size seqnb;
77  cpl_image * img;
78  cpl_propertylist * plist;
79  visir_plane_type type;
80  /* original detected or set beam position */
81  double x;
82  double y;
83  /* detected shift on original position */
84  double xshift;
85  double yshift;
86  double correlation;
87  double median;
88  double mad;
89  double stdev;
90  double fwhmx;
91  double fwhmy;
92  double min;
93  double max;
94  double crval1;
95  double crval2;
96 } visir_plane;
97 
98 
99 typedef struct {
100  double min_cor;
101  double max_shift;
102  double max_mad;
103 } stats_limits;
104 
105 static
106 cpl_error_code visir_util_detect_shift_one(cpl_frameset *,
107  irplib_framelist * , int,
108  const cpl_parameterlist *,
109  cpl_mask *,
110  visir_plane **);
111 
112 #ifdef _OPENMP
113 static double t_loading = 0.;
114 static double t_loading_blocked = 0.;
115 static double t_handling_blocked_in = 0.;
116 static double t_handling = 0.;
117 static double t_handling_blocked_out = 0.;
118 static double t_writing = 0.;
119 static double t_writing_blocked = 0.;
120 #endif
121 static size_t nbytes_load = 0;
122 static size_t nbytes_load_eff = 0;
123 static size_t nbytes_save = 0;
124 
125 static int is_aqu_data = -1;
126 
127 #ifdef VISIR_CHAIN
128 #define cpl_plugin_get_info visir_util_detect_shift_get_info
129 #endif
130 cpl_recipe_define(visir_util_detect_shift, VISIR_BINARY_VERSION,
131  "Find beam positions and reject bad images",
132  "Julian Tayor", PACKAGE_BUGREPORT, "2012",
133  "Detects beams in a background subtracted multi-beam image,"
134  " determines the shifts between different images and their "
135  "correlation. Images can be rejected if their correlation "
136  "is bad.\n"
137  "The files listed in the Set Of Frames (sof-file) "
138  "must be tagged: \n"
139  "VISIR-chopnod-corrected-file.fits "
140  VISIR_UTIL_CORRECTED
141  "\nbad-pixel-mask.fits " VISIR_CALIB_STATIC_MASK
142  "\nThe output consists of single beam images of the size "
143  "of the throw. The shifts will be written int CRPIX[12].");
144 
145 /*----------------------------------------------------------------------------*/
149 /*----------------------------------------------------------------------------*/
150 
151 /*-----------------------------------------------------------------------------
152  Functions code
153  -----------------------------------------------------------------------------*/
154 
155 static visir_plane * visir_plane_new(void)
156 {
157  return cpl_calloc(1, sizeof(visir_plane));
158 }
159 
160 static void visir_plane_delete(visir_plane * pl)
161 {
162  if (pl) {
163  cpl_image_delete(pl->img);
164  cpl_propertylist_delete(pl->plist);
165  cpl_free(pl);
166  }
167 }
168 
169 static inline int
170 pl_compare_seqnb(const visir_plane * a, const visir_plane * b)
171 {
172  if (a->seqnb < b->seqnb)
173  return -1;
174  if (a->seqnb > b->seqnb)
175  return 1;
176  return 0;
177 }
178 
179 static void planelist_delete(cx_list * l)
180 {
181  if (l)
182  cx_list_destroy(l, (cx_free_func)&visir_plane_delete);
183 }
184 
185 static inline cpl_image *
186 window_extract(const cpl_image * img,
187  const cpl_size x, const cpl_size y, const cpl_size window)
188 {
189  const cpl_size nx = cpl_image_get_size_x(img);
190  const cpl_size ny = cpl_image_get_size_y(img);
191  if (x > nx || y > ny || window <= 0)
192  return NULL;
193  return cpl_image_extract(img,
194  CX_MAX(1, x - window / 2),
195  CX_MAX(1, y - window / 2),
196  CX_MIN(nx, x + window / 2),
197  CX_MIN(ny, y + window / 2));
198 }
199 
200 
201 static inline void
202 window_extract_p(visir_plane * pl, const cpl_image * img,
203  const double x, const double y, const cpl_size window)
204 {
205  pl->img = window_extract(img, x, y, window);
206  pl->x = x - CX_MAX(1, (cpl_size)x - window / 2) + 1;
207  pl->y = y - CX_MAX(1, (cpl_size)y - window / 2) + 1;
208 }
209 
210 /* ---------------------------------------------------------------------------*/
216 /* ---------------------------------------------------------------------------*/
217 static void cut_aqu_illuminated(cpl_image ** img)
218 {
219  assert(is_aqu_data >= 0);
220  if (!is_aqu_data) {
221  return;
222  }
223  /* only cut full sized aqu images, skip e.g. smaller aquisition images */
224  cpl_size llx = cpl_image_get_size_x(*img) == 1024 ? 22 : 1;
225  cpl_size lly = cpl_image_get_size_y(*img) == 1024 ? 92 : 1;
226  cpl_size urx = cpl_image_get_size_x(*img) == 1024 ?
227  878 : cpl_image_get_size_x(*img);
228  cpl_size ury = cpl_image_get_size_y(*img) == 1024 ?
229  948 : cpl_image_get_size_y(*img);
230 
231  cpl_image * tmp = cpl_image_extract(*img, llx, lly, urx, ury);
232  cpl_image_delete(*img);
233  *img = tmp;
234 }
235 
236 static inline cpl_boolean
237 is_image_bad(const visir_plane * pl, const stats_limits * limits)
238 {
239  return pl->correlation < limits->min_cor ||
240  hypot(pl->xshift, pl->yshift) > limits->max_shift ||
241  (limits->max_mad > 0 && pl->mad > limits->max_mad);
242 }
243 
244 
245 /*----------------------------------------------------------------------------*/
253 /*----------------------------------------------------------------------------*/
254 static cpl_error_code
255 visir_util_detect_shift_fill_parameterlist(cpl_parameterlist * self)
256 {
257  const char * context = PACKAGE "." RECIPE_STRING;
258  cpl_error_code err;
259 
260  cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);
261 
262  /* Fill the parameters list */
263 
264  /* --template */
265  err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
266  "template", VISIR_STR_PAR_EMPTY, NULL,
267  context, "Correlation template used "
268  "to detect shifts and reject bad "
269  "images.\n If none given an "
270  "averaged image is used");
271  cpl_ensure_code(!err, err);
272 
273  /* --min-correlation */
274  err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
275  "min-correlation", 0.5, NULL, context,
276  "Minimal correlation to the template "
277  "required to be accepted");
278  cpl_ensure_code(!err, err);
279 
280  /* --max-shift */
281  err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
282  "max-shift", 10., NULL, context,
283  "Maximal allowed object shift for an "
284  "image to be accepted");
285  cpl_ensure_code(!err, err);
286 
287  /* --max-mad */
288  err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
289  "max-mad", 0., NULL, context,
290  "Maximal allowed median absolute "
291  "deviation for an image to be "
292  "accepted. <= 0 equals no limit");
293  cpl_ensure_code(!err, err);
294 
295  /* --naverage */
296  err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
297  "naverage", 1, NULL, context,
298  "Number of planes to average before "
299  "attempting to detect the shifts.");
300  cpl_ensure_code(!err, err);
301 
302  /* --method */
303  err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
304  "method", "xcorrelate", NULL, context,
305  "Method to determine beam shifts.\n"
306  " xcorrelate: use the cross "
307  "correlation shift.\n"
308  " brightest: use the position of "
309  "the brightest pixel");
310  cpl_ensure_code(!err, err);
311 
312  /* --no-shift */
313  err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
314  "no-shift", CPL_FALSE, NULL, context,
315  "Sets whether to apply the "
316  "determined shifts in following "
317  "recipes. Set to TRUE of shifts "
318  "appear too large.");
319  cpl_ensure_code(!err, err);
320 
321  /* --no-reject */
322  err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
323  "no-reject", CPL_TRUE, NULL, context,
324  "If true images with bad "
325  "statistics will not be rejected "
326  "and no shift correction is done");
327  cpl_ensure_code(!err, err);
328 
329  /* --beampos */
330  err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
331  "beampos", VISIR_STR_PAR_EMPTY,
332  NULL, context,
333  "Define the positions of the beams. "
334  "These positions are cut by the "
335  "window to extract the single "
336  "beam images.\n Format: "
337  "sign:x,y,window;sign:x,y,window;... "
338  "\n where sign is \"pos\" or \"neg\" "
339  "depending on whether the beam is "
340  "positive or negative.\n "
341  "The window is optional and defines "
342  "the size of the cut image around "
343  "the beam. The default window is "
344  "the chop throw.\n E.g.: "
345  "pos:50,60;neg:50,160;\n By default "
346  "autodetection is attempted.\n");
347  cpl_ensure_code(!err, err);
348 
349  err =
350  irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING, "eccmax",
351  0.25, NULL, context,
352  "The maximum eccentricity allowed in the combination "
353  "of the three (in parallel nod/chopping) or four (in "
354  "perpendicular nod/chopping) beams. In parallel mode, "
355  "three perfectly aligned points spaced with the "
356  "chopnod throw will have eccentricity 0, while in "
357  "perpedicular mode a square with the chopnod throw as "
358  "the side length will have eccentricity 0"
359  );
360  cpl_ensure_code(!err, err);
361 
362  return CPL_ERROR_NONE;
363 }
364 
365 
366 /*----------------------------------------------------------------------------*/
373 /*----------------------------------------------------------------------------*/
374 static int visir_util_detect_shift(cpl_frameset * framelist,
375  const cpl_parameterlist * parlist)
376 {
377  irplib_framelist * allframes = NULL;
378  irplib_framelist * rawframes = NULL;
379  int n;
380  cpl_mask * bpm = NULL;
381  visir_plane * template = NULL;
382 
383 #ifdef _OPENMP
384  omp_set_num_threads(visir_get_num_threads(CPL_FALSE));
385 #endif
386 
387  /* Identify the RAW and CALIB frames in the input frameset */
388  skip_if (visir_dfs_set_groups(framelist));
389  cpl_fits_set_mode(CPL_FITS_START_CACHING);
390 
391  /* Objects observation */
392  allframes = irplib_framelist_cast(framelist);
393  skip_if(allframes == NULL);
394  rawframes = irplib_framelist_extract_regexp(allframes, "^("
395  VISIR_UTIL_CORRECTED
396  ")$", CPL_FALSE);
397  skip_if (rawframes == NULL);
398  n = irplib_framelist_get_size(rawframes);
399 
400 
401  if (cpl_frameset_find_const(framelist, VISIR_CALIB_STATIC_MASK)) {
402  irplib_framelist * bpmframes =
403  irplib_framelist_extract(allframes, VISIR_CALIB_STATIC_MASK);
404  if (irplib_framelist_get_size(bpmframes) == 1) {
405  const cpl_frame * bpmframe =
406  irplib_framelist_get_const(bpmframes, 0);
407  const char * bpmname = cpl_frame_get_filename(bpmframe);
408  cpl_image * bpmimg =
409  cpl_image_load(bpmname, CPL_TYPE_UNSPECIFIED, 0, 0);
410  cpl_error_code errori;
411 
412  bpm = cpl_mask_threshold_image_create(bpmimg, 0.1, DBL_MAX);
413  cpl_image_delete(bpmimg);
414 
415  errori = cpl_error_set_where(cpl_func);
416  error_if(errori, errori, "Could not load bad pixel map");
417  cpl_msg_info(cpl_func, "Loaded bad pixel map");
418  }
419  else
420  error_if(1, CPL_ERROR_ILLEGAL_INPUT,
421  "Unexpected number of bad pixel maps");
422  irplib_framelist_delete(bpmframes);
423  }
424 
425  for (cpl_size i = 0; i < n; i++) {
426  skip_if(visir_util_detect_shift_one(framelist, rawframes, i,
427  parlist, bpm, & template));
428  cpl_fits_set_mode(CPL_FITS_RESTART_CACHING);
429  }
430 
431  end_skip;
432 
433  irplib_framelist_delete(allframes);
434  irplib_framelist_delete(rawframes);
435  cpl_mask_delete(bpm);
436  visir_plane_delete(template);
437 
438  return cpl_error_get_code();
439 }
440 
441 /* ---------------------------------------------------------------------------*/
455 /* ---------------------------------------------------------------------------*/
456 static void
457 parallel_load(const char * filename, cpl_size iext, cpl_size lim,
458  cpl_image * imgbuf[], cpl_size nimgbuf)
459 {
460  assert(nimgbuf % 2 == 0);
461  for (cpl_size i = 0; i < nimgbuf; i+=2) {
462  OMP3_PRAGMA(omp task firstprivate(i) if (iext + i < lim))
463  {
464  cpl_errorstate prestate = cpl_errorstate_get();
465  if (iext + i < lim) {
466  imgbuf[i] = cpl_image_load(filename, CPL_TYPE_FLOAT, 0,
467  iext + i);
468  }
469  if (iext + i + 1 < lim) {
470  imgbuf[i + 1] = cpl_image_load(filename, CPL_TYPE_FLOAT, 0,
471  iext + i + 1);
472  }
473  cpl_errorstate_set(prestate);
474  }
475  }
476  OMP3_PRAGMA(omp taskwait)
477 }
478 
479 
480 static visir_imglist *
481 load_range(const cpl_frame *frame, cpl_size * start, cpl_size nplanes,
482  const cpl_size naverage, cpl_mask * bpm)
483 {
484  const cpl_size next = cpl_frame_get_nextensions(frame);
485  visir_imglist * range = visir_imglist_new(nplanes, NULL);
486  const char * filename = cpl_frame_get_filename(frame);
487  cpl_size iext;
488  cpl_propertylist * plist = NULL;
489  size_t *pbpm = NULL;
490  size_t ndata;
491  /* image fifo for parallel loading */
492  cpl_image * imgbuf[20] = {NULL};
493  size_t nimgbuf = sizeof(imgbuf) / sizeof(imgbuf[0]);
494 
495  nplanes = CX_MAX(nplanes, naverage);
496 
497  cpl_ensure(naverage > 0, CPL_ERROR_ILLEGAL_INPUT, NULL);
498  cpl_imagelist * av = cpl_imagelist_new();
499 
500  for (iext = *start; iext < 1 + next; iext++) {
501  cpl_errorstate prestate = cpl_errorstate_get();
502  size_t nbytes;
503 
504  cpl_propertylist_delete(plist);
505 
506  plist = cpl_propertylist_load(filename, iext);
507  if (plist == NULL) {
508  cpl_msg_info(cpl_func, "No propertylist in extension %d",
509  (int)iext);
510  cpl_errorstate_set(prestate);
511  continue;
512  }
513 
514 
515  /* load more images if our fifo is empty
516  * as the input can be compressed and is often still in the cache from
517  * repack we profit from loading with more then one thread */
518  if (imgbuf[0] == NULL) {
519  parallel_load(filename, iext, next + 1, imgbuf, nimgbuf);
520  }
521 
522  /* pop one image of the buffer */
523  cpl_image * img = imgbuf[0];
524  memmove(imgbuf, imgbuf + 1, sizeof(imgbuf) - sizeof(imgbuf[0]));
525  imgbuf[nimgbuf - 1] = NULL;
526 
527  if (img == NULL) {
528  cpl_msg_debug(cpl_func, "No image-data in extension %d",
529  (int)iext);
530  cpl_propertylist_delete(plist);
531  plist = NULL;
532  cpl_errorstate_set(prestate);
533  continue;
534  }
535 
536  nbytes = visir_get_nbytes_plist(plist);
537 
538  /* readahead the file for optimal disk IO
539  * start + 1 as first extension is normally empty */
540  if (iext == *start + 1) {
541  visir_readahead(filename, iext * nbytes, nplanes * nbytes);
542  }
543 
544  nbytes_load += nbytes;
545  /* uncompressed data loaded assuming bitpix = float */
546  nbytes_load_eff += cpl_image_get_size_x(img) *
547  cpl_image_get_size_y(img) * 4;
548  cpl_imagelist_set(av, img, cpl_imagelist_get_size(av));
549 
550  if (cpl_imagelist_get_size(av) == naverage) {
551  cpl_image * avimg = naverage > 1 ? cpl_imagelist_collapse_create(av)
552  : cpl_imagelist_unset(av, 0);
553  if (bpm) {
554  cpl_image_reject_from_mask(avimg, bpm);
555  /* cpl_detector_interpolate_rejected(avimg); too slow */
556  visir_interpolate_rejected(avimg, &pbpm, &ndata);
557  }
558 
559  cut_aqu_illuminated(&avimg);
560  visir_imglist_append(range, avimg, plist);
561 
562  plist = NULL;
563  cpl_imagelist_delete(av);
564  av = cpl_imagelist_new();
565  }
566 
567  if (visir_imglist_get_size(range) == nplanes/naverage)
568  break;
569  }
570 
571  if (cpl_imagelist_get_size(av) != 0) {
572  cpl_image * avimg = cpl_imagelist_collapse_create(av);
573  if (bpm) {
574  cpl_image_reject_from_mask(avimg, bpm);
575  /* cpl_detector_interpolate_rejected(avimg); */
576  visir_interpolate_rejected(avimg, &pbpm, &ndata);
577  }
578  cut_aqu_illuminated(&avimg);
579  visir_imglist_append(range, avimg, plist);
580  }
581 
582  cpl_free(pbpm);
583  cpl_imagelist_delete(av);
584  *start = iext + 1;
585 
586  /* we don't need the input in the page cache anymore
587  * TODO: we might not want to do this when called from reflex */
588  if (visir_imglist_get_size(range) > 1) {
589  /* approximation assuming non padded header and all extensions have
590  * similar compression, take first extension as primary has no data */
591  cpl_propertylist * plist = visir_imglist_get_data(range, 1);
592  size_t nbytes = visir_get_nbytes_plist(plist);
593  visir_drop_cache(filename, 0, nbytes * (*start));
594  }
595 
596  return range;
597 }
598 
599 typedef struct {
600  double x;
601  double y;
602  cpl_size window;
603  int sign;
604  /* weight of beam image compared to the original input beams (4 or 2)
605  * in parallel mode one beam is the sum of two input beams -> weight = 2
606  * parallel mode images in util_qc */
607  float weight;
608  int type;
609 } beam_info;
610 
611 
612 static cx_list *
613 parse_beampos_cmd(const char * bpstr)
614 {
615  char *str, *token, *istr = cpl_strdup(bpstr);
616  cx_list * bpl = cx_list_new();
617 
618  for (str = istr; ; str = NULL) {
619  beam_info * bi;
620  double w = 0;
621  int ret;
622 
623  token = strtok(str, ";");
624  if (token == NULL)
625  break;
626 
627  bi = cpl_calloc(1, sizeof(beam_info));
628  bi->type = cx_list_size(bpl);
629  cx_list_push_back(bpl, bi);
630 
631  bi->weight = 1;
632  if (strncmp(token, "pos", 3) == 0)
633  bi->sign=1;
634  else if (strncmp(token, "neg", 3) == 0)
635  bi->sign=-1;
636  else
637  error_if(1, CPL_ERROR_ILLEGAL_INPUT,
638  "Invalid beam sign: %s", bpstr);
639 
640  ret = sscanf(token+4, "%lf,%lf,%lf", &bi->x, &bi->y, &w);
641  if ((ret != 2 && ret != 3) || bi->x < 0 || bi->y < 0 || w < 0)
642  error_if(1, CPL_ERROR_ILLEGAL_INPUT,
643  "Invalid beam position: %s", bpstr);
644 
645  if (ret == 3)
646  bi->window = ceil(w);
647  else
648  bi->window = 0;
649  }
650 
651  error_if(cx_list_size(bpl) < 1, CPL_ERROR_ILLEGAL_INPUT,
652  "Invalid beam position: %s", bpstr);
653  {
654  beam_info * pos = NULL;
655  int npos = 0, nneg = 0;
656  FOR_EACH_T(beam_info * bi, bpl) {
657  if (bi->sign > 0) {
658  pos = bi;
659  npos++;
660  }
661  else
662  nneg--;
663  }
664  /* give positive beam in parallel case double weight */
665  if (nneg == 2 && npos == 1)
666  pos->weight *= 2.;
667  }
668  end_skip;
669 
670  cpl_free(istr);
671 
672  return bpl;
673 }
674 
675 static cx_list *
676 get_beam_positions(cpl_image * average, const cpl_parameterlist * parlist,
677  cpl_propertylist * plist)
678 {
679  cpl_errorstate cleanstate = cpl_errorstate_get();
680  cpl_propertylist * qclist = cpl_propertylist_new();
681  cpl_image * inverse = cpl_image_duplicate(average);
682  double x[4], y[4];
683  const cpl_boolean chop_on =
684  cpl_propertylist_get_bool(plist, VISIR_PFITS_BOOL_CHOP_STATUS);
685  cx_list * beamposl = cx_list_new();
686  int n;
687 
688  skip_if(0);
689 
690  cpl_image_multiply_scalar(inverse, -1);
691 
692  visir_chopnod_mode mode =
693  visir_img_find_beam(qclist, average, inverse,
694  plist, parlist, RECIPE_STRING,
695  x, y);
696  if (cpl_error_get_code() != CPL_ERROR_NONE) {
697  irplib_error_recover(cleanstate, "Could not detect objects,"
698  " using full image. Photometry "
699  "will not work without object positions.");
700  beam_info * bi = cpl_malloc(sizeof(beam_info));
701  const cpl_size nx = cpl_image_get_size_x(average);
702  const cpl_size ny = cpl_image_get_size_y(average);
703  bi->x = nx / 2;
704  bi->y = ny / 2;
705  bi->window = CX_MAX(nx, ny);
706  bi->sign = 1;
707  bi->weight = 1;
708  bi->type = VISIR_PLANE_UNDEFINED;
709  cx_list_push_back(beamposl, bi);
710  }
711  else {
712 
713  if (mode == VISIR_CHOPNOD_PERPENDICULAR)
714  n = 4;
715  else if (mode == VISIR_CHOPNOD_PARALLEL)
716  n = 3;
717  else
718  error_if(1, CPL_ERROR_UNSUPPORTED_MODE, "Unsupported chop mode");
719 
720  if (chop_on == CPL_FALSE)
721  n = 2;
722 
723  for (int i = 0; i < n; i++) {
724  beam_info * bi = cpl_calloc(1, sizeof(beam_info));
725  bi->x = x[i];
726  bi->y = y[i];
727  /* the first are always the positive apertures */
728  if (n == 4 || n == 2) {
729  bi->sign = i < n / 2 ? 1 : -1;
730  bi->weight = 1.;
731  }
732  else {
733  bug_if(n != 3);
734  /* give positive beam in parallel case double weight */
735  bi->sign = i < 1 ? 1 : -1;
736  bi->weight = i < 1 ? 2. : 1.;
737  }
738  bi->type = i;
739  cx_list_push_back(beamposl, bi);
740  }
741  }
742 
743  end_skip;
744  cpl_image_delete(inverse);
745  cpl_propertylist_delete(qclist);
746 
747  return beamposl;
748 }
749 
750 static cx_list *
751 cut_image(const cpl_image * img, const cpl_propertylist * plist,
752  const cx_list * beampos, cpl_size window)
753 {
754  cx_list * l = cx_list_new();
755 
756  FOR_EACH_T(const beam_info * bi, beampos) {
757  const cpl_size win = bi->window > 0 ? bi->window : window;
758 
759  visir_plane * pl = visir_plane_new();
760  bug_if(0);
761 
762  pl->plist = plist ?
763  cpl_propertylist_duplicate(plist) : cpl_propertylist_new();
764 
765  window_extract_p(pl, img, bi->x, bi->y, win);
766  skip_if(pl->img == NULL);
767 
768  pl->type = bi->type;
769 
770  if (bi->sign != 1)
771  cpl_image_multiply_scalar(pl->img, bi->sign);
772 
773  cx_list_push_back(l, pl);
774  }
775 
776  end_skip;
777 
778  return l;
779 }
780 
781 
782 static void
783 generate_statistics(visir_plane * pl)
784 {
785  cpl_size mx, my;
786  cpl_errorstate prestate = cpl_errorstate_get();
787  cpl_stats * stat =
788  cpl_stats_new_from_image(pl->img, CPL_STATS_MAD |
789  CPL_STATS_MEDIAN | CPL_STATS_MAXPOS |
790  CPL_STATS_MAX | CPL_STATS_MIN |
791  CPL_STATS_STDEV);
792  pl->stdev = cpl_stats_get_stdev(stat);
793  pl->median = cpl_stats_get_median(stat);
794  pl->mad = cpl_stats_get_mad(stat);
795  pl->max = cpl_stats_get_max(stat);
796  pl->min = cpl_stats_get_min(stat);
797  mx = cpl_stats_get_max_x(stat);
798  my = cpl_stats_get_max_y(stat);
799  if (cpl_image_get_fwhm(pl->img, mx, my,
800  &pl->fwhmx, &pl->fwhmy) != CPL_ERROR_NONE) {
801  pl->fwhmx = 0;
802  pl->fwhmy = 0;
803  cpl_msg_warning(cpl_func, "Could not determine fwhm.");
804  cpl_errorstate_set(prestate);
805  }
806 
807  cpl_stats_delete(stat);
808 }
809 
810 
811 static cx_list *
812 handle_chunk(visir_imglist * chunk, const visir_plane * template,
813  const cx_list * beampos,
814  const visir_shift_method method,
815  const stats_limits * limits)
816 {
817  const cpl_size n = visir_imglist_get_size(chunk);
818  cpl_propertylist * mplist = visir_imglist_get_mplist(chunk);
819  double pthrow = visir_pfits_get_chop_pthrow(mplist);
820  cx_list * res = cx_list_new();
821 
822  /* prepare the fft transformed template, one per beam */
823  visir_fftx_cache * fft_cache[MAX_BEAMS];
824  for (size_t i = 0; i < MAX_BEAMS; i++) {
825  fft_cache[i] = visir_new_fftx_cache();
826  }
827  if (n > 0) {
828  cx_list * beams = cut_image(visir_imglist_get_img(chunk, 0),
829  NULL, beampos, pthrow);
830  skip_if(0);
831  for (cpl_size j = 0, m = cx_list_size(beams); j < m; j++) {
832  visir_plane * pl = cx_list_pop_front(beams);
833  visir_fftxcorrelate(template->img, pl->img, CPL_TRUE,
834  NULL, NULL, NULL, fft_cache[j]);
835  visir_plane_delete(pl);
836  }
837  cx_list_delete(beams);
838  }
839 
840  skip_if(0);
841 
842  /* tasks are used instead of a parallel for to yield resources to the
843  * reader thread if necessary (e.g. to decompress data)
844  * with 1k x 1k burst data the handling is already starved for data
845  * with ~8 threads */
846  for (cpl_size i = 0; i < n; i++)
847 OMP3_PRAGMA(omp task firstprivate(i))
848  {
849  cx_list * beams = cut_image(visir_imglist_get_img(chunk, i),
850  visir_imglist_get_data(chunk, i),
851  beampos, pthrow);
852 
853  for (cpl_size j = 0, m = cx_list_size(beams); j < m; j++) {
854  double xshift = 0, yshift = 0, max_cor = 1;
855  visir_plane * pl = cx_list_pop_front(beams);
856 
857  if (limits->min_cor > 0 || method == VISIR_SHIFT_XCORRELATE) {
858  /* FIXME: cropping around beampos might improve robustness */
859  visir_fftxcorrelate(template->img, pl->img, CPL_TRUE,
860  &xshift, &yshift, &max_cor, fft_cache[j]);
861  xshift = template->x + xshift;
862  yshift = template->y + yshift;
863  }
864 
865  if (method == VISIR_SHIFT_BRIGHTEST) {
866  cpl_size ixshift, iyshift;
867  double xsub, ysub;
868  cpl_image_get_maxpos(pl->img, &ixshift, &iyshift);
869  visir_get_subpixel_maxpos(pl->img, ixshift, iyshift,
870  &xsub, &ysub);
871  xshift = ixshift + xsub;
872  yshift = iyshift + ysub;
873  }
874 
875  cpl_msg_debug(cpl_func, "%4s: correlation %5.3f shift %5.2f %5.2f",
876  type2string[pl->type], max_cor, xshift, yshift);
877 
878  pl->seqnb = i;
879  pl->xshift = xshift - pl->x;
880  pl->yshift = yshift - pl->y;
881  pl->correlation = max_cor;
882  /* with shift the wcs are not same as the template */
883  pl->crval1 = template->crval1;
884  pl->crval2 = template->crval2;
885 
886  generate_statistics(pl);
887 OMP_PRAGMA(omp critical(chunk_lock))
888  cx_list_push_back(res, pl);
889  }
890 
891  cx_list_delete(beams);
892  }
893 OMP3_PRAGMA(omp taskwait)
894 
895 #ifdef _OPENMP
896  /* a priority queue or per thread lists would be more efficient
897  * but time spent here is negligible */
898  cx_list_sort(res, (cx_compare_func)pl_compare_seqnb);
899 #endif
900 
901  end_skip;
902 
903  for (size_t i = 0; i < sizeof(fft_cache)/sizeof(fft_cache[0]); i++)
904  visir_delete_fftx_cache(fft_cache[i]);
905 
906  return res;
907 }
908 
909 static visir_plane *
910 improve_template(const cx_list *imgs, const stats_limits * limits)
911 {
912  cpl_imagelist * l = cpl_imagelist_new();
913  visir_plane * template = visir_plane_new();
914  cx_list_iterator it = cx_list_begin(imgs);
915  const cx_list_iterator end = cx_list_end(imgs);
916  cpl_size nx = 0;
917  cpl_size ny = 0;
918 
919  skip_if(cx_list_empty(imgs));
920 
921  while (it != end) {
922  visir_plane * pl = cx_list_get(imgs, it);
923  cpl_image * img = cpl_image_duplicate(pl->img);
924  nx = nx == 0 ? cpl_image_get_size_x(img) : nx;
925  ny = ny == 0 ? cpl_image_get_size_y(img) : ny;
926 
927  /* sizes of different beams can diverge if they are close to the
928  * detector edge */
929  if (cpl_image_get_size_x(img) != nx ||
930  cpl_image_get_size_y(img) != ny) {
931  it = cx_list_next(imgs, it);
932  continue;
933  }
934 
935  // FIXME: use cpl_geom_img_offset_saa for subpixel shifting
936  if (is_image_bad(pl, limits) == CPL_FALSE)
937  cpl_image_shift(img, -visir_round_to_int(pl->xshift),
938  -visir_round_to_int(pl->yshift));
939 
940  cpl_imagelist_set(l, img, cpl_imagelist_get_size(l));
941 
942  it = cx_list_next(imgs, it);
943  skip_if(0);
944  }
945 
946  {
947  /* average and shrink it a bit around the beam for better correlation*/
948  cpl_image * tmp = visir_parallel_median_collapse(l);
949  visir_plane * firstpl = cx_list_get(imgs, cx_list_begin(imgs));
950  template->x = firstpl->x;
951  template->y = firstpl->y;
952  window_extract_p(template, tmp, firstpl->x, firstpl->y,
953  cpl_image_get_size_x(tmp) / 2);
954  cpl_image_delete(tmp);
955  }
956 
957  end_skip;
958  cpl_imagelist_delete(l);
959 
960  return template;
961 }
962 
963 static cpl_error_code
964 save_images(cx_list * imgs, const cpl_propertylist * mplist,
965  const cpl_parameterlist * parlist, const int iframe,
966  const stats_limits * limits)
967 {
968  const cpl_boolean noshift =
969  irplib_parameterlist_get_bool(parlist, PACKAGE,
970  RECIPE_STRING, "no-shift");
971  const cpl_boolean noreject =
972  irplib_parameterlist_get_bool(parlist, PACKAGE,
973  RECIPE_STRING, "no-reject");
974  const int naverage =
975  irplib_parameterlist_get_int(parlist, PACKAGE,
976  RECIPE_STRING, "naverage");
977 
978  for (cpl_size j = 0, n = cx_list_size(imgs); j < n; j++) {
979  char buffer[128];
980  visir_plane * pl = cx_list_pop_front(imgs);
981 
982  cpl_boolean bad = is_image_bad(pl, limits);
983  cpl_type save_type = CPL_TYPE_FLOAT;
984 
985  if (noreject == CPL_FALSE && bad) {
986  visir_plane_delete(pl);
987  continue;
988  }
989 
990  if (pl->max <= CX_MAXSHORT && pl->min >= CX_MINSHORT &&
991  naverage == 1 && cpl_propertylist_get_int(pl->plist, "BITPIX") > 0)
992  save_type = CPL_TYPE_SHORT;
993 
994  cpl_propertylist_copy_property_regexp(pl->plist, mplist, "^("
995  IRPLIB_PFITS_WCS_REGEXP ")$",
996  CPL_FALSE);
997 
998  /* don't correct for shift if the stats are bad in no-reject mode */
999  if (noshift || (noreject && bad)) {
1000  cpl_propertylist_append_double(pl->plist, "CRPIX1", pl->x);
1001  cpl_propertylist_append_double(pl->plist, "CRPIX2", pl->y);
1002  } else {
1003  cpl_propertylist_append_double(pl->plist, "CRPIX1",
1004  pl->x + pl->xshift);
1005  cpl_propertylist_append_double(pl->plist, "CRPIX2",
1006  pl->y + pl->yshift);
1007  }
1008  cpl_propertylist_append_double(pl->plist, "CRVAL1", pl->crval1);
1009  cpl_propertylist_append_double(pl->plist, "CRVAL2", pl->crval2);
1010 
1011  cpl_propertylist_append_double(pl->plist, "ESO QC CORR",
1012  pl->correlation);
1013 
1014  cpl_propertylist_append_double(pl->plist, "ESO QC MEDIAN",
1015  pl->median);
1016  cpl_propertylist_append_double(pl->plist, "ESO QC MAD",
1017  pl->mad);
1018  cpl_propertylist_append_double(pl->plist, "ESO QC STDEV",
1019  pl->stdev);
1020  cpl_propertylist_append_double(pl->plist, "ESO QC FWHMX",
1021  pl->fwhmx);
1022  cpl_propertylist_append_double(pl->plist, "ESO QC FWHMY",
1023  pl->fwhmy);
1024 
1025  sprintf(buffer, RECIPE_STRING"_%03d_%s"CPL_DFS_FITS,
1026  iframe+1, type2string[pl->type]);
1027 
1028  cpl_image_save(pl->img, buffer, save_type,
1029  pl->plist, CPL_IO_EXTEND);
1030  nbytes_save += cpl_image_get_size_x(pl->img) *
1031  cpl_image_get_size_y(pl->img) * cpl_type_get_sizeof(save_type);
1032 
1033  visir_plane_delete(pl);
1034  skip_if(0);
1035  }
1036 
1037  end_skip;
1038 
1039  return cpl_error_get_code();
1040 }
1041 
1042 
1043 #ifdef VISIR_OMP3
1044 static cpl_error_code
1045 read_data(visir_queue * readq, cpl_propertylist * mplist,
1046  cpl_frame * frame, const cpl_size chunksize,
1047  const cpl_size naverage, cpl_mask * bpm, cpl_size * cur)
1048 {
1049  visir_imglist * range = NULL;
1050  do {
1051  double t = cpl_test_get_walltime(), t2;
1052  range = load_range(frame, cur, chunksize,
1053  naverage, bpm);
1054  skip_if(0);
1055 
1056  if (visir_imglist_get_size(range) == 0) {
1057  visir_imglist_delete(range, (visir_free)cpl_propertylist_delete);
1058  break;
1059  }
1060 
1061  visir_imglist_set_mplist(range, mplist);
1062  t2 = cpl_test_get_walltime();
1063  t_loading += t2 - t;
1064  skip_if(visir_queue_put(readq, range));
1065  t_loading_blocked += cpl_test_get_walltime() - t2;
1066  } while (visir_imglist_get_size(range) != 0);
1067 
1068  cpl_msg_info(cpl_func,
1069  "Loading thread done, %.3g MB/s (%.3g MB/s uncompressed)",
1070  (nbytes_load / 1024. / 1024.) / t_loading,
1071  (nbytes_load_eff / 1024. / 1024.) / t_loading);
1072 
1073  end_skip;
1074  /* signal end of data */
1075  visir_queue_put(readq, NULL);
1076 
1077  cpl_error_code err = cpl_error_get_code();
1078  if (err != CPL_ERROR_NONE)
1079  visir_queue_set_error(readq, err);
1080 
1081  return err;
1082 }
1083 
1084 static cpl_error_code
1085 handle_data(visir_queue * readq, visir_queue * writeq,
1086  const visir_plane * template,
1087  cx_list * beampos, const visir_shift_method method,
1088  const stats_limits * limits)
1089 {
1090  visir_imglist * range = NULL;
1091  do {
1092  double t = cpl_test_get_walltime(), t2;
1093  range = visir_queue_pop(readq);
1094  t2 = cpl_test_get_walltime();
1095  t_handling_blocked_in += t2 - t;
1096  if (range == NULL)
1097  break;
1098  cx_list * res = handle_chunk(range, template, beampos,
1099  method, limits);
1100 
1101  visir_imglist_delete(range, (visir_free)cpl_propertylist_delete);
1102 
1103  t = cpl_test_get_walltime();
1104  t_handling += t - t2;
1105  skip_if(visir_queue_put(writeq, res));
1106  t_handling_blocked_out += cpl_test_get_walltime() - t;
1107  } while(range != NULL);
1108 
1109  end_skip;
1110  /* signal end of data */
1111  visir_queue_put(writeq, NULL);
1112 
1113  cpl_error_code err = cpl_error_get_code();
1114  if (err != CPL_ERROR_NONE) {
1115  visir_queue_set_error(writeq, err);
1116  visir_queue_set_error(readq, err);
1117  }
1118 
1119  return err;
1120 }
1121 
1122 
1123 static cpl_error_code
1124 write_data(visir_queue * writeq, cpl_propertylist * mplist,
1125  const cpl_parameterlist * parlist,
1126  const int iframe, const stats_limits * limits,
1127  const cpl_size chunksize, const cpl_size next)
1128 {
1129  cx_list * res = NULL;
1130  cpl_size count = 1;
1131  do {
1132  double t = cpl_test_get_walltime(), t2;
1133  res = visir_queue_pop(writeq);
1134  t2 = cpl_test_get_walltime();
1135  t_writing_blocked += t2 - t;
1136  if (res == NULL)
1137  break;
1138 
1139  save_images(res, mplist, parlist, iframe, limits);
1140  planelist_delete(res);
1141  t_writing += cpl_test_get_walltime() - t2;
1142  skip_if(0);
1143  cpl_msg_info(cpl_func, "done: %d/%d",
1144  (int)CX_MIN((count * chunksize), next + 1),
1145  (int)next + 1);
1146  count++;
1147  } while(res != NULL);
1148 
1149  cpl_msg_info(cpl_func, "Writing thread done, %.3g MB /s",
1150  (nbytes_save / 1024. / 1024.) / t_writing);
1151 
1152  end_skip;
1153 
1154  cpl_error_code err = cpl_error_get_code();
1155  if (err != CPL_ERROR_NONE)
1156  visir_queue_set_error(writeq, err);
1157 
1158  return err;
1159 }
1160 #endif
1161 
1162 /*----------------------------------------------------------------------------*/
1171 /*----------------------------------------------------------------------------*/
1172 static
1173 cpl_error_code visir_util_detect_shift_one(cpl_frameset * framelist,
1174  irplib_framelist * rawframes,
1175  int iframe,
1176  const cpl_parameterlist * parlist,
1177  cpl_mask * bpm,
1178  visir_plane ** ptemplate)
1179 {
1180  cpl_frameset * products = cpl_frameset_new();
1181  visir_plane * template = NULL;
1182  cpl_image * av = NULL;
1183  cpl_boolean bfirst = CPL_FALSE;
1184  cx_list * res = NULL;
1185  cpl_frame * frame = irplib_framelist_get(rawframes, iframe);
1186  const char * filename = cpl_frame_get_filename(frame);
1187  const char * fn_template =
1188  irplib_parameterlist_get_string(parlist, PACKAGE,
1189  RECIPE_STRING, "template");
1190  const int naverage =
1191  irplib_parameterlist_get_int(parlist, PACKAGE,
1192  RECIPE_STRING, "naverage");
1193  cpl_size cur = 0;
1194  const cpl_size chunksize = 200 * naverage;
1195  cpl_frame_set_group(frame, CPL_FRAME_GROUP_RAW);
1196 
1197  /* load the first chunk for the template */
1198  cpl_size next;
1199  cpl_propertylist * mplist = cpl_propertylist_load(filename, 0);
1200  {
1201  visir_data_type datatype;
1202  visir_get_data_type(frame, mplist, &datatype, &next);
1203  is_aqu_data = visir_data_is_aqu(datatype);
1204  }
1205  visir_imglist * range = load_range(frame, &cur, chunksize,
1206  naverage, bpm);
1207  visir_imglist_set_mplist(range, mplist);
1208  double pthrow = visir_pfits_get_chop_pthrow(mplist);
1209  const char * sbeampos =
1210  irplib_parameterlist_get_string(parlist, PACKAGE,
1211  RECIPE_STRING, "beampos");
1212  cx_list * beampos = NULL;
1213 
1214  skip_if(range == NULL);
1215 
1216  if (cpl_frameset_find(framelist, "MEAN")) {
1217  cpl_frame * fn = cpl_frameset_find(framelist, "MEAN");
1218  av = cpl_image_load(cpl_frame_get_filename(fn),
1219  CPL_TYPE_FLOAT, 0,0);
1220  cut_aqu_illuminated(&av);
1221  skip_if(0);
1222 
1223  if (bpm) {
1224  cpl_image * ibpm = cpl_image_new_from_mask(bpm);
1225  cut_aqu_illuminated(&ibpm);
1226  cpl_mask * nbpm = cpl_mask_threshold_image_create(ibpm, 0.1, DBL_MAX);
1227 
1228  cpl_image_reject_from_mask(av, nbpm);
1229 
1230  cpl_image_delete(ibpm);
1231  cpl_mask_delete(nbpm);
1232 
1233  skip_if(visir_interpolate_rejected(av, NULL, NULL));
1234  }
1235  }
1236  else {
1237  const cpl_imagelist * tmpl = visir_imglist_get_imglist(range);
1238  av = visir_parallel_median_collapse(tmpl);
1239  skip_if(0);
1240  }
1241 
1242  if (!visir_str_par_is_empty(sbeampos))
1243  beampos = parse_beampos_cmd(sbeampos);
1244  else
1245  beampos = get_beam_positions(av, parlist, mplist);
1246 
1247  skip_if(0);
1248 
1249  error_if(cx_list_size(beampos) > MAX_BEAMS, CPL_ERROR_ILLEGAL_INPUT,
1250  "Too many beams, maximum supported: %d", MAX_BEAMS);
1251  error_if(ptemplate == NULL, CPL_ERROR_NULL_INPUT, "Bug: ptemplate == NULL");
1252 
1253  /* when beam positions is determined once for all datasets we need to adapt
1254  * for the jitter */
1255  if ((!visir_str_par_is_empty(sbeampos) ||
1256  cpl_frameset_find(framelist, "MEAN")) &&
1257  cpl_propertylist_has(mplist, VISIR_DRS_CUMOFFSETXA)) {
1258  double dxa = cpl_propertylist_get_double(mplist, VISIR_DRS_CUMOFFSETXA);
1259  double dya = cpl_propertylist_get_double(mplist, VISIR_DRS_CUMOFFSETYA);
1260  double dxb = cpl_propertylist_get_double(mplist, VISIR_DRS_CUMOFFSETXB);
1261  double dyb = cpl_propertylist_get_double(mplist, VISIR_DRS_CUMOFFSETYB);
1262  FOR_EACH_T(beam_info * bi, beampos) {
1263  if (bi->type == VISIR_PLANE_AOFF || bi->type == VISIR_PLANE_AON) {
1264  bi->x += dxa;
1265  bi->y += dya;
1266  }
1267  else if (bi->type == VISIR_PLANE_BOFF || bi->type == VISIR_PLANE_BON) {
1268  bi->x += dxb;
1269  bi->y += dyb;
1270  }
1271  }
1272  }
1273 
1274  FOR_EACH_T(const beam_info * bi, beampos) {
1275  double window = bi->window > 0 ? bi->window : pthrow;
1276  cpl_msg_info(cpl_func, "Beam position: x %5.1f, y %5.1f, window %4.0f,"
1277  " sign %2d", bi->x, bi->y, window, bi->sign);
1278  }
1279 
1280  stats_limits limits;
1281  limits.min_cor =
1282  irplib_parameterlist_get_double(parlist, PACKAGE,
1283  RECIPE_STRING, "min-correlation");
1284  limits.max_shift =
1285  irplib_parameterlist_get_double(parlist, PACKAGE,
1286  RECIPE_STRING, "max-shift");
1287  limits.max_mad =
1288  irplib_parameterlist_get_double(parlist, PACKAGE,
1289  RECIPE_STRING, "max-mad");
1290 
1291  const char * smethod =
1292  irplib_parameterlist_get_string(parlist, PACKAGE,
1293  RECIPE_STRING, "method");
1294  visir_shift_method method = VISIR_SHIFT_XCORRELATE;
1295 
1296  if (strncmp(smethod, "xcorrelate", sizeof("xcorrelate")) == 0)
1297  method = VISIR_SHIFT_XCORRELATE;
1298  else if (strncmp(smethod, "brightest", sizeof("brightest")) == 0)
1299  method = VISIR_SHIFT_BRIGHTEST;
1300  else
1301  cpl_msg_warning(cpl_func, "Unknown method %s. Using xcorrelate",
1302  smethod);
1303 
1304  /* FIXME: add option for template per file but make sure
1305  * center is the same in all versions */
1306  if (*ptemplate != NULL)
1307  template = *ptemplate;
1308  else if (!visir_str_par_is_empty(fn_template)) {
1309  cpl_msg_info(cpl_func, "Loading %s as template", fn_template);
1310  template = visir_plane_new();
1311  template->plist = cpl_propertylist_load(fn_template, 0);
1312  template->img = cpl_image_load(fn_template, CPL_TYPE_UNSPECIFIED, 0, 0);
1313  skip_if(template->img == NULL || template->plist == NULL);
1314  if (cpl_propertylist_has(template->plist, "CRPIX1") &&
1315  cpl_propertylist_has(template->plist, "CRPIX2") &&
1316  cpl_propertylist_has(template->plist, "CRVAL1") &&
1317  cpl_propertylist_has(template->plist, "CRVAL2")) {
1318  template->x = cpl_propertylist_get_double(template->plist, "CRPIX1");
1319  template->y = cpl_propertylist_get_double(template->plist, "CRPIX2");
1320  template->crval1 = cpl_propertylist_get_double(template->plist, "CRVAL1");
1321  template->crval2 = cpl_propertylist_get_double(template->plist, "CRVAL2");
1322  }
1323  else {
1324  error_if(1, CPL_ERROR_ILLEGAL_INPUT,
1325  "Input template must have WCS coordinates");
1326 
1327  }
1328  *ptemplate = template;
1329  bfirst = CPL_TRUE;
1330  }
1331  else {
1332  template = visir_plane_new();
1333  cpl_msg_info(cpl_func, "Using average as template");
1334  const cpl_boolean noshift =
1335  irplib_parameterlist_get_bool(parlist, PACKAGE,
1336  RECIPE_STRING, "no-shift");
1337  beam_info * bi = cx_list_get(beampos, cx_list_begin(beampos));
1338  /* crops a little extra to get less background in the template */
1339  const cpl_size window = bi->window > 0 ?
1340  bi->window * 2 / 3. : pthrow * 2 / 3.;
1341  window_extract_p(template, av, bi->x, bi->y, window);
1342  cpl_image_multiply_scalar(template->img, bi->sign);
1343 
1344  if (!noshift &&
1345  (limits.min_cor > 0 || method == VISIR_SHIFT_XCORRELATE)) {
1346  /* improve the template by doing a coarse shift correction */
1347  stats_limits lim;
1348  lim.min_cor = 0;
1349  lim.max_shift = 5000;
1350  lim.max_mad = 0;
1351 OMP_PRAGMA(omp parallel)
1352 OMP_PRAGMA(omp single)
1353  res = handle_chunk(range, template, beampos, method, &lim);
1354  visir_plane_delete(template);
1355  template = improve_template(res, &limits);
1356 
1357  planelist_delete(res);
1358  res = NULL;
1359  }
1360  template->plist = cpl_propertylist_new();
1361  cpl_propertylist_append_double(template->plist, "CRPIX1", template->x);
1362  cpl_propertylist_append_double(template->plist, "CRPIX2", template->y);
1363  *ptemplate = template;
1364 
1365  /* get world coordinate of template center */
1366  cpl_wcs * wcs = cpl_wcs_new_from_propertylist(mplist);
1367  cpl_matrix * from = cpl_matrix_new(1, 2);
1368  cpl_matrix * to = NULL;
1369  cpl_matrix_set(from, 0, 0, bi->x);
1370  cpl_matrix_set(from, 0, 1, bi->y);
1371  cpl_array * status = NULL;
1372  skip_if(cpl_wcs_convert(wcs, from, &to, &status, CPL_WCS_PHYS2WORLD));
1373  template->crval1 = cpl_matrix_get(to, 0, 0);
1374  template->crval2 = cpl_matrix_get(to, 0, 1);
1375  cpl_matrix_delete(from);
1376  cpl_matrix_delete(to);
1377  cpl_array_delete(status);
1378 
1379  bfirst = CPL_TRUE;
1380  }
1381  skip_if(0);
1382 
1383  if (bpm) {
1384  int i = 0;
1385  cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_STATIC_MASK);
1386  cpl_image * ibpm = cpl_image_new_from_mask(bpm);
1387  cut_aqu_illuminated(&ibpm);
1388  cx_list * cbpml = cut_image(ibpm, NULL, beampos, pthrow);
1389  cpl_frameset * usedframes = cpl_frameset_new();
1390  cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
1391 
1392  FOR_EACH(it, cbpml) {
1393  char buffer[128];
1394  cpl_image * img = ((visir_plane*)cx_list_get(cbpml, it))->img;
1395  cpl_image_abs(img);
1396  sprintf(buffer, RECIPE_STRING"_bpm_%03d_%s"CPL_DFS_FITS,
1397  iframe+1, type2string[i++]);
1398  irplib_dfs_save_image(products, parlist,
1399  usedframes, img, CPL_BPP_8_UNSIGNED,
1400  RECIPE_STRING,
1401  VISIR_CALIB_BPM,
1402  NULL, NULL, visir_pipe_id,
1403  buffer);
1404  }
1405  planelist_delete(cbpml);
1406  cpl_image_delete(ibpm);
1407  cpl_frameset_delete(usedframes);
1408  }
1409 
1410  skip_if(0);
1411 
1412  if (bfirst) {
1413  char buffer[128];
1414  cpl_frameset * usedframes = cpl_frameset_new();
1415  sprintf(buffer, RECIPE_STRING"_template_%03d"CPL_DFS_FITS,
1416  iframe+1);
1417 
1418  cpl_frameset_insert(usedframes, cpl_frame_duplicate(frame));
1419  skip_if(irplib_dfs_save_image(products, parlist,
1420  usedframes, template->img, CPL_TYPE_FLOAT,
1421  RECIPE_STRING,
1422  VISIR_UTIL_DETECT_SHIFT_TEMPLATE_PROCATG,
1423  template->plist, NULL, visir_pipe_id,
1424  buffer));
1425  cpl_frameset_delete(usedframes);
1426  }
1427 
1428  FOR_EACH_T(const beam_info * bi, beampos) {
1429  char buffer[128];
1430  cpl_frameset * usedframes = cpl_frameset_new();
1431  cpl_propertylist * _mplist = cpl_propertylist_new();
1432  cpl_propertylist_copy_property_regexp(_mplist, mplist, ".*", 0);
1433 
1434  if (bi->type != VISIR_PLANE_UNDEFINED)
1435  sprintf(buffer, "%01d", bi->type);
1436  else
1437  sprintf(buffer, "UNDEFINED");
1438  cpl_propertylist_append_string(_mplist, "ESO QC BEAMID", buffer);
1439  cpl_propertylist_append_float(_mplist, "ESO DRS IMGWGT", bi->weight);
1440  cpl_propertylist_append_double(_mplist, "ESO QC BEAMX", bi->x);
1441  cpl_propertylist_append_double(_mplist, "ESO QC BEAMY", bi->y);
1442 
1443  sprintf(buffer, RECIPE_STRING"_%03d_%s"CPL_DFS_FITS,
1444  iframe+1, type2string[bi->type]);
1445 
1446  cpl_frameset_insert(usedframes, cpl_frame_duplicate(frame));
1447  irplib_dfs_save_propertylist(products, parlist,
1448  usedframes, RECIPE_STRING,
1449  VISIR_UTIL_DETECT_SHIFT_PROCATG,
1450  _mplist, NULL, visir_pipe_id, buffer);
1451  cpl_frameset_delete(usedframes);
1452  cpl_propertylist_delete(_mplist);
1453  skip_if(0);
1454  }
1455 
1456  cpl_msg_info(cpl_func, "Working on frame %d", iframe);
1457  /* first range already loaded
1458  * process it and load next range in parallel*/
1459 #ifdef VISIR_OMP3
1460  cpl_error_code serr, werr, lerr;
1461  serr = werr = lerr = CPL_ERROR_NONE;
1462  visir_queue * readq = visir_queue_init(3);
1463  visir_queue * writeq = visir_queue_init(3);
1464  visir_queue_put(readq, range);
1465  range = NULL;
1466  /* Start a thread to read data, a thread to handle the data and a thread to
1467  * write it. Interleaving of reads and write not a big issue as modern
1468  * file systems will delay the actual disk write for a very long time.
1469  * The reader and worker parallize by openmp tasks so no nested
1470  * parallization is required.
1471  * While the scheduling is more expensive than the simple static of the
1472  * nested case, this has the advantage that when the worker thread is
1473  * stalled for data the reader thread can take all resources it needs to
1474  * load new data as fast as possible. This is very relevant when reading
1475  * cached or compressed data which can be CPU bound on fast disks */
1476 
1477  /* oversubscribe by two so the queues can't deadlock */
1478  int nthreads = CX_MAX(visir_get_num_threads(CPL_FALSE) + 2, 3);
1479 #pragma omp parallel num_threads(nthreads)
1480  {
1481 #pragma omp single nowait
1482  lerr = read_data(readq, mplist, frame, chunksize, naverage, bpm, &cur);
1483 #pragma omp single nowait
1484  serr = handle_data(readq, writeq, template, beampos, method, &limits);
1485 #pragma omp single nowait
1486  werr = write_data(writeq, mplist, parlist, iframe, &limits,
1487  chunksize, next);
1488  }
1489  visir_queue_delete(readq);
1490  visir_queue_delete(writeq);
1491 
1492  error_if(serr, serr, "Failed to save data");
1493  error_if(werr, werr, "Failed to handle data");
1494  error_if(lerr, lerr, "Failed to load data");
1495 
1496  cpl_msg_info(cpl_func, "Time spent loading data: %3.3gs",
1497  t_loading);
1498  cpl_msg_info(cpl_func, "Time loading thread was blocked on output: %3.3gs",
1499  t_loading_blocked < 1e-3 ? 0. : t_loading_blocked);
1500  cpl_msg_info(cpl_func, "Time working thread was blocked on input: %3.3gs",
1501  t_handling_blocked_in < 1e-3 ? 0. : t_handling_blocked_in);
1502  cpl_msg_info(cpl_func, "Time working on data: %3.3gs",
1503  t_handling);
1504  cpl_msg_info(cpl_func, "Time working thread was blocked on output: %3.3gs",
1505  t_handling_blocked_out < 1e-3 ? 0. : t_handling_blocked_out);
1506  cpl_msg_info(cpl_func, "Time saving thread was blocked on input: %3.3gs",
1507  t_writing_blocked < 1e-3 ? 0. : t_writing_blocked);
1508  cpl_msg_info(cpl_func, "Time spent saving data: %3.3gs",
1509  t_writing);
1510 #else
1511  /* without omp 3.0 just do it synchrounous */
1512  do {
1513  res = handle_chunk(range, template, beampos,
1514  method, &limits);
1515 
1516  skip_if(0);
1517  cpl_msg_info(cpl_func, "done: %d/%d",
1518  (int)cur - 1, (int)next + 1);
1519  visir_imglist_delete(range,
1520  (visir_free)cpl_propertylist_delete);
1521  skip_if(0);
1522 
1523  range = load_range(frame, &cur, chunksize,
1524  naverage, bpm);
1525  skip_if(0);
1526  visir_imglist_set_mplist(range, mplist);
1527 
1528  save_images(res, mplist, parlist, iframe, &limits);
1529  planelist_delete(res);
1530  res = NULL;
1531  skip_if(0);
1532  } while (visir_imglist_get_size(range) != 0);
1533 #endif
1534 
1535  /* only output products that have accepted images */
1536  FOR_EACH_FRAMESET_C(frm, products) {
1537  if (cpl_frame_get_nextensions(frm) > 0 ||
1538  strcmp(cpl_frame_get_tag(frm),
1539  VISIR_UTIL_DETECT_SHIFT_TEMPLATE_PROCATG) == 0 ||
1540  strcmp(cpl_frame_get_tag(frm),
1541  VISIR_CALIB_BPM) == 0 ||
1542  cpl_frame_get_group(frm) != CPL_FRAME_GROUP_PRODUCT) {
1543  cpl_frameset_insert(framelist, cpl_frame_duplicate(frm));
1544  }
1545  }
1546 
1547  end_skip;
1548  planelist_delete(res);
1549  cpl_image_delete(av);
1550  cx_list_destroy(beampos, (visir_free)cpl_free);
1551  visir_imglist_delete(range, (visir_free)cpl_propertylist_delete);
1552  cpl_propertylist_delete(mplist);
1553  cpl_frameset_delete(products);
1554 
1555  return cpl_error_get_code();
1556 }
1557 
double visir_pfits_get_chop_pthrow(const cpl_propertylist *self)
The chopping throw in pixels.
Definition: visir_pfits.c:214
cpl_error_code irplib_dfs_save_propertylist(cpl_frameset *allframes, const cpl_parameterlist *parlist, const cpl_frameset *usedframes, const char *recipe, const char *procat, const cpl_propertylist *applist, const char *remregexp, const char *pipe_id, const char *filename)
Save a propertylist as a DFS-compliant pipeline product.
Definition: irplib_utils.c:240
irplib_framelist * irplib_framelist_extract_regexp(const irplib_framelist *self, const char *regexp, cpl_boolean invert)
Extract the frames with the given tag from a framelist.
cpl_error_code irplib_dfs_save_image(cpl_frameset *allframes, const cpl_parameterlist *parlist, const cpl_frameset *usedframes, const cpl_image *image, cpl_type_bpp bpp, const char *recipe, const char *procat, const cpl_propertylist *applist, const char *remregexp, const char *pipe_id, const char *filename)
Save an image as a DFS-compliant pipeline product.
Definition: irplib_utils.c:192
cpl_frame * irplib_framelist_get(irplib_framelist *self, int pos)
Get the specified frame from the framelist.
irplib_framelist * irplib_framelist_extract(const irplib_framelist *self, const char *tag)
Extract the frames with the given tag from a framelist.
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.
Definition: visir_dfs.c:72
const cpl_frame * irplib_framelist_get_const(const irplib_framelist *self, int pos)
Get the specified frame from the framelist.
void irplib_framelist_delete(irplib_framelist *self)
Deallocate an irplib_framelist with its frames and properties.
irplib_framelist * irplib_framelist_cast(const cpl_frameset *frameset)
Create an irplib_framelist from a cpl_framelist.
int irplib_framelist_get_size(const irplib_framelist *self)
Get the size of a framelist.