36#include "visir_recipe.h"
49#define RECIPE_STRING "visir_util_clip"
51#ifndef VISIR_UTIL_CLIP_KEEPFRAC
52#define VISIR_UTIL_CLIP_KEEPFRAC 0.9
54#ifndef VISIR_UTIL_CLIP_KAPPA
55#define VISIR_UTIL_CLIP_KAPPA 5.0
58#ifndef VISIR_UTIL_CLIP_MAXITE
59#define VISIR_UTIL_CLIP_MAXITE 3
62#define VISIR_UTIL_CLIP_QC_MAP_MAX "ESO QC CONTRIBUTION MAX"
63#define VISIR_UTIL_CLIP_QC_MAP_MEAN "ESO QC CONTRIBUTION MEAN"
64#define VISIR_UTIL_CLIP_QC_MAP_MEDIAN "ESO QC CONTRIBUTION MEDIAN"
67 VISIR_ERROR_SRC_IMG_STDEV,
68 VISIR_ERROR_SRC_TIMESERIES,
72static int prnok_nz = 0;
73static double * prnok = NULL;
78static double median_abs_dev(
const cpl_image * img)
81 const size_t npix = cpl_image_get_size_x(img) * cpl_image_get_size_y(img);
82 const double median = cpl_image_get_median(img);
83 const cpl_mask * bpm = cpl_image_get_bpm_const(img);
84 const cpl_binary * bpm_ = cpl_mask_get_data_const(bpm);
85 const float * data = cpl_image_get_data_float_const(img);
86 cpl_array * tmp = cpl_array_new(npix, CPL_TYPE_FLOAT);
88 for (
size_t i = 0; i < npix; i++)
89 if (bpm_[i] != CPL_BINARY_0)
90 cpl_array_set_invalid(tmp, i);
92 cpl_array_set_float(tmp, i, fabs(data[i] - median));
94 double mad = cpl_array_get_median(tmp);
95 cpl_array_delete(tmp);
100cpl_error_code visir_util_clip_kappa_sigma_double(cpl_imagelist *,
105cpl_error_code visir_util_clip_kappa_sigma_float(cpl_imagelist *,
110cpl_error_code visir_util_clip_kappa_sigma_int(cpl_imagelist *,
115static cpl_error_code visir_util_clip_one(cpl_frameset *,
120 const cpl_parameterlist *);
122static cpl_error_code visir_util_clip_kappa_sigma(cpl_imagelist *,
124 const cpl_parameterlist *,
129#define cpl_plugin_get_info visir_util_clip_get_info
131cpl_recipe_define(visir_util_clip, VISIR_BINARY_VERSION,
132 "Lars Lundin", PACKAGE_BUGREPORT,
"2011",
133 "Kappa-sigma clipping of outliers for each pixel",
134 "The files listed in the Set Of Frames (sof-file) "
135 "must be tagged pair-wise:\n"
136 "VISIR-raw-file.fits " VISIR_UTIL_INPUTS_RAW
"\n"
137 "VISIR-bpm-file.fits " VISIR_CALIB_BPM
"\n"
138 "\nThe product(s) will have a FITS card\n"
139 "'HIERARCH ESO PRO CATG' with a value of:\n"
140 VISIR_IMG_CLIPPED_PROCATG
"\n"
141 "The outliers are marked as rejected in the matching\n"
164cpl_error_code visir_util_clip_fill_parameterlist(cpl_parameterlist * self)
167 const char * context = PACKAGE
"." RECIPE_STRING;
170 cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);
175 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
176 "keepfrac", VISIR_UTIL_CLIP_KEEPFRAC,
177 NULL, context,
"The fraction of "
178 "pixels to keep for the initial"
180 cpl_ensure_code(!err, err);
184 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
185 "kappa", VISIR_UTIL_CLIP_KAPPA,
186 NULL, context,
"Clip outside "
187 "+ or - kappa * sigma "
188 "(the standard deviation)");
189 cpl_ensure_code(!err, err);
193 err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
194 "maxite", VISIR_UTIL_CLIP_MAXITE, NULL,
195 context,
"Max number of kappa-sigma "
196 "clipping iterations");
197 cpl_ensure_code(!err, err);
200 err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
201 "shift-beams", CPL_TRUE, NULL,
202 context,
"Account for movements of the "
203 "object defined in CRPIX[12]");
204 cpl_ensure_code(!err, err);
209 err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
210 "error-source",
"img-stdev", NULL,
211 context,
"Defines the way errors "
213 " img-stdev: stdev of image\n"
214 " timeseries: stdev of each pixel "
215 "over the time axis of the cube\n"
216 " none: no error generation");
217 cpl_ensure_code(!err, err);
220 err = irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
221 "error-out-type",
"error", NULL,
222 context,
"Output clipped error as "
223 "error, variance, weight or none");
224 cpl_ensure_code(!err, err);
227 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
228 "badimage", 0.2, NULL,
229 context,
"If percentage of clipped "
230 "pixels above this value the whole "
231 "image is considered bad");
232 cpl_ensure_code(!err, err);
234 return CPL_ERROR_NONE;
246static int visir_util_clip(cpl_frameset * framelist,
247 const cpl_parameterlist * parlist)
249 cpl_errorstate cleanstate = cpl_errorstate_get();
250 cpl_error_code didfail = CPL_ERROR_NONE;
251 irplib_framelist * allframes = NULL;
252 irplib_framelist * rawframes = NULL;
253 irplib_framelist * bpmframes = NULL;
254 cpl_size n, nbad = 0;
255 cpl_mask * static_mask = NULL;
259 omp_set_num_threads(visir_get_num_threads(CPL_FALSE));
264 cpl_fits_set_mode(CPL_FITS_START_CACHING);
267 allframes = irplib_framelist_cast(framelist);
268 skip_if(allframes == NULL);
270 irplib_framelist_extract_regexp(allframes,
"^("VISIR_UTIL_INPUTS_RAW
271 "|"VISIR_UTIL_CORRECTED
")$",
273 skip_if (rawframes == NULL);
274 bpmframes = irplib_framelist_extract_regexp(allframes,
"^(" VISIR_CALIB_BPM
277 if (bpmframes == NULL)
278 cpl_errorstate_set(cleanstate);
280 nbad = irplib_framelist_get_size(bpmframes);
283 cpl_frame * static_frm = cpl_frameset_find(framelist,
284 VISIR_CALIB_STATIC_MASK);
286 const char * fn = cpl_frame_get_filename(static_frm);
287 static_mask = cpl_mask_load(fn, 0, 0);
289 cpl_errorstate_set(cleanstate);
290 static_mask = cpl_mask_load(fn, 0, 1);
292 cpl_errorstate_set(cleanstate);
297 n = irplib_framelist_get_size(rawframes);
298 error_if(nbad != n && nbad != 1 && nbad != 0, CPL_ERROR_INCOMPATIBLE_INPUT,
299 "%d raw-frames <=> %d bpm frames", (
int)n, (
int)nbad);
302#if defined _OPENMP && \
303 defined CPL_VERSION_CODE && CPL_VERSION_CODE >= CPL_VERSION(6, 1, 0)
304#pragma omp parallel for
306 for (cpl_size i = 0; i < n; i++) {
316 if (visir_util_clip_one(framelist, rawframes, bpmframes,
318 nbad == 1, parlist)) {
319 const cpl_error_code errori = cpl_error_set_where(cpl_func);
323 cpl_errorstate_dump(cleanstate, CPL_FALSE, NULL);
324 cpl_errorstate_set(cleanstate);
325#pragma omp critical(visir_util_clip)
332 error_if(didfail, didfail,
"Failed to clip %d frame(s)", (
int)n);
336 irplib_framelist_delete(allframes);
337 irplib_framelist_delete(rawframes);
338 irplib_framelist_delete(bpmframes);
342 return cpl_error_get_code();
359static cpl_error_code visir_util_clip_one(cpl_frameset * framelist,
360 irplib_framelist * rawframes,
361 irplib_framelist * bpmframes,
362 const cpl_mask * static_mask,
363 int iframe, cpl_boolean bshared,
364 const cpl_parameterlist * parlist)
367 cpl_frameset * products = cpl_frameset_new();
368 cpl_frameset * usedframes = cpl_frameset_new();
370 char * bpmname = cpl_sprintf(RECIPE_STRING
"_bpm_%03d" CPL_DFS_FITS, iframe);
371 char * mapname = cpl_sprintf(RECIPE_STRING
"_map_%03d" CPL_DFS_FITS, iframe);
372 char * errname = cpl_sprintf(RECIPE_STRING
"_error_%03d" CPL_DFS_FITS, iframe);
374 const int n = irplib_framelist_get_size(rawframes);
375 cpl_frameset * rawone = cpl_frameset_new();
376 cpl_frameset * bpmone = cpl_frameset_new();
377 cpl_frame * rawframe =
378 cpl_frame_duplicate(irplib_framelist_get_const(rawframes, iframe));
379 cpl_frame * bpmframe = bpmframes ?
380 cpl_frame_duplicate(irplib_framelist_get_const(bpmframes,
381 bshared ? 0 : iframe))
383 const cpl_error_code errr = cpl_frameset_insert(rawone, rawframe);
384 const cpl_error_code errb = bpmframe ?
385 cpl_frameset_insert(bpmone, bpmframe) : CPL_ERROR_NONE;
387 const char * serr = irplib_parameterlist_get_string(parlist, PACKAGE,
390 cpl_boolean berr = serr && strcasecmp(serr,
"none") != 0;
392 const cpl_boolean bshifts = irplib_parameterlist_get_bool(parlist, PACKAGE,
396 const double badimage = irplib_parameterlist_get_double(parlist, PACKAGE,
400 const char * serr_src =
401 irplib_parameterlist_get_string(parlist, PACKAGE, RECIPE_STRING,
404 cpl_imagelist * devlist = cpl_imagelist_new();
405 cpl_imagelist * rawlist =
406 cpl_imagelist_load_frameset(rawone, CPL_TYPE_UNSPECIFIED, 0, -1);
407 cpl_imagelist * bpmlist = bpmframe ?
408 cpl_imagelist_load_frameset(bpmone, CPL_TYPE_INT, 0, -1) : NULL;
410 cpl_mask * bpm = NULL;
411 cpl_image * map = NULL;
412 cpl_propertylist * qclist = cpl_propertylist_new();
413 cpl_propertylist * xtlist = cpl_propertylist_new();
415 const int m = rawlist ? cpl_imagelist_get_size(rawlist) : 0;
416 const int nbpm = bpmlist ? cpl_imagelist_get_size(bpmlist) : 0;
417 int mapmax, mapmean, mapmedian;
418 int * shifts = cpl_calloc(m * 2,
sizeof(
int));
419 double fx = 0, fy = 0;
421 const char * err_procatg =
"ERROR_MAP";
422 visir_error_type terr = VISIR_ERROR;
423 visir_error_source err_src = VISIR_ERROR_SRC_IMG_STDEV;
424 if (berr && strcmp(serr,
"variance") == 0) {
425 terr = VISIR_VARIANCE;
426 err_procatg =
"VARIANCE_MAP";
428 else if (berr && strcmp(serr,
"weight") == 0) {
430 err_procatg =
"WEIGHT_MAP";
433 if (serr_src && !strcmp(serr_src,
"img-stdev"))
434 err_src = VISIR_ERROR_SRC_IMG_STDEV;
435 else if (serr_src && !strcmp(serr_src,
"timeseries"))
436 err_src = VISIR_ERROR_SRC_TIMESERIES;
437 else if (serr_src && !strcasecmp(serr_src,
"none"))
440 error_if(1, CPL_ERROR_ILLEGAL_INPUT,
"Unknown error-source: %s",
445 for (
int e = 0; bshifts && e < m; e++) {
446 const cpl_propertylist * plist;
447 double crpix1, crpix2;
448 cpl_errorstate prestate = cpl_errorstate_get();
450 irplib_framelist_load_propertylist(rawframes, iframe, e + 1,
"^("
451 IRPLIB_PFITS_WCS_REGEXP
")$",
453 plist = irplib_framelist_get_propertylist_const(rawframes, iframe);
454 if (!cpl_propertylist_has(plist,
"CRPIX1") ||
455 !cpl_propertylist_has(plist,
"CRPIX2")) {
456 cpl_errorstate_set(prestate);
460 crpix1 = irplib_pfits_get_double(plist,
"CRPIX1");
461 crpix2 = irplib_pfits_get_double(plist,
"CRPIX2");
469 shifts[e * 2] = visir_round_to_int(-(fx - crpix1));
470 shifts[e * 2 + 1] = visir_round_to_int(-(fy - crpix2));
472 cpl_msg_debug(cpl_func,
"CRPIX shifts %d %d, %f %f", shifts[e * 2],
473 shifts[e * 2 + 1], crpix1 - fx, crpix2 - fy);
476 skip_if(rawlist == NULL);
481 error_if(nbpm != 1 && nbpm != m && nbpm != 0, CPL_ERROR_INCOMPATIBLE_INPUT,
482 "Frame-pair %d/%d: %d image(s) <=> %d bad pixel map(s)",
483 1+iframe, n, m, (
int)cpl_imagelist_get_size(bpmlist));
485 bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(rawframe)));
487 bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(bpmframe)));
489 for (
int j = 0; bpmframe && j < m; j++) {
490 const cpl_image * bpmimg =
491 cpl_imagelist_get_const(bpmlist, nbpm > 1 ? j : 0);
492 cpl_image * rawimg = cpl_imagelist_get(rawlist, j);
494 cpl_mask_delete(bpm);
495 bpm = cpl_mask_threshold_image_create(bpmimg, 0.5, FLT_MAX);
499 cpl_mask_or(bpm, static_mask);
501 bug_if(cpl_image_reject_from_mask(rawimg, bpm));
506 skip_if(visir_util_clip_kappa_sigma(rawlist, devlist, parlist, shifts));
510 cpl_image * img = cpl_imagelist_get(rawlist, 0);
511 const int total_size =
512 cpl_image_get_size_x(img) * cpl_image_get_size_y(img);
513 cpl_mask * allbad = cpl_mask_new(cpl_image_get_size_x(img),
514 cpl_image_get_size_y(img));
515 cpl_mask_not(allbad);
517 for (cpl_size i = 0; i < cpl_imagelist_get_size(rawlist); i++) {
518 cpl_mask * mask = cpl_image_get_bpm(cpl_imagelist_get(rawlist, i));
519 img = cpl_imagelist_get(rawlist, i);
520 if (badimage < (
double)cpl_mask_count(mask) / total_size)
521 cpl_image_reject_from_mask(img, allbad);
524 cpl_mask_delete(allbad);
528 bug_if(cpl_imagelist_get_size(devlist) != 2);
530 map = cpl_image_new_from_accepted(rawlist);
531 mapmax = cpl_image_get_max(map);
532 mapmean = cpl_image_get_mean(map);
533 mapmedian = cpl_image_get_median(map);
534 bug_if(cpl_propertylist_append_int(qclist, VISIR_UTIL_CLIP_QC_MAP_MAX,
536 bug_if(cpl_propertylist_append_int(qclist, VISIR_UTIL_CLIP_QC_MAP_MEAN,
538 bug_if(cpl_propertylist_append_int(qclist, VISIR_UTIL_CLIP_QC_MAP_MEDIAN,
540 bug_if(cpl_propertylist_set_comment(qclist, VISIR_UTIL_CLIP_QC_MAP_MAX,
541 "The maximum contribution on a pixel"));
542 bug_if(cpl_propertylist_set_comment(qclist, VISIR_UTIL_CLIP_QC_MAP_MEAN,
543 "The mean contribution on a pixel"));
544 bug_if(cpl_propertylist_set_comment(qclist, VISIR_UTIL_CLIP_QC_MAP_MEDIAN,
545 "The median contribution on a pixel"));
547 skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
549 VISIR_IMG_CLIPPED_PROCATG, qclist,
550 NULL, visir_pipe_id, bpmname));
553 skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
556 NULL, visir_pipe_id, errname));
558 for (
int j = 0; j < m; j++) {
559 cpl_image * rawimg = cpl_imagelist_get(rawlist, j);
560 const cpl_mask * newbpm = cpl_image_get_bpm_const(rawimg);
561 const cpl_size npix =
562 cpl_image_get_size_x(rawimg) * cpl_image_get_size_y(rawimg);
565 skip_if(cpl_mask_save(newbpm, bpmname, NULL, CPL_IO_EXTEND));
567 if (berr == CPL_FALSE)
570 if (err_src == VISIR_ERROR_SRC_IMG_STDEV) {
572 err = cpl_image_new(cpl_image_get_size_x(rawimg),
573 cpl_image_get_size_y(rawimg),
575 if (cpl_mask_count(cpl_image_get_bpm_const(rawimg)) == npix)
578 cpl_image * dupl = cpl_image_cast(rawimg, CPL_TYPE_FLOAT);
579 const double median = cpl_image_get_median(dupl);
580 const double sigma = cpl_image_get_stdev(dupl);
582 irplib_parameterlist_get_double(parlist, PACKAGE,
583 RECIPE_STRING,
"kappa");
586 cpl_mask_threshold_image_create(dupl, median - kappa * sigma,
587 median + kappa * sigma);
590 if (!cpl_mask_is_empty(mask))
592 cpl_mask_or(mask, cpl_image_get_bpm(dupl));
593 cpl_image_reject_from_mask(dupl, mask);
595 bkgsigma = median_abs_dev(dupl) * 1.4826;
596 cpl_msg_debug(cpl_func,
"%d: sigma %.4f", j, bkgsigma);
597 cpl_image_delete(dupl);
598 cpl_mask_delete(mask);
601 cpl_image_add_scalar(err, bkgsigma);
603 else if (err_src == VISIR_ERROR_SRC_TIMESERIES)
604 err = cpl_image_duplicate(cpl_imagelist_get_const(devlist, 1));
608 cpl_image_reject_from_mask(err, newbpm);
609 if (terr == VISIR_ERROR)
610 cpl_image_fill_rejected(err, INFINITY);
611 else if (terr == VISIR_VARIANCE) {
612 cpl_image_power(err, 2);
613 cpl_image_fill_rejected(err, INFINITY);
615 else if (terr == VISIR_WEIGHT) {
616 cpl_image_power(err, -2);
617 cpl_image_fill_rejected(err, 0);
619 cpl_image_save(err, errname, CPL_TYPE_FLOAT, NULL, CPL_IO_EXTEND);
620 cpl_image_delete(err);
624 skip_if(irplib_dfs_save_image(products, parlist, usedframes,
625 map, m < 256 ? CPL_BPP_8_UNSIGNED
626 : (m < 65536 ? CPL_BPP_16_UNSIGNED
627 : CPL_BPP_32_SIGNED), RECIPE_STRING,
628 VISIR_IMG_CLIPPED_MAP_PROCATG, qclist,
629 NULL, visir_pipe_id, mapname));
631 bug_if(cpl_propertylist_append_string(xtlist,
"EXTNAME",
"NO CLIP STANDARD "
634 skip_if(cpl_image_save(cpl_imagelist_get_const(devlist, 0), mapname,
635 CPL_TYPE_FLOAT, xtlist, CPL_IO_EXTEND));
637 bug_if(cpl_propertylist_update_string(xtlist,
"EXTNAME",
"CLIPPED STANDARD "
640 skip_if(cpl_image_save(cpl_imagelist_get_const(devlist, 1), mapname,
641 CPL_TYPE_FLOAT, xtlist, CPL_IO_EXTEND));
644#pragma omp critical(visir_util_clip_one)
646 FOR_EACH_FRAMESET_C(frame, products)
647 cpl_frameset_insert(framelist, cpl_frame_duplicate(frame));
656 cpl_frameset_delete(rawone);
657 cpl_frameset_delete(bpmone);
658 cpl_frameset_delete(usedframes);
659 cpl_frameset_delete(products);
661 cpl_propertylist_delete(qclist);
662 cpl_propertylist_delete(xtlist);
664 cpl_mask_delete(bpm);
665 cpl_image_delete(map);
666 cpl_imagelist_delete(rawlist);
667 cpl_imagelist_delete(bpmlist);
668 cpl_imagelist_delete(devlist);
670 return cpl_error_get_code();
685cpl_error_code visir_util_clip_kappa_sigma(cpl_imagelist * self,
686 cpl_imagelist * devlist,
687 const cpl_parameterlist * parlist,
691 const double keepfrac = irplib_parameterlist_get_double(parlist, PACKAGE,
694 const double kappa = irplib_parameterlist_get_double(parlist, PACKAGE,
697 const int maxite = irplib_parameterlist_get_int(parlist, PACKAGE,
700 const cpl_image * img = cpl_imagelist_get_const(self, 0);
702 switch (cpl_image_get_type(img)) {
703 case CPL_TYPE_DOUBLE:
704 skip_if(visir_util_clip_kappa_sigma_double(self, devlist, keepfrac,
705 kappa, maxite, shifts));
708 skip_if(visir_util_clip_kappa_sigma_float(self, devlist, keepfrac,
709 kappa, maxite, shifts));
712 skip_if(visir_util_clip_kappa_sigma_int(self, devlist, keepfrac,
713 kappa, maxite, shifts));
721 return cpl_error_get_code();
728#define CONCAT(a,b) a ## _ ## b
729#define CONCAT2X(a,b) CONCAT(a,b)
731#define PIXEL_TYPE double
732#define STDEV_TYPE CPL_TYPE_DOUBLE
733#define PIXEL_TYPE_CPL CPL_TYPE_DOUBLE
734#include "visir_util_clip_body.c"
739#define PIXEL_TYPE float
740#define PIXEL_TYPE_CPL CPL_TYPE_FLOAT
741#define STDEV_TYPE CPL_TYPE_FLOAT
742#include "visir_util_clip_body.c"
747#define PIXEL_TYPE int
748#define PIXEL_TYPE_CPL CPL_TYPE_INT
749#define STDEV_TYPE CPL_TYPE_FLOAT
750#include "visir_util_clip_body.c"
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.