IIINSTRUMENT Pipeline Reference Manual 4.4.3
visir_util_run_swarp.c
1/*
2 * This file is part of the VISIR Pipeline
3 * Copyright (C) 2012,2013,2014 European Southern Observatory
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
18 */
19
20#ifdef HAVE_CONFIG_H
21#include <config.h>
22#endif
23
24/*-----------------------------------------------------------------------------
25 Includes
26 -----------------------------------------------------------------------------*/
27
28#include "visir_recipe.h"
29#include "visir_utils.h"
30#include "irplib_framelist.h"
31#include <cxlist.h>
32#include <stdlib.h>
33#include <string.h>
34#include <strings.h>
35#include <sys/types.h>
36#include <sys/wait.h>
37/* for access */
38#include <unistd.h>
39
40#include <fcntl.h>
41#include <semaphore.h>
42
43/*-----------------------------------------------------------------------------
44 Defines
45 -----------------------------------------------------------------------------*/
46
47#define RECIPE_STRING "visir_util_run_swarp"
48#define DEFAULT_CONFIG VISIR_CONFIG_PATH "/visir_default.swarp"
49#define SWARP_COADD_HEAD RECIPE_STRING "_coadd.head"
50#define SWARP_DATA_FILES RECIPE_STRING "_data_files"
51
52/*-----------------------------------------------------------------------------
53 Private Functions prototypes
54 -----------------------------------------------------------------------------*/
55
56#ifdef VISIR_CHAIN
57#define cpl_plugin_get_info visir_util_run_swarp_get_info
58#endif
59cpl_recipe_define(visir_util_run_swarp, VISIR_BINARY_VERSION,
60 "Julian Taylor", PACKAGE_BUGREPORT, "2012",
61 "Run swarp program to coadd images.",
62 "WARNING: this recipe is intented to be run in a "
63 "temporary directory, it does not clean up behind itself\n"
64 "The files listed in the Set Of Frames (sof-file) "
65 "must be tagged:\n"
66 "VISIR-chopnod-corrected-file.fits "VISIR_UTIL_CORRECTED"\n"
67 "VISIR-error-map.fits "VISIR_UTIL_ERROR_MAP
68 "\nIt will produce the coaddition of each input frame "
69 "separately and of all frames together.\n"
70 "The single frame product(s) will have a FITS card\n"
71 "'HIERARCH ESO PRO CATG' with a value of: COADDED_IMAGE and "
72 "COADDED_WEIGHT\nThe total coaddition will have the tags "
73 VISIR_IMG_COADDED_IMG" and "VISIR_IMG_COADDED_WGT);
74
75/*----------------------------------------------------------------------------*/
79/*----------------------------------------------------------------------------*/
80
81/*-----------------------------------------------------------------------------
82 Functions code
83 -----------------------------------------------------------------------------*/
84
85typedef enum {
86 VISIR_AVG_COMB,
87 VISIR_SUM_COMB
88} visir_comb_type;
89
90
91/*----------------------------------------------------------------------------*/
99/*----------------------------------------------------------------------------*/
100static cpl_error_code
101visir_util_run_swarp_fill_parameterlist(cpl_parameterlist * self)
102{
103 const char * context = PACKAGE ".visir_util_run_swarp";
104
105 skip_if(irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
106 "output-all", CPL_FALSE, NULL,
107 context, "Output a coadded image "
108 "for each input file in addition "
109 "to the complete coaddition."));
110
111 skip_if(irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
112 "extra_config", "NONE", NULL, context,
113 "Additional configuration parameters"));
114
115 skip_if(irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
116 "config_fname", DEFAULT_CONFIG, NULL,
117 context, "Swarp configure file name."));
118
119 skip_if(irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
120 "nprocs", -1, NULL, context,
121 "Maximum number of swarp processes "
122 "that can be spawned simultaneously."));
123 end_skip;
124
125 return cpl_error_get_code();
126}
127
128
129static cpl_error_code
130check_swarp(void)
131{
132 int ret = system(VISIR_SWARP_BIN " -v > /dev/null 2>/dev/null");
133 if (WEXITSTATUS(ret) != 0) {
134 cpl_error_set_message(cpl_func, CPL_ERROR_UNSUPPORTED_MODE,
135 "swarp not found in PATH");
136 return CPL_ERROR_UNSUPPORTED_MODE;
137 }
138 return CPL_ERROR_NONE;
139}
140
141
142static cx_list *
143sort_framelist(const cpl_frameset * allframes, const char * tag)
144{
145 cx_list * frames = cx_list_new();
146
147 FOR_EACH_FRAMESET_C(frm, allframes) {
148 if (tag == NULL || strcmp(cpl_frame_get_tag(frm), tag) == 0)
149 cx_list_push_back(frames, frm);
150 }
151
152 cx_list_sort(frames, (cx_compare_func)visir_cmp_frm_fn);
153
154 return frames;
155}
156
157
158static cpl_error_code
159create_swarp_frames(cx_list * img_list, cx_list * err_list,
160 const cpl_frameset * frameset)
161{
162 cx_list * imgframes = sort_framelist(frameset, VISIR_UTIL_CORRECTED);
163 cx_list * errframes = sort_framelist(frameset, VISIR_UTIL_ERROR_MAP);
164 skip_if(0);
165
166 FOR_EACH_T(cpl_frame * frm, imgframes) {
167 cpl_frame_set_group(frm, CPL_FRAME_GROUP_RAW);
168 cx_list_push_back(img_list, cpl_frame_duplicate(frm));
169 }
170
171 FOR_EACH_T(cpl_frame * frm, errframes) {
172 cpl_frame_set_group(frm, CPL_FRAME_GROUP_RAW);
173 cx_list_push_back(err_list, cpl_frame_duplicate(frm));
174 }
175
176 /* FIXME: check #ext of frames */
177 error_if(cx_list_size(err_list) != 0 &&
178 cx_list_size(img_list) != cx_list_size(err_list),
179 CPL_ERROR_UNSUPPORTED_MODE,
180 "Number of error frames does not match image frames");
181
182 end_skip;
183
184 cx_list_delete(imgframes);
185 cx_list_delete(errframes);
186
187 return cpl_error_get_code();
188}
189
190
191/*----------------------------------------------------------------------------*/
206/*----------------------------------------------------------------------------*/
207static pid_t
208run_swarp_resample(cpl_frame * imgfrm, cpl_frame * errfrm,
209 const cpl_parameterlist * parlist,
210 char * basedir,
211 int ext_low, int ext_high,
212 sem_t * sem_resamp)
213{
214 cpl_msg_debug(cpl_func, "start swarp %s: %d %d", cpl_frame_get_filename(imgfrm), ext_low, ext_high);
215 pid_t pid = fork();
216 if (pid == 0) {
217 char * cmd;
218 char datafile[128];
219 char errorfile[128];
220 const char * config =
221 irplib_parameterlist_get_string(parlist, PACKAGE,
222 RECIPE_STRING, "config_fname");
223 const char * extra =
224 irplib_parameterlist_get_string(parlist, PACKAGE,
225 RECIPE_STRING, "extra_config");
226 if (strcasecmp(extra, "NONE") == 0)
227 extra = "";
228 pid_t cpid = getpid();
229 sprintf(datafile, "swarp_data_%d.list", (int)cpid);
230 sprintf(errorfile, "swarp_error_%d.list", (int)cpid);
231
232 FILE * data = fopen(datafile, "w");
233 if (cpl_frame_get_filename(imgfrm)[0] != '/') {
234 fprintf(data, "%s/%s[%d:%d]\n", basedir,
235 cpl_frame_get_filename(imgfrm), ext_low, ext_high);
236 }
237 else {
238 fprintf(data, "%s[%d:%d]\n", cpl_frame_get_filename(imgfrm),
239 ext_low, ext_high);
240 }
241 fclose(data);
242
243 FILE * error = fopen(errorfile, "w");
244 if (cpl_frame_get_filename(errfrm)[0] != '/') {
245 fprintf(error, "%s/%s[%d:%d]\n", basedir,
246 cpl_frame_get_filename(errfrm), ext_low, ext_high);
247 }
248 else {
249 fprintf(error, "%s[%d:%d]\n", cpl_frame_get_filename(errfrm),
250 ext_low, ext_high);
251 }
252 fclose(error);
253
254 cmd = cpl_sprintf(VISIR_SWARP_BIN " -c %s @%s -NTHREADS 1 "
255 "-BLANK_BADPIXELS Y "
256 "-WEIGHT_IMAGE @%s "
257 "-WEIGHT_TYPE MAP_RMS %s "
258 "-COMBINE N "
259 "> \"%s/swarp-resamp_%d.log\" 2>&1",
260 config, datafile, errorfile, extra, basedir, (int)cpid);
261 cpl_msg_debug(cpl_func, "RESAMPLE: %s", cmd);
262 int status = system(cmd);
263 cpl_free(cmd);
264
265 /* notify parent that a resample has ended */
266 sem_post(sem_resamp);
267 /* close our forks copy of the semaphore */
268 sem_close(sem_resamp);
269 _exit(WEXITSTATUS(status));
270 }
271 else {
272 return pid;
273 }
274}
275
276/* ---------------------------------------------------------------------------*/
291/* ---------------------------------------------------------------------------*/
292static pid_t
293run_swarp_coadd(cpl_frame * imgfrm, cpl_frame * errfrm,
294 const cpl_parameterlist * parlist,
295 sem_t * sem_coadd, sem_t * sem_resamp)
296{
297 pid_t pid = fork();
298 if (pid == 0) {
299 /* start multiple resampling processes on slices of the extensions of
300 * the frame */
301 cpl_size next = cpl_frame_get_nextensions(imgfrm);
302 cpl_size bs = 300;
303 pid_t lpids[next / bs + ((next % bs) != 0)];
304 int l = 0;
305 char tmpdir[] = RECIPE_STRING"_XXXXXX";
306 char * basedir = visir_get_cwd();
307 if (basedir == NULL ||
308 visir_get_tempdir(tmpdir) != CPL_TRUE || chdir(tmpdir) != 0) {
309 cpl_msg_error(cpl_func, "Creating temporary directory failed");
310 /* notify parent that coadd job failed */
311 sem_post(sem_coadd);
312 /* close our forks copy of the semaphores */
313 sem_close(sem_coadd);
314 sem_close(sem_resamp);
315 cpl_free(basedir);
316 _exit(EXIT_FAILURE);
317 }
318
319 /* duplicate the header defining the final projection
320 * to each output file */
321 char * cmd = cpl_sprintf("cp \"%s/"SWARP_COADD_HEAD"\" coadd.head", basedir);
322 int status = system(cmd);
323 cpl_free(cmd);
324 if (WEXITSTATUS(status) != 0) {
325 cpl_msg_error(cpl_func, "copy of coaddition header failed");
326 /* notify parent that coadd job failed */
327 sem_post(sem_coadd);
328 /* close our forks copy of the semaphores */
329 sem_close(sem_coadd);
330 sem_close(sem_resamp);
331 cpl_free(basedir);
332 _exit(EXIT_FAILURE);
333 }
334
335 for (cpl_size b = 1; b < next + 1; b += bs) {
336 /* wait to not start too many resamplings */
337 sem_wait(sem_resamp);
338 pid_t resamp_pid = run_swarp_resample(imgfrm, errfrm, parlist,
339 basedir, b,
340 CX_MIN(b + bs, next + 1),
341 sem_resamp);
342 lpids[l++] = resamp_pid;
343 }
344 /* notify parent that the next resample->coadd can start, but the
345 * sem_resamp blocks them until enough of this coadd are done */
346 sem_post(sem_coadd);
347 /* close our forks copy of the semaphores */
348 sem_close(sem_coadd);
349 sem_close(sem_resamp);
350
351 for (int k = 0; k < l; k++) {
352 pid_t r = waitpid(lpids[k], &status, 0);
353 if (r == -1 || !WIFEXITED(status) || WEXITSTATUS(status))
354 _exit(WEXITSTATUS(status));
355 cpl_msg_debug(cpl_func, "resamp %s %d done",
356 cpl_frame_get_filename(imgfrm), k);
357 }
358
359 const char * config =
360 irplib_parameterlist_get_string(parlist, PACKAGE,
361 RECIPE_STRING, "config_fname");
362 const char * extra =
363 irplib_parameterlist_get_string(parlist, PACKAGE,
364 RECIPE_STRING, "extra_config");
365 if (strcasecmp(extra, "NONE") == 0)
366 extra = "";
367 pid_t cpid = getpid();
368 char * coadd_img = cpl_sprintf("%s/tmp_coadd_%d.fits",
369 basedir, (int)cpid);
370 char * coadd_weight = cpl_sprintf("%s/tmp_coadd_wgt_%d.fits",
371 basedir, (int)cpid);
372
373 /* duplicate the header defining the final projection
374 * to each output file , searches for IMAGEOUT_NAME.head */
375 cmd = cpl_sprintf("cp \"%s/"SWARP_COADD_HEAD"\" \"%s/tmp_coadd_%d.head\"",
376 basedir, basedir, (int)cpid);
377 status = system(cmd);
378 cpl_free(cmd);
379 if (WEXITSTATUS(status) != 0) {
380 cpl_msg_error(cpl_func, "copy of coaddition header failed");
381 cpl_free(coadd_img);
382 cpl_free(coadd_weight);
383 cpl_free(basedir);
384 _exit(WEXITSTATUS(status));
385 }
386
387 cmd = cpl_sprintf(VISIR_SWARP_BIN" -c %s $(ls *resamp.fits) "
388 "-NTHREADS 1 -BLANK_BADPIXELS Y "
389 "-WEIGHT_TYPE MAP_WEIGHT %s "
390 "-IMAGEOUT_NAME %s "
391 "-WEIGHTOUT_NAME %s "
392 "-RESAMPLE N -COMBINE Y "
393 "> %s/swarp-coadd-%d.log 2>&1",
394 config, extra,
395 coadd_img, coadd_weight, basedir, (int)cpid);
396 cpl_msg_debug(cpl_func, "COADD: %s", cmd);
397 status = system(cmd);
398 cpl_free(cmd);
399 if (chdir(basedir) == 0) {
400 cmd = cpl_sprintf("rm -rf \"%s\"", tmpdir);
401 status = system(cmd);
402 cpl_free(cmd);
403 }
404 cpl_free(coadd_img);
405 cpl_free(coadd_weight);
406
407 _exit(WEXITSTATUS(status));
408 }
409 else {
410 return pid;
411 }
412}
413
414
415static cpl_error_code
416generate_coaddition_header(const cx_list * imglist, const char * config, const char * extra)
417{
418 /* generate coaddition header which is read by each coaddition
419 * process to get the correct final output projection */
420 FILE * data = fopen(SWARP_DATA_FILES, "w");
421 char * cmd = cpl_sprintf(VISIR_SWARP_BIN " -c %s @"SWARP_DATA_FILES
422 " -IMAGEOUT_NAME "SWARP_COADD_HEAD
423 " -HEADER_ONLY Y %s > /dev/null 2>/dev/null",
424 config, extra);
425 int status;
426
427 FOR_EACH_T(cpl_frame * frm, imglist) {
428 int ret;
429 cpl_propertylist * plist =
430 cpl_propertylist_load(cpl_frame_get_filename(frm), 0);
431 cpl_errorstate cleanstate = cpl_errorstate_get();
432 cpl_boolean invalid =
433 (cpl_propertylist_get_double(plist, "CD1_1") == 0. &&
434 cpl_propertylist_get_double(plist, "CD1_2") == 0.) ||
435 (cpl_propertylist_get_double(plist, "CD2_1") == 0. &&
436 cpl_propertylist_get_double(plist, "CD2_2") == 0.);
437 invalid = invalid || (cpl_error_get_code() != CPL_ERROR_NONE);
438 cpl_errorstate_set(cleanstate);
439 cpl_propertylist_delete(plist);
440 error_if (invalid, CPL_ERROR_ILLEGAL_INPUT,
441 "CDX_Y WCS key missing or zero");
442
443 cpl_msg_info(cpl_func, "%s", cpl_frame_get_filename(frm));
444 ret = fprintf(data, "%s\n", cpl_frame_get_filename(frm));
445 error_if(ret < 0, CPL_ERROR_FILE_IO, "Failed writting swarp file list");
446 }
447 fclose(data);
448
449 cpl_msg_info(cpl_func, "Generating coaddition header");
450 status = system(cmd);
451
452 error_if(WEXITSTATUS(status) != 0, CPL_ERROR_FILE_IO, "Swarp failed");
453
454 end_skip;
455 cpl_free(cmd);
456
457 return cpl_error_get_code();
458}
459
460
461static cpl_error_code
462coadd_images(pid_t * pids, unsigned long * beamids, float * imgwgts,
463 cpl_frameset * usedframes,
464 const cx_list * imglist, const cx_list * errlist,
465 const int nprocs, const cpl_parameterlist * parlist)
466{
467 const int nfiles = cx_list_size(imglist);
468 int i = 0;
469 /* setup tiered load balancing semaphores
470 * start multiple coadding jobs at once
471 * coadding jobs launch resampling jobs until the resampling semaphore blocks
472 * the coadding semaphore blocks at one but is incremented when all
473 * resampling jobs are posted so the other coadd jobs can queue in their
474 * resampling jobs.
475 * That way we are always resampling but only one IO bound coaddition is
476 * running at a time as long as the resampling queue is full. */
477 char * sem_resamp_name = cpl_sprintf("/visir_swarp_resamp_%d_%d",
478 getuid(), getpid());
479 char * sem_coadd_name = cpl_sprintf("/visir_swarp_coadd_%d_%d",
480 getuid(), getpid());
481 sem_t * sem_resamp = sem_open(sem_resamp_name , O_CREAT, 0644, nprocs);
482 sem_t * sem_coadd = sem_open(sem_coadd_name , O_CREAT, 0644, 1);
483 sem_unlink(sem_coadd_name);
484 sem_unlink(sem_resamp_name);
485
486 for(cx_list_iterator iit = cx_list_begin(imglist),
487 eit = cx_list_begin(errlist);
488 iit != cx_list_end(imglist);
489 iit = cx_list_next(imglist, iit),
490 eit = cx_list_next(errlist, eit)) {
491 const char * beamid;
492 cpl_frame * ifrm = cx_list_get(imglist, iit);
493 cpl_frame * efrm = cx_list_get(errlist, eit);
494 cpl_propertylist * plist =
495 cpl_propertylist_load(cpl_frame_get_filename(ifrm), 0);
496 skip_if(0);
497
498 beamid = cpl_propertylist_get_string(plist, "ESO QC BEAMID");
499 beamids[i] = strtoul(beamid, NULL, 10);
500 imgwgts[i] = visir_pfits_get_img_weight(plist);
501 cpl_propertylist_delete(plist);
502
503 skip_if(0);
504
505 cpl_frameset_insert(usedframes, cpl_frame_duplicate(ifrm));
506 cpl_frameset_insert(usedframes, cpl_frame_duplicate(efrm));
507 /* wait for child to send of all resampling jobs before starting the next
508 * this is to assign as many cpus as possible to handling one input
509 * file to reduce disk IO and load balance the coaddition phase */
510 sem_wait(sem_coadd);
511 pids[i++] = run_swarp_coadd(ifrm, efrm, parlist,
512 sem_coadd, sem_resamp);
513 cpl_msg_info(cpl_func, "Coadding file (%d/%d)", i, nfiles);
514 }
515 for (int j = 0; j < i; j++) {
516 int status;
517 pid_t r = waitpid(-1, &status, 0);
518 error_if(r == -1 || !WIFEXITED(status) || WEXITSTATUS(status),
519 CPL_ERROR_FILE_IO, "Coaddition failed: %d, %s",
520 WEXITSTATUS(status), r == -1 ? strerror(errno) : "");
521 }
522
523 end_skip;
524 sem_close(sem_coadd);
525 sem_close(sem_resamp);
526 cpl_free(sem_resamp_name);
527 cpl_free(sem_coadd_name);
528
529 return cpl_error_get_code();
530}
531
532
533static cpl_error_code
534save_single_images(cpl_frameset * framelist, const cpl_size nfiles,
535 const pid_t * pids, const cpl_frameset * usedframes,
536 const cpl_parameterlist * parlist)
537{
538 cpl_image * img = NULL;
539 cpl_image * wgt = NULL;
540 cpl_propertylist * plist = NULL;
541
542 for (cpl_size j = 0; j < nfiles; j++) {
543 char coadd_img[200];
544 char coadd_wgt[200];
545
546 sprintf(coadd_img, "tmp_coadd_%d.fits", (int)pids[j]);
547 sprintf(coadd_wgt, "tmp_coadd_wgt_%d.fits", (int)pids[j]);
548
549 cpl_image_delete(img);
550 cpl_image_delete(wgt);
551 img = cpl_image_load(coadd_img, CPL_TYPE_UNSPECIFIED, 0, 0);
552 wgt = cpl_image_load(coadd_wgt, CPL_TYPE_UNSPECIFIED, 0, 0);
553 skip_if(0);
554
555 cpl_propertylist_delete(plist);
556 plist = cpl_propertylist_load(coadd_img, 0);
557 skip_if(0);
558
559 sprintf(coadd_img, "swarp_coadd_img_%03d.fits", (int)j);
560 sprintf(coadd_wgt, "swarp_coadd_wgt_%03d.fits", (int)j);
561
562 skip_if(irplib_dfs_save_image(framelist, parlist, usedframes,
563 img, CPL_BPP_IEEE_FLOAT,
564 RECIPE_STRING, "COADDED_IMAGE",
565 plist, NULL, visir_pipe_id,
566 coadd_img));
567 skip_if(irplib_dfs_save_image(framelist, parlist, usedframes,
568 wgt, CPL_BPP_IEEE_FLOAT,
569 RECIPE_STRING, "COADDED_WEIGHT",
570 plist, NULL, visir_pipe_id,
571 coadd_wgt));
572 }
573
574 end_skip;
575 cpl_image_delete(img);
576 cpl_image_delete(wgt);
577 cpl_propertylist_delete(plist);
578
579 return cpl_error_get_code();
580}
581
582
583static cpl_propertylist *
584generate_beam_averages(cpl_image ** av, cpl_image ** wav, float * imgwgts,
585 const cpl_size nfiles,
586 const pid_t * pids, const unsigned long * beamids)
587{
588 /* all files are in the same projection so simply add them together */
589 cpl_image * img = NULL;
590 cpl_image * wgt = NULL;
591 cpl_propertylist * plist = NULL;
592 unsigned long max_beamid = 0;
593 float tmpwgts[nfiles];
594
595 cpl_ensure(nfiles > 0, CPL_ERROR_ILLEGAL_INPUT, NULL);
596
597 for (cpl_size j = 0; j < nfiles; j++) {
598 const unsigned long idx = beamids[j];
599 char coadd_img[200];
600 char coadd_wgt[200];
601
602 cpl_image_delete(img);
603 cpl_image_delete(wgt);
604 sprintf(coadd_img, "tmp_coadd_%d.fits", (int)pids[j]);
605 sprintf(coadd_wgt, "tmp_coadd_wgt_%d.fits", (int)pids[j]);
606 img = cpl_image_load(coadd_img, CPL_TYPE_UNSPECIFIED, 0, 0);
607 wgt = cpl_image_load(coadd_wgt, CPL_TYPE_UNSPECIFIED, 0, 0);
608
609 skip_if(0);
610
611 if (plist == NULL)
612 plist = cpl_propertylist_load(coadd_img, 0);
613
614 /* PIXEL mode is significantly faster and accurate enough for the
615 * fov of VISIR, set it back to TAN for other tools,
616 * note that swarp will flip the image and CD1_1 will have a different
617 * sign to account for that */
618 if (strcmp(cpl_propertylist_get_string(plist, "CTYPE1"),
619 "PIXEL") == 0) {
620 cpl_propertylist_set_string(plist, "CTYPE1", "RA---TAN");
621 cpl_propertylist_set_string(plist, "CTYPE2", "DEC--TAN");
622 cpl_propertylist_set_string(plist, "CUNIT1", "deg");
623 cpl_propertylist_set_string(plist, "CUNIT2", "deg");
624 }
625
626 skip_if(0);
627
628 cpl_image_multiply(img, wgt);
629 if (av[idx] == NULL) {
630 av[idx] = cpl_image_duplicate(img);
631 wav[idx] = cpl_image_duplicate(wgt);
632 }
633 else {
634 skip_if(cpl_image_add(av[idx], img));
635 skip_if(cpl_image_add(wav[idx], wgt));
636 }
637 max_beamid = CX_MAX(max_beamid, idx);
638 /* weights always the same for a single beam */
639 tmpwgts[idx] = imgwgts[j];
640 }
641
642 for (unsigned long i = 0; i < max_beamid + 1; i++) {
643 cpl_image_divide(av[i], wav[i]);
644 imgwgts[i] = tmpwgts[i];
645 }
646
647 end_skip;
648
649 cpl_image_delete(img);
650 cpl_image_delete(wgt);
651
652 return plist;
653}
654
655
656/*----------------------------------------------------------------------------*/
670/*----------------------------------------------------------------------------*/
671static cpl_error_code
672generate_combined_image(cpl_image ** avg, cpl_image ** wgt,
673 const float * imgwgts, const unsigned long n,
674 cpl_image ** oimg, cpl_image ** owgt,
675 cpl_image ** octr, visir_comb_type ctype)
676{
677 /* summing combine mode will lead to ugly images in case of bad pixels */
678 cpl_image * cimg = cpl_image_new(cpl_image_get_size_x(avg[0]),
679 cpl_image_get_size_y(avg[0]),
680 CPL_TYPE_FLOAT);
681 cpl_image * cwgt = cpl_image_new(cpl_image_get_size_x(wgt[0]),
682 cpl_image_get_size_y(wgt[0]),
683 CPL_TYPE_FLOAT);
684 cpl_image * contrib = cpl_image_new(cpl_image_get_size_x(wgt[0]),
685 cpl_image_get_size_y(wgt[0]),
686 CPL_TYPE_INT);
687 int wgtsum = 0;
688
689 skip_if(0);
690 /* e_sum = sqrt(sum(sig^2)) / n :sig = sqrt(1/w)
691 w_sum = 1 / ((sum(1/w) / n ^ 2)) */
692 for (unsigned long i = 0; i < n; i++) {
693 cpl_mask * m = cpl_mask_threshold_image_create(wgt[i], -1, FLT_EPSILON);
694 cpl_image * bad = cpl_image_new_from_mask(m);
695 cpl_image_reject_from_mask(avg[i], m);
696 /* the beams have a non zero background which averages out over
697 * all beams, but in the case of bad pixels some values are missing
698 * which leads to artefacts if we don't correct the background */
699 cpl_image_subtract_scalar(avg[i], cpl_image_get_median(avg[i]));
700 cpl_image_fill_rejected(avg[i], 0);
701 cpl_mask_delete(m);
702
703 /* will leave zero weight bad pixels at zero */
704 cpl_image_power(wgt[i], -1);
705 cpl_image_accept_all(wgt[i]);
706
707 cpl_image_multiply_scalar(bad, imgwgts[i]);
708 cpl_image_subtract(contrib, bad);
709 cpl_image_delete(bad);
710
711 skip_if(0);
712
713 cpl_image_add(cimg, avg[i]);
714 cpl_image_add(cwgt, wgt[i]);
715
716 wgtsum += imgwgts[i];
717 }
718
719 cpl_image_add_scalar(contrib, wgtsum);
720 if (ctype == VISIR_AVG_COMB) {
721 cpl_image_divide(cimg, contrib);
722
723 cpl_image_divide(cwgt, contrib);
724 cpl_image_divide(cwgt, contrib);
725 }
726 cpl_image_power(cwgt, -1);
727
728 *oimg = cimg;
729 *owgt = cwgt;
730 *octr = contrib;
731
732 end_skip;
733
734 return cpl_error_get_code();
735}
736
737
738/* ---------------------------------------------------------------------------*/
750/* ---------------------------------------------------------------------------*/
751static cpl_error_code
752cut_bad_edges(cpl_image ** img, cpl_image ** wgt, cpl_image ** ctr,
753 cpl_propertylist * plist)
754{
755 const cpl_size nx = cpl_image_get_size_x(*ctr);
756 const cpl_size ny = cpl_image_get_size_y(*ctr);
757 const cpl_size max_contrib = cpl_image_get_max(*ctr);
758 const double ctr_cutoff = max_contrib == 2 ? 0.4 : 0.7;
759 cpl_size cutly = 0;
760 cpl_size cutuy = 0;
761 cpl_size cutlx = 0;
762 cpl_size cutux = 0;
763 for (cpl_size i = 0; i < ny / 2; i++) {
764 cpl_vector * row = cpl_vector_new_from_image_row(*ctr, i + 1);
765 double sum = cpl_vector_get_sum(row);
766 cpl_vector_delete(row);
767 if (sum >= max_contrib * nx * ctr_cutoff) {
768 break;
769 }
770 cutly = i + 1;
771 }
772 for (cpl_size i = ny - 1; i >= ny / 2; i--) {
773 cpl_vector * row = cpl_vector_new_from_image_row(*ctr, i + 1);
774 double sum = cpl_vector_get_sum(row);
775 cpl_vector_delete(row);
776 if (sum >= max_contrib * nx * ctr_cutoff) {
777 break;
778 }
779 cutuy = i + 1;
780 }
781 for (cpl_size i = 0; i < nx / 2; i++) {
782 cpl_vector * col = cpl_vector_new_from_image_column(*ctr, i + 1);
783 double sum = cpl_vector_get_sum(col);
784 cpl_vector_delete(col);
785 if (sum >= max_contrib * ny * ctr_cutoff) {
786 break;
787 }
788 cutlx = i + 1;
789 }
790 for (cpl_size i = nx - 1; i >= nx / 2; i--) {
791 cpl_vector * col = cpl_vector_new_from_image_column(*ctr, i + 1);
792 double sum = cpl_vector_get_sum(col);
793 cpl_vector_delete(col);
794 if (sum >= max_contrib * ny * ctr_cutoff) {
795 break;
796 }
797 cutux = i + 1;
798 }
799 cutlx += 1;
800 cutly += 1;
801 cutux -= 1;
802 cutuy -= 1;
803 cpl_msg_info(cpl_func, "Cutting low contribution edges "
804 "llx: %lld, lly: %lld, urx: %lld, ury: %lld",
805 cutlx, cutly, cutux, cutuy);
806
807 /* make sure to only cut if something remains */
808 cpl_image * nimg, * nwgt, * nctr;
809 if (cutlx < cutux && cutly < cutuy) {
810 cpl_propertylist_set_double(plist, "CRPIX1",
811 cpl_propertylist_get_double(plist, "CRPIX1") - cutlx);
812 cpl_propertylist_set_double(plist, "CRPIX2",
813 cpl_propertylist_get_double(plist, "CRPIX2") - cutly);
814 nimg = cpl_image_extract(*img, cutlx, cutly, cutux, cutuy);
815 nwgt = cpl_image_extract(*wgt, cutlx, cutly, cutux, cutuy);
816 nctr = cpl_image_extract(*ctr, cutlx, cutly, cutux, cutuy);
817 }
818 else {
819 nimg = cpl_image_duplicate(*img);
820 nwgt = cpl_image_duplicate(*wgt);
821 nctr = cpl_image_duplicate(*ctr);
822 }
823 cpl_image_delete(*img);
824 cpl_image_delete(*wgt);
825 cpl_image_delete(*ctr);
826 *img = nimg;
827 *wgt = nwgt;
828 *ctr = nctr;
829
830 return cpl_error_get_code();
831}
832
833
834static cpl_error_code
835check_inputs(const cpl_frameset * framelist, const char * config)
836{
837 irplib_framelist * alldata = irplib_framelist_cast(framelist);
838 irplib_framelist * data = irplib_framelist_extract(alldata,
839 VISIR_UTIL_CORRECTED);
840 irplib_framelist * errors = irplib_framelist_extract(alldata, "ERROR_MAP");
841 irplib_framelist_load_propertylist_all(data, 0, "ESO QC", CPL_FALSE);
842 error_if(irplib_framelist_get_size(data) !=
843 irplib_framelist_get_size(errors), CPL_ERROR_ILLEGAL_INPUT,
844 "Unequal number of data and error frames");
845 skip_if(irplib_framelist_contains(data, "ESO QC BEAMID", CPL_TYPE_STRING,
846 CPL_FALSE, 0.0));
847
848 error_if(access(config, R_OK) != 0, CPL_ERROR_FILE_NOT_FOUND,
849 "Swarp configuration not found: %s", config);
850 end_skip;
851
852 irplib_framelist_delete(alldata);
853 irplib_framelist_delete(data);
854 irplib_framelist_delete(errors);
855 return cpl_error_get_code();
856}
857
858
859/*----------------------------------------------------------------------------*/
866/*----------------------------------------------------------------------------*/
867static int visir_util_run_swarp(cpl_frameset * framelist,
868 const cpl_parameterlist * parlist)
869{
870 cpl_frameset * usedframes = cpl_frameset_new();
871 cpl_propertylist * plist = NULL;
872 cx_list * imglist = cx_list_new();
873 cx_list * errlist = cx_list_new();
874
875 skip_if(check_swarp());
876 /* only reading single extension files, no cache needed */
877 cpl_fits_set_mode(CPL_FITS_STOP_CACHING);
878
879 const cpl_boolean output_all =
880 irplib_parameterlist_get_bool(parlist, PACKAGE,
881 RECIPE_STRING, "output-all");
882 const char * config =
883 irplib_parameterlist_get_string(parlist, PACKAGE,
884 RECIPE_STRING, "config_fname");
885 const char * extra =
886 irplib_parameterlist_get_string(parlist, PACKAGE,
887 RECIPE_STRING, "extra_config");
888 int nmax_procs =
889 irplib_parameterlist_get_int(parlist, PACKAGE,
890 RECIPE_STRING, "nprocs");
891 if (visir_str_par_is_empty(extra))
892 extra = "";
893
894 if (nmax_procs <= 0) {
895 nmax_procs = visir_get_num_threads(CPL_TRUE);
896 }
897
898 error_if(visir_str_par_is_empty(config), CPL_ERROR_FILE_NOT_FOUND,
899 "A swarp configuration file is required. "
900 "You can create a default configuration with: swarp -d");
901
902 skip_if(check_inputs(framelist, config));
903
904 unlink(SWARP_DATA_FILES);
905 unlink(SWARP_COADD_HEAD);
906
907 skip_if(0);
908
909 create_swarp_frames(imglist, errlist, framelist);
910 error_if(cx_list_size(imglist) == 0 || cx_list_size(errlist) == 0,
911 CPL_ERROR_ILLEGAL_INPUT, "No frames tagged "
912 VISIR_UTIL_CORRECTED" and "VISIR_UTIL_ERROR_MAP" found");
913
914 skip_if(generate_coaddition_header(imglist, config, extra));
915
916 {
917 const cpl_size nfiles = cx_list_size(imglist);
918 pid_t pids[nfiles];
919 unsigned long beamids[nfiles];
920 float imgwgts[nfiles];
921 unsigned long nmax_beamid = 0;
922 cpl_image * av[nfiles];
923 cpl_image * wav[nfiles];
924 cpl_image * cimg = NULL;
925 cpl_image * cwgt = NULL;
926 cpl_image * cctr = NULL;
927
928 for (cpl_size j = 0; j < nfiles; j++)
929 av[j] = wav[j] = NULL;
930
931 skip_if(coadd_images(pids, beamids, imgwgts, usedframes,
932 imglist, errlist, nmax_procs, parlist));
933
934 for (cpl_size i = 0; i < nfiles; i++)
935 nmax_beamid = CX_MAX(nmax_beamid, beamids[i] + 1);
936
937 error_if((cpl_size)nmax_beamid - 1 > nfiles,
938 CPL_ERROR_UNSUPPORTED_MODE,
939 "Only consecutive BEAMID's supported");
940
941 if (output_all && nfiles > 1)
942 save_single_images(framelist, nfiles, pids, usedframes, parlist);
943
944 skip_if(0);
945
946 plist = generate_beam_averages(av, wav, imgwgts, nfiles, pids, beamids);
947 skip_if(plist == NULL);
948
949 generate_combined_image(av, wav, imgwgts, nmax_beamid, &cimg, &cwgt,
950 &cctr, VISIR_AVG_COMB);
951 for (cpl_size i = 0; i < nfiles; i++) {
952 cpl_image_delete(av[i]);
953 cpl_image_delete(wav[i]);
954 }
955
956 cut_bad_edges(&cimg, &cwgt, &cctr, plist);
957
958 irplib_dfs_save_image(framelist, parlist, usedframes, cimg,
959 CPL_BPP_IEEE_FLOAT, RECIPE_STRING,
960 VISIR_IMG_COADDED_IMG, plist, NULL,
961 visir_pipe_id, "swarp_coadd_img_all.fits");
962 irplib_dfs_save_image(framelist, parlist, usedframes, cwgt,
963 CPL_BPP_IEEE_FLOAT, RECIPE_STRING,
964 VISIR_IMG_COADDED_WGT, plist, NULL,
965 visir_pipe_id, "swarp_coadd_wgt_all.fits");
966 irplib_dfs_save_image(framelist, parlist, usedframes, cctr,
967 CPL_TYPE_INT, RECIPE_STRING,
968 VISIR_IMG_COADDED_CTR, plist, NULL,
969 visir_pipe_id, "swarp_coadd_ctr_all.fits");
970 cpl_image_delete(cimg);
971 cpl_image_delete(cwgt);
972 cpl_image_delete(cctr);
973 skip_if(0);
974 }
975
976 end_skip;
977
978 unlink(SWARP_DATA_FILES);
979 unlink(SWARP_COADD_HEAD);
980 cx_list_destroy(imglist, (cx_free_func)cpl_frame_delete);
981 cx_list_destroy(errlist, (cx_free_func)cpl_frame_delete);
982 cpl_frameset_delete(usedframes);
983 cpl_propertylist_delete(plist);
984 return cpl_error_get_code();
985}
double visir_pfits_get_img_weight(const cpl_propertylist *self)
The relative weight of the image compared to the other images.
Definition: visir_pfits.c:913