IIINSTRUMENT Pipeline Reference Manual 1.3.15
detmon_lg_mr.c
1/*
2 * This file is part of the DETMON Pipeline
3 * Copyright (C) 2013 European Southern Observatory
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
18 */
19
20/*
21 * $Author: jtaylor $
22 * $Date: 2013-08-07 09:32:50 $
23 * $Revision: 1.6 $
24 * $Name: not supported by cvs2svn $
25 */
26#ifdef HAVE_CONFIG_H
27#include <config.h>
28#endif
29
30/*----------------------------------------------------------------------------
31 Includes
32 ----------------------------------------------------------------------------*/
33
34
35#include "detmon_lg.h"
36#include "detmon.h"
37
38#include "irplib_plugin.h"
39#include "irplib_utils.h"
40
41#include <string.h>
42#include <errno.h>
43#include <stdlib.h>
44#include <assert.h>
45
46/*----------------------------------------------------------------------------
47 Private Functions prototypes
48 ----------------------------------------------------------------------------*/
49int
50detmon_lg_dfs_set_groups(cpl_frameset * set,
51 const char *tag_on, const char *tag_off);
52
53static int check_regions(const cpl_size *, const cpl_size *,
54 const cpl_size *, const cpl_size *,
55 cpl_size) CPL_ATTR_NONNULL;
56
57static int compare_tags(const cpl_frame *, const cpl_frame *) CPL_ATTR_NONNULL;
58
59static const cpl_parameter *
60get_parameter_const(const cpl_parameterlist * parlist,
61 const char * recipe, const char * name)
62{
63 char * parname = cpl_sprintf("detmon.%s.%s", recipe, name);
64 const cpl_parameter * p = cpl_parameterlist_find_const(parlist, parname);
65 cpl_free(parname);
66 return p;
67}
68
69static cpl_parameter *
70get_parameter(cpl_parameterlist * parlist,
71 const char * recipe, const char * name)
72{
73 char * parname = cpl_sprintf("detmon.%s.%s", recipe, name);
74 cpl_parameter * p = cpl_parameterlist_find(parlist, parname);
75 cpl_free(parname);
76 return p;
77}
78
79static int compare_tags(const cpl_frame * a, const cpl_frame * b)
80{
81 return strcmp(cpl_frame_get_tag(a), cpl_frame_get_tag(b)) == 0;
82}
83
86/* ---------------------------------------------------------------------------*/
97/* ---------------------------------------------------------------------------*/
98static cpl_error_code merge_keys(cpl_propertylist * mplist,
99 const cpl_propertylist * plist_,
100 const int pos,
101 const char * rmregexp)
102{
103 char buffer[256];
104 /* no copy_regex, so duplicate and use erase_regex on the copy */
105 cpl_propertylist * plist = cpl_propertylist_duplicate(plist_);
106
107 if (rmregexp) {
108 cpl_propertylist_erase_regexp(plist, rmregexp, 0);
109 cpl_propertylist_erase_regexp(mplist, rmregexp, 0);
110 }
111
112 for (cpl_size i = 0; i < cpl_propertylist_get_size(plist); i++) {
113 cpl_property * p = cpl_propertylist_get(plist, i);
114 const char * key = cpl_property_get_name(p);
115 /* extend all qc keywords by a region prefix */
116 if (!strncmp(key, "ESO QC ", 7)) {
117 const char * com;
118 sprintf(buffer, "ESO QC REG%d %s", pos, key + 7);
119 cpl_msg_debug(cpl_func, "merging %s to %s", key, buffer);
120 if (cpl_property_get_type(p) == CPL_TYPE_INT) {
121 int val = cpl_property_get_int(p);
122 cpl_propertylist_append_int(mplist, buffer, val);
123 } else if (cpl_property_get_type(p) == CPL_TYPE_DOUBLE) {
124 double val = cpl_property_get_double(p);
125 cpl_propertylist_append_double(mplist, buffer, val);
126 } else if (cpl_property_get_type(p) == CPL_TYPE_STRING) {
127 const char * val = cpl_property_get_string(p);
128 cpl_propertylist_append_string(mplist, buffer, val);
129 }
130 else
131 bug_if(1);
132 com = cpl_property_get_comment(p);
133 cpl_propertylist_set_comment(mplist, buffer, com);
134
135 /* remove unprefixed key */
136 if (cpl_propertylist_has(mplist, key))
137 cpl_propertylist_erase(mplist, key);
138 }
139 }
140 skip_if(0);
141
142 end_skip;
143 cpl_propertylist_delete(plist);
144
145 return cpl_error_get_code();
146}
147
148
149/* ---------------------------------------------------------------------------*/
157/* ---------------------------------------------------------------------------*/
158static cpl_error_code
159merge_images(cpl_imagelist * mimgl, const cpl_imagelist * img)
160{
161 cpl_ensure_code(cpl_imagelist_get_size(img) ==
162 cpl_imagelist_get_size(mimgl), CPL_ERROR_ILLEGAL_INPUT);
163 for (int i = 0; i < cpl_imagelist_get_size(img); i++) {
164 cpl_image * mimg = cpl_imagelist_get(mimgl, i);
165 const cpl_image * part = cpl_imagelist_get_const(img, i);
166 cpl_image_add(mimg, part);
167 }
168
169 return cpl_error_get_code();
170}
171
172
173/* ---------------------------------------------------------------------------*/
181/* ---------------------------------------------------------------------------*/
182static cpl_error_code
183merge_table(cpl_table * mtab, cpl_table * tab, const int pos)
184{
185 cpl_table_new_column(tab, "REGION", CPL_TYPE_INT);
186 cpl_table_fill_column_window(tab, "REGION", 0, cpl_table_get_nrow(tab), pos);
187 skip_if(cpl_table_insert(mtab, tab, cpl_table_get_nrow(mtab)));
188
189 end_skip;
190
191 return cpl_error_get_code();
192}
193
194
195static cpl_error_code
196parse_token(const char * token, cpl_size * llx, cpl_size * lly,
197 cpl_size * urx, cpl_size * ury)
198{
199 assert(sizeof(*llx) == 8);
200 if (sscanf(token, " %lld , %lld , %lld , %lld ", llx, lly, urx, ury) == 4) {
201 return CPL_ERROR_NONE;
202 }
203 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
204 "Data not formated correctly: %s; "
205 "expecting: name,llx,lly,urx,ury",
206 token);
207}
208
209
210/* ---------------------------------------------------------------------------*/
223/* ---------------------------------------------------------------------------*/
224static cpl_error_code
225read_regions_file(const char * filename, cpl_size ** rllx,
226 cpl_size ** rlly, cpl_size ** rurx, cpl_size ** rury,
227 cpl_size * n)
228{
229 FILE * f = fopen(filename, "r");
230 char line[1024];
231 cpl_size i = 0;
232 cpl_size nmax = 100;
233 if (f == NULL) {
234 return cpl_error_set_message(cpl_func, CPL_ERROR_FILE_IO,
235 "Could not open file %s: %s", filename,
236 strerror(errno));
237 }
238 cpl_msg_info(cpl_func, "Reading %s", filename);
239
240 *n = 0;
241 *rllx = cpl_malloc(sizeof(**rllx) * nmax);
242 *rlly = cpl_malloc(sizeof(**rllx) * nmax);
243 *rurx = cpl_malloc(sizeof(**rllx) * nmax);
244 *rury = cpl_malloc(sizeof(**rllx) * nmax);
245
246 while (fgets(line, 1024, f) != NULL) {
247 const size_t nline = strlen(line);
248 if (line[nline - 1] != '\n') {
249 cpl_error_set_message(cpl_func, CPL_ERROR_FILE_IO,
250 "File %s contained too long line",
251 filename);
252 skip_if(0);
253 }
254
255 if (line[0] == '#')
256 continue;
257
258 skip_if(parse_token(line, &(*rllx)[i], &(*rlly)[i],
259 &(*rurx)[i], &(*rury)[i]));
260 *n = ++i;
261 if (i >= nmax) {
262 nmax *= 2;
263 *rllx = cpl_realloc(*rllx, sizeof(**rllx) * nmax);
264 *rlly = cpl_realloc(*rlly, sizeof(**rllx) * nmax);
265 *rurx = cpl_realloc(*rurx, sizeof(**rllx) * nmax);
266 *rury = cpl_realloc(*rury, sizeof(**rllx) * nmax);
267 skip_if(0);
268 }
269 }
270
271 end_skip;
272 if (f)
273 fclose(f);
274 return CPL_ERROR_NONE;
275}
276
277
278/* ---------------------------------------------------------------------------*/
291/* ---------------------------------------------------------------------------*/
292static cpl_error_code
293read_regions_str(const char * istr, cpl_size ** rllx,
294 cpl_size ** rlly, cpl_size ** rurx, cpl_size ** rury,
295 cpl_size * n)
296{
297 cpl_size nmax = 100;
298 char * str_ = cpl_strdup(istr);
299 char * str = str_;
300 *n = 0;
301 *rllx = cpl_malloc(sizeof(**rllx) * nmax);
302 *rlly = cpl_malloc(sizeof(**rllx) * nmax);
303 *rurx = cpl_malloc(sizeof(**rllx) * nmax);
304 *rury = cpl_malloc(sizeof(**rllx) * nmax);
305
306 for (int i = 0; ;i++) {
307 const char * token = strtok(str, ":");
308 if (token == NULL)
309 break;
310
311 skip_if(parse_token(token, &(*rllx)[i], &(*rlly)[i],
312 &(*rurx)[i], &(*rury)[i]));
313 (*n)++;
314 if (i >= nmax - 1) {
315 nmax *= 2;
316 *rllx = cpl_realloc(*rllx, sizeof(**rllx) * nmax);
317 *rlly = cpl_realloc(*rlly, sizeof(**rllx) * nmax);
318 *rurx = cpl_realloc(*rurx, sizeof(**rllx) * nmax);
319 *rury = cpl_realloc(*rury, sizeof(**rllx) * nmax);
320 skip_if(0);
321 }
322 str = NULL;
323 }
324
325 end_skip;
326 cpl_free(str_);
327
328 return CPL_ERROR_NONE;
329}
330
331static int check_regions(const cpl_size * llx, const cpl_size * lly,
332 const cpl_size * urx, const cpl_size * ury,
333 cpl_size nregions)
334{
335 for (cpl_size i = 0; i < nregions; i++) {
336 if (llx[i] < 1 || lly[i] < 1 ||
337 urx[i] < 1 || ury[i] < 1) {
338 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
339 "Region %lld (%lld,%lld,%lld,%lld) has entry smaller one",
340 i, llx[i], lly[i], urx[i], ury[i]);
341 }
342 if ((urx[i] - llx[i] < 1) || (ury[i] - lly[i] < 1)) {
343 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
344 "Region %lld (%lld,%lld,%lld,%lld) has size zero",
345 i, llx[i], lly[i], urx[i], ury[i]);
346 }
347 for (cpl_size j = i + 1; j < nregions; j++) {
348 if (!(llx[i] > urx[j] || urx[i] < llx[j] ||
349 lly[i] > ury[j] || ury[i] < lly[j])) {
350 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
351 "Regions %lld (%lld,%lld,%lld,%lld) and "
352 "%lld (%lld,%lld,%lld,%lld) overlap (llx,lly,urx,ury)",
353 i, llx[i], lly[i], urx[i], ury[i],
354 j, llx[j], lly[j], urx[j], ury[j]);
355 }
356 }
357 }
358
359 return CPL_ERROR_NONE;
360}
361
362
363/* ---------------------------------------------------------------------------*/
376/* ---------------------------------------------------------------------------*/
377static cpl_error_code
378merge_lingain(cpl_propertylist * mplist[], cpl_table * mtab[],
379 const char * fn, const cpl_size next,
380 const char * rmregexp, int pos)
381{
382 cpl_errorstate cleanstate = cpl_errorstate_get();
383
384 {
385 cpl_propertylist * plist = cpl_propertylist_load(fn, 0);
386 skip_if(plist == NULL);
387 if (mplist[0] == NULL) {
388 mplist[0] = plist;
389 merge_keys(mplist[0], plist, pos, rmregexp);
390 }
391 else {
392 merge_keys(mplist[0], plist, pos, rmregexp);
393 cpl_propertylist_delete(plist);
394 }
395 }
396 skip_if(0);
397
398 for (int e = 1; e < next + 1; e++) {
399 cpl_propertylist * plist = cpl_propertylist_load(fn, e);
400 cpl_table * tab = cpl_table_load(fn, e, 0);
401 /* empty extensions may exist */
402 if (tab == NULL) {
403 cpl_errorstate_set(cleanstate);
404 cpl_propertylist_delete(plist);
405 continue;
406 }
407 skip_if(plist == NULL);
408
409 if (mplist[e] == NULL) {
410 merge_keys(plist, plist, pos, rmregexp);
411 cpl_table_new_column(tab, "REGION", CPL_TYPE_INT);
412 cpl_table_fill_column_window(tab, "REGION", 0,
413 cpl_table_get_nrow(tab), 1);
414 skip_if(0);
415 mplist[e] = plist;
416 mtab[e] = tab;
417 }
418 else {
419 merge_keys(mplist[e], plist, pos, rmregexp);
420 merge_table(mtab[e], tab, pos);
421 skip_if(0);
422 cpl_propertylist_delete(plist);
423 cpl_table_delete(tab);
424 }
425 skip_if(0);
426 }
427
428 end_skip;
429
430 return cpl_error_get_code();
431}
432
433
434/* ---------------------------------------------------------------------------*/
447/* ---------------------------------------------------------------------------*/
448static cpl_error_code
449merge_bpmcoeff(cpl_propertylist * mplist[], cpl_imagelist * mimg[],
450 const char * fn, const cpl_size next,
451 const char * rmregexp, int pos)
452{
453 cpl_errorstate cleanstate = cpl_errorstate_get();
454
455 for (int e = 0; e < next + 1; e++) {
456 cpl_propertylist * plist = cpl_propertylist_load(fn, e);
457 cpl_imagelist * img = cpl_imagelist_load(fn, CPL_TYPE_UNSPECIFIED, e);
458 /* empty extensions may exist */
459 if (img == NULL) {
460 cpl_errorstate_set(cleanstate);
461 cpl_propertylist_delete(plist);
462 continue;
463 }
464 skip_if(plist == NULL);
465
466 if (mimg[e] == NULL) {
467 merge_keys(plist, plist, pos, rmregexp);
468 mplist[e] = plist;
469 mimg[e] = img;
470 }
471 else {
472 merge_keys(mplist[e], plist, pos, rmregexp);
473 merge_images(mimg[e], img);
474 skip_if(0);
475 cpl_propertylist_delete(plist);
476 cpl_imagelist_delete(img);
477 }
478 skip_if(0);
479 }
480
481 end_skip;
482
483 return cpl_error_get_code();
484}
485
486typedef enum {
487 DETMON_BP_MAP_NL_IDX = 0,
488 DETMON_LIN_INFO_IDX = 1,
489 DETMON_GAIN_INFO_IDX = 2,
490 DETMON_BPM_IDX = 3,
491 DETMON_COEFF_IDX = 4,
492 DETMON_MAX_IDX,
493} detmon_idx;
494
495static cpl_error_code disable_contamination(cpl_parameterlist * parlist,
496 const char * recipe)
497{
498 /* "disable" by setting it to a region of size 1 */
499 cpl_parameter * par;
500 for (int i = 1; i < 6; i++) {
501 char buffer[100];
502 sprintf(buffer, "detmon.%s.llx%d", recipe, i);
503 par = cpl_parameterlist_find(parlist, buffer);
504 cpl_parameter_set_int(par, 1);
505 sprintf(buffer, "detmon.%s.lly%d", recipe, i);
506 par = cpl_parameterlist_find(parlist, buffer);
507 cpl_parameter_set_int(par, 1);
508 sprintf(buffer, "detmon.%s.urx%d", recipe, i);
509 par = cpl_parameterlist_find(parlist, buffer);
510 cpl_parameter_set_int(par, 2);
511 sprintf(buffer, "detmon.%s.ury%d", recipe, i);
512 par = cpl_parameterlist_find(parlist, buffer);
513 cpl_parameter_set_int(par, 2);
514 }
515
516 return cpl_error_get_code();
517}
518
519static cpl_error_code detmon_lg_mr_phu_qc_reg_kw(
520 cpl_propertylist** mplist[],
521 int idx,
522 cpl_size next,
523 cpl_size nregions,
524 const char * kw_segment,
525 cpl_type kw_type
526 ) {
527
528 char * kw_input = NULL;
529 char* kw_output_avg = NULL ;
530 char* kw_output_rms = NULL;
531 cpl_array* kw_vals = cpl_array_new(next, kw_type);
532
533 // Need to ensure the PHU propertylist actually exists - create one
534 // if it's NULL
535 if (mplist[idx][0] == NULL) {
536 mplist[idx][0] = cpl_propertylist_new();
537 }
538
539 for (int i=0; i < nregions; i++) {
540 kw_input = cpl_sprintf("ESO QC REG%d %s", i+1, kw_segment);
541 cpl_msg_debug(cpl_func, "Look for header kw %s", kw_input);
542 kw_output_avg = cpl_sprintf("ESO QC REG%d %s AVG", i+1, kw_segment);
543 kw_output_rms = cpl_sprintf("ESO QC REG%d %s RMS", i+1, kw_segment);
544 cpl_array_fill_window_invalid(kw_vals, 0, next);
545
546 for (cpl_size e = 1 ; e < next + 1 ; ++e) {
547 if (cpl_propertylist_has(mplist[idx][e], kw_input)) {
548 cpl_msg_debug(
549 cpl_func,
550 " Found value %s in ext %" CPL_SIZE_FORMAT "",
551 kw_input, e);
552 if (kw_type == CPL_TYPE_DOUBLE) {
553 cpl_array_set_double(kw_vals, e-1,
554 cpl_propertylist_get_double(
555 mplist[idx][e], kw_input));
556 } else if (kw_type == CPL_TYPE_INT) {
557 cpl_array_set_int(kw_vals, e-1,
558 cpl_propertylist_get_int(
559 mplist[idx][e], kw_input));
560 }
561 }
562 }
563
564 cpl_propertylist_append_double(mplist[idx][0],
565 kw_output_avg,
566 cpl_array_get_mean(kw_vals));
567 cpl_msg_debug(cpl_func, " Set mean");
568 cpl_propertylist_set_comment(
569 mplist[idx][0],
570 kw_output_avg,
571 cpl_sprintf("Avg of QC REG%d %s over exts",
572 i+1, kw_segment));
573 cpl_msg_debug(cpl_func, " Set mean comment");
574 cpl_propertylist_append_double(mplist[idx][0],
575 kw_output_rms,
576 cpl_array_get_stdev(kw_vals));
577 cpl_msg_debug(cpl_func, " Set RMS");
578 cpl_propertylist_set_comment(
579 mplist[idx][0],
580 kw_output_rms,
581 cpl_sprintf("RMS of QC REG%d %s over exts",
582 i+1, kw_segment));
583 if (cpl_error_get_code() != CPL_ERROR_NONE) {
584 cpl_msg_error(__func__, "Error found in detmon_lg_mr_phu_qc_reg_kw!");
585 cpl_msg_error(__func__, "In %s, L%u", cpl_error_get_file(),
586 cpl_error_get_line());
587 cpl_msg_error(__func__, "%s", cpl_error_get_message());
588 cpl_msg_error(__func__, "Values ingested:");
589 cpl_array_dump(kw_vals, 0, next, stdout);
590 cpl_msg_error(__func__, "There were %d extensions", next);
591 cpl_msg_error(__func__, "There were %d regions detected", nregions);
592 }
593 }
594
595 cpl_free(kw_input);
596 cpl_free(kw_output_rms);
597 cpl_free(kw_output_avg);
598 cpl_array_delete(kw_vals);
599
600 return cpl_error_get_code();
601}
602
603/*---------------------------------------------------------------------------*/
604/*
605 @brief Interpret the command line options and execute the data processing
606 @param frameset the frames list
607 @param parlist_ the parameters list
608 @param recipe recipe name
609 @param is_nir switch to know if opt or nir
610 @return 0 if everything is ok
611 */
612/*---------------------------------------------------------------------------*/
613
614int detmon_lg_mr(cpl_frameset * frameset,
615 const cpl_parameterlist * parlist_,
616 const char * recipe,
617 cpl_boolean is_nir)
618{
619 const char * tag_on=NULL;
620 const char * tag_off=NULL;
621 IRPLIB_DIAG_PRAGMA_PUSH_IGN(-Wcast-qual);
622 cpl_parameterlist * parlist = (cpl_parameterlist *)parlist_;
623 IRPLIB_DIAG_PRAGMA_POP;
624 cpl_propertylist ** mplist[DETMON_MAX_IDX] = {NULL};
625 cpl_imagelist ** mimg[DETMON_MAX_IDX] = {NULL};
626 cpl_table ** mtab[DETMON_MAX_IDX] = {NULL};
627 cpl_size next[DETMON_MAX_IDX] = {0};
628 char * outname[DETMON_MAX_IDX] = {NULL};
629 char * outtag[DETMON_MAX_IDX] = {NULL};
630 cpl_size nregions = 0;
631
632 cpl_frameset * frameset_copy = NULL;
633
634 cpl_propertylist * lintbl =
635 detmon_fill_prolist("DET_LIN_INFO", "REDUCED", "TECH", CPL_FALSE);
636
637 cpl_propertylist * gaintbl =
638 detmon_fill_prolist("GAIN_INFO", "REDUCED", "TECH", CPL_FALSE);
639
640 cpl_propertylist * coeffscube =
641 detmon_fill_prolist("COEFFS_CUBE", "REDUCED", "TECH", CPL_FALSE);
642
643 cpl_propertylist * bpm =
644 detmon_fill_prolist("BP_MAP_NL", "REDUCED", "TECH", CPL_FALSE);
645
646 cpl_propertylist * corr =
647 detmon_fill_prolist("AUTOCORR", "REDUCED", "TECH", CPL_FALSE);
648
649 cpl_propertylist * diff_flat =
650 detmon_fill_prolist("DIFF_FLAT", "REDUCED", "TECH", CPL_FALSE);
651 const char * regions_file = NULL;
652 const char * regions_str = NULL;
653 cpl_error_code error = CPL_ERROR_NONE;
654 cpl_size * llx = NULL, * lly = NULL, * urx = NULL, * ury = NULL;
655 const cpl_parameter * p =
656 get_parameter_const(parlist, recipe, "regions-file");
657 if (p) {
658 regions_file = cpl_parameter_get_string(p);
659 if (strlen(regions_file) == 0)
660 regions_file = NULL;
661 }
662
663 p = get_parameter_const(parlist, recipe, "regions");
664 if (p) {
665 regions_str = cpl_parameter_get_string(p);
666 if (strlen(regions_str) == 0)
667 regions_str = NULL;
668 else {
669 if (regions_file) {
670 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
671 "Please only provide either "
672 "--regions or "
673 "--regions-file, not both");
674 }
675 }
676 }
677
678 p = get_parameter(parlist, recipe, "intermediate");
679 if (p) {
680 error_if(cpl_parameter_get_bool(p), CPL_ERROR_UNSUPPORTED_MODE,
681 "--intermediate not supported by multi region recipe");
682 }
683
684 skip_if (detmon_lg_set_tag(frameset, &tag_on, &tag_off));
685 skip_if (detmon_lg_dfs_set_groups(frameset, tag_on, tag_off));
686
687 if (regions_file) {
688 read_regions_file(regions_file, &llx, &lly, &urx, &ury,
689 &nregions);
690 }
691 else if (regions_str) {
692 read_regions_str(regions_str, &llx, &lly, &urx, &ury,
693 &nregions);
694 }
695 else {
696 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
697 "Empty value in --regions or "
698 "--regions-file");
699 }
700 skip_if(0);
701
702 skip_if(check_regions(llx, lly, urx, ury, nregions));
703
704 if (!is_nir) {
705 /* contamination not supported */
706 skip_if(disable_contamination(parlist, recipe));
707 }
708
709 /* additionally iterating on extensions would improve IO caching and
710 * reduce memory usage for pix2pix but requires checks if detmon really
711 * gives the same result in this case */
712 for (int i = 0; i < nregions; i++) {
713 cpl_size nlabels;
714 cpl_size * selection;
715 cpl_parameter * par;
716 par = get_parameter(parlist, recipe, "llx");
717 cpl_parameter_set_int(par, llx[i]);
718 par = get_parameter(parlist, recipe, "lly");
719 cpl_parameter_set_int(par, lly[i]);
720 par = get_parameter(parlist, recipe, "urx");
721 cpl_parameter_set_int(par, urx[i]);
722 par = get_parameter(parlist, recipe, "ury");
723 cpl_parameter_set_int(par, ury[i]);
724 cpl_msg_info(cpl_func, "working on region %d/%d: %lld %lld, "
725 "%lld %lld", i + 1, (int)nregions,
726 llx[i], lly[i], urx[i], ury[i]);
727
728 /* copy and throw away so we don't accumulate products
729 * for each call */
730 cpl_frameset_delete(frameset_copy);
731 frameset_copy = cpl_frameset_duplicate(frameset);
732 error = detmon_lg(frameset_copy,
733 parlist,
734 tag_on,
735 tag_off,
736 recipe,
737 PACKAGE_TARNAME,
738 REGEXP,
739 lintbl, gaintbl, coeffscube,
740 bpm, corr, diff_flat,
741 PACKAGE "/" PACKAGE_VERSION,
742 NULL, NULL, is_nir);
743 skip_if(error);
744
745 selection = cpl_frameset_labelise(frameset_copy, compare_tags,
746 &nlabels);
747
748 for (cpl_size j = 0; j < nlabels; j++) {
749 cpl_frameset * a = cpl_frameset_extract(frameset_copy, selection, j);
750 cpl_frame * frm = cpl_frameset_get_position(a, 0);
751 const char * fn = cpl_frame_get_filename(frm);
752 int idx;
753
754 if (strcmp(cpl_frame_get_tag(frm), "DET_LIN_INFO") == 0)
755 idx = DETMON_LIN_INFO_IDX;
756 else if (strcmp(cpl_frame_get_tag(frm), "GAIN_INFO") == 0)
757 idx = DETMON_GAIN_INFO_IDX;
758 else if (strcmp(cpl_frame_get_tag(frm), "BP_MAP_NL") == 0)
759 idx = DETMON_BPM_IDX;
760 else if (strcmp(cpl_frame_get_tag(frm), "COEFFS_CUBE") == 0)
761 idx = DETMON_COEFF_IDX;
762 else {
763 cpl_frameset_delete(a);
764 continue;
765 }
766
767 /* prepare storage */
768 if (mplist[idx] == NULL) {
769 next[idx] = cpl_frame_get_nextensions(frm);
770 outname[idx] = cpl_strdup(cpl_frame_get_filename(frm));
771 outtag[idx] = cpl_strdup(cpl_frame_get_tag(frm));
772 mplist[idx] = cpl_calloc(sizeof(cpl_propertylist*), next[idx] + 1);
773 mimg[idx] = cpl_calloc(sizeof(cpl_imagelist*), next[idx] + 1);
774 mtab[idx] = cpl_calloc(sizeof(cpl_table*), next[idx] + 1);
775 }
776
777 if (idx == DETMON_LIN_INFO_IDX) {
778 const char * rmregexp = "ESO QC CONTAM.*";
779 merge_lingain(mplist[idx], mtab[idx], fn, next[idx],
780 rmregexp, i + 1);
781 }
782 else if (idx == DETMON_GAIN_INFO_IDX) {
783 merge_lingain(mplist[idx], mtab[idx], fn, next[idx],
784 NULL, i + 1);
785 }
786 else if (idx == DETMON_BPM_IDX) {
787 merge_bpmcoeff(mplist[idx], mimg[idx], fn, next[idx],
788 NULL, i + 1);
789 }
790 else if (idx == DETMON_COEFF_IDX) {
791 merge_bpmcoeff(mplist[idx], mimg[idx], fn, next[idx],
792 NULL, i + 1);
793 }
794 skip_if(0);
795
796 for (int e = 0; e < next[idx] + 1; e++) {
797 char buffer[300];
798 cpl_propertylist * copy_plist;
799 cpl_size len;
800 if (!mplist[idx][e])
801 continue;
802
803 /* don't add region if extension has no QC parameters */
804 copy_plist = cpl_propertylist_duplicate(mplist[idx][e]);
805 cpl_propertylist_erase_regexp(copy_plist, "ESO QC.*", 1);
806 len = cpl_propertylist_get_size(copy_plist);
807 cpl_propertylist_delete(copy_plist);
808 if (len == 0) {
809 continue;
810 }
811
812 sprintf(buffer, "ESO QC REG%d LLX", i + 1);
813 cpl_propertylist_append_int(mplist[idx][e], buffer, llx[i]);
814 cpl_propertylist_set_comment(mplist[idx][e], buffer,
815 "Lower left X of region");
816 sprintf(buffer, "ESO QC REG%d LLY", i + 1);
817 cpl_propertylist_append_int(mplist[idx][e], buffer, lly[i]);
818 cpl_propertylist_set_comment(mplist[idx][e], buffer,
819 "Lower left Y of region");
820 sprintf(buffer, "ESO QC REG%d URX", i + 1);
821 cpl_propertylist_append_int(mplist[idx][e], buffer, urx[i]);
822 cpl_propertylist_set_comment(mplist[idx][e], buffer,
823 "Upper right X of region");
824 sprintf(buffer, "ESO QC REG%d URY", i + 1);
825 cpl_propertylist_append_int(mplist[idx][e], buffer, ury[i]);
826 cpl_propertylist_set_comment(mplist[idx][e], buffer,
827 "Upper right Y of region");
828 }
829
830 cpl_frameset_delete(a);
831 skip_if(0);
832 }
833 cpl_free(selection);
834 }
835 cpl_msg_debug(__func__, "Completed looping over regions");
836
837 {
838 cpl_frameset * usedframes = cpl_frameset_new();
839 const cpl_size nframes = cpl_frameset_get_size(frameset);
840
841 cpl_frameset * rawframes = cpl_frameset_new();
842
843 for (cpl_size idx = 0; idx < nframes; idx++) {
844 const cpl_frame * frm = cpl_frameset_get_position_const(frameset,
845 idx);
846 if (cpl_frame_get_group(frm) == CPL_FRAME_GROUP_RAW ||
847 cpl_frame_get_group(frm) == CPL_FRAME_GROUP_CALIB) {
848 cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
849 }
850 if (cpl_frame_get_group(frm) == CPL_FRAME_GROUP_RAW) {
851 cpl_frameset_insert(rawframes, cpl_frame_duplicate(frm));
852 }
853 }
854
855 cpl_array* reg_values = NULL;
856 char * var_part = NULL;
857
858 for (int idx = 0; idx < DETMON_MAX_IDX; idx++) {
859 cpl_msg_debug(__func__, "Doing idx loop %d of %d", idx,
860 DETMON_MAX_IDX-1);
861 if (!mplist[idx])
862 continue;
863
864 if ((idx == DETMON_LIN_INFO_IDX || idx == DETMON_GAIN_INFO_IDX)) {
865
866 // PIPE-10123 - Compute additional parameters here for
867 // these two tables
868 double exptime = 0.0 ;
869 reg_values = cpl_array_new(nregions,
870 CPL_TYPE_DOUBLE);
871
872 if (!is_nir && next[idx] > 0) {
873 cpl_msg_debug(cpl_func, "Doing PIPE-10123 lin/gain kw for %s",
874 recipe);
875
876 // Compute EXPTIME MAX from RAWFRAMES
877 for (cpl_size r=0;
878 r < cpl_frameset_get_size(rawframes);
879 ++r) {
880 const cpl_frame * frm = cpl_frameset_get_position_const(
881 rawframes, r
882 );
883 const char * rfn = cpl_frame_get_filename(frm);
884 const cpl_propertylist * rpl = cpl_propertylist_load(
885 rfn, 0
886 );
887 if (rpl != NULL &&
888 cpl_propertylist_has(rpl, "EXPTIME")) {
889 if (cpl_propertylist_get_double(rpl, "EXPTIME") >
890 exptime) {
891 exptime = cpl_propertylist_get_double(
892 rpl, "EXPTIME");
893 }
894 }
895 }
896 cpl_propertylist_append_double(mplist[idx][0],
897 "ESO QC EXPTIME MAX",
898 exptime);
899 cpl_propertylist_set_comment(
900 mplist[idx][0],
901 "ESO QC EXPTIME MAX",
902 "Max. EXPTIME from input raws");
903
904 if (idx == DETMON_GAIN_INFO_IDX) {
905 var_part = "GAIN";
906 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
907 nregions, "CONAD",
908 CPL_TYPE_DOUBLE);
909 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
910 nregions, "GAIN",
911 CPL_TYPE_DOUBLE);
912 } else {
913 var_part = "LIN EFF";
914 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
915 nregions, "GAIN ERR",
916 CPL_TYPE_DOUBLE);
917 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
918 nregions, "FPN",
919 CPL_TYPE_DOUBLE);
920 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
921 nregions, "COUNTS MIN",
922 CPL_TYPE_DOUBLE);
923 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
924 nregions, "COUNTS MAX",
925 CPL_TYPE_DOUBLE);
926 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
927 nregions, "LIN EFF",
928 CPL_TYPE_DOUBLE);
929 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
930 nregions, "LIN EFF FLUX",
931 CPL_TYPE_DOUBLE);
932 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
933 nregions, "LLX",
934 CPL_TYPE_INT);
935 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
936 nregions, "LLY",
937 CPL_TYPE_INT);
938 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
939 nregions, "URX",
940 CPL_TYPE_INT);
941 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
942 nregions, "URY",
943 CPL_TYPE_INT);
944 }
945
946 for (cpl_size e = 1; e < next[idx] + 1; e++) {
947
948 for (int i=0 ; i < nregions ; i++) {
949 if (cpl_propertylist_has(mplist[idx][e],
950 cpl_sprintf("ESO QC REG%d %s",
951 i+1, var_part))) {
952 cpl_array_set_double(
953 reg_values, i,
954 cpl_propertylist_get_double(mplist[idx][e],
955 cpl_sprintf("ESO QC REG%d %s",
956 i+1, var_part)
957 ));
958 }
959
960 }
961
962 cpl_propertylist_append_double(mplist[idx][e],
963 cpl_sprintf("ESO QC %s AVG",
964 var_part),
965 cpl_array_get_mean(
966 reg_values));
967 cpl_propertylist_set_comment(mplist[idx][e],
968 cpl_sprintf("ESO QC %s AVG",
969 var_part),
970 cpl_sprintf(
971 "Avg of QC.REGn.%s",
972 var_part));
973 cpl_propertylist_append_double(mplist[idx][e],
974 cpl_sprintf("ESO QC %s RMS",
975 var_part),
976 cpl_array_get_stdev(
977 reg_values));
978 cpl_propertylist_set_comment(mplist[idx][e],
979 cpl_sprintf("ESO QC %s RMS",
980 var_part),
981 cpl_sprintf(
982 "RMS of QC.REGn.%s",
983 var_part));
984 cpl_msg_debug(__func__, "Wrote ESO QC RMS");
985 }
986
987 }
988 cpl_msg_debug(__func__, "Doing a GAIN/LIN save (var_part=%s)",
989 var_part);
990 irplib_dfs_save_propertylist(frameset, parlist, usedframes,
991 recipe, outtag[idx],
992 mplist[idx][0], NULL, PACKAGE "/"
993 PACKAGE_VERSION, outname[idx]);
994
995 skip_if(0);
996 for (cpl_size e = 1; e < next[idx] + 1; e++) {
997 if (mtab[idx][e]) {
998 cpl_table_save(mtab[idx][e], NULL, mplist[idx][e],
999 outname[idx], CPL_IO_EXTEND);
1000 }
1001 else {
1002 cpl_propertylist_save(mplist[idx][e], outname[idx],
1003 CPL_IO_EXTEND);
1004 }
1005 skip_if(0);
1006 }
1007 }
1008 // No need to alter _mr BPM after all for PIPE-10123
1009 else if (idx == DETMON_BPM_IDX) {
1010 cpl_msg_debug(__func__, "Doing a BPM save");
1011 if (!mimg[idx][0]) {
1012 irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1013 recipe, outtag[idx],
1014 mplist[idx][0], NULL, PACKAGE
1015 "/" PACKAGE_VERSION,
1016 outname[idx]);
1017 }
1018 else {
1019 irplib_dfs_save_image(frameset, parlist, usedframes,
1020 cpl_imagelist_get(mimg[idx][0], 0),
1021 CPL_TYPE_UNSPECIFIED, recipe,
1022 outtag[idx], mplist[idx][0], NULL,
1023 PACKAGE "/" PACKAGE_VERSION,
1024 outname[idx]);
1025 cpl_imagelist_delete(mimg[idx][0]);
1026 mimg[idx][0] = NULL;
1027 }
1028 skip_if(0);
1029 for (cpl_size e = 1; e < next[idx] + 1; e++) {
1030 if (mimg[idx][e]) {
1031 cpl_image * img = cpl_imagelist_get(mimg[idx][e], 0);
1032 cpl_image_save(img, outname[idx], CPL_TYPE_UNSPECIFIED,
1033 mplist[idx][e],
1034 CPL_IO_EXTEND);
1035 cpl_imagelist_delete(mimg[idx][e]);
1036 mimg[idx][e] = NULL;
1037 }
1038 else {
1039 cpl_propertylist_save(mplist[idx][e], outname[idx],
1040 CPL_IO_EXTEND);
1041 }
1042 skip_if(0);
1043 }
1044 }
1045 else if (idx == DETMON_COEFF_IDX) {
1046 // PIPE-10123 Add the additional QC keywords
1047 if (!is_nir && next[idx] > 0) {
1048 cpl_msg_debug(cpl_func, "Doing PIPE-10123 COEFF kw for %s",
1049 recipe);
1050 for (int j=0; j < 4 ; ++j ) {
1051 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
1052 nregions,
1053 cpl_sprintf("LIN COEF%d", j),
1054 CPL_TYPE_DOUBLE);
1055 }
1056 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
1057 nregions, "ERRFIT",
1058 CPL_TYPE_DOUBLE);
1059 }
1060
1061 cpl_msg_debug(__func__, "Doing a COEFFS save");
1062 if (!mimg[idx][0]) {
1063 irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1064 recipe, outtag[idx],
1065 mplist[idx][0], NULL, PACKAGE
1066 "/" PACKAGE_VERSION,
1067 outname[idx]);
1068 }
1069 else {
1070 irplib_dfs_save_imagelist(frameset, parlist, usedframes,
1071 mimg[idx][0],
1072 CPL_TYPE_UNSPECIFIED,
1073 recipe, outtag[idx],
1074 mplist[idx][0], NULL, PACKAGE "/"
1075 PACKAGE_VERSION, outname[idx]);
1076 cpl_imagelist_delete(mimg[idx][0]);
1077 mimg[idx][0] = NULL;
1078 }
1079 skip_if(0);
1080 for (cpl_size e = 1; e < next[idx] + 1; e++) {
1081 if (mimg[idx][e]) {
1082 cpl_imagelist_save(mimg[idx][e], outname[idx],
1083 CPL_TYPE_UNSPECIFIED,
1084 mplist[idx][e], CPL_IO_EXTEND);
1085 cpl_imagelist_delete(mimg[idx][e]);
1086 mimg[idx][e] = NULL;
1087 }
1088 else {
1089 cpl_propertylist_save(mplist[idx][e], outname[idx],
1090 CPL_IO_EXTEND);
1091 }
1092 skip_if(0);
1093 }
1094 }
1095 }
1096 // PIPE-10123 cleanup
1097 cpl_array_delete(reg_values);
1098
1099 cpl_frameset_delete(usedframes);
1100 cpl_frameset_delete(rawframes);
1101 skip_if(0);
1102 }
1103
1104 end_skip;
1105 cpl_frameset_delete(frameset_copy);
1106 cpl_propertylist_delete(lintbl);
1107 cpl_propertylist_delete(gaintbl);
1108 cpl_propertylist_delete(coeffscube);
1109 cpl_propertylist_delete(bpm);
1110 cpl_propertylist_delete(corr);
1111 cpl_propertylist_delete(diff_flat);
1112 for (int i = 0; i < DETMON_MAX_IDX; i++) {
1113 for (int e = 0; e < next[i] + 1; e++) {
1114 if (mplist[i])
1115 cpl_propertylist_delete(mplist[i][e]);
1116 if (mimg[i])
1117 cpl_imagelist_delete(mimg[i][e]);
1118 if (mtab[i])
1119 cpl_table_delete(mtab[i][e]);
1120 }
1121 cpl_free(mplist[i]);
1122 cpl_free(mimg[i]);
1123 cpl_free(mtab[i]);
1124 cpl_free(outname[i]);
1125 cpl_free(outtag[i]);
1126 }
1127 cpl_free(llx);
1128 cpl_free(lly);
1129 cpl_free(urx);
1130 cpl_free(ury);
1131
1132 return cpl_error_get_code();
1133}