IIINSTRUMENT Pipeline Reference Manual 1.3.13
detmon_lg_mr.c
1/*
2 * This file is part of the DETMON Pipeline
3 * Copyright (C) 2013 European Southern Observatory
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
18 */
19
20/*
21 * $Author: jtaylor $
22 * $Date: 2013-08-07 09:32:50 $
23 * $Revision: 1.6 $
24 * $Name: not supported by cvs2svn $
25 */
26#ifdef HAVE_CONFIG_H
27#include <config.h>
28#endif
29
30/*----------------------------------------------------------------------------
31 Includes
32 ----------------------------------------------------------------------------*/
33
34
35#include "detmon_lg.h"
36#include "detmon.h"
37
38#include "irplib_plugin.h"
39#include "irplib_utils.h"
40
41#include <string.h>
42#include <errno.h>
43#include <stdlib.h>
44#include <assert.h>
45
46/*----------------------------------------------------------------------------
47 Private Functions prototypes
48 ----------------------------------------------------------------------------*/
49int
50detmon_lg_dfs_set_groups(cpl_frameset * set,
51 const char *tag_on, const char *tag_off);
52
53static int check_regions(const cpl_size *, const cpl_size *,
54 const cpl_size *, const cpl_size *,
55 cpl_size) CPL_ATTR_NONNULL;
56
57static int compare_tags(const cpl_frame *, const cpl_frame *) CPL_ATTR_NONNULL;
58
59static const cpl_parameter *
60get_parameter_const(const cpl_parameterlist * parlist,
61 const char * recipe, const char * name)
62{
63 char * parname = cpl_sprintf("detmon.%s.%s", recipe, name);
64 const cpl_parameter * p = cpl_parameterlist_find_const(parlist, parname);
65 cpl_free(parname);
66 return p;
67}
68
69static cpl_parameter *
70get_parameter(cpl_parameterlist * parlist,
71 const char * recipe, const char * name)
72{
73 char * parname = cpl_sprintf("detmon.%s.%s", recipe, name);
74 cpl_parameter * p = cpl_parameterlist_find(parlist, parname);
75 cpl_free(parname);
76 return p;
77}
78
79static int compare_tags(const cpl_frame * a, const cpl_frame * b)
80{
81 return strcmp(cpl_frame_get_tag(a), cpl_frame_get_tag(b)) == 0;
82}
83
86/* ---------------------------------------------------------------------------*/
97/* ---------------------------------------------------------------------------*/
98static cpl_error_code merge_keys(cpl_propertylist * mplist,
99 const cpl_propertylist * plist_,
100 const int pos,
101 const char * rmregexp)
102{
103 char buffer[256];
104 /* no copy_regex, so duplicate and use erase_regex on the copy */
105 cpl_propertylist * plist = cpl_propertylist_duplicate(plist_);
106
107 if (rmregexp) {
108 cpl_propertylist_erase_regexp(plist, rmregexp, 0);
109 cpl_propertylist_erase_regexp(mplist, rmregexp, 0);
110 }
111
112 for (cpl_size i = 0; i < cpl_propertylist_get_size(plist); i++) {
113 cpl_property * p = cpl_propertylist_get(plist, i);
114 const char * key = cpl_property_get_name(p);
115 /* extend all qc keywords by a region prefix */
116 if (!strncmp(key, "ESO QC ", 7)) {
117 const char * com;
118 sprintf(buffer, "ESO QC REG%d %s", pos, key + 7);
119 cpl_msg_debug(cpl_func, "merging %s to %s", key, buffer);
120 if (cpl_property_get_type(p) == CPL_TYPE_INT) {
121 int val = cpl_property_get_int(p);
122 cpl_propertylist_append_int(mplist, buffer, val);
123 } else if (cpl_property_get_type(p) == CPL_TYPE_DOUBLE) {
124 double val = cpl_property_get_double(p);
125 cpl_propertylist_append_double(mplist, buffer, val);
126 } else if (cpl_property_get_type(p) == CPL_TYPE_STRING) {
127 const char * val = cpl_property_get_string(p);
128 cpl_propertylist_append_string(mplist, buffer, val);
129 }
130 else
131 bug_if(1);
132 com = cpl_property_get_comment(p);
133 cpl_propertylist_set_comment(mplist, buffer, com);
134
135 /* remove unprefixed key */
136 if (cpl_propertylist_has(mplist, key))
137 cpl_propertylist_erase(mplist, key);
138 }
139 }
140 skip_if(0);
141
142 end_skip;
143 cpl_propertylist_delete(plist);
144
145 return cpl_error_get_code();
146}
147
148
149/* ---------------------------------------------------------------------------*/
157/* ---------------------------------------------------------------------------*/
158static cpl_error_code
159merge_images(cpl_imagelist * mimgl, const cpl_imagelist * img)
160{
161 cpl_ensure_code(cpl_imagelist_get_size(img) ==
162 cpl_imagelist_get_size(mimgl), CPL_ERROR_ILLEGAL_INPUT);
163 for (int i = 0; i < cpl_imagelist_get_size(img); i++) {
164 cpl_image * mimg = cpl_imagelist_get(mimgl, i);
165 const cpl_image * part = cpl_imagelist_get_const(img, i);
166 cpl_image_add(mimg, part);
167 }
168
169 return cpl_error_get_code();
170}
171
172
173/* ---------------------------------------------------------------------------*/
181/* ---------------------------------------------------------------------------*/
182static cpl_error_code
183merge_table(cpl_table * mtab, cpl_table * tab, const int pos)
184{
185 cpl_table_new_column(tab, "REGION", CPL_TYPE_INT);
186 cpl_table_fill_column_window(tab, "REGION", 0, cpl_table_get_nrow(tab), pos);
187 skip_if(cpl_table_insert(mtab, tab, cpl_table_get_nrow(mtab)));
188
189 end_skip;
190
191 return cpl_error_get_code();
192}
193
194
195static cpl_error_code
196parse_token(const char * token, cpl_size * llx, cpl_size * lly,
197 cpl_size * urx, cpl_size * ury)
198{
199 assert(sizeof(*llx) == 8);
200 if (sscanf(token, " %lld , %lld , %lld , %lld ", llx, lly, urx, ury) == 4) {
201 return CPL_ERROR_NONE;
202 }
203 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
204 "Data not formated correctly: %s; "
205 "expecting: name,llx,lly,urx,ury",
206 token);
207}
208
209
210/* ---------------------------------------------------------------------------*/
223/* ---------------------------------------------------------------------------*/
224static cpl_error_code
225read_regions_file(const char * filename, cpl_size ** rllx,
226 cpl_size ** rlly, cpl_size ** rurx, cpl_size ** rury,
227 cpl_size * n)
228{
229 FILE * f = fopen(filename, "r");
230 char line[1024];
231 cpl_size i = 0;
232 cpl_size nmax = 100;
233 if (f == NULL) {
234 return cpl_error_set_message(cpl_func, CPL_ERROR_FILE_IO,
235 "Could not open file %s: %s", filename,
236 strerror(errno));
237 }
238 cpl_msg_info(cpl_func, "Reading %s", filename);
239
240 *n = 0;
241 *rllx = cpl_malloc(sizeof(**rllx) * nmax);
242 *rlly = cpl_malloc(sizeof(**rllx) * nmax);
243 *rurx = cpl_malloc(sizeof(**rllx) * nmax);
244 *rury = cpl_malloc(sizeof(**rllx) * nmax);
245
246 while (fgets(line, 1024, f) != NULL) {
247 const size_t nline = strlen(line);
248 if (line[nline - 1] != '\n') {
249 cpl_error_set_message(cpl_func, CPL_ERROR_FILE_IO,
250 "File %s contained too long line",
251 filename);
252 skip_if(0);
253 }
254
255 if (line[0] == '#')
256 continue;
257
258 skip_if(parse_token(line, &(*rllx)[i], &(*rlly)[i],
259 &(*rurx)[i], &(*rury)[i]));
260 *n = ++i;
261 if (i >= nmax) {
262 nmax *= 2;
263 *rllx = cpl_realloc(*rllx, sizeof(**rllx) * nmax);
264 *rlly = cpl_realloc(*rlly, sizeof(**rllx) * nmax);
265 *rurx = cpl_realloc(*rurx, sizeof(**rllx) * nmax);
266 *rury = cpl_realloc(*rury, sizeof(**rllx) * nmax);
267 skip_if(0);
268 }
269 }
270
271 end_skip;
272 if (f)
273 fclose(f);
274 return CPL_ERROR_NONE;
275}
276
277
278/* ---------------------------------------------------------------------------*/
291/* ---------------------------------------------------------------------------*/
292static cpl_error_code
293read_regions_str(const char * istr, cpl_size ** rllx,
294 cpl_size ** rlly, cpl_size ** rurx, cpl_size ** rury,
295 cpl_size * n)
296{
297 cpl_size nmax = 100;
298 char * str_ = cpl_strdup(istr);
299 char * str = str_;
300 *n = 0;
301 *rllx = cpl_malloc(sizeof(**rllx) * nmax);
302 *rlly = cpl_malloc(sizeof(**rllx) * nmax);
303 *rurx = cpl_malloc(sizeof(**rllx) * nmax);
304 *rury = cpl_malloc(sizeof(**rllx) * nmax);
305
306 for (int i = 0; ;i++) {
307 const char * token = strtok(str, ":");
308 if (token == NULL)
309 break;
310
311 skip_if(parse_token(token, &(*rllx)[i], &(*rlly)[i],
312 &(*rurx)[i], &(*rury)[i]));
313 (*n)++;
314 if (i >= nmax - 1) {
315 nmax *= 2;
316 *rllx = cpl_realloc(*rllx, sizeof(**rllx) * nmax);
317 *rlly = cpl_realloc(*rlly, sizeof(**rllx) * nmax);
318 *rurx = cpl_realloc(*rurx, sizeof(**rllx) * nmax);
319 *rury = cpl_realloc(*rury, sizeof(**rllx) * nmax);
320 skip_if(0);
321 }
322 str = NULL;
323 }
324
325 end_skip;
326 cpl_free(str_);
327
328 return CPL_ERROR_NONE;
329}
330
331static int check_regions(const cpl_size * llx, const cpl_size * lly,
332 const cpl_size * urx, const cpl_size * ury,
333 cpl_size nregions)
334{
335 for (cpl_size i = 0; i < nregions; i++) {
336 if (llx[i] < 1 || lly[i] < 1 ||
337 urx[i] < 1 || ury[i] < 1) {
338 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
339 "Region %lld (%lld,%lld,%lld,%lld) has entry smaller one",
340 i, llx[i], lly[i], urx[i], ury[i]);
341 }
342 if ((urx[i] - llx[i] < 1) || (ury[i] - lly[i] < 1)) {
343 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
344 "Region %lld (%lld,%lld,%lld,%lld) has size zero",
345 i, llx[i], lly[i], urx[i], ury[i]);
346 }
347 for (cpl_size j = i + 1; j < nregions; j++) {
348 if (!(llx[i] > urx[j] || urx[i] < llx[j] ||
349 lly[i] > ury[j] || ury[i] < lly[j])) {
350 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
351 "Regions %lld (%lld,%lld,%lld,%lld) and "
352 "%lld (%lld,%lld,%lld,%lld) overlap (llx,lly,urx,ury)",
353 i, llx[i], lly[i], urx[i], ury[i],
354 j, llx[j], lly[j], urx[j], ury[j]);
355 }
356 }
357 }
358
359 return CPL_ERROR_NONE;
360}
361
362
363/* ---------------------------------------------------------------------------*/
376/* ---------------------------------------------------------------------------*/
377static cpl_error_code
378merge_lingain(cpl_propertylist * mplist[], cpl_table * mtab[],
379 const char * fn, const cpl_size next,
380 const char * rmregexp, int pos)
381{
382 cpl_errorstate cleanstate = cpl_errorstate_get();
383
384 {
385 cpl_propertylist * plist = cpl_propertylist_load(fn, 0);
386 skip_if(plist == NULL);
387 if (mplist[0] == NULL) {
388 mplist[0] = plist;
389 merge_keys(mplist[0], plist, pos, rmregexp);
390 }
391 else {
392 merge_keys(mplist[0], plist, pos, rmregexp);
393 cpl_propertylist_delete(plist);
394 }
395 }
396 skip_if(0);
397
398 for (int e = 1; e < next + 1; e++) {
399 cpl_propertylist * plist = cpl_propertylist_load(fn, e);
400 cpl_table * tab = cpl_table_load(fn, e, 0);
401 /* empty extensions may exist */
402 if (tab == NULL) {
403 cpl_errorstate_set(cleanstate);
404 cpl_propertylist_delete(plist);
405 continue;
406 }
407 skip_if(plist == NULL);
408
409 if (mplist[e] == NULL) {
410 merge_keys(plist, plist, pos, rmregexp);
411 cpl_table_new_column(tab, "REGION", CPL_TYPE_INT);
412 cpl_table_fill_column_window(tab, "REGION", 0,
413 cpl_table_get_nrow(tab), 1);
414 skip_if(0);
415 mplist[e] = plist;
416 mtab[e] = tab;
417 }
418 else {
419 merge_keys(mplist[e], plist, pos, rmregexp);
420 merge_table(mtab[e], tab, pos);
421 skip_if(0);
422 cpl_propertylist_delete(plist);
423 cpl_table_delete(tab);
424 }
425 skip_if(0);
426 }
427
428 end_skip;
429
430 return cpl_error_get_code();
431}
432
433
434/* ---------------------------------------------------------------------------*/
447/* ---------------------------------------------------------------------------*/
448static cpl_error_code
449merge_bpmcoeff(cpl_propertylist * mplist[], cpl_imagelist * mimg[],
450 const char * fn, const cpl_size next,
451 const char * rmregexp, int pos)
452{
453 cpl_errorstate cleanstate = cpl_errorstate_get();
454
455 for (int e = 0; e < next + 1; e++) {
456 cpl_propertylist * plist = cpl_propertylist_load(fn, e);
457 cpl_imagelist * img = cpl_imagelist_load(fn, CPL_TYPE_UNSPECIFIED, e);
458 /* empty extensions may exist */
459 if (img == NULL) {
460 cpl_errorstate_set(cleanstate);
461 cpl_propertylist_delete(plist);
462 continue;
463 }
464 skip_if(plist == NULL);
465
466 if (mimg[e] == NULL) {
467 merge_keys(plist, plist, pos, rmregexp);
468 mplist[e] = plist;
469 mimg[e] = img;
470 }
471 else {
472 merge_keys(mplist[e], plist, pos, rmregexp);
473 merge_images(mimg[e], img);
474 skip_if(0);
475 cpl_propertylist_delete(plist);
476 cpl_imagelist_delete(img);
477 }
478 skip_if(0);
479 }
480
481 end_skip;
482
483 return cpl_error_get_code();
484}
485
486typedef enum {
487 DETMON_BP_MAP_NL_IDX = 0,
488 DETMON_LIN_INFO_IDX = 1,
489 DETMON_GAIN_INFO_IDX = 2,
490 DETMON_BPM_IDX = 3,
491 DETMON_COEFF_IDX = 4,
492 DETMON_MAX_IDX,
493} detmon_idx;
494
495static cpl_error_code disable_contamination(cpl_parameterlist * parlist,
496 const char * recipe)
497{
498 /* "disable" by setting it to a region of size 1 */
499 cpl_parameter * par;
500 for (int i = 1; i < 6; i++) {
501 char buffer[100];
502 sprintf(buffer, "detmon.%s.llx%d", recipe, i);
503 par = cpl_parameterlist_find(parlist, buffer);
504 cpl_parameter_set_int(par, 1);
505 sprintf(buffer, "detmon.%s.lly%d", recipe, i);
506 par = cpl_parameterlist_find(parlist, buffer);
507 cpl_parameter_set_int(par, 1);
508 sprintf(buffer, "detmon.%s.urx%d", recipe, i);
509 par = cpl_parameterlist_find(parlist, buffer);
510 cpl_parameter_set_int(par, 2);
511 sprintf(buffer, "detmon.%s.ury%d", recipe, i);
512 par = cpl_parameterlist_find(parlist, buffer);
513 cpl_parameter_set_int(par, 2);
514 }
515
516 return cpl_error_get_code();
517}
518
519static cpl_error_code detmon_lg_mr_phu_qc_reg_kw(
520 cpl_propertylist** mplist[],
521 int idx,
522 cpl_size next,
523 cpl_size nregions,
524 const char * kw_segment,
525 cpl_type kw_type
526 ) {
527
528// cpl_msg_info(cpl_func, "Within detmon_lg_mr_phu_qc_reg_kw for %s",
529// kw_segment);
530 char * kw_input = NULL;
531 char* kw_output_avg = NULL ;
532 char* kw_output_rms = NULL;
533 cpl_array* kw_vals = cpl_array_new(next, kw_type);
534// cpl_msg_info(cpl_func, " Variables instantiated");
535
536 // Need to ensure the PHU propertylist actually exists - create one
537 // if it's NULL
538 if (mplist[idx][0] == NULL) {
539 mplist[idx][0] = cpl_propertylist_new();
540// cpl_msg_info(cpl_func, " PHU propertylist instantiated");
541 }
542
543 for (int i=0; i < nregions; i++) {
544// cpl_msg_info(cpl_func, " Region %d of %d", i+1, nregions);
545 kw_input = cpl_sprintf("ESO QC REG%d %s", i+1, kw_segment);
546 cpl_msg_info(cpl_func, "Look for header kw %s", kw_input);
547 kw_output_avg = cpl_sprintf("ESO QC REG%d %s AVG", i+1, kw_segment);
548 kw_output_rms = cpl_sprintf("ESO QC REG%d %s RMS", i+1, kw_segment);
549 cpl_array_fill_window_invalid(kw_vals, 0, next);
550// cpl_msg_info(cpl_func, " Variables set");
551// cpl_msg_info(cpl_func, " %s", kw_input);
552// cpl_msg_info(cpl_func, " %s", kw_output_avg);
553// cpl_msg_info(cpl_func, " %s", kw_output_rms);
554
555 for (cpl_size e = 1 ; e < next + 1 ; ++e) {
556 if (cpl_propertylist_has(mplist[idx][e], kw_input)) {
557 cpl_msg_info(
558 cpl_func,
559 " Found value %s in ext %" CPL_SIZE_FORMAT "",
560 kw_input, e);
561 if (kw_type == CPL_TYPE_DOUBLE) {
562 cpl_array_set_double(kw_vals, e-1,
563 cpl_propertylist_get_double(
564 mplist[idx][e], kw_input));
565 } else if (kw_type == CPL_TYPE_INT) {
566 cpl_array_set_int(kw_vals, e-1,
567 cpl_propertylist_get_int(
568 mplist[idx][e], kw_input));
569 }
570 }
571 }
572 cpl_msg_info(cpl_func, " Array populated");
573
574 cpl_propertylist_append_double(mplist[idx][0],
575 kw_output_avg,
576 cpl_array_get_mean(kw_vals));
577 cpl_msg_info(cpl_func, " Set mean");
578 cpl_propertylist_set_comment(
579 mplist[idx][0],
580 kw_output_avg,
581 cpl_sprintf("Avg of QC REG%d %s over exts",
582 i+1, kw_segment));
583 cpl_msg_info(cpl_func, " Set mean comment");
584 cpl_propertylist_append_double(mplist[idx][0],
585 kw_output_rms,
586 cpl_array_get_stdev(kw_vals));
587 cpl_msg_info(cpl_func, " Set RMS");
588 cpl_propertylist_set_comment(
589 mplist[idx][0],
590 kw_output_rms,
591 cpl_sprintf("RMS of QC REG%d %s over exts",
592 i+1, kw_segment));
593// cpl_msg_info(cpl_func, " Set RMS comment");
594// cpl_msg_info(cpl_func, " KW set for region %d", i+1);
595 if (cpl_error_get_code() != CPL_ERROR_NONE) {
596 cpl_msg_error(__func__, "Error found in detmon_lg_mr_phu_qc_reg_kw!");
597 cpl_msg_error(__func__, "In %s, L%u", cpl_error_get_file(),
598 cpl_error_get_line());
599 cpl_msg_error(__func__, "%s", cpl_error_get_message());
600 cpl_msg_error(__func__, "Values ingested:");
601 cpl_array_dump(kw_vals, 0, next, stdout);
602 cpl_msg_error(__func__, "There were %d extensions", next);
603 cpl_msg_error(__func__, "There were %d regions detected", nregions);
604// cpl_msg_error(__func__, "Propertylist:");
605// cpl_propertylist_dump(mplist[idx][0], stdout);
606// getchar();
607 }
608 }
609
610// cpl_msg_info(cpl_func, " Proceeding to clean-up");
611 cpl_free(kw_input);
612 cpl_free(kw_output_rms);
613 cpl_free(kw_output_avg);
614 cpl_array_delete(kw_vals);
615
616// cpl_msg_info(cpl_func, "Returning!");
617 return cpl_error_get_code();
618}
619
620/*---------------------------------------------------------------------------*/
621/*
622 @brief Interpret the command line options and execute the data processing
623 @param frameset the frames list
624 @param parlist_ the parameters list
625 @param recipe recipe name
626 @param is_nir switch to know if opt or nir
627 @return 0 if everything is ok
628 */
629/*---------------------------------------------------------------------------*/
630
631int detmon_lg_mr(cpl_frameset * frameset,
632 const cpl_parameterlist * parlist_,
633 const char * recipe,
634 cpl_boolean is_nir)
635{
636 const char * tag_on=NULL;
637 const char * tag_off=NULL;
638 IRPLIB_DIAG_PRAGMA_PUSH_IGN(-Wcast-qual);
639 cpl_parameterlist * parlist = (cpl_parameterlist *)parlist_;
640 IRPLIB_DIAG_PRAGMA_POP;
641 cpl_propertylist ** mplist[DETMON_MAX_IDX] = {NULL};
642 cpl_imagelist ** mimg[DETMON_MAX_IDX] = {NULL};
643 cpl_table ** mtab[DETMON_MAX_IDX] = {NULL};
644 cpl_size next[DETMON_MAX_IDX] = {0};
645 char * outname[DETMON_MAX_IDX] = {NULL};
646 char * outtag[DETMON_MAX_IDX] = {NULL};
647 cpl_size nregions = 0;
648
649 cpl_frameset * frameset_copy = NULL;
650
651 cpl_propertylist * lintbl =
652 detmon_fill_prolist("DET_LIN_INFO", "REDUCED", "TECH", CPL_FALSE);
653
654 cpl_propertylist * gaintbl =
655 detmon_fill_prolist("GAIN_INFO", "REDUCED", "TECH", CPL_FALSE);
656
657 cpl_propertylist * coeffscube =
658 detmon_fill_prolist("COEFFS_CUBE", "REDUCED", "TECH", CPL_FALSE);
659
660 cpl_propertylist * bpm =
661 detmon_fill_prolist("BP_MAP_NL", "REDUCED", "TECH", CPL_FALSE);
662
663 cpl_propertylist * corr =
664 detmon_fill_prolist("AUTOCORR", "REDUCED", "TECH", CPL_FALSE);
665
666 cpl_propertylist * diff_flat =
667 detmon_fill_prolist("DIFF_FLAT", "REDUCED", "TECH", CPL_FALSE);
668 const char * regions_file = NULL;
669 const char * regions_str = NULL;
670 cpl_error_code error = CPL_ERROR_NONE;
671 cpl_size * llx = NULL, * lly = NULL, * urx = NULL, * ury = NULL;
672 const cpl_parameter * p =
673 get_parameter_const(parlist, recipe, "regions-file");
674 if (p) {
675 regions_file = cpl_parameter_get_string(p);
676 if (strlen(regions_file) == 0)
677 regions_file = NULL;
678 }
679
680 p = get_parameter_const(parlist, recipe, "regions");
681 if (p) {
682 regions_str = cpl_parameter_get_string(p);
683 if (strlen(regions_str) == 0)
684 regions_str = NULL;
685 else {
686 if (regions_file) {
687 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
688 "Please only provide either "
689 "--regions or "
690 "--regions-file, not both");
691 }
692 }
693 }
694
695 p = get_parameter(parlist, recipe, "intermediate");
696 if (p) {
697 error_if(cpl_parameter_get_bool(p), CPL_ERROR_UNSUPPORTED_MODE,
698 "--intermediate not supported by multi region recipe");
699 }
700
701 skip_if (detmon_lg_set_tag(frameset, &tag_on, &tag_off));
702 skip_if (detmon_lg_dfs_set_groups(frameset, tag_on, tag_off));
703
704 if (regions_file) {
705 read_regions_file(regions_file, &llx, &lly, &urx, &ury,
706 &nregions);
707 }
708 else if (regions_str) {
709 read_regions_str(regions_str, &llx, &lly, &urx, &ury,
710 &nregions);
711 }
712 else {
713 return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
714 "Empty value in --regions or "
715 "--regions-file");
716 }
717 skip_if(0);
718
719 skip_if(check_regions(llx, lly, urx, ury, nregions));
720
721 if (!is_nir) {
722 /* contamination not supported */
723 skip_if(disable_contamination(parlist, recipe));
724 }
725
726 /* additionally iterating on extensions would improve IO caching and
727 * reduce memory usage for pix2pix but requires checks if detmon really
728 * gives the same result in this case */
729 for (int i = 0; i < nregions; i++) {
730 cpl_size nlabels;
731 cpl_size * selection;
732 cpl_parameter * par;
733 par = get_parameter(parlist, recipe, "llx");
734 cpl_parameter_set_int(par, llx[i]);
735 par = get_parameter(parlist, recipe, "lly");
736 cpl_parameter_set_int(par, lly[i]);
737 par = get_parameter(parlist, recipe, "urx");
738 cpl_parameter_set_int(par, urx[i]);
739 par = get_parameter(parlist, recipe, "ury");
740 cpl_parameter_set_int(par, ury[i]);
741 cpl_msg_info(cpl_func, "working on region %d/%d: %lld %lld, "
742 "%lld %lld", i + 1, (int)nregions,
743 llx[i], lly[i], urx[i], ury[i]);
744
745 /* copy and throw away so we don't accumulate products
746 * for each call */
747 cpl_frameset_delete(frameset_copy);
748 frameset_copy = cpl_frameset_duplicate(frameset);
749 error = detmon_lg(frameset_copy,
750 parlist,
751 tag_on,
752 tag_off,
753 recipe,
754 PACKAGE_TARNAME,
755 REGEXP,
756 lintbl, gaintbl, coeffscube,
757 bpm, corr, diff_flat,
758 PACKAGE "/" PACKAGE_VERSION,
759 NULL, NULL, is_nir);
760 skip_if(error);
761
762 selection = cpl_frameset_labelise(frameset_copy, compare_tags,
763 &nlabels);
764
765 for (cpl_size j = 0; j < nlabels; j++) {
766 cpl_frameset * a = cpl_frameset_extract(frameset_copy, selection, j);
767 cpl_frame * frm = cpl_frameset_get_position(a, 0);
768 const char * fn = cpl_frame_get_filename(frm);
769 int idx;
770
771 if (strcmp(cpl_frame_get_tag(frm), "DET_LIN_INFO") == 0)
772 idx = DETMON_LIN_INFO_IDX;
773 else if (strcmp(cpl_frame_get_tag(frm), "GAIN_INFO") == 0)
774 idx = DETMON_GAIN_INFO_IDX;
775 else if (strcmp(cpl_frame_get_tag(frm), "BP_MAP_NL") == 0)
776 idx = DETMON_BPM_IDX;
777 else if (strcmp(cpl_frame_get_tag(frm), "COEFFS_CUBE") == 0)
778 idx = DETMON_COEFF_IDX;
779 else {
780 cpl_frameset_delete(a);
781 continue;
782 }
783
784 /* prepare storage */
785 if (mplist[idx] == NULL) {
786 next[idx] = cpl_frame_get_nextensions(frm);
787 outname[idx] = cpl_strdup(cpl_frame_get_filename(frm));
788 outtag[idx] = cpl_strdup(cpl_frame_get_tag(frm));
789 mplist[idx] = cpl_calloc(sizeof(cpl_propertylist*), next[idx] + 1);
790 mimg[idx] = cpl_calloc(sizeof(cpl_imagelist*), next[idx] + 1);
791 mtab[idx] = cpl_calloc(sizeof(cpl_table*), next[idx] + 1);
792 }
793
794 if (idx == DETMON_LIN_INFO_IDX) {
795 const char * rmregexp = "ESO QC CONTAM.*";
796 merge_lingain(mplist[idx], mtab[idx], fn, next[idx],
797 rmregexp, i + 1);
798 }
799 else if (idx == DETMON_GAIN_INFO_IDX) {
800 merge_lingain(mplist[idx], mtab[idx], fn, next[idx],
801 NULL, i + 1);
802 }
803 else if (idx == DETMON_BPM_IDX) {
804 merge_bpmcoeff(mplist[idx], mimg[idx], fn, next[idx],
805 NULL, i + 1);
806 }
807 else if (idx == DETMON_COEFF_IDX) {
808 merge_bpmcoeff(mplist[idx], mimg[idx], fn, next[idx],
809 NULL, i + 1);
810 }
811 skip_if(0);
812
813 for (int e = 0; e < next[idx] + 1; e++) {
814 char buffer[300];
815 cpl_propertylist * copy_plist;
816 cpl_size len;
817 if (!mplist[idx][e])
818 continue;
819
820 /* don't add region if extension has no QC parameters */
821 copy_plist = cpl_propertylist_duplicate(mplist[idx][e]);
822 cpl_propertylist_erase_regexp(copy_plist, "ESO QC.*", 1);
823 len = cpl_propertylist_get_size(copy_plist);
824 cpl_propertylist_delete(copy_plist);
825 if (len == 0) {
826 continue;
827 }
828
829 sprintf(buffer, "ESO QC REG%d LLX", i + 1);
830 cpl_propertylist_append_int(mplist[idx][e], buffer, llx[i]);
831 cpl_propertylist_set_comment(mplist[idx][e], buffer,
832 "Lower left X of region");
833 sprintf(buffer, "ESO QC REG%d LLY", i + 1);
834 cpl_propertylist_append_int(mplist[idx][e], buffer, lly[i]);
835 cpl_propertylist_set_comment(mplist[idx][e], buffer,
836 "Lower left Y of region");
837 sprintf(buffer, "ESO QC REG%d URX", i + 1);
838 cpl_propertylist_append_int(mplist[idx][e], buffer, urx[i]);
839 cpl_propertylist_set_comment(mplist[idx][e], buffer,
840 "Upper right X of region");
841 sprintf(buffer, "ESO QC REG%d URY", i + 1);
842 cpl_propertylist_append_int(mplist[idx][e], buffer, ury[i]);
843 cpl_propertylist_set_comment(mplist[idx][e], buffer,
844 "Upper right Y of region");
845 }
846
847 cpl_frameset_delete(a);
848 skip_if(0);
849 }
850 cpl_free(selection);
851 }
852 cpl_msg_info(__func__, "Completed looping over regions");
853// getchar();
854
855 {
856 cpl_frameset * usedframes = cpl_frameset_new();
857 const cpl_size nframes = cpl_frameset_get_size(frameset);
858
859 cpl_frameset * rawframes = cpl_frameset_new();
860
861 for (cpl_size idx = 0; idx < nframes; idx++) {
862 const cpl_frame * frm = cpl_frameset_get_position_const(frameset,
863 idx);
864 if (cpl_frame_get_group(frm) == CPL_FRAME_GROUP_RAW ||
865 cpl_frame_get_group(frm) == CPL_FRAME_GROUP_CALIB) {
866 cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
867 }
868 if (cpl_frame_get_group(frm) == CPL_FRAME_GROUP_RAW) {
869 cpl_frameset_insert(rawframes, cpl_frame_duplicate(frm));
870 }
871 }
872
873 cpl_array* reg_values = NULL;
874 char * var_part = NULL;
875// cpl_frame* frm = NULL ;
876// char * rfn = NULL ;
877// cpl_propertylist* rpl = NULL ;
878 for (int idx = 0; idx < DETMON_MAX_IDX; idx++) {
879 cpl_msg_info(__func__, "Doing idx loop %d of %d", idx,
880 DETMON_MAX_IDX-1);
881 if (!mplist[idx])
882 continue;
883
884 if ((idx == DETMON_LIN_INFO_IDX || idx == DETMON_GAIN_INFO_IDX)) {
885
886 // PIPE-10123 - Compute additional parameters here for
887 // these two tables
888 double exptime = 0.0 ;
889 reg_values = cpl_array_new(nregions,
890 CPL_TYPE_DOUBLE);
891
892 if (!is_nir && next[idx] > 0) {
893 cpl_msg_info(cpl_func, "Doing PIPE-10123 lin/gain kw for %s",
894 recipe);
895
896 // Compute EXPTIME MAX from RAWFRAMES
897 for (cpl_size r=0;
898 r < cpl_frameset_get_size(rawframes);
899 ++r) {
900 const cpl_frame * frm = cpl_frameset_get_position_const(
901 rawframes, r
902 );
903 const char * rfn = cpl_frame_get_filename(frm);
904 const cpl_propertylist * rpl = cpl_propertylist_load(
905 rfn, 0
906 );
907 if (rpl != NULL &&
908 cpl_propertylist_has(rpl, "EXPTIME")) {
909 if (cpl_propertylist_get_double(rpl, "EXPTIME") >
910 exptime) {
911 exptime = cpl_propertylist_get_double(
912 rpl, "EXPTIME");
913 }
914 }
915 }
916 cpl_propertylist_append_double(mplist[idx][0],
917 "ESO QC EXPTIME MAX",
918 exptime);
919 cpl_propertylist_set_comment(
920 mplist[idx][0],
921 "ESO QC EXPTIME MAX",
922 "Max. EXPTIME from input raws");
923
924 if (idx == DETMON_GAIN_INFO_IDX) {
925 var_part = "GAIN";
926 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
927 nregions, "CONAD",
928 CPL_TYPE_DOUBLE);
929 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
930 nregions, "GAIN",
931 CPL_TYPE_DOUBLE);
932 } else {
933 var_part = "LIN EFF";
934 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
935 nregions, "GAIN ERR",
936 CPL_TYPE_DOUBLE);
937 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
938 nregions, "FPN",
939 CPL_TYPE_DOUBLE);
940 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
941 nregions, "COUNTS MIN",
942 CPL_TYPE_DOUBLE);
943 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
944 nregions, "COUNTS MAX",
945 CPL_TYPE_DOUBLE);
946 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
947 nregions, "LIN EFF",
948 CPL_TYPE_DOUBLE);
949 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
950 nregions, "LIN EFF FLUX",
951 CPL_TYPE_DOUBLE);
952 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
953 nregions, "LLX",
954 CPL_TYPE_INT);
955 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
956 nregions, "LLY",
957 CPL_TYPE_INT);
958 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
959 nregions, "URX",
960 CPL_TYPE_INT);
961 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
962 nregions, "URY",
963 CPL_TYPE_INT);
964 }
965
966 for (cpl_size e = 1; e < next[idx] + 1; e++) {
967
968 for (int i=0 ; i < nregions ; i++) {
969 if (cpl_propertylist_has(mplist[idx][e],
970 cpl_sprintf("ESO QC REG%d %s",
971 i+1, var_part))) {
972 cpl_array_set_double(
973 reg_values, i,
974 cpl_propertylist_get_double(mplist[idx][e],
975 cpl_sprintf("ESO QC REG%d %s",
976 i+1, var_part)
977 ));
978 }
979
980 }
981
982 cpl_propertylist_append_double(mplist[idx][e],
983 cpl_sprintf("ESO QC %s AVG",
984 var_part),
985 cpl_array_get_mean(
986 reg_values));
987 cpl_propertylist_set_comment(mplist[idx][e],
988 cpl_sprintf("ESO QC %s AVG",
989 var_part),
990 cpl_sprintf(
991 "Avg of QC.REGn.%s",
992 var_part));
993 cpl_propertylist_append_double(mplist[idx][e],
994 cpl_sprintf("ESO QC %s RMS",
995 var_part),
996 cpl_array_get_stdev(
997 reg_values));
998 cpl_propertylist_set_comment(mplist[idx][e],
999 cpl_sprintf("ESO QC %s RMS",
1000 var_part),
1001 cpl_sprintf(
1002 "RMS of QC.REGn.%s",
1003 var_part));
1004 cpl_msg_info(__func__, "Wrote ESO QC RMS");
1005 }
1006
1007 }
1008 cpl_msg_info(__func__, "Doing a GAIN/LIN save (var_part=%s)",
1009 var_part);
1010 irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1011 recipe, outtag[idx],
1012 mplist[idx][0], NULL, PACKAGE "/"
1013 PACKAGE_VERSION, outname[idx]);
1014
1015 skip_if(0);
1016 for (cpl_size e = 1; e < next[idx] + 1; e++) {
1017 if (mtab[idx][e]) {
1018 cpl_table_save(mtab[idx][e], NULL, mplist[idx][e],
1019 outname[idx], CPL_IO_EXTEND);
1020 }
1021 else {
1022 cpl_propertylist_save(mplist[idx][e], outname[idx],
1023 CPL_IO_EXTEND);
1024 }
1025 skip_if(0);
1026 }
1027 }
1028 // No need to alter _mr BPM after all for PIPE-10123
1029 else if (idx == DETMON_BPM_IDX) {
1030 cpl_msg_info(__func__, "Doing a BPM save");
1031 if (!mimg[idx][0]) {
1032 irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1033 recipe, outtag[idx],
1034 mplist[idx][0], NULL, PACKAGE
1035 "/" PACKAGE_VERSION,
1036 outname[idx]);
1037 }
1038 else {
1039 irplib_dfs_save_image(frameset, parlist, usedframes,
1040 cpl_imagelist_get(mimg[idx][0], 0),
1041 CPL_TYPE_UNSPECIFIED, recipe,
1042 outtag[idx], mplist[idx][0], NULL,
1043 PACKAGE "/" PACKAGE_VERSION,
1044 outname[idx]);
1045 cpl_imagelist_delete(mimg[idx][0]);
1046 mimg[idx][0] = NULL;
1047 }
1048 skip_if(0);
1049 for (cpl_size e = 1; e < next[idx] + 1; e++) {
1050 if (mimg[idx][e]) {
1051 cpl_image * img = cpl_imagelist_get(mimg[idx][e], 0);
1052 cpl_image_save(img, outname[idx], CPL_TYPE_UNSPECIFIED,
1053 mplist[idx][e],
1054 CPL_IO_EXTEND);
1055 cpl_imagelist_delete(mimg[idx][e]);
1056 mimg[idx][e] = NULL;
1057 }
1058 else {
1059 cpl_propertylist_save(mplist[idx][e], outname[idx],
1060 CPL_IO_EXTEND);
1061 }
1062 skip_if(0);
1063 }
1064 }
1065 else if (idx == DETMON_COEFF_IDX) {
1066 // PIPE-10123 Add the additional QC keywords
1067 if (!is_nir && next[idx] > 0) {
1068 cpl_msg_info(cpl_func, "Doing PIPE-10123 COEFF kw for %s",
1069 recipe);
1070 for (int j=0; j < 4 ; ++j ) {
1071 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
1072 nregions,
1073 cpl_sprintf("LIN COEF%d", j),
1074 CPL_TYPE_DOUBLE);
1075 }
1076 detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
1077 nregions, "ERRFIT",
1078 CPL_TYPE_DOUBLE);
1079 }
1080
1081 cpl_msg_info(__func__, "Doing a COEFFS save");
1082 if (!mimg[idx][0]) {
1083 irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1084 recipe, outtag[idx],
1085 mplist[idx][0], NULL, PACKAGE
1086 "/" PACKAGE_VERSION,
1087 outname[idx]);
1088 }
1089 else {
1090 irplib_dfs_save_imagelist(frameset, parlist, usedframes,
1091 mimg[idx][0],
1092 CPL_TYPE_UNSPECIFIED,
1093 recipe, outtag[idx],
1094 mplist[idx][0], NULL, PACKAGE "/"
1095 PACKAGE_VERSION, outname[idx]);
1096 cpl_imagelist_delete(mimg[idx][0]);
1097 mimg[idx][0] = NULL;
1098 }
1099 skip_if(0);
1100 for (cpl_size e = 1; e < next[idx] + 1; e++) {
1101 if (mimg[idx][e]) {
1102 cpl_imagelist_save(mimg[idx][e], outname[idx],
1103 CPL_TYPE_UNSPECIFIED,
1104 mplist[idx][e], CPL_IO_EXTEND);
1105 cpl_imagelist_delete(mimg[idx][e]);
1106 mimg[idx][e] = NULL;
1107 }
1108 else {
1109 cpl_propertylist_save(mplist[idx][e], outname[idx],
1110 CPL_IO_EXTEND);
1111 }
1112 skip_if(0);
1113 }
1114 }
1115 }
1116 // PIPE-10123 cleanup
1117 cpl_array_delete(reg_values);
1118
1119 cpl_frameset_delete(usedframes);
1120 cpl_frameset_delete(rawframes);
1121 skip_if(0);
1122 }
1123
1124 end_skip;
1125 cpl_frameset_delete(frameset_copy);
1126 cpl_propertylist_delete(lintbl);
1127 cpl_propertylist_delete(gaintbl);
1128 cpl_propertylist_delete(coeffscube);
1129 cpl_propertylist_delete(bpm);
1130 cpl_propertylist_delete(corr);
1131 cpl_propertylist_delete(diff_flat);
1132 for (int i = 0; i < DETMON_MAX_IDX; i++) {
1133 for (int e = 0; e < next[i] + 1; e++) {
1134 if (mplist[i])
1135 cpl_propertylist_delete(mplist[i][e]);
1136 if (mimg[i])
1137 cpl_imagelist_delete(mimg[i][e]);
1138 if (mtab[i])
1139 cpl_table_delete(mtab[i][e]);
1140 }
1141 cpl_free(mplist[i]);
1142 cpl_free(mimg[i]);
1143 cpl_free(mtab[i]);
1144 cpl_free(outname[i]);
1145 cpl_free(outtag[i]);
1146 }
1147 cpl_free(llx);
1148 cpl_free(lly);
1149 cpl_free(urx);
1150 cpl_free(ury);
1151
1152 return cpl_error_get_code();
1153}