IIINSTRUMENT Pipeline Reference Manual  1.3.12b
detmon_lg_mr.c
1 /*
2  * This file is part of the DETMON Pipeline
3  * Copyright (C) 2013 European Southern Observatory
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
18  */
19 
20 /*
21  * $Author: jtaylor $
22  * $Date: 2013-08-07 09:32:50 $
23  * $Revision: 1.6 $
24  * $Name: not supported by cvs2svn $
25  */
26 #ifdef HAVE_CONFIG_H
27 #include <config.h>
28 #endif
29 
30 /*----------------------------------------------------------------------------
31  Includes
32  ----------------------------------------------------------------------------*/
33 
34 
35 #include "detmon_lg.h"
36 #include "detmon.h"
37 
38 #include "irplib_plugin.h"
39 #include "irplib_utils.h"
40 
41 #include <string.h>
42 #include <errno.h>
43 #include <stdlib.h>
44 #include <assert.h>
45 
46 /*----------------------------------------------------------------------------
47  Private Functions prototypes
48  ----------------------------------------------------------------------------*/
49 int
50 detmon_lg_dfs_set_groups(cpl_frameset * set,
51  const char *tag_on, const char *tag_off);
52 
53 static int check_regions(const cpl_size *, const cpl_size *,
54  const cpl_size *, const cpl_size *,
55  cpl_size) CPL_ATTR_NONNULL;
56 
57 static int compare_tags(const cpl_frame *, const cpl_frame *) CPL_ATTR_NONNULL;
58 
59 static const cpl_parameter *
60 get_parameter_const(const cpl_parameterlist * parlist,
61  const char * recipe, const char * name)
62 {
63  char * parname = cpl_sprintf("detmon.%s.%s", recipe, name);
64  const cpl_parameter * p = cpl_parameterlist_find_const(parlist, parname);
65  cpl_free(parname);
66  return p;
67 }
68 
69 static cpl_parameter *
70 get_parameter(cpl_parameterlist * parlist,
71  const char * recipe, const char * name)
72 {
73  char * parname = cpl_sprintf("detmon.%s.%s", recipe, name);
74  cpl_parameter * p = cpl_parameterlist_find(parlist, parname);
75  cpl_free(parname);
76  return p;
77 }
78 
79 static int compare_tags(const cpl_frame * a, const cpl_frame * b)
80 {
81  return strcmp(cpl_frame_get_tag(a), cpl_frame_get_tag(b)) == 0;
82 }
83 
86 /* ---------------------------------------------------------------------------*/
97 /* ---------------------------------------------------------------------------*/
98 static cpl_error_code merge_keys(cpl_propertylist * mplist,
99  const cpl_propertylist * plist_,
100  const int pos,
101  const char * rmregexp)
102 {
103  char buffer[256];
104  /* no copy_regex, so duplicate and use erase_regex on the copy */
105  cpl_propertylist * plist = cpl_propertylist_duplicate(plist_);
106 
107  if (rmregexp) {
108  cpl_propertylist_erase_regexp(plist, rmregexp, 0);
109  cpl_propertylist_erase_regexp(mplist, rmregexp, 0);
110  }
111 
112  for (cpl_size i = 0; i < cpl_propertylist_get_size(plist); i++) {
113  cpl_property * p = cpl_propertylist_get(plist, i);
114  const char * key = cpl_property_get_name(p);
115  /* extend all qc keywords by a region prefix */
116  if (!strncmp(key, "ESO QC ", 7)) {
117  const char * com;
118  sprintf(buffer, "ESO QC REG%d %s", pos, key + 7);
119  cpl_msg_debug(cpl_func, "merging %s to %s", key, buffer);
120  if (cpl_property_get_type(p) == CPL_TYPE_INT) {
121  int val = cpl_property_get_int(p);
122  cpl_propertylist_append_int(mplist, buffer, val);
123  } else if (cpl_property_get_type(p) == CPL_TYPE_DOUBLE) {
124  double val = cpl_property_get_double(p);
125  cpl_propertylist_append_double(mplist, buffer, val);
126  } else if (cpl_property_get_type(p) == CPL_TYPE_STRING) {
127  const char * val = cpl_property_get_string(p);
128  cpl_propertylist_append_string(mplist, buffer, val);
129  }
130  else
131  bug_if(1);
132  com = cpl_property_get_comment(p);
133  cpl_propertylist_set_comment(mplist, buffer, com);
134 
135  /* remove unprefixed key */
136  if (cpl_propertylist_has(mplist, key))
137  cpl_propertylist_erase(mplist, key);
138  }
139  }
140  skip_if(0);
141 
142  end_skip;
143  cpl_propertylist_delete(plist);
144 
145  return cpl_error_get_code();
146 }
147 
148 
149 /* ---------------------------------------------------------------------------*/
157 /* ---------------------------------------------------------------------------*/
158 static cpl_error_code
159 merge_images(cpl_imagelist * mimgl, const cpl_imagelist * img)
160 {
161  cpl_ensure_code(cpl_imagelist_get_size(img) ==
162  cpl_imagelist_get_size(mimgl), CPL_ERROR_ILLEGAL_INPUT);
163  for (int i = 0; i < cpl_imagelist_get_size(img); i++) {
164  cpl_image * mimg = cpl_imagelist_get(mimgl, i);
165  const cpl_image * part = cpl_imagelist_get_const(img, i);
166  cpl_image_add(mimg, part);
167  }
168 
169  return cpl_error_get_code();
170 }
171 
172 
173 /* ---------------------------------------------------------------------------*/
181 /* ---------------------------------------------------------------------------*/
182 static cpl_error_code
183 merge_table(cpl_table * mtab, cpl_table * tab, const int pos)
184 {
185  cpl_table_new_column(tab, "REGION", CPL_TYPE_INT);
186  cpl_table_fill_column_window(tab, "REGION", 0, cpl_table_get_nrow(tab), pos);
187  skip_if(cpl_table_insert(mtab, tab, cpl_table_get_nrow(mtab)));
188 
189  end_skip;
190 
191  return cpl_error_get_code();
192 }
193 
194 
195 static cpl_error_code
196 parse_token(const char * token, cpl_size * llx, cpl_size * lly,
197  cpl_size * urx, cpl_size * ury)
198 {
199  assert(sizeof(*llx) == 8);
200  if (sscanf(token, " %lld , %lld , %lld , %lld ", llx, lly, urx, ury) == 4) {
201  return CPL_ERROR_NONE;
202  }
203  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
204  "Data not formated correctly: %s; "
205  "expecting: name,llx,lly,urx,ury",
206  token);
207 }
208 
209 
210 /* ---------------------------------------------------------------------------*/
223 /* ---------------------------------------------------------------------------*/
224 static cpl_error_code
225 read_regions_file(const char * filename, cpl_size ** rllx,
226  cpl_size ** rlly, cpl_size ** rurx, cpl_size ** rury,
227  cpl_size * n)
228 {
229  FILE * f = fopen(filename, "r");
230  char line[1024];
231  cpl_size i = 0;
232  cpl_size nmax = 100;
233  if (f == NULL) {
234  return cpl_error_set_message(cpl_func, CPL_ERROR_FILE_IO,
235  "Could not open file %s: %s", filename,
236  strerror(errno));
237  }
238  cpl_msg_info(cpl_func, "Reading %s", filename);
239 
240  *n = 0;
241  *rllx = cpl_malloc(sizeof(**rllx) * nmax);
242  *rlly = cpl_malloc(sizeof(**rllx) * nmax);
243  *rurx = cpl_malloc(sizeof(**rllx) * nmax);
244  *rury = cpl_malloc(sizeof(**rllx) * nmax);
245 
246  while (fgets(line, 1024, f) != NULL) {
247  const size_t nline = strlen(line);
248  if (line[nline - 1] != '\n') {
249  cpl_error_set_message(cpl_func, CPL_ERROR_FILE_IO,
250  "File %s contained too long line",
251  filename);
252  skip_if(0);
253  }
254 
255  if (line[0] == '#')
256  continue;
257 
258  skip_if(parse_token(line, &(*rllx)[i], &(*rlly)[i],
259  &(*rurx)[i], &(*rury)[i]));
260  *n = ++i;
261  if (i >= nmax) {
262  nmax *= 2;
263  *rllx = cpl_realloc(*rllx, sizeof(**rllx) * nmax);
264  *rlly = cpl_realloc(*rlly, sizeof(**rllx) * nmax);
265  *rurx = cpl_realloc(*rurx, sizeof(**rllx) * nmax);
266  *rury = cpl_realloc(*rury, sizeof(**rllx) * nmax);
267  skip_if(0);
268  }
269  }
270 
271  end_skip;
272  if (f)
273  fclose(f);
274  return CPL_ERROR_NONE;
275 }
276 
277 
278 /* ---------------------------------------------------------------------------*/
291 /* ---------------------------------------------------------------------------*/
292 static cpl_error_code
293 read_regions_str(const char * istr, cpl_size ** rllx,
294  cpl_size ** rlly, cpl_size ** rurx, cpl_size ** rury,
295  cpl_size * n)
296 {
297  cpl_size nmax = 100;
298  char * str_ = cpl_strdup(istr);
299  char * str = str_;
300  *n = 0;
301  *rllx = cpl_malloc(sizeof(**rllx) * nmax);
302  *rlly = cpl_malloc(sizeof(**rllx) * nmax);
303  *rurx = cpl_malloc(sizeof(**rllx) * nmax);
304  *rury = cpl_malloc(sizeof(**rllx) * nmax);
305 
306  for (int i = 0; ;i++) {
307  const char * token = strtok(str, ":");
308  if (token == NULL)
309  break;
310 
311  skip_if(parse_token(token, &(*rllx)[i], &(*rlly)[i],
312  &(*rurx)[i], &(*rury)[i]));
313  (*n)++;
314  if (i >= nmax - 1) {
315  nmax *= 2;
316  *rllx = cpl_realloc(*rllx, sizeof(**rllx) * nmax);
317  *rlly = cpl_realloc(*rlly, sizeof(**rllx) * nmax);
318  *rurx = cpl_realloc(*rurx, sizeof(**rllx) * nmax);
319  *rury = cpl_realloc(*rury, sizeof(**rllx) * nmax);
320  skip_if(0);
321  }
322  str = NULL;
323  }
324 
325  end_skip;
326  cpl_free(str_);
327 
328  return CPL_ERROR_NONE;
329 }
330 
331 static int check_regions(const cpl_size * llx, const cpl_size * lly,
332  const cpl_size * urx, const cpl_size * ury,
333  cpl_size nregions)
334 {
335  for (cpl_size i = 0; i < nregions; i++) {
336  if (llx[i] < 1 || lly[i] < 1 ||
337  urx[i] < 1 || ury[i] < 1) {
338  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
339  "Region %lld (%lld,%lld,%lld,%lld) has entry smaller one",
340  i, llx[i], lly[i], urx[i], ury[i]);
341  }
342  if ((urx[i] - llx[i] < 1) || (ury[i] - lly[i] < 1)) {
343  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
344  "Region %lld (%lld,%lld,%lld,%lld) has size zero",
345  i, llx[i], lly[i], urx[i], ury[i]);
346  }
347  for (cpl_size j = i + 1; j < nregions; j++) {
348  if (!(llx[i] > urx[j] || urx[i] < llx[j] ||
349  lly[i] > ury[j] || ury[i] < lly[j])) {
350  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
351  "Regions %lld (%lld,%lld,%lld,%lld) and "
352  "%lld (%lld,%lld,%lld,%lld) overlap (llx,lly,urx,ury)",
353  i, llx[i], lly[i], urx[i], ury[i],
354  j, llx[j], lly[j], urx[j], ury[j]);
355  }
356  }
357  }
358 
359  return CPL_ERROR_NONE;
360 }
361 
362 
363 /* ---------------------------------------------------------------------------*/
376 /* ---------------------------------------------------------------------------*/
377 static cpl_error_code
378 merge_lingain(cpl_propertylist * mplist[], cpl_table * mtab[],
379  const char * fn, const cpl_size next,
380  const char * rmregexp, int pos)
381 {
382  cpl_errorstate cleanstate = cpl_errorstate_get();
383 
384  {
385  cpl_propertylist * plist = cpl_propertylist_load(fn, 0);
386  skip_if(plist == NULL);
387  if (mplist[0] == NULL) {
388  mplist[0] = plist;
389  merge_keys(mplist[0], plist, pos, rmregexp);
390  }
391  else {
392  merge_keys(mplist[0], plist, pos, rmregexp);
393  cpl_propertylist_delete(plist);
394  }
395  }
396  skip_if(0);
397 
398  for (int e = 1; e < next + 1; e++) {
399  cpl_propertylist * plist = cpl_propertylist_load(fn, e);
400  cpl_table * tab = cpl_table_load(fn, e, 0);
401  /* empty extensions may exist */
402  if (tab == NULL) {
403  cpl_errorstate_set(cleanstate);
404  cpl_propertylist_delete(plist);
405  continue;
406  }
407  skip_if(plist == NULL);
408 
409  if (mplist[e] == NULL) {
410  merge_keys(plist, plist, pos, rmregexp);
411  cpl_table_new_column(tab, "REGION", CPL_TYPE_INT);
412  cpl_table_fill_column_window(tab, "REGION", 0,
413  cpl_table_get_nrow(tab), 1);
414  skip_if(0);
415  mplist[e] = plist;
416  mtab[e] = tab;
417  }
418  else {
419  merge_keys(mplist[e], plist, pos, rmregexp);
420  merge_table(mtab[e], tab, pos);
421  skip_if(0);
422  cpl_propertylist_delete(plist);
423  cpl_table_delete(tab);
424  }
425  skip_if(0);
426  }
427 
428  end_skip;
429 
430  return cpl_error_get_code();
431 }
432 
433 
434 /* ---------------------------------------------------------------------------*/
447 /* ---------------------------------------------------------------------------*/
448 static cpl_error_code
449 merge_bpmcoeff(cpl_propertylist * mplist[], cpl_imagelist * mimg[],
450  const char * fn, const cpl_size next,
451  const char * rmregexp, int pos)
452 {
453  cpl_errorstate cleanstate = cpl_errorstate_get();
454 
455  for (int e = 0; e < next + 1; e++) {
456  cpl_propertylist * plist = cpl_propertylist_load(fn, e);
457  cpl_imagelist * img = cpl_imagelist_load(fn, CPL_TYPE_UNSPECIFIED, e);
458  /* empty extensions may exist */
459  if (img == NULL) {
460  cpl_errorstate_set(cleanstate);
461  cpl_propertylist_delete(plist);
462  continue;
463  }
464  skip_if(plist == NULL);
465 
466  if (mimg[e] == NULL) {
467  merge_keys(plist, plist, pos, rmregexp);
468  mplist[e] = plist;
469  mimg[e] = img;
470  }
471  else {
472  merge_keys(mplist[e], plist, pos, rmregexp);
473  merge_images(mimg[e], img);
474  skip_if(0);
475  cpl_propertylist_delete(plist);
476  cpl_imagelist_delete(img);
477  }
478  skip_if(0);
479  }
480 
481  end_skip;
482 
483  return cpl_error_get_code();
484 }
485 
486 typedef enum {
487  DETMON_BP_MAP_NL_IDX = 0,
488  DETMON_LIN_INFO_IDX = 1,
489  DETMON_GAIN_INFO_IDX = 2,
490  DETMON_BPM_IDX = 3,
491  DETMON_COEFF_IDX = 4,
492  DETMON_MAX_IDX,
493 } detmon_idx;
494 
495 static cpl_error_code disable_contamination(cpl_parameterlist * parlist,
496  const char * recipe)
497 {
498  /* "disable" by setting it to a region of size 1 */
499  cpl_parameter * par;
500  for (int i = 1; i < 6; i++) {
501  char buffer[100];
502  sprintf(buffer, "detmon.%s.llx%d", recipe, i);
503  par = cpl_parameterlist_find(parlist, buffer);
504  cpl_parameter_set_int(par, 1);
505  sprintf(buffer, "detmon.%s.lly%d", recipe, i);
506  par = cpl_parameterlist_find(parlist, buffer);
507  cpl_parameter_set_int(par, 1);
508  sprintf(buffer, "detmon.%s.urx%d", recipe, i);
509  par = cpl_parameterlist_find(parlist, buffer);
510  cpl_parameter_set_int(par, 2);
511  sprintf(buffer, "detmon.%s.ury%d", recipe, i);
512  par = cpl_parameterlist_find(parlist, buffer);
513  cpl_parameter_set_int(par, 2);
514  }
515 
516  return cpl_error_get_code();
517 }
518 
519 static cpl_error_code detmon_lg_mr_phu_qc_reg_kw(
520  cpl_propertylist** mplist[],
521  int idx,
522  cpl_size next,
523  cpl_size nregions,
524  const char * kw_segment,
525  cpl_type kw_type
526  ) {
527 
528 // cpl_msg_info(cpl_func, "Within detmon_lg_mr_phu_qc_reg_kw for %s",
529 // kw_segment);
530  char * kw_input = NULL;
531  char* kw_output_avg = NULL ;
532  char* kw_output_rms = NULL;
533  cpl_array* kw_vals = cpl_array_new(next, kw_type);
534 // cpl_msg_info(cpl_func, " Variables instantiated");
535 
536  // Need to ensure the PHU propertylist actually exists - create one
537  // if it's NULL
538  if (mplist[idx][0] == NULL) {
539  mplist[idx][0] = cpl_propertylist_new();
540 // cpl_msg_info(cpl_func, " PHU propertylist instantiated");
541  }
542 
543  for (int i=0; i < nregions; i++) {
544 // cpl_msg_info(cpl_func, " Region %d of %d", i+1, nregions);
545  kw_input = cpl_sprintf("ESO QC REG%d %s", i+1, kw_segment);
546  kw_output_avg = cpl_sprintf("ESO QC REG%d %s AVG", i+1, kw_segment);
547  kw_output_rms = cpl_sprintf("ESO QC REG%d %s RMS", i+1, kw_segment);
548  cpl_array_fill_window_invalid(kw_vals, 0, next);
549 // cpl_msg_info(cpl_func, " Variables set");
550 // cpl_msg_info(cpl_func, " %s", kw_input);
551 // cpl_msg_info(cpl_func, " %s", kw_output_avg);
552 // cpl_msg_info(cpl_func, " %s", kw_output_rms);
553 
554  for (cpl_size e = 1 ; e < next + 1 ; ++e) {
555  if (cpl_propertylist_has(mplist[idx][e], kw_input)) {
556  cpl_msg_info(
557  cpl_func,
558  " Found value %s in ext %" CPL_SIZE_FORMAT "",
559  kw_input, e);
560  if (kw_type == CPL_TYPE_DOUBLE) {
561  cpl_array_set_double(kw_vals, e-1,
562  cpl_propertylist_get_double(
563  mplist[idx][e], kw_input));
564  } else if (kw_type == CPL_TYPE_INT) {
565  cpl_array_set_int(kw_vals, e-1,
566  cpl_propertylist_get_int(
567  mplist[idx][e], kw_input));
568  }
569  }
570  }
571  cpl_msg_info(cpl_func, " Array populated");
572 
573  cpl_propertylist_append_double(mplist[idx][0],
574  kw_output_avg,
575  cpl_array_get_mean(kw_vals));
576  cpl_msg_info(cpl_func, " Set mean");
577  cpl_propertylist_set_comment(
578  mplist[idx][0],
579  kw_output_avg,
580  cpl_sprintf("Avg of QC REG%d %s over exts",
581  i+1, kw_segment));
582  cpl_msg_info(cpl_func, " Set mean comment");
583  cpl_propertylist_append_double(mplist[idx][0],
584  kw_output_rms,
585  cpl_array_get_stdev(kw_vals));
586  cpl_msg_info(cpl_func, " Set RMS");
587  cpl_propertylist_set_comment(
588  mplist[idx][0],
589  kw_output_rms,
590  cpl_sprintf("RMS of QC REG%d %s over exts",
591  i+1, kw_segment));
592 // cpl_msg_info(cpl_func, " Set RMS comment");
593 // cpl_msg_info(cpl_func, " KW set for region %d", i+1);
594  if (cpl_error_get_code() != CPL_ERROR_NONE) {
595  cpl_msg_error(__func__, "Error found in detmon_lg_mr_phu_qc_reg_kw!");
596  cpl_msg_error(__func__, "In %s, L%u", cpl_error_get_file(),
597  cpl_error_get_line());
598  cpl_msg_error(__func__, "%s", cpl_error_get_message());
599  cpl_msg_error(__func__, "Values ingested:");
600  cpl_array_dump(kw_vals, 0, next, stdout);
601  cpl_msg_error(__func__, "There were %d extensions", next);
602  cpl_msg_error(__func__, "There were %d regions detected", nregions);
603  cpl_msg_error(__func__, "Propertylist:");
604  cpl_propertylist_dump(mplist[idx][0], stdout);
605 // getchar();
606  }
607  }
608 
609 // cpl_msg_info(cpl_func, " Proceeding to clean-up");
610  cpl_free(kw_input);
611  cpl_free(kw_output_rms);
612  cpl_free(kw_output_avg);
613  cpl_array_delete(kw_vals);
614 
615 // cpl_msg_info(cpl_func, "Returning!");
616  return cpl_error_get_code();
617 }
618 
619 /*---------------------------------------------------------------------------*/
620 /*
621  @brief Interpret the command line options and execute the data processing
622  @param frameset the frames list
623  @param parlist_ the parameters list
624  @param recipe recipe name
625  @param is_nir switch to know if opt or nir
626  @return 0 if everything is ok
627  */
628 /*---------------------------------------------------------------------------*/
629 
630 int detmon_lg_mr(cpl_frameset * frameset,
631  const cpl_parameterlist * parlist_,
632  const char * recipe,
633  cpl_boolean is_nir)
634 {
635  const char * tag_on=NULL;
636  const char * tag_off=NULL;
637  IRPLIB_DIAG_PRAGMA_PUSH_IGN(-Wcast-qual);
638  cpl_parameterlist * parlist = (cpl_parameterlist *)parlist_;
639  IRPLIB_DIAG_PRAGMA_POP;
640  cpl_propertylist ** mplist[DETMON_MAX_IDX] = {NULL};
641  cpl_imagelist ** mimg[DETMON_MAX_IDX] = {NULL};
642  cpl_table ** mtab[DETMON_MAX_IDX] = {NULL};
643  cpl_size next[DETMON_MAX_IDX] = {0};
644  char * outname[DETMON_MAX_IDX] = {NULL};
645  char * outtag[DETMON_MAX_IDX] = {NULL};
646  cpl_size nregions = 0;
647 
648  cpl_frameset * frameset_copy = NULL;
649 
650  cpl_propertylist * lintbl =
651  detmon_fill_prolist("DET_LIN_INFO", "REDUCED", "TECH", CPL_FALSE);
652 
653  cpl_propertylist * gaintbl =
654  detmon_fill_prolist("GAIN_INFO", "REDUCED", "TECH", CPL_FALSE);
655 
656  cpl_propertylist * coeffscube =
657  detmon_fill_prolist("COEFFS_CUBE", "REDUCED", "TECH", CPL_FALSE);
658 
659  cpl_propertylist * bpm =
660  detmon_fill_prolist("BP_MAP_NL", "REDUCED", "TECH", CPL_FALSE);
661 
662  cpl_propertylist * corr =
663  detmon_fill_prolist("AUTOCORR", "REDUCED", "TECH", CPL_FALSE);
664 
665  cpl_propertylist * diff_flat =
666  detmon_fill_prolist("DIFF_FLAT", "REDUCED", "TECH", CPL_FALSE);
667  const char * regions_file = NULL;
668  const char * regions_str = NULL;
669  cpl_error_code error = CPL_ERROR_NONE;
670  cpl_size * llx = NULL, * lly = NULL, * urx = NULL, * ury = NULL;
671  const cpl_parameter * p =
672  get_parameter_const(parlist, recipe, "regions-file");
673  if (p) {
674  regions_file = cpl_parameter_get_string(p);
675  if (strlen(regions_file) == 0)
676  regions_file = NULL;
677  }
678 
679  p = get_parameter_const(parlist, recipe, "regions");
680  if (p) {
681  regions_str = cpl_parameter_get_string(p);
682  if (strlen(regions_str) == 0)
683  regions_str = NULL;
684  else {
685  if (regions_file) {
686  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
687  "Please only provide either "
688  "--regions or "
689  "--regions-file, not both");
690  }
691  }
692  }
693 
694  p = get_parameter(parlist, recipe, "intermediate");
695  if (p) {
696  error_if(cpl_parameter_get_bool(p), CPL_ERROR_UNSUPPORTED_MODE,
697  "--intermediate not supported by multi region recipe");
698  }
699 
700  skip_if (detmon_lg_set_tag(frameset, &tag_on, &tag_off));
701  skip_if (detmon_lg_dfs_set_groups(frameset, tag_on, tag_off));
702 
703  if (regions_file) {
704  read_regions_file(regions_file, &llx, &lly, &urx, &ury,
705  &nregions);
706  }
707  else if (regions_str) {
708  read_regions_str(regions_str, &llx, &lly, &urx, &ury,
709  &nregions);
710  }
711  else {
712  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
713  "Empty value in --regions or "
714  "--regions-file");
715  }
716  skip_if(0);
717 
718  skip_if(check_regions(llx, lly, urx, ury, nregions));
719 
720  if (!is_nir) {
721  /* contamination not supported */
722  skip_if(disable_contamination(parlist, recipe));
723  }
724 
725  /* additionally iterating on extensions would improve IO caching and
726  * reduce memory usage for pix2pix but requires checks if detmon really
727  * gives the same result in this case */
728  for (int i = 0; i < nregions; i++) {
729  cpl_size nlabels;
730  cpl_size * selection;
731  cpl_parameter * par;
732  par = get_parameter(parlist, recipe, "llx");
733  cpl_parameter_set_int(par, llx[i]);
734  par = get_parameter(parlist, recipe, "lly");
735  cpl_parameter_set_int(par, lly[i]);
736  par = get_parameter(parlist, recipe, "urx");
737  cpl_parameter_set_int(par, urx[i]);
738  par = get_parameter(parlist, recipe, "ury");
739  cpl_parameter_set_int(par, ury[i]);
740  cpl_msg_info(cpl_func, "working on region %d/%d: %lld %lld, "
741  "%lld %lld", i + 1, (int)nregions,
742  llx[i], lly[i], urx[i], ury[i]);
743 
744  /* copy and throw away so we don't accumulate products
745  * for each call */
746  cpl_frameset_delete(frameset_copy);
747  frameset_copy = cpl_frameset_duplicate(frameset);
748  error = detmon_lg(frameset_copy,
749  parlist,
750  tag_on,
751  tag_off,
752  recipe,
753  PACKAGE_TARNAME,
754  REGEXP,
755  lintbl, gaintbl, coeffscube,
756  bpm, corr, diff_flat,
757  PACKAGE "/" PACKAGE_VERSION,
758  NULL, NULL, is_nir);
759  skip_if(error);
760 
761  selection = cpl_frameset_labelise(frameset_copy, compare_tags,
762  &nlabels);
763 
764  for (cpl_size j = 0; j < nlabels; j++) {
765  cpl_frameset * a = cpl_frameset_extract(frameset_copy, selection, j);
766  cpl_frame * frm = cpl_frameset_get_position(a, 0);
767  const char * fn = cpl_frame_get_filename(frm);
768  int idx;
769 
770  if (strcmp(cpl_frame_get_tag(frm), "DET_LIN_INFO") == 0)
771  idx = DETMON_LIN_INFO_IDX;
772  else if (strcmp(cpl_frame_get_tag(frm), "GAIN_INFO") == 0)
773  idx = DETMON_GAIN_INFO_IDX;
774  else if (strcmp(cpl_frame_get_tag(frm), "BP_MAP_NL") == 0)
775  idx = DETMON_BPM_IDX;
776  else if (strcmp(cpl_frame_get_tag(frm), "COEFFS_CUBE") == 0)
777  idx = DETMON_COEFF_IDX;
778  else {
779  cpl_frameset_delete(a);
780  continue;
781  }
782 
783  /* prepare storage */
784  if (mplist[idx] == NULL) {
785  next[idx] = cpl_frame_get_nextensions(frm);
786  outname[idx] = cpl_strdup(cpl_frame_get_filename(frm));
787  outtag[idx] = cpl_strdup(cpl_frame_get_tag(frm));
788  mplist[idx] = cpl_calloc(sizeof(cpl_propertylist*), next[idx] + 1);
789  mimg[idx] = cpl_calloc(sizeof(cpl_imagelist*), next[idx] + 1);
790  mtab[idx] = cpl_calloc(sizeof(cpl_table*), next[idx] + 1);
791  }
792 
793  if (idx == DETMON_LIN_INFO_IDX) {
794  const char * rmregexp = "ESO QC CONTAM.*";
795  merge_lingain(mplist[idx], mtab[idx], fn, next[idx],
796  rmregexp, i + 1);
797  }
798  else if (idx == DETMON_GAIN_INFO_IDX) {
799  merge_lingain(mplist[idx], mtab[idx], fn, next[idx],
800  NULL, i + 1);
801  }
802  else if (idx == DETMON_BPM_IDX) {
803  merge_bpmcoeff(mplist[idx], mimg[idx], fn, next[idx],
804  NULL, i + 1);
805  }
806  else if (idx == DETMON_COEFF_IDX) {
807  merge_bpmcoeff(mplist[idx], mimg[idx], fn, next[idx],
808  NULL, i + 1);
809  }
810  skip_if(0);
811 
812  for (int e = 0; e < next[idx] + 1; e++) {
813  char buffer[300];
814  cpl_propertylist * copy_plist;
815  cpl_size len;
816  if (!mplist[idx][e])
817  continue;
818 
819  /* don't add region if extension has no QC parameters */
820  copy_plist = cpl_propertylist_duplicate(mplist[idx][e]);
821  cpl_propertylist_erase_regexp(copy_plist, "ESO QC.*", 1);
822  len = cpl_propertylist_get_size(copy_plist);
823  cpl_propertylist_delete(copy_plist);
824  if (len == 0) {
825  continue;
826  }
827 
828  sprintf(buffer, "ESO QC REG%d LLX", i + 1);
829  cpl_propertylist_append_int(mplist[idx][e], buffer, llx[i]);
830  cpl_propertylist_set_comment(mplist[idx][e], buffer,
831  "Lower left X of region");
832  sprintf(buffer, "ESO QC REG%d LLY", i + 1);
833  cpl_propertylist_append_int(mplist[idx][e], buffer, lly[i]);
834  cpl_propertylist_set_comment(mplist[idx][e], buffer,
835  "Lower left Y of region");
836  sprintf(buffer, "ESO QC REG%d URX", i + 1);
837  cpl_propertylist_append_int(mplist[idx][e], buffer, urx[i]);
838  cpl_propertylist_set_comment(mplist[idx][e], buffer,
839  "Upper right X of region");
840  sprintf(buffer, "ESO QC REG%d URY", i + 1);
841  cpl_propertylist_append_int(mplist[idx][e], buffer, ury[i]);
842  cpl_propertylist_set_comment(mplist[idx][e], buffer,
843  "Upper right Y of region");
844  }
845 
846  cpl_frameset_delete(a);
847  skip_if(0);
848  }
849  cpl_free(selection);
850  }
851  cpl_msg_info(__func__, "Completed looping over regions");
852 // getchar();
853 
854  {
855  cpl_frameset * usedframes = cpl_frameset_new();
856  const cpl_size nframes = cpl_frameset_get_size(frameset);
857 
858  cpl_frameset * rawframes = cpl_frameset_new();
859 
860  for (cpl_size idx = 0; idx < nframes; idx++) {
861  const cpl_frame * frm = cpl_frameset_get_position_const(frameset,
862  idx);
863  if (cpl_frame_get_group(frm) == CPL_FRAME_GROUP_RAW ||
864  cpl_frame_get_group(frm) == CPL_FRAME_GROUP_CALIB) {
865  cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
866  }
867  if (cpl_frame_get_group(frm) == CPL_FRAME_GROUP_RAW) {
868  cpl_frameset_insert(rawframes, cpl_frame_duplicate(frm));
869  }
870  }
871 
872  cpl_array* reg_values = NULL;
873  char * var_part = NULL;
874 // cpl_frame* frm = NULL ;
875 // char * rfn = NULL ;
876 // cpl_propertylist* rpl = NULL ;
877  for (int idx = 0; idx < DETMON_MAX_IDX; idx++) {
878  cpl_msg_info(__func__, "Doing idx loop %d of %d", idx,
879  DETMON_MAX_IDX-1);
880  if (!mplist[idx])
881  continue;
882 
883  if ((idx == DETMON_LIN_INFO_IDX || idx == DETMON_GAIN_INFO_IDX)
884  && next[idx] > 0) {
885 
886  // PIPE-10123 - Compute additional parameters here for
887  // these two tables
888  double exptime = 0.0 ;
889  reg_values = cpl_array_new(nregions,
890  CPL_TYPE_DOUBLE);
891 
892  if (!is_nir) {
893  cpl_msg_info(cpl_func, "Doing PIPE-10123 lin/gain kw for %s",
894  recipe);
895 
896  // Compute EXPTIME MAX from RAWFRAMES
897  for (cpl_size r=0;
898  r < cpl_frameset_get_size(rawframes);
899  ++r) {
900  const cpl_frame * frm = cpl_frameset_get_position_const(
901  rawframes, r
902  );
903  const char * rfn = cpl_frame_get_filename(frm);
904  const cpl_propertylist * rpl = cpl_propertylist_load(
905  rfn, 0
906  );
907  if (rpl != NULL &&
908  cpl_propertylist_has(rpl, "EXPTIME")) {
909  if (cpl_propertylist_get_double(rpl, "EXPTIME") >
910  exptime) {
911  exptime = cpl_propertylist_get_double(
912  rpl, "EXPTIME");
913  }
914  }
915  }
916  cpl_propertylist_append_double(mplist[idx][0],
917  "ESO QC EXPTIME MAX",
918  exptime);
919  cpl_propertylist_set_comment(
920  mplist[idx][0],
921  "ESO QC EXPTIME MAX",
922  "Max. EXPTIME from input raws");
923 
924  if (idx == DETMON_GAIN_INFO_IDX) {
925  var_part = "GAIN";
926  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
927  nregions, "CONAD",
928  CPL_TYPE_DOUBLE);
929  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
930  nregions, "GAIN",
931  CPL_TYPE_DOUBLE);
932  } else {
933  var_part = "LIN EFF";
934  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
935  nregions, "GAIN ERR",
936  CPL_TYPE_DOUBLE);
937  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
938  nregions, "FPN",
939  CPL_TYPE_DOUBLE);
940  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
941  nregions, "COUNTS MIN",
942  CPL_TYPE_DOUBLE);
943  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
944  nregions, "COUNTS MAX",
945  CPL_TYPE_DOUBLE);
946  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
947  nregions, "LIN EFF",
948  CPL_TYPE_DOUBLE);
949  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
950  nregions, "LIN EFF FLUX",
951  CPL_TYPE_DOUBLE);
952  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
953  nregions, "LLX",
954  CPL_TYPE_INT);
955  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
956  nregions, "LLY",
957  CPL_TYPE_INT);
958  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
959  nregions, "URX",
960  CPL_TYPE_INT);
961  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
962  nregions, "URY",
963  CPL_TYPE_INT);
964  }
965 
966  for (cpl_size e = 1; e < next[idx] + 1; e++) {
967 
968  for (int i=0 ; i < nregions ; i++) {
969  if (cpl_propertylist_has(mplist[idx][e],
970  cpl_sprintf("ESO QC REG%d %s",
971  i+1, var_part))) {
972  cpl_array_set_double(
973  reg_values, i,
974  cpl_propertylist_get_double(mplist[idx][e],
975  cpl_sprintf("ESO QC REG%d %s",
976  i+1, var_part)
977  ));
978  }
979 
980  }
981 
982  cpl_propertylist_append_double(mplist[idx][e],
983  cpl_sprintf("ESO QC %s AVG",
984  var_part),
985  cpl_array_get_mean(
986  reg_values));
987  cpl_propertylist_set_comment(mplist[idx][e],
988  cpl_sprintf("ESO QC %s AVG",
989  var_part),
990  cpl_sprintf(
991  "Avg of QC.REGn.%s",
992  var_part));
993  cpl_propertylist_append_double(mplist[idx][e],
994  cpl_sprintf("ESO QC %s RMS",
995  var_part),
996  cpl_array_get_stdev(
997  reg_values));
998  cpl_propertylist_set_comment(mplist[idx][e],
999  cpl_sprintf("ESO QC %s RMS",
1000  var_part),
1001  cpl_sprintf(
1002  "RMS of QC.REGn.%s",
1003  var_part));
1004  cpl_msg_info(__func__, "Wrote ESO QC RMS");
1005  }
1006 
1007  }
1008  cpl_msg_info(__func__, "Doing a GAIN/LIN save (var_part=%s)",
1009  var_part);
1010  irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1011  recipe, outtag[idx],
1012  mplist[idx][0], NULL, PACKAGE "/"
1013  PACKAGE_VERSION, outname[idx]);
1014 
1015  skip_if(0);
1016  for (cpl_size e = 1; e < next[idx] + 1; e++) {
1017  if (mtab[idx][e]) {
1018  cpl_table_save(mtab[idx][e], NULL, mplist[idx][e],
1019  outname[idx], CPL_IO_EXTEND);
1020  }
1021  else {
1022  cpl_propertylist_save(mplist[idx][e], outname[idx],
1023  CPL_IO_EXTEND);
1024  }
1025  skip_if(0);
1026  }
1027  }
1028  // No need to alter _mr BPM after all for PIPE-10123
1029  else if (idx == DETMON_BPM_IDX && next[idx] > 0) {
1030  cpl_msg_info(__func__, "Doing a BPM save");
1031  if (!mimg[idx][0]) {
1032  irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1033  recipe, outtag[idx],
1034  mplist[idx][0], NULL, PACKAGE
1035  "/" PACKAGE_VERSION,
1036  outname[idx]);
1037  }
1038  else {
1039  irplib_dfs_save_image(frameset, parlist, usedframes,
1040  cpl_imagelist_get(mimg[idx][0], 0),
1041  CPL_TYPE_UNSPECIFIED, recipe,
1042  outtag[idx], mplist[idx][0], NULL,
1043  PACKAGE "/" PACKAGE_VERSION,
1044  outname[idx]);
1045  cpl_imagelist_delete(mimg[idx][0]);
1046  mimg[idx][0] = NULL;
1047  }
1048  skip_if(0);
1049  for (cpl_size e = 1; e < next[idx] + 1; e++) {
1050  if (mimg[idx][e]) {
1051  cpl_image * img = cpl_imagelist_get(mimg[idx][e], 0);
1052  cpl_image_save(img, outname[idx], CPL_TYPE_UNSPECIFIED,
1053  mplist[idx][e],
1054  CPL_IO_EXTEND);
1055  cpl_imagelist_delete(mimg[idx][e]);
1056  mimg[idx][e] = NULL;
1057  }
1058  else {
1059  cpl_propertylist_save(mplist[idx][e], outname[idx],
1060  CPL_IO_EXTEND);
1061  }
1062  skip_if(0);
1063  }
1064  }
1065  else if (idx == DETMON_COEFF_IDX && next[idx] > 0) {
1066  // PIPE-10123 Add the additional QC keywords
1067  if (!is_nir) {
1068  cpl_msg_info(cpl_func, "Doing PIPE-10123 coeff kw for %s",
1069  recipe);
1070  for (int j=0; j < 4 ; ++j ) {
1071  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
1072  nregions,
1073  cpl_sprintf("LIN COEF%d", j),
1074  CPL_TYPE_DOUBLE);
1075  }
1076  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
1077  nregions, "ERRFIT",
1078  CPL_TYPE_DOUBLE);
1079  }
1080 
1081  cpl_msg_info(__func__, "Doing a COEFFS save");
1082  if (!mimg[idx][0]) {
1083  irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1084  recipe, outtag[idx],
1085  mplist[idx][0], NULL, PACKAGE
1086  "/" PACKAGE_VERSION,
1087  outname[idx]);
1088  }
1089  else {
1090  irplib_dfs_save_imagelist(frameset, parlist, usedframes,
1091  mimg[idx][0],
1092  CPL_TYPE_UNSPECIFIED,
1093  recipe, outtag[idx],
1094  mplist[idx][0], NULL, PACKAGE "/"
1095  PACKAGE_VERSION, outname[idx]);
1096  cpl_imagelist_delete(mimg[idx][0]);
1097  mimg[idx][0] = NULL;
1098  }
1099  skip_if(0);
1100  for (cpl_size e = 1; e < next[idx] + 1; e++) {
1101  if (mimg[idx][e]) {
1102  cpl_imagelist_save(mimg[idx][e], outname[idx],
1103  CPL_TYPE_UNSPECIFIED,
1104  mplist[idx][e], CPL_IO_EXTEND);
1105  cpl_imagelist_delete(mimg[idx][e]);
1106  mimg[idx][e] = NULL;
1107  }
1108  else {
1109  cpl_propertylist_save(mplist[idx][e], outname[idx],
1110  CPL_IO_EXTEND);
1111  }
1112  skip_if(0);
1113  }
1114  }
1115  }
1116  // PIPE-10123 cleanup
1117  cpl_array_delete(reg_values);
1118 
1119  cpl_frameset_delete(usedframes);
1120  cpl_frameset_delete(rawframes);
1121  skip_if(0);
1122  }
1123 
1124  end_skip;
1125  cpl_frameset_delete(frameset_copy);
1126  cpl_propertylist_delete(lintbl);
1127  cpl_propertylist_delete(gaintbl);
1128  cpl_propertylist_delete(coeffscube);
1129  cpl_propertylist_delete(bpm);
1130  cpl_propertylist_delete(corr);
1131  cpl_propertylist_delete(diff_flat);
1132  for (int i = 0; i < DETMON_MAX_IDX; i++) {
1133  for (int e = 0; e < next[i] + 1; e++) {
1134  if (mplist[i])
1135  cpl_propertylist_delete(mplist[i][e]);
1136  if (mimg[i])
1137  cpl_imagelist_delete(mimg[i][e]);
1138  if (mtab[i])
1139  cpl_table_delete(mtab[i][e]);
1140  }
1141  cpl_free(mplist[i]);
1142  cpl_free(mimg[i]);
1143  cpl_free(mtab[i]);
1144  cpl_free(outname[i]);
1145  cpl_free(outtag[i]);
1146  }
1147  cpl_free(llx);
1148  cpl_free(lly);
1149  cpl_free(urx);
1150  cpl_free(ury);
1151 
1152  return cpl_error_get_code();
1153 }