IIINSTRUMENT Pipeline Reference Manual  1.3.13
detmon_lg_mr.c
1 /*
2  * This file is part of the DETMON Pipeline
3  * Copyright (C) 2013 European Southern Observatory
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
18  */
19 
20 /*
21  * $Author: jtaylor $
22  * $Date: 2013-08-07 09:32:50 $
23  * $Revision: 1.6 $
24  * $Name: not supported by cvs2svn $
25  */
26 #ifdef HAVE_CONFIG_H
27 #include <config.h>
28 #endif
29 
30 /*----------------------------------------------------------------------------
31  Includes
32  ----------------------------------------------------------------------------*/
33 
34 
35 #include "detmon_lg.h"
36 #include "detmon.h"
37 
38 #include "irplib_plugin.h"
39 #include "irplib_utils.h"
40 
41 #include <string.h>
42 #include <errno.h>
43 #include <stdlib.h>
44 #include <assert.h>
45 
46 /*----------------------------------------------------------------------------
47  Private Functions prototypes
48  ----------------------------------------------------------------------------*/
49 int
50 detmon_lg_dfs_set_groups(cpl_frameset * set,
51  const char *tag_on, const char *tag_off);
52 
53 static int check_regions(const cpl_size *, const cpl_size *,
54  const cpl_size *, const cpl_size *,
55  cpl_size) CPL_ATTR_NONNULL;
56 
57 static int compare_tags(const cpl_frame *, const cpl_frame *) CPL_ATTR_NONNULL;
58 
59 static const cpl_parameter *
60 get_parameter_const(const cpl_parameterlist * parlist,
61  const char * recipe, const char * name)
62 {
63  char * parname = cpl_sprintf("detmon.%s.%s", recipe, name);
64  const cpl_parameter * p = cpl_parameterlist_find_const(parlist, parname);
65  cpl_free(parname);
66  return p;
67 }
68 
69 static cpl_parameter *
70 get_parameter(cpl_parameterlist * parlist,
71  const char * recipe, const char * name)
72 {
73  char * parname = cpl_sprintf("detmon.%s.%s", recipe, name);
74  cpl_parameter * p = cpl_parameterlist_find(parlist, parname);
75  cpl_free(parname);
76  return p;
77 }
78 
79 static int compare_tags(const cpl_frame * a, const cpl_frame * b)
80 {
81  return strcmp(cpl_frame_get_tag(a), cpl_frame_get_tag(b)) == 0;
82 }
83 
86 /* ---------------------------------------------------------------------------*/
97 /* ---------------------------------------------------------------------------*/
98 static cpl_error_code merge_keys(cpl_propertylist * mplist,
99  const cpl_propertylist * plist_,
100  const int pos,
101  const char * rmregexp)
102 {
103  char buffer[256];
104  /* no copy_regex, so duplicate and use erase_regex on the copy */
105  cpl_propertylist * plist = cpl_propertylist_duplicate(plist_);
106 
107  if (rmregexp) {
108  cpl_propertylist_erase_regexp(plist, rmregexp, 0);
109  cpl_propertylist_erase_regexp(mplist, rmregexp, 0);
110  }
111 
112  for (cpl_size i = 0; i < cpl_propertylist_get_size(plist); i++) {
113  cpl_property * p = cpl_propertylist_get(plist, i);
114  const char * key = cpl_property_get_name(p);
115  /* extend all qc keywords by a region prefix */
116  if (!strncmp(key, "ESO QC ", 7)) {
117  const char * com;
118  sprintf(buffer, "ESO QC REG%d %s", pos, key + 7);
119  cpl_msg_debug(cpl_func, "merging %s to %s", key, buffer);
120  if (cpl_property_get_type(p) == CPL_TYPE_INT) {
121  int val = cpl_property_get_int(p);
122  cpl_propertylist_append_int(mplist, buffer, val);
123  } else if (cpl_property_get_type(p) == CPL_TYPE_DOUBLE) {
124  double val = cpl_property_get_double(p);
125  cpl_propertylist_append_double(mplist, buffer, val);
126  } else if (cpl_property_get_type(p) == CPL_TYPE_STRING) {
127  const char * val = cpl_property_get_string(p);
128  cpl_propertylist_append_string(mplist, buffer, val);
129  }
130  else
131  bug_if(1);
132  com = cpl_property_get_comment(p);
133  cpl_propertylist_set_comment(mplist, buffer, com);
134 
135  /* remove unprefixed key */
136  if (cpl_propertylist_has(mplist, key))
137  cpl_propertylist_erase(mplist, key);
138  }
139  }
140  skip_if(0);
141 
142  end_skip;
143  cpl_propertylist_delete(plist);
144 
145  return cpl_error_get_code();
146 }
147 
148 
149 /* ---------------------------------------------------------------------------*/
157 /* ---------------------------------------------------------------------------*/
158 static cpl_error_code
159 merge_images(cpl_imagelist * mimgl, const cpl_imagelist * img)
160 {
161  cpl_ensure_code(cpl_imagelist_get_size(img) ==
162  cpl_imagelist_get_size(mimgl), CPL_ERROR_ILLEGAL_INPUT);
163  for (int i = 0; i < cpl_imagelist_get_size(img); i++) {
164  cpl_image * mimg = cpl_imagelist_get(mimgl, i);
165  const cpl_image * part = cpl_imagelist_get_const(img, i);
166  cpl_image_add(mimg, part);
167  }
168 
169  return cpl_error_get_code();
170 }
171 
172 
173 /* ---------------------------------------------------------------------------*/
181 /* ---------------------------------------------------------------------------*/
182 static cpl_error_code
183 merge_table(cpl_table * mtab, cpl_table * tab, const int pos)
184 {
185  cpl_table_new_column(tab, "REGION", CPL_TYPE_INT);
186  cpl_table_fill_column_window(tab, "REGION", 0, cpl_table_get_nrow(tab), pos);
187  skip_if(cpl_table_insert(mtab, tab, cpl_table_get_nrow(mtab)));
188 
189  end_skip;
190 
191  return cpl_error_get_code();
192 }
193 
194 
195 static cpl_error_code
196 parse_token(const char * token, cpl_size * llx, cpl_size * lly,
197  cpl_size * urx, cpl_size * ury)
198 {
199  assert(sizeof(*llx) == 8);
200  if (sscanf(token, " %lld , %lld , %lld , %lld ", llx, lly, urx, ury) == 4) {
201  return CPL_ERROR_NONE;
202  }
203  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
204  "Data not formated correctly: %s; "
205  "expecting: name,llx,lly,urx,ury",
206  token);
207 }
208 
209 
210 /* ---------------------------------------------------------------------------*/
223 /* ---------------------------------------------------------------------------*/
224 static cpl_error_code
225 read_regions_file(const char * filename, cpl_size ** rllx,
226  cpl_size ** rlly, cpl_size ** rurx, cpl_size ** rury,
227  cpl_size * n)
228 {
229  FILE * f = fopen(filename, "r");
230  char line[1024];
231  cpl_size i = 0;
232  cpl_size nmax = 100;
233  if (f == NULL) {
234  return cpl_error_set_message(cpl_func, CPL_ERROR_FILE_IO,
235  "Could not open file %s: %s", filename,
236  strerror(errno));
237  }
238  cpl_msg_info(cpl_func, "Reading %s", filename);
239 
240  *n = 0;
241  *rllx = cpl_malloc(sizeof(**rllx) * nmax);
242  *rlly = cpl_malloc(sizeof(**rllx) * nmax);
243  *rurx = cpl_malloc(sizeof(**rllx) * nmax);
244  *rury = cpl_malloc(sizeof(**rllx) * nmax);
245 
246  while (fgets(line, 1024, f) != NULL) {
247  const size_t nline = strlen(line);
248  if (line[nline - 1] != '\n') {
249  cpl_error_set_message(cpl_func, CPL_ERROR_FILE_IO,
250  "File %s contained too long line",
251  filename);
252  skip_if(0);
253  }
254 
255  if (line[0] == '#')
256  continue;
257 
258  skip_if(parse_token(line, &(*rllx)[i], &(*rlly)[i],
259  &(*rurx)[i], &(*rury)[i]));
260  *n = ++i;
261  if (i >= nmax) {
262  nmax *= 2;
263  *rllx = cpl_realloc(*rllx, sizeof(**rllx) * nmax);
264  *rlly = cpl_realloc(*rlly, sizeof(**rllx) * nmax);
265  *rurx = cpl_realloc(*rurx, sizeof(**rllx) * nmax);
266  *rury = cpl_realloc(*rury, sizeof(**rllx) * nmax);
267  skip_if(0);
268  }
269  }
270 
271  end_skip;
272  if (f)
273  fclose(f);
274  return CPL_ERROR_NONE;
275 }
276 
277 
278 /* ---------------------------------------------------------------------------*/
291 /* ---------------------------------------------------------------------------*/
292 static cpl_error_code
293 read_regions_str(const char * istr, cpl_size ** rllx,
294  cpl_size ** rlly, cpl_size ** rurx, cpl_size ** rury,
295  cpl_size * n)
296 {
297  cpl_size nmax = 100;
298  char * str_ = cpl_strdup(istr);
299  char * str = str_;
300  *n = 0;
301  *rllx = cpl_malloc(sizeof(**rllx) * nmax);
302  *rlly = cpl_malloc(sizeof(**rllx) * nmax);
303  *rurx = cpl_malloc(sizeof(**rllx) * nmax);
304  *rury = cpl_malloc(sizeof(**rllx) * nmax);
305 
306  for (int i = 0; ;i++) {
307  const char * token = strtok(str, ":");
308  if (token == NULL)
309  break;
310 
311  skip_if(parse_token(token, &(*rllx)[i], &(*rlly)[i],
312  &(*rurx)[i], &(*rury)[i]));
313  (*n)++;
314  if (i >= nmax - 1) {
315  nmax *= 2;
316  *rllx = cpl_realloc(*rllx, sizeof(**rllx) * nmax);
317  *rlly = cpl_realloc(*rlly, sizeof(**rllx) * nmax);
318  *rurx = cpl_realloc(*rurx, sizeof(**rllx) * nmax);
319  *rury = cpl_realloc(*rury, sizeof(**rllx) * nmax);
320  skip_if(0);
321  }
322  str = NULL;
323  }
324 
325  end_skip;
326  cpl_free(str_);
327 
328  return CPL_ERROR_NONE;
329 }
330 
331 static int check_regions(const cpl_size * llx, const cpl_size * lly,
332  const cpl_size * urx, const cpl_size * ury,
333  cpl_size nregions)
334 {
335  for (cpl_size i = 0; i < nregions; i++) {
336  if (llx[i] < 1 || lly[i] < 1 ||
337  urx[i] < 1 || ury[i] < 1) {
338  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
339  "Region %lld (%lld,%lld,%lld,%lld) has entry smaller one",
340  i, llx[i], lly[i], urx[i], ury[i]);
341  }
342  if ((urx[i] - llx[i] < 1) || (ury[i] - lly[i] < 1)) {
343  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
344  "Region %lld (%lld,%lld,%lld,%lld) has size zero",
345  i, llx[i], lly[i], urx[i], ury[i]);
346  }
347  for (cpl_size j = i + 1; j < nregions; j++) {
348  if (!(llx[i] > urx[j] || urx[i] < llx[j] ||
349  lly[i] > ury[j] || ury[i] < lly[j])) {
350  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
351  "Regions %lld (%lld,%lld,%lld,%lld) and "
352  "%lld (%lld,%lld,%lld,%lld) overlap (llx,lly,urx,ury)",
353  i, llx[i], lly[i], urx[i], ury[i],
354  j, llx[j], lly[j], urx[j], ury[j]);
355  }
356  }
357  }
358 
359  return CPL_ERROR_NONE;
360 }
361 
362 
363 /* ---------------------------------------------------------------------------*/
376 /* ---------------------------------------------------------------------------*/
377 static cpl_error_code
378 merge_lingain(cpl_propertylist * mplist[], cpl_table * mtab[],
379  const char * fn, const cpl_size next,
380  const char * rmregexp, int pos)
381 {
382  cpl_errorstate cleanstate = cpl_errorstate_get();
383 
384  {
385  cpl_propertylist * plist = cpl_propertylist_load(fn, 0);
386  skip_if(plist == NULL);
387  if (mplist[0] == NULL) {
388  mplist[0] = plist;
389  merge_keys(mplist[0], plist, pos, rmregexp);
390  }
391  else {
392  merge_keys(mplist[0], plist, pos, rmregexp);
393  cpl_propertylist_delete(plist);
394  }
395  }
396  skip_if(0);
397 
398  for (int e = 1; e < next + 1; e++) {
399  cpl_propertylist * plist = cpl_propertylist_load(fn, e);
400  cpl_table * tab = cpl_table_load(fn, e, 0);
401  /* empty extensions may exist */
402  if (tab == NULL) {
403  cpl_errorstate_set(cleanstate);
404  cpl_propertylist_delete(plist);
405  continue;
406  }
407  skip_if(plist == NULL);
408 
409  if (mplist[e] == NULL) {
410  merge_keys(plist, plist, pos, rmregexp);
411  cpl_table_new_column(tab, "REGION", CPL_TYPE_INT);
412  cpl_table_fill_column_window(tab, "REGION", 0,
413  cpl_table_get_nrow(tab), 1);
414  skip_if(0);
415  mplist[e] = plist;
416  mtab[e] = tab;
417  }
418  else {
419  merge_keys(mplist[e], plist, pos, rmregexp);
420  merge_table(mtab[e], tab, pos);
421  skip_if(0);
422  cpl_propertylist_delete(plist);
423  cpl_table_delete(tab);
424  }
425  skip_if(0);
426  }
427 
428  end_skip;
429 
430  return cpl_error_get_code();
431 }
432 
433 
434 /* ---------------------------------------------------------------------------*/
447 /* ---------------------------------------------------------------------------*/
448 static cpl_error_code
449 merge_bpmcoeff(cpl_propertylist * mplist[], cpl_imagelist * mimg[],
450  const char * fn, const cpl_size next,
451  const char * rmregexp, int pos)
452 {
453  cpl_errorstate cleanstate = cpl_errorstate_get();
454 
455  for (int e = 0; e < next + 1; e++) {
456  cpl_propertylist * plist = cpl_propertylist_load(fn, e);
457  cpl_imagelist * img = cpl_imagelist_load(fn, CPL_TYPE_UNSPECIFIED, e);
458  /* empty extensions may exist */
459  if (img == NULL) {
460  cpl_errorstate_set(cleanstate);
461  cpl_propertylist_delete(plist);
462  continue;
463  }
464  skip_if(plist == NULL);
465 
466  if (mimg[e] == NULL) {
467  merge_keys(plist, plist, pos, rmregexp);
468  mplist[e] = plist;
469  mimg[e] = img;
470  }
471  else {
472  merge_keys(mplist[e], plist, pos, rmregexp);
473  merge_images(mimg[e], img);
474  skip_if(0);
475  cpl_propertylist_delete(plist);
476  cpl_imagelist_delete(img);
477  }
478  skip_if(0);
479  }
480 
481  end_skip;
482 
483  return cpl_error_get_code();
484 }
485 
486 typedef enum {
487  DETMON_BP_MAP_NL_IDX = 0,
488  DETMON_LIN_INFO_IDX = 1,
489  DETMON_GAIN_INFO_IDX = 2,
490  DETMON_BPM_IDX = 3,
491  DETMON_COEFF_IDX = 4,
492  DETMON_MAX_IDX,
493 } detmon_idx;
494 
495 static cpl_error_code disable_contamination(cpl_parameterlist * parlist,
496  const char * recipe)
497 {
498  /* "disable" by setting it to a region of size 1 */
499  cpl_parameter * par;
500  for (int i = 1; i < 6; i++) {
501  char buffer[100];
502  sprintf(buffer, "detmon.%s.llx%d", recipe, i);
503  par = cpl_parameterlist_find(parlist, buffer);
504  cpl_parameter_set_int(par, 1);
505  sprintf(buffer, "detmon.%s.lly%d", recipe, i);
506  par = cpl_parameterlist_find(parlist, buffer);
507  cpl_parameter_set_int(par, 1);
508  sprintf(buffer, "detmon.%s.urx%d", recipe, i);
509  par = cpl_parameterlist_find(parlist, buffer);
510  cpl_parameter_set_int(par, 2);
511  sprintf(buffer, "detmon.%s.ury%d", recipe, i);
512  par = cpl_parameterlist_find(parlist, buffer);
513  cpl_parameter_set_int(par, 2);
514  }
515 
516  return cpl_error_get_code();
517 }
518 
519 static cpl_error_code detmon_lg_mr_phu_qc_reg_kw(
520  cpl_propertylist** mplist[],
521  int idx,
522  cpl_size next,
523  cpl_size nregions,
524  const char * kw_segment,
525  cpl_type kw_type
526  ) {
527 
528 // cpl_msg_info(cpl_func, "Within detmon_lg_mr_phu_qc_reg_kw for %s",
529 // kw_segment);
530  char * kw_input = NULL;
531  char* kw_output_avg = NULL ;
532  char* kw_output_rms = NULL;
533  cpl_array* kw_vals = cpl_array_new(next, kw_type);
534 // cpl_msg_info(cpl_func, " Variables instantiated");
535 
536  // Need to ensure the PHU propertylist actually exists - create one
537  // if it's NULL
538  if (mplist[idx][0] == NULL) {
539  mplist[idx][0] = cpl_propertylist_new();
540 // cpl_msg_info(cpl_func, " PHU propertylist instantiated");
541  }
542 
543  for (int i=0; i < nregions; i++) {
544 // cpl_msg_info(cpl_func, " Region %d of %d", i+1, nregions);
545  kw_input = cpl_sprintf("ESO QC REG%d %s", i+1, kw_segment);
546  cpl_msg_info(cpl_func, "Look for header kw %s", kw_input);
547  kw_output_avg = cpl_sprintf("ESO QC REG%d %s AVG", i+1, kw_segment);
548  kw_output_rms = cpl_sprintf("ESO QC REG%d %s RMS", i+1, kw_segment);
549  cpl_array_fill_window_invalid(kw_vals, 0, next);
550 // cpl_msg_info(cpl_func, " Variables set");
551 // cpl_msg_info(cpl_func, " %s", kw_input);
552 // cpl_msg_info(cpl_func, " %s", kw_output_avg);
553 // cpl_msg_info(cpl_func, " %s", kw_output_rms);
554 
555  for (cpl_size e = 1 ; e < next + 1 ; ++e) {
556  if (cpl_propertylist_has(mplist[idx][e], kw_input)) {
557  cpl_msg_info(
558  cpl_func,
559  " Found value %s in ext %" CPL_SIZE_FORMAT "",
560  kw_input, e);
561  if (kw_type == CPL_TYPE_DOUBLE) {
562  cpl_array_set_double(kw_vals, e-1,
563  cpl_propertylist_get_double(
564  mplist[idx][e], kw_input));
565  } else if (kw_type == CPL_TYPE_INT) {
566  cpl_array_set_int(kw_vals, e-1,
567  cpl_propertylist_get_int(
568  mplist[idx][e], kw_input));
569  }
570  }
571  }
572  cpl_msg_info(cpl_func, " Array populated");
573 
574  cpl_propertylist_append_double(mplist[idx][0],
575  kw_output_avg,
576  cpl_array_get_mean(kw_vals));
577  cpl_msg_info(cpl_func, " Set mean");
578  cpl_propertylist_set_comment(
579  mplist[idx][0],
580  kw_output_avg,
581  cpl_sprintf("Avg of QC REG%d %s over exts",
582  i+1, kw_segment));
583  cpl_msg_info(cpl_func, " Set mean comment");
584  cpl_propertylist_append_double(mplist[idx][0],
585  kw_output_rms,
586  cpl_array_get_stdev(kw_vals));
587  cpl_msg_info(cpl_func, " Set RMS");
588  cpl_propertylist_set_comment(
589  mplist[idx][0],
590  kw_output_rms,
591  cpl_sprintf("RMS of QC REG%d %s over exts",
592  i+1, kw_segment));
593 // cpl_msg_info(cpl_func, " Set RMS comment");
594 // cpl_msg_info(cpl_func, " KW set for region %d", i+1);
595  if (cpl_error_get_code() != CPL_ERROR_NONE) {
596  cpl_msg_error(__func__, "Error found in detmon_lg_mr_phu_qc_reg_kw!");
597  cpl_msg_error(__func__, "In %s, L%u", cpl_error_get_file(),
598  cpl_error_get_line());
599  cpl_msg_error(__func__, "%s", cpl_error_get_message());
600  cpl_msg_error(__func__, "Values ingested:");
601  cpl_array_dump(kw_vals, 0, next, stdout);
602  cpl_msg_error(__func__, "There were %d extensions", next);
603  cpl_msg_error(__func__, "There were %d regions detected", nregions);
604 // cpl_msg_error(__func__, "Propertylist:");
605 // cpl_propertylist_dump(mplist[idx][0], stdout);
606 // getchar();
607  }
608  }
609 
610 // cpl_msg_info(cpl_func, " Proceeding to clean-up");
611  cpl_free(kw_input);
612  cpl_free(kw_output_rms);
613  cpl_free(kw_output_avg);
614  cpl_array_delete(kw_vals);
615 
616 // cpl_msg_info(cpl_func, "Returning!");
617  return cpl_error_get_code();
618 }
619 
620 /*---------------------------------------------------------------------------*/
621 /*
622  @brief Interpret the command line options and execute the data processing
623  @param frameset the frames list
624  @param parlist_ the parameters list
625  @param recipe recipe name
626  @param is_nir switch to know if opt or nir
627  @return 0 if everything is ok
628  */
629 /*---------------------------------------------------------------------------*/
630 
631 int detmon_lg_mr(cpl_frameset * frameset,
632  const cpl_parameterlist * parlist_,
633  const char * recipe,
634  cpl_boolean is_nir)
635 {
636  const char * tag_on=NULL;
637  const char * tag_off=NULL;
638  IRPLIB_DIAG_PRAGMA_PUSH_IGN(-Wcast-qual);
639  cpl_parameterlist * parlist = (cpl_parameterlist *)parlist_;
640  IRPLIB_DIAG_PRAGMA_POP;
641  cpl_propertylist ** mplist[DETMON_MAX_IDX] = {NULL};
642  cpl_imagelist ** mimg[DETMON_MAX_IDX] = {NULL};
643  cpl_table ** mtab[DETMON_MAX_IDX] = {NULL};
644  cpl_size next[DETMON_MAX_IDX] = {0};
645  char * outname[DETMON_MAX_IDX] = {NULL};
646  char * outtag[DETMON_MAX_IDX] = {NULL};
647  cpl_size nregions = 0;
648 
649  cpl_frameset * frameset_copy = NULL;
650 
651  cpl_propertylist * lintbl =
652  detmon_fill_prolist("DET_LIN_INFO", "REDUCED", "TECH", CPL_FALSE);
653 
654  cpl_propertylist * gaintbl =
655  detmon_fill_prolist("GAIN_INFO", "REDUCED", "TECH", CPL_FALSE);
656 
657  cpl_propertylist * coeffscube =
658  detmon_fill_prolist("COEFFS_CUBE", "REDUCED", "TECH", CPL_FALSE);
659 
660  cpl_propertylist * bpm =
661  detmon_fill_prolist("BP_MAP_NL", "REDUCED", "TECH", CPL_FALSE);
662 
663  cpl_propertylist * corr =
664  detmon_fill_prolist("AUTOCORR", "REDUCED", "TECH", CPL_FALSE);
665 
666  cpl_propertylist * diff_flat =
667  detmon_fill_prolist("DIFF_FLAT", "REDUCED", "TECH", CPL_FALSE);
668  const char * regions_file = NULL;
669  const char * regions_str = NULL;
670  cpl_error_code error = CPL_ERROR_NONE;
671  cpl_size * llx = NULL, * lly = NULL, * urx = NULL, * ury = NULL;
672  const cpl_parameter * p =
673  get_parameter_const(parlist, recipe, "regions-file");
674  if (p) {
675  regions_file = cpl_parameter_get_string(p);
676  if (strlen(regions_file) == 0)
677  regions_file = NULL;
678  }
679 
680  p = get_parameter_const(parlist, recipe, "regions");
681  if (p) {
682  regions_str = cpl_parameter_get_string(p);
683  if (strlen(regions_str) == 0)
684  regions_str = NULL;
685  else {
686  if (regions_file) {
687  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
688  "Please only provide either "
689  "--regions or "
690  "--regions-file, not both");
691  }
692  }
693  }
694 
695  p = get_parameter(parlist, recipe, "intermediate");
696  if (p) {
697  error_if(cpl_parameter_get_bool(p), CPL_ERROR_UNSUPPORTED_MODE,
698  "--intermediate not supported by multi region recipe");
699  }
700 
701  skip_if (detmon_lg_set_tag(frameset, &tag_on, &tag_off));
702  skip_if (detmon_lg_dfs_set_groups(frameset, tag_on, tag_off));
703 
704  if (regions_file) {
705  read_regions_file(regions_file, &llx, &lly, &urx, &ury,
706  &nregions);
707  }
708  else if (regions_str) {
709  read_regions_str(regions_str, &llx, &lly, &urx, &ury,
710  &nregions);
711  }
712  else {
713  return cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
714  "Empty value in --regions or "
715  "--regions-file");
716  }
717  skip_if(0);
718 
719  skip_if(check_regions(llx, lly, urx, ury, nregions));
720 
721  if (!is_nir) {
722  /* contamination not supported */
723  skip_if(disable_contamination(parlist, recipe));
724  }
725 
726  /* additionally iterating on extensions would improve IO caching and
727  * reduce memory usage for pix2pix but requires checks if detmon really
728  * gives the same result in this case */
729  for (int i = 0; i < nregions; i++) {
730  cpl_size nlabels;
731  cpl_size * selection;
732  cpl_parameter * par;
733  par = get_parameter(parlist, recipe, "llx");
734  cpl_parameter_set_int(par, llx[i]);
735  par = get_parameter(parlist, recipe, "lly");
736  cpl_parameter_set_int(par, lly[i]);
737  par = get_parameter(parlist, recipe, "urx");
738  cpl_parameter_set_int(par, urx[i]);
739  par = get_parameter(parlist, recipe, "ury");
740  cpl_parameter_set_int(par, ury[i]);
741  cpl_msg_info(cpl_func, "working on region %d/%d: %lld %lld, "
742  "%lld %lld", i + 1, (int)nregions,
743  llx[i], lly[i], urx[i], ury[i]);
744 
745  /* copy and throw away so we don't accumulate products
746  * for each call */
747  cpl_frameset_delete(frameset_copy);
748  frameset_copy = cpl_frameset_duplicate(frameset);
749  error = detmon_lg(frameset_copy,
750  parlist,
751  tag_on,
752  tag_off,
753  recipe,
754  PACKAGE_TARNAME,
755  REGEXP,
756  lintbl, gaintbl, coeffscube,
757  bpm, corr, diff_flat,
758  PACKAGE "/" PACKAGE_VERSION,
759  NULL, NULL, is_nir);
760  skip_if(error);
761 
762  selection = cpl_frameset_labelise(frameset_copy, compare_tags,
763  &nlabels);
764 
765  for (cpl_size j = 0; j < nlabels; j++) {
766  cpl_frameset * a = cpl_frameset_extract(frameset_copy, selection, j);
767  cpl_frame * frm = cpl_frameset_get_position(a, 0);
768  const char * fn = cpl_frame_get_filename(frm);
769  int idx;
770 
771  if (strcmp(cpl_frame_get_tag(frm), "DET_LIN_INFO") == 0)
772  idx = DETMON_LIN_INFO_IDX;
773  else if (strcmp(cpl_frame_get_tag(frm), "GAIN_INFO") == 0)
774  idx = DETMON_GAIN_INFO_IDX;
775  else if (strcmp(cpl_frame_get_tag(frm), "BP_MAP_NL") == 0)
776  idx = DETMON_BPM_IDX;
777  else if (strcmp(cpl_frame_get_tag(frm), "COEFFS_CUBE") == 0)
778  idx = DETMON_COEFF_IDX;
779  else {
780  cpl_frameset_delete(a);
781  continue;
782  }
783 
784  /* prepare storage */
785  if (mplist[idx] == NULL) {
786  next[idx] = cpl_frame_get_nextensions(frm);
787  outname[idx] = cpl_strdup(cpl_frame_get_filename(frm));
788  outtag[idx] = cpl_strdup(cpl_frame_get_tag(frm));
789  mplist[idx] = cpl_calloc(sizeof(cpl_propertylist*), next[idx] + 1);
790  mimg[idx] = cpl_calloc(sizeof(cpl_imagelist*), next[idx] + 1);
791  mtab[idx] = cpl_calloc(sizeof(cpl_table*), next[idx] + 1);
792  }
793 
794  if (idx == DETMON_LIN_INFO_IDX) {
795  const char * rmregexp = "ESO QC CONTAM.*";
796  merge_lingain(mplist[idx], mtab[idx], fn, next[idx],
797  rmregexp, i + 1);
798  }
799  else if (idx == DETMON_GAIN_INFO_IDX) {
800  merge_lingain(mplist[idx], mtab[idx], fn, next[idx],
801  NULL, i + 1);
802  }
803  else if (idx == DETMON_BPM_IDX) {
804  merge_bpmcoeff(mplist[idx], mimg[idx], fn, next[idx],
805  NULL, i + 1);
806  }
807  else if (idx == DETMON_COEFF_IDX) {
808  merge_bpmcoeff(mplist[idx], mimg[idx], fn, next[idx],
809  NULL, i + 1);
810  }
811  skip_if(0);
812 
813  for (int e = 0; e < next[idx] + 1; e++) {
814  char buffer[300];
815  cpl_propertylist * copy_plist;
816  cpl_size len;
817  if (!mplist[idx][e])
818  continue;
819 
820  /* don't add region if extension has no QC parameters */
821  copy_plist = cpl_propertylist_duplicate(mplist[idx][e]);
822  cpl_propertylist_erase_regexp(copy_plist, "ESO QC.*", 1);
823  len = cpl_propertylist_get_size(copy_plist);
824  cpl_propertylist_delete(copy_plist);
825  if (len == 0) {
826  continue;
827  }
828 
829  sprintf(buffer, "ESO QC REG%d LLX", i + 1);
830  cpl_propertylist_append_int(mplist[idx][e], buffer, llx[i]);
831  cpl_propertylist_set_comment(mplist[idx][e], buffer,
832  "Lower left X of region");
833  sprintf(buffer, "ESO QC REG%d LLY", i + 1);
834  cpl_propertylist_append_int(mplist[idx][e], buffer, lly[i]);
835  cpl_propertylist_set_comment(mplist[idx][e], buffer,
836  "Lower left Y of region");
837  sprintf(buffer, "ESO QC REG%d URX", i + 1);
838  cpl_propertylist_append_int(mplist[idx][e], buffer, urx[i]);
839  cpl_propertylist_set_comment(mplist[idx][e], buffer,
840  "Upper right X of region");
841  sprintf(buffer, "ESO QC REG%d URY", i + 1);
842  cpl_propertylist_append_int(mplist[idx][e], buffer, ury[i]);
843  cpl_propertylist_set_comment(mplist[idx][e], buffer,
844  "Upper right Y of region");
845  }
846 
847  cpl_frameset_delete(a);
848  skip_if(0);
849  }
850  cpl_free(selection);
851  }
852  cpl_msg_info(__func__, "Completed looping over regions");
853 // getchar();
854 
855  {
856  cpl_frameset * usedframes = cpl_frameset_new();
857  const cpl_size nframes = cpl_frameset_get_size(frameset);
858 
859  cpl_frameset * rawframes = cpl_frameset_new();
860 
861  for (cpl_size idx = 0; idx < nframes; idx++) {
862  const cpl_frame * frm = cpl_frameset_get_position_const(frameset,
863  idx);
864  if (cpl_frame_get_group(frm) == CPL_FRAME_GROUP_RAW ||
865  cpl_frame_get_group(frm) == CPL_FRAME_GROUP_CALIB) {
866  cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
867  }
868  if (cpl_frame_get_group(frm) == CPL_FRAME_GROUP_RAW) {
869  cpl_frameset_insert(rawframes, cpl_frame_duplicate(frm));
870  }
871  }
872 
873  cpl_array* reg_values = NULL;
874  char * var_part = NULL;
875 // cpl_frame* frm = NULL ;
876 // char * rfn = NULL ;
877 // cpl_propertylist* rpl = NULL ;
878  for (int idx = 0; idx < DETMON_MAX_IDX; idx++) {
879  cpl_msg_info(__func__, "Doing idx loop %d of %d", idx,
880  DETMON_MAX_IDX-1);
881  if (!mplist[idx])
882  continue;
883 
884  if ((idx == DETMON_LIN_INFO_IDX || idx == DETMON_GAIN_INFO_IDX)) {
885 
886  // PIPE-10123 - Compute additional parameters here for
887  // these two tables
888  double exptime = 0.0 ;
889  reg_values = cpl_array_new(nregions,
890  CPL_TYPE_DOUBLE);
891 
892  if (!is_nir && next[idx] > 0) {
893  cpl_msg_info(cpl_func, "Doing PIPE-10123 lin/gain kw for %s",
894  recipe);
895 
896  // Compute EXPTIME MAX from RAWFRAMES
897  for (cpl_size r=0;
898  r < cpl_frameset_get_size(rawframes);
899  ++r) {
900  const cpl_frame * frm = cpl_frameset_get_position_const(
901  rawframes, r
902  );
903  const char * rfn = cpl_frame_get_filename(frm);
904  const cpl_propertylist * rpl = cpl_propertylist_load(
905  rfn, 0
906  );
907  if (rpl != NULL &&
908  cpl_propertylist_has(rpl, "EXPTIME")) {
909  if (cpl_propertylist_get_double(rpl, "EXPTIME") >
910  exptime) {
911  exptime = cpl_propertylist_get_double(
912  rpl, "EXPTIME");
913  }
914  }
915  }
916  cpl_propertylist_append_double(mplist[idx][0],
917  "ESO QC EXPTIME MAX",
918  exptime);
919  cpl_propertylist_set_comment(
920  mplist[idx][0],
921  "ESO QC EXPTIME MAX",
922  "Max. EXPTIME from input raws");
923 
924  if (idx == DETMON_GAIN_INFO_IDX) {
925  var_part = "GAIN";
926  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
927  nregions, "CONAD",
928  CPL_TYPE_DOUBLE);
929  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
930  nregions, "GAIN",
931  CPL_TYPE_DOUBLE);
932  } else {
933  var_part = "LIN EFF";
934  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
935  nregions, "GAIN ERR",
936  CPL_TYPE_DOUBLE);
937  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
938  nregions, "FPN",
939  CPL_TYPE_DOUBLE);
940  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
941  nregions, "COUNTS MIN",
942  CPL_TYPE_DOUBLE);
943  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
944  nregions, "COUNTS MAX",
945  CPL_TYPE_DOUBLE);
946  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
947  nregions, "LIN EFF",
948  CPL_TYPE_DOUBLE);
949  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
950  nregions, "LIN EFF FLUX",
951  CPL_TYPE_DOUBLE);
952  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
953  nregions, "LLX",
954  CPL_TYPE_INT);
955  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
956  nregions, "LLY",
957  CPL_TYPE_INT);
958  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
959  nregions, "URX",
960  CPL_TYPE_INT);
961  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
962  nregions, "URY",
963  CPL_TYPE_INT);
964  }
965 
966  for (cpl_size e = 1; e < next[idx] + 1; e++) {
967 
968  for (int i=0 ; i < nregions ; i++) {
969  if (cpl_propertylist_has(mplist[idx][e],
970  cpl_sprintf("ESO QC REG%d %s",
971  i+1, var_part))) {
972  cpl_array_set_double(
973  reg_values, i,
974  cpl_propertylist_get_double(mplist[idx][e],
975  cpl_sprintf("ESO QC REG%d %s",
976  i+1, var_part)
977  ));
978  }
979 
980  }
981 
982  cpl_propertylist_append_double(mplist[idx][e],
983  cpl_sprintf("ESO QC %s AVG",
984  var_part),
985  cpl_array_get_mean(
986  reg_values));
987  cpl_propertylist_set_comment(mplist[idx][e],
988  cpl_sprintf("ESO QC %s AVG",
989  var_part),
990  cpl_sprintf(
991  "Avg of QC.REGn.%s",
992  var_part));
993  cpl_propertylist_append_double(mplist[idx][e],
994  cpl_sprintf("ESO QC %s RMS",
995  var_part),
996  cpl_array_get_stdev(
997  reg_values));
998  cpl_propertylist_set_comment(mplist[idx][e],
999  cpl_sprintf("ESO QC %s RMS",
1000  var_part),
1001  cpl_sprintf(
1002  "RMS of QC.REGn.%s",
1003  var_part));
1004  cpl_msg_info(__func__, "Wrote ESO QC RMS");
1005  }
1006 
1007  }
1008  cpl_msg_info(__func__, "Doing a GAIN/LIN save (var_part=%s)",
1009  var_part);
1010  irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1011  recipe, outtag[idx],
1012  mplist[idx][0], NULL, PACKAGE "/"
1013  PACKAGE_VERSION, outname[idx]);
1014 
1015  skip_if(0);
1016  for (cpl_size e = 1; e < next[idx] + 1; e++) {
1017  if (mtab[idx][e]) {
1018  cpl_table_save(mtab[idx][e], NULL, mplist[idx][e],
1019  outname[idx], CPL_IO_EXTEND);
1020  }
1021  else {
1022  cpl_propertylist_save(mplist[idx][e], outname[idx],
1023  CPL_IO_EXTEND);
1024  }
1025  skip_if(0);
1026  }
1027  }
1028  // No need to alter _mr BPM after all for PIPE-10123
1029  else if (idx == DETMON_BPM_IDX) {
1030  cpl_msg_info(__func__, "Doing a BPM save");
1031  if (!mimg[idx][0]) {
1032  irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1033  recipe, outtag[idx],
1034  mplist[idx][0], NULL, PACKAGE
1035  "/" PACKAGE_VERSION,
1036  outname[idx]);
1037  }
1038  else {
1039  irplib_dfs_save_image(frameset, parlist, usedframes,
1040  cpl_imagelist_get(mimg[idx][0], 0),
1041  CPL_TYPE_UNSPECIFIED, recipe,
1042  outtag[idx], mplist[idx][0], NULL,
1043  PACKAGE "/" PACKAGE_VERSION,
1044  outname[idx]);
1045  cpl_imagelist_delete(mimg[idx][0]);
1046  mimg[idx][0] = NULL;
1047  }
1048  skip_if(0);
1049  for (cpl_size e = 1; e < next[idx] + 1; e++) {
1050  if (mimg[idx][e]) {
1051  cpl_image * img = cpl_imagelist_get(mimg[idx][e], 0);
1052  cpl_image_save(img, outname[idx], CPL_TYPE_UNSPECIFIED,
1053  mplist[idx][e],
1054  CPL_IO_EXTEND);
1055  cpl_imagelist_delete(mimg[idx][e]);
1056  mimg[idx][e] = NULL;
1057  }
1058  else {
1059  cpl_propertylist_save(mplist[idx][e], outname[idx],
1060  CPL_IO_EXTEND);
1061  }
1062  skip_if(0);
1063  }
1064  }
1065  else if (idx == DETMON_COEFF_IDX) {
1066  // PIPE-10123 Add the additional QC keywords
1067  if (!is_nir && next[idx] > 0) {
1068  cpl_msg_info(cpl_func, "Doing PIPE-10123 COEFF kw for %s",
1069  recipe);
1070  for (int j=0; j < 4 ; ++j ) {
1071  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
1072  nregions,
1073  cpl_sprintf("LIN COEF%d", j),
1074  CPL_TYPE_DOUBLE);
1075  }
1076  detmon_lg_mr_phu_qc_reg_kw(mplist, idx, next[idx],
1077  nregions, "ERRFIT",
1078  CPL_TYPE_DOUBLE);
1079  }
1080 
1081  cpl_msg_info(__func__, "Doing a COEFFS save");
1082  if (!mimg[idx][0]) {
1083  irplib_dfs_save_propertylist(frameset, parlist, usedframes,
1084  recipe, outtag[idx],
1085  mplist[idx][0], NULL, PACKAGE
1086  "/" PACKAGE_VERSION,
1087  outname[idx]);
1088  }
1089  else {
1090  irplib_dfs_save_imagelist(frameset, parlist, usedframes,
1091  mimg[idx][0],
1092  CPL_TYPE_UNSPECIFIED,
1093  recipe, outtag[idx],
1094  mplist[idx][0], NULL, PACKAGE "/"
1095  PACKAGE_VERSION, outname[idx]);
1096  cpl_imagelist_delete(mimg[idx][0]);
1097  mimg[idx][0] = NULL;
1098  }
1099  skip_if(0);
1100  for (cpl_size e = 1; e < next[idx] + 1; e++) {
1101  if (mimg[idx][e]) {
1102  cpl_imagelist_save(mimg[idx][e], outname[idx],
1103  CPL_TYPE_UNSPECIFIED,
1104  mplist[idx][e], CPL_IO_EXTEND);
1105  cpl_imagelist_delete(mimg[idx][e]);
1106  mimg[idx][e] = NULL;
1107  }
1108  else {
1109  cpl_propertylist_save(mplist[idx][e], outname[idx],
1110  CPL_IO_EXTEND);
1111  }
1112  skip_if(0);
1113  }
1114  }
1115  }
1116  // PIPE-10123 cleanup
1117  cpl_array_delete(reg_values);
1118 
1119  cpl_frameset_delete(usedframes);
1120  cpl_frameset_delete(rawframes);
1121  skip_if(0);
1122  }
1123 
1124  end_skip;
1125  cpl_frameset_delete(frameset_copy);
1126  cpl_propertylist_delete(lintbl);
1127  cpl_propertylist_delete(gaintbl);
1128  cpl_propertylist_delete(coeffscube);
1129  cpl_propertylist_delete(bpm);
1130  cpl_propertylist_delete(corr);
1131  cpl_propertylist_delete(diff_flat);
1132  for (int i = 0; i < DETMON_MAX_IDX; i++) {
1133  for (int e = 0; e < next[i] + 1; e++) {
1134  if (mplist[i])
1135  cpl_propertylist_delete(mplist[i][e]);
1136  if (mimg[i])
1137  cpl_imagelist_delete(mimg[i][e]);
1138  if (mtab[i])
1139  cpl_table_delete(mtab[i][e]);
1140  }
1141  cpl_free(mplist[i]);
1142  cpl_free(mimg[i]);
1143  cpl_free(mtab[i]);
1144  cpl_free(outname[i]);
1145  cpl_free(outtag[i]);
1146  }
1147  cpl_free(llx);
1148  cpl_free(lly);
1149  cpl_free(urx);
1150  cpl_free(ury);
1151 
1152  return cpl_error_get_code();
1153 }