ERIS Pipeline Reference Manual 1.9.2
eris_persistence_monitor.c
1/* $Id: eris_persistence_monitor.c,v 1.6 2013-03-26 17:17:32 jtaylor Exp $
2 *
3 * This file is part of the ERIS Pipeline
4 * Copyright (C) 2002,2003 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21/*
22 * $Author: jtaylor $
23 * $Date: 2013-03-26 17:17:32 $
24 * $Revision: 1.6 $
25 * $Name: not supported by cvs2svn $
26 */
27
28#ifdef HAVE_CONFIG_H
29#include <config.h>
30#endif
31
32/*-----------------------------------------------------------------------------
33 Includes
34 -----------------------------------------------------------------------------*/
35
36#include "eris_utils.h"
37#include "eris_pfits.h"
38#include "eris_dfs.h"
39
40#include <cpl.h>
41
42#include <string.h>
43#include <eris_ifu_dfs.h>
44#include <eris_nix_dfs.h>
45/*-----------------------------------------------------------------------------
46 Static variables
47 -----------------------------------------------------------------------------*/
48typedef enum {SLOW=0, FAST=1, UP_TO_RAMP=2} read_out_mode;
49static char eris_persistence_monitor_description[] =
50"This recipe counts the number of pixel above user defined thresholds. \n"
51"Inputs are ERIS SPIFFIER or NIX PERSISTENCE cubes and their corresponding\n"
52"The recipe applies the check only on the 1st input raw frame, thus the set\n"
53"of frames should contain only two input frames, the raw data and the \n"
54"Master dark (same DIT). The corresponding associated tags are: \n"
55"PERSISTENCE_IMA \n"
56"MASTER_DARK_IFU (or MASTER_DARK_IMA for NIX data) \n"
57"\n"
58"The number of pixels above saturation or threshold values are stored as QC\n"
59"In the output product that has PRO.CATG PERSISTENCE_HDR. Optional product \n"
60"when the user sets debug to true is the image on which the pixel are computed"
61"\n";
62
63/*-----------------------------------------------------------------------------
64 Private function prototypes
65 -----------------------------------------------------------------------------*/
66
67cpl_recipe_define(eris_persistence_monitor, ERIS_BINARY_VERSION, "Firstname Lastname",
68 PACKAGE_BUGREPORT, "2017",
69 "Monitor PERSISTENCE image sat. & threshold pixels",
70 eris_persistence_monitor_description);
71
72/*-----------------------------------------------------------------------------
73 Function code
74 -----------------------------------------------------------------------------*/
75
76
77/*----------------------------------------------------------------------------*/
85/*----------------------------------------------------------------------------*/
86static
87cpl_error_code eris_persistence_monitor_fill_parameterlist(cpl_parameterlist *self)
88{
89 cpl_errorstate prestate = cpl_errorstate_get();
90 cpl_parameter * p;
91
92 /* Fill the parameters list */
93 /* --saturation */
94 p = cpl_parameter_new_value("eris.eris_persistence_monitor.saturation",
95 CPL_TYPE_DOUBLE, "saturation level",
96 "eris.eris_persistence_monitor", 60000.);
97 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "saturation");
98 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
99 cpl_parameterlist_append(self, p);
100
101 p = cpl_parameter_new_value("eris.eris_persistence_monitor.saturation_neg",
102 CPL_TYPE_DOUBLE, "negative saturation level",
103 "eris.eris_persistence_monitor", -4.5e7);
104 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "saturation_neg");
105 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
106 cpl_parameterlist_append(self, p);
107
108 p = cpl_parameter_new_value("eris.eris_persistence_monitor.threshold",
109 CPL_TYPE_DOUBLE, "threshold level",
110 "eris.eris_persistence_monitor",300.);
111 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "threshold");
112 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
113 cpl_parameterlist_append(self, p);
114
115 /* --boolopt */
116 p = cpl_parameter_new_value("eris.eris_persistence_monitor.debug",
117 CPL_TYPE_BOOL, "If set to 'true' extra products are created",
118 "eris.eris_persistence_monitor", CPL_FALSE);
119 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "debug");
120 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
121 cpl_parameterlist_append(self, p);
122
123 return cpl_errorstate_is_equal(prestate) ? CPL_ERROR_NONE
124 : cpl_error_set_where(cpl_func);
125}
126
127/*----------------------------------------------------------------------------*/
134/*----------------------------------------------------------------------------*/
135static int eris_persistence_monitor(cpl_frameset * frameset,
136 const cpl_parameterlist * parameters)
137{
138 cpl_parameter * param = NULL;
139 cpl_parameterlist * parlist = (cpl_parameterlist *) parameters;
140 cpl_boolean debug = CPL_FALSE;
141 cpl_frameset * rawframes = NULL;
142 const cpl_frame * firstframe = NULL;
143
144 cpl_propertylist * plist = NULL;
145 cpl_propertylist * applist = NULL;
146
147 int nraw;
148
149 double saturation = 0;
150 double threshold = 0;
151 double saturation_negative = - 4.5e7;
152
153 const char* recipe = "eris_persistence_monitor";
154 /* Use the errorstate to detect an error in a function that does not
155 return an error code. */
156 cpl_errorstate prestate = cpl_errorstate_get();
157
158 /* HOW TO RETRIEVE INPUT PARAMETERS */
159 /* --double */
160 param = cpl_parameterlist_find(parlist,
161 "eris.eris_persistence_monitor.saturation");
162 int saturation_sw_set = cpl_parameter_get_default_flag(param);
163 saturation = cpl_parameter_get_double(param);
164
165
166 param = cpl_parameterlist_find(parlist,
167 "eris.eris_persistence_monitor.saturation_neg");
168 //int saturation_neg_sw_set = cpl_parameter_get_default_flag(param);
169 saturation_negative = cpl_parameter_get_double(param);
170
171 param = cpl_parameterlist_find(parlist,
172 "eris.eris_persistence_monitor.threshold");
173 int threshold_sw_set = cpl_parameter_get_default_flag(param);
174 threshold = cpl_parameter_get_double(param);
175 /* --boolopt */
176 param = cpl_parameterlist_find(parlist,
177 "eris.eris_persistence_monitor.debug");
178 debug = cpl_parameter_get_bool(param);
179
180 if (!cpl_errorstate_is_equal(prestate)) {
181 return (int)cpl_error_set_message(cpl_func, cpl_error_get_code(),
182 "Could not retrieve the input "
183 "parameters");
184 }
185
186 /* Identify the RAW and CALIB frames in the input frameset */
187 /* We use NIX classifications also for IFU data, where we added support for
188 * MASTER_DARK_IFU, as NIX does not do too many checks (that input frame is
189 * of the same SEQ.ARM) not needed with unique PRO.CATGs
190 */
191
192
193 cpl_ensure_code(eris_nix_dfs_set_groups(frameset) == CPL_ERROR_NONE,
194 cpl_error_get_code());
195
196
197 /* HOW TO ACCESS INPUT DATA */
198 /* - A required file */
199 rawframes = cpl_frameset_new();
200 nraw = 0;
201 /* TODO: here we could use +eris_dfs_extract_raw_frames */
202 for (cpl_size j = 0; j<cpl_frameset_get_size(frameset); j++) {
203 const cpl_frame * current_frame;
204 current_frame = cpl_frameset_get_position_const(frameset, j);
205 if(!strcmp(cpl_frame_get_tag(current_frame), ERIS_PERSISTENCE_IMA) ||
206 !strcmp(cpl_frame_get_tag(current_frame), ERIS_PERSISTENCE_CUBE)) {
207 cpl_frame * new_frame = cpl_frame_duplicate(current_frame);
208 cpl_frameset_insert(rawframes, new_frame);
209 nraw++;
210 }
211 }
212
213 if (nraw == 0) {
214 cpl_frameset_delete(rawframes);
215 return (int)cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND,
216 "SOF does not have any file tagged "
217 "with %s or %s", ERIS_PERSISTENCE_IMA,
218 ERIS_PERSISTENCE_CUBE);
219 }
220
221 cpl_propertylist_delete(plist);
222
223 /* Check for a change in the CPL error state */
224 /* - if it did change then propagate the error and return */
225 cpl_ensure_code(cpl_errorstate_is_equal(prestate), cpl_error_get_code());
226
227 /* NOW PERFORMING THE DATA REDUCTION */
228 /* Let's just load an image for the example */
229 cpl_size nframes = cpl_frameset_get_size(rawframes);
230
231 firstframe = cpl_frameset_get_position_const(rawframes, 0);
232 const char* fname = cpl_frame_get_filename(firstframe);
233 plist = cpl_propertylist_load(fname, 0);
234 const char* arm = cpl_propertylist_get_string(plist,FHDR_E_ARM);
235 int naxis = cpl_propertylist_get_int(plist,"NAXIS");
236
237 cpl_frame* frm_mdark = NULL;
238 const char* read_curname = cpl_propertylist_get_string(plist,
239 "ESO DET READ CURNAME");
240 read_out_mode rom = SLOW;
241
242 if (read_curname == NULL) {
243 cpl_propertylist_delete(plist);
244 cpl_frameset_delete(rawframes);
245 return cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND,
246 "No 'ESO DET READ CURNAME' in raw "
247 "frame 1/%d", nraw);
248 }
249 if (strcmp(read_curname,"FAST_UNCORR") == 0) {
250 rom = FAST;
251 } else if (strcmp(read_curname,"SLOW_GR_UTR") == 0) {
252 rom = SLOW;
253 } else if (strcmp(read_curname,"Normal_UpTheRamp") == 0) {
254 rom = UP_TO_RAMP;
255 }
256 if(strcmp(arm,"SPIFFIER") == 0) {
257 if(saturation_sw_set == 1) {
258 // do nothing
259 } else {
260 saturation = 60000;
261 }
262 if(threshold_sw_set == 1) {
263 // do nothing
264 } else {
265 threshold = 300;
266 }
267 frm_mdark = cpl_frameset_find(frameset,ERIS_IFU_PRO_DARK);
268 } else {
269 if(saturation_sw_set == 1) {
270 // do nothing
271 } else {
272 if (rom == FAST) {
273 saturation = 35000;
274 } else {
275 saturation = 15000;
276 }
277 }
278 if(threshold_sw_set == 1) {
279 // do nothing
280 } else {
281 threshold = 300;
282 }
283 frm_mdark = cpl_frameset_find(frameset,ERIS_NIX_MASTER_DARK_IMG_PRO_CATG);
284 }
285 cpl_propertylist_delete(plist);
286 /* make sure the parameters are updated to the actual values set to have
287 * proper signature in the products */
288 param = cpl_parameterlist_find(parlist,
289 "eris.eris_persistence_monitor.saturation");
290 cpl_parameter_set_double(param, saturation);
291 param = cpl_parameterlist_find(parlist,
292 "eris.eris_persistence_monitor.threshold");
293 cpl_parameter_set_double(param, threshold);
294 cpl_image* img_persistence = NULL;
295 cpl_imagelist* iml_persistence = NULL;
296
297 /* Add the product category */
298 applist = cpl_propertylist_new();
299 cpl_propertylist_append_string(applist, CPL_DFS_PRO_CATG,
300 ERIS_PERSISTENCE_HDR);
301
302 nframes = cpl_frameset_get_size(rawframes);
303
304
305 //double nsat_avg = 0;
306 //double nsat_frac = 0;
307 int nsat_max = 0;
308 //int nsat_tot = 0;
309 int frame_nsat_max = 0;
310
311 //double nthresh_avg = 0;
312 //double nthresh_frac = 0;
313 int nthresh_max = 0;
314 //int nthresh_tot = 0;
315 int frame_nthresh_max = 0;
316
317
318 double nsat_avg_tmp = 0;
319 double nsat_frac_tmp = 0;
320 int nsat_max_tmp = 0;
321 int nsat_tot_tmp = 0;
322 //int frame_nsat_max_tmp = 0;
323
324
325 double nthresh_avg_tmp = 0;
326 double nthresh_frac_tmp = 0;
327 int nthresh_max_tmp = 0;
328 int nthresh_tot_tmp = 0;
329 //int frame_nthresh_max_tmp = 0;
330
331
332 char* keyname;
333 for(cpl_size i = 0; i < nframes; i++) {
334
335 const cpl_frame * frame = cpl_frameset_get_position_const(rawframes, i);
336 fname = cpl_frame_get_filename(frame);
337 plist = cpl_propertylist_load(fname, 0);
338 arm = cpl_propertylist_get_string(plist,FHDR_E_ARM);
339 naxis = cpl_propertylist_get_int(plist,"NAXIS");
340 cpl_propertylist_delete(plist);
341 cpl_msg_info(cpl_func,"process frame: %lld %s", i, fname);
342
343 if(naxis == 2) {
344
345 eris_get_sat_qc_for_image(fname, frm_mdark, saturation,
346 saturation_negative, threshold, i, applist);
347
348
349 keyname = cpl_sprintf("ESO QC FRM%lld SLIC0 NSAT MAX",i);
350 nsat_max_tmp = cpl_propertylist_get_int(applist, keyname);
351 cpl_free(keyname);
352
353 keyname = cpl_sprintf("ESO QC FRM%lld SLIC0 NSAT TOT",i);
354 nsat_tot_tmp += cpl_propertylist_get_int(applist, keyname);
355 cpl_free(keyname);
356
357 keyname = cpl_sprintf("ESO QC FRM%lld SLIC0 NSAT AVG",i);
358 nsat_avg_tmp += cpl_propertylist_get_int(applist, keyname);
359 cpl_free(keyname);
360
361 keyname = cpl_sprintf("ESO QC FRM%lld SLIC0 BPSAT FRAC", i);
362 nsat_frac_tmp += cpl_propertylist_get_double(applist, keyname);
363 cpl_free(keyname);
364
365 keyname = cpl_sprintf("ESO QC FRM%lld SLIC0 NTHRESH MAX",i);
366 nthresh_max_tmp = cpl_propertylist_get_int(applist, keyname);
367 cpl_free(keyname);
368
369 keyname = cpl_sprintf("ESO QC FRM%lld SLIC0 NTHRESH TOT",i);
370 nthresh_tot_tmp += cpl_propertylist_get_int(applist, keyname);
371 cpl_free(keyname);
372
373 keyname = cpl_sprintf("ESO QC FRM%lld SLIC0 NTHRESH AVG",i);
374 nthresh_avg_tmp += cpl_propertylist_get_int(applist, keyname);
375 cpl_free(keyname);
376
377 keyname = cpl_sprintf("ESO QC FRM%lld SLIC0 BPTHRESH FRAC", i);
378 nthresh_frac_tmp += cpl_propertylist_get_double(applist, keyname);
379 cpl_free(keyname);
380
381 if(nsat_max_tmp > nsat_max) {
382 nsat_max = nsat_max_tmp;
383 frame_nsat_max = i;
384 }
385
386 if(nthresh_max_tmp > nthresh_max) {
387 nthresh_max = nthresh_max_tmp;
388 frame_nthresh_max = i;
389 }
390
391
392 } else {
393
394 eris_get_sat_qc_for_cube(fname, frm_mdark, saturation, saturation_negative,
395 threshold, i, applist);
396
397 keyname = cpl_sprintf("ESO QC FRM%lld NSAT MAX", i);
398 nsat_max_tmp = cpl_propertylist_get_int(applist, keyname);
399 cpl_free(keyname);
400
401 keyname = cpl_sprintf("ESO QC FRM%lld NSAT TOT", i);
402 nsat_tot_tmp += cpl_propertylist_get_int(applist, keyname);
403 cpl_free(keyname);
404
405 keyname = cpl_sprintf("ESO QC FRM%lld NSAT AVG", i);
406 nsat_avg_tmp += cpl_propertylist_get_double(applist, keyname);
407 cpl_free(keyname);
408
409 keyname = cpl_sprintf("ESO QC FRM%lld BPSAT FRAC", i);
410 nsat_frac_tmp += cpl_propertylist_get_double(applist, keyname);
411 cpl_free(keyname);
412
413 keyname = cpl_sprintf("ESO QC FRM%lld NTHRESH MAX", i);
414 nthresh_max_tmp = cpl_propertylist_get_int(applist, keyname);
415 cpl_free(keyname);
416
417 keyname = cpl_sprintf("ESO QC FRM%lld NTHRESH TOT", i);
418 nthresh_tot_tmp += cpl_propertylist_get_int(applist, keyname);
419 cpl_free(keyname);
420
421 keyname = cpl_sprintf("ESO QC FRM%lld NTHRESH AVG", i);
422 nthresh_avg_tmp += cpl_propertylist_get_double(applist, keyname);
423 cpl_free(keyname);
424
425 keyname = cpl_sprintf("ESO QC FRM%lld BPTHRESH FRAC", i);
426 nthresh_frac_tmp += cpl_propertylist_get_double(applist, keyname);
427 cpl_free(keyname);
428
429 if(nsat_max_tmp > nsat_max) {
430 nsat_max = nsat_max_tmp;
431 frame_nsat_max = i;
432 }
433
434 if(nthresh_max_tmp > nthresh_max) {
435 nthresh_max = nthresh_max_tmp;
436 frame_nthresh_max = i;
437 }
438
439 }
440
441
442 /* HOW TO SAVE A DFS-COMPLIANT PRODUCT TO DISK */
443 if(debug) {
444 cpl_propertylist_update_string(applist, CPL_DFS_PRO_CATG,
445 ERIS_PERSISTENCE_DBG);
446 if(naxis == 2) {
447 if (cpl_dfs_save_image(frameset, NULL, parlist, frameset, NULL,
448 img_persistence, CPL_BPP_IEEE_FLOAT, ERIS_PERSISTENCE_DBG,
449 applist, NULL, PACKAGE "/" PACKAGE_VERSION,
450 ERIS_PERSISTENCE_DBG_FN)) {
451 /* Propagate the error */
452 (void)cpl_error_set_where(cpl_func);
453 }
454 } else {
455 if (cpl_dfs_save_imagelist(frameset, NULL, parlist, frameset, NULL,
456 iml_persistence, CPL_BPP_IEEE_FLOAT, ERIS_PERSISTENCE_DBG,
457 applist, NULL, PACKAGE "/" PACKAGE_VERSION,
458 ERIS_PERSISTENCE_DBG_FN)) {
459 /* Propagate the error */
460 (void)cpl_error_set_where(cpl_func);
461 }
462 }
463 }
464
465 if(naxis == 2) {
466 cpl_image_delete(img_persistence);
467 } else {
468 cpl_imagelist_delete(iml_persistence);
469 }
470
471 }// end loop over frames
472 keyname = cpl_sprintf("ESO QC NSAT MAX");
473 cpl_propertylist_append_int(applist, keyname, nsat_max);
474 cpl_free(keyname);
475
476 keyname = cpl_sprintf("ESO QC NSAT TOT");
477 cpl_propertylist_append_int(applist, keyname, nsat_tot_tmp);
478 cpl_free(keyname);
479
480 keyname = cpl_sprintf("ESO QC NSAT AVG");
481 cpl_propertylist_append_double(applist, keyname, (nsat_avg_tmp / nframes));
482 cpl_free(keyname);
483
484 keyname = cpl_sprintf("ESO QC FRM NSAT MAX");
485 cpl_propertylist_append_int(applist, keyname, frame_nsat_max);
486 cpl_free(keyname);
487
488 keyname = cpl_sprintf("ESO QC BPSAT FRAC");
489 cpl_propertylist_append_double(applist, keyname,
490 (double)(nsat_frac_tmp / nframes));
491 cpl_free(keyname);
492
493 keyname = cpl_sprintf("ESO QC NTHRESH MAX");
494 cpl_propertylist_append_int(applist, keyname, nthresh_max);
495 cpl_free(keyname);
496
497 keyname = cpl_sprintf("ESO QC NTHRESH TOT");
498 cpl_propertylist_append_int(applist, keyname, nthresh_tot_tmp);
499 cpl_free(keyname);
500
501 keyname = cpl_sprintf("ESO QC NTHRESH AVG");
502 cpl_propertylist_append_double(applist, keyname, (nthresh_avg_tmp / nframes));
503 cpl_free(keyname);
504
505 keyname = cpl_sprintf("ESO QC FRM NTHRESH MAX");
506 cpl_propertylist_append_int(applist, keyname, frame_nthresh_max);
507 cpl_free(keyname);
508
509 keyname = cpl_sprintf("ESO QC BPTHRESH FRAC");
510 cpl_propertylist_append_double(applist, keyname, (nthresh_frac_tmp / nframes));
511 cpl_free(keyname);
512
513 cpl_dfs_save_propertylist(frameset, NULL, parlist, frameset, NULL,
514 recipe, applist, NULL,
515 PACKAGE "/" PACKAGE_VERSION, ERIS_PERSISTENCE_HDR_FN);
516
517
518 cpl_propertylist_delete(applist);
519 cpl_frameset_delete(rawframes);
520
521
522 return (int)cpl_error_get_code();
523}
eris_nix_dfs_set_groups
cpl_error_code eris_nix_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.
Definition: eris_nix_dfs.c:58
Generated on Tue Jan 27 2026 21:31:13 for ERIS Pipeline Reference Manual by doxygen 1.9.6