37 #include "crires_recipe.h"
38 #include "irplib_utils.h"
40 #include "crires_combine.h"
46 #define RECIPE_STRING "crires_spec_flat"
52 static cpl_imagelist * crires_spec_flat_reduce(cpl_frameset *,
const char *,
54 static cpl_imagelist * crires_spec_flat_bpm(cpl_imagelist *,
double,
56 static int crires_spec_flat_save(
const cpl_imagelist *,
const cpl_imagelist *,
57 int, cpl_frameset *,
const cpl_parameterlist *, cpl_frameset *) ;
58 static int crires_spec_flat_compare(
const cpl_frame *,
const cpl_frame *) ;
60 static char crires_spec_flat_description[] =
61 "crires_spec_flat -- Flat-field recipe\n"
62 "The files listed in the Set Of Frames (sof-file) must be tagged:\n"
63 "raw-file.fits "CRIRES_SPEC_FLAT_RAW
" or\n"
64 "dark-file.fits "CRIRES_CALPRO_DARK
" or\n"
65 "detlin-file.fits "CRIRES_CALPRO_COEFFS_CUBE
".\n" ;
67 CRIRES_RECIPE_DEFINE(crires_spec_flat,
68 CRIRES_PARAM_THRESHOLDS |
69 CRIRES_PARAM_BPM_RATE |
70 CRIRES_PARAM_REPLACE |
71 CRIRES_PARAM_CLEAN_FLAT |
72 CRIRES_PARAM_NORM_STARTY |
73 CRIRES_PARAM_NORM_STOPY |
74 CRIRES_PARAM_KAPPA_SIGCLIP |
75 CRIRES_PARAM_COLLAPSE_METH,
77 crires_spec_flat_description) ;
87 double bpm_lines_ratio ;
92 double kappa_sigclip ;
93 crires_collapse_method coll_meth ;
95 crires_illum_period period ;
96 int bpm_nb[CRIRES_NB_DETECTORS] ;
97 double flat_mean[CRIRES_NB_DETECTORS] ;
98 double flat_stdev[CRIRES_NB_DETECTORS] ;
99 double flat_flux[CRIRES_NB_DETECTORS] ;
100 double flat_master_rms[CRIRES_NB_DETECTORS] ;
101 } crires_spec_flat_config ;
115 static int crires_spec_flat(
116 cpl_frameset * frameset,
117 const cpl_parameterlist * parlist)
122 cpl_frameset * rawframes ;
124 const char * detlin ;
125 cpl_imagelist * flat ;
126 cpl_frameset * flat_one ;
127 cpl_imagelist * bpm ;
131 setlocale(LC_NUMERIC,
"C");
135 for (i=0 ; i<CRIRES_NB_DETECTORS ; i++) {
136 crires_spec_flat_config.bpm_nb[i] = -1 ;
137 crires_spec_flat_config.flat_mean[i] = -1.0 ;
138 crires_spec_flat_config.flat_stdev[i] = -1.0 ;
139 crires_spec_flat_config.flat_flux[i] = -1.0 ;
140 crires_spec_flat_config.flat_master_rms[i] = -1.0 ;
144 sval = crires_parameterlist_get_string(parlist, RECIPE_STRING,
145 CRIRES_PARAM_THRESHOLDS) ;
147 if (sscanf(sval,
"%lg,%lg",
148 &crires_spec_flat_config.bpm_low,
149 &crires_spec_flat_config.bpm_high)!=2) {
152 crires_spec_flat_config.replace_flag = crires_parameterlist_get_bool(
153 parlist, RECIPE_STRING, CRIRES_PARAM_REPLACE) ;
154 crires_spec_flat_config.clean_flat_flag = crires_parameterlist_get_bool(
155 parlist, RECIPE_STRING, CRIRES_PARAM_CLEAN_FLAT) ;
156 crires_spec_flat_config.bpm_lines_ratio = crires_parameterlist_get_double(
157 parlist, RECIPE_STRING, CRIRES_PARAM_BPM_RATE) ;
158 crires_spec_flat_config.starty = crires_parameterlist_get_int(parlist,
159 RECIPE_STRING, CRIRES_PARAM_NORM_STARTY) ;
160 crires_spec_flat_config.stopy = crires_parameterlist_get_int(parlist,
161 RECIPE_STRING, CRIRES_PARAM_NORM_STOPY) ;
162 crires_spec_flat_config.kappa_sigclip = crires_parameterlist_get_double(
163 parlist, RECIPE_STRING, CRIRES_PARAM_KAPPA_SIGCLIP) ;
164 sval = crires_parameterlist_get_string(parlist, RECIPE_STRING,
165 CRIRES_PARAM_COLLAPSE_METH) ;
166 if (!strcmp(sval,
"avg"))
167 crires_spec_flat_config.coll_meth = CRIRES_COLLAPSE_AVG ;
168 else if (!strcmp(sval,
"med"))
169 crires_spec_flat_config.coll_meth = CRIRES_COLLAPSE_MED ;
170 else if (!strcmp(sval,
"sig"))
171 crires_spec_flat_config.coll_meth = CRIRES_COLLAPSE_SIG ;
173 cpl_msg_error(__func__,
"Invalid collapse method specified");
178 if (crires_dfs_set_groups(frameset,
"crires_spec_flat")) {
179 cpl_msg_error(__func__,
"Cannot identify RAW and CALIB frames") ;
184 dark = crires_extract_filename(frameset, CRIRES_CALPRO_DARK) ;
185 detlin = crires_extract_filename(frameset, CRIRES_CALPRO_COEFFS_CUBE) ;
188 if ((rawframes = crires_extract_frameset(frameset,
189 CRIRES_SPEC_FLAT_RAW)) == NULL) {
190 cpl_msg_error(__func__,
"No raw frame in input") ;
195 crires_spec_flat_config.period =
196 crires_get_detector_illum_period(
197 cpl_frame_get_filename(cpl_frameset_get_position(rawframes, 0))) ;
198 if (crires_spec_flat_config.period == CRIRES_ILLUM_UNKNOWN) {
199 cpl_msg_error(__func__,
200 "Cannot determine the detector illumination period") ;
201 cpl_frameset_delete(rawframes) ;
204 crires_display_detector_illum(crires_spec_flat_config.period) ;
208 if ((labels = cpl_frameset_labelise(rawframes, crires_spec_flat_compare,
209 &nlabels)) == NULL) {
210 cpl_msg_error(__func__,
"Cannot labelise input frames") ;
211 cpl_frameset_delete(rawframes) ;
216 for (i=0 ; i<(int)nlabels ; i++) {
218 cpl_msg_info(__func__,
"Reduce data set %d / %"CPL_SIZE_FORMAT,
220 cpl_msg_indent_more() ;
221 flat_one = cpl_frameset_extract(rawframes, labels, (cpl_size)i) ;
222 flat = crires_spec_flat_reduce(flat_one, dark, detlin) ;
223 cpl_msg_indent_less() ;
226 cpl_msg_info(__func__,
"Save the products") ;
227 cpl_msg_indent_more() ;
229 cpl_msg_warning(__func__,
"Cannot reduce set nb %d", i+1) ;
231 if ((bpm = crires_spec_flat_bpm(flat,
232 crires_spec_flat_config.bpm_low,
233 crires_spec_flat_config.bpm_high,
234 crires_spec_flat_config.bpm_lines_ratio)) == NULL) {
235 cpl_msg_warning(__func__,
"Cannot create bad pixels map") ;
237 crires_spec_flat_save(flat, bpm, i+1, flat_one, parlist, frameset) ;
238 cpl_imagelist_delete(flat) ;
239 cpl_imagelist_delete(bpm) ;
241 cpl_msg_indent_less() ;
242 cpl_frameset_delete(flat_one) ;
244 cpl_frameset_delete(rawframes) ;
248 if (cpl_error_get_code())
return -1 ;
261 static cpl_imagelist * crires_spec_flat_reduce(
262 cpl_frameset * flatframes,
266 cpl_propertylist * plist ;
267 cpl_frame * ref_frame ;
270 double dit_frame, dit_dark ;
272 cpl_imagelist * out ;
273 cpl_vector * medians ;
276 int nx, ny, starty, stopy, ly ;
280 if (flatframes == NULL)
return NULL ;
283 nframes = cpl_frameset_get_size(flatframes) ;
287 ref_frame = cpl_frameset_get_position(flatframes, 0) ;
288 fname = cpl_frame_get_filename(ref_frame) ;
289 if ((plist=cpl_propertylist_load(cpl_frame_get_filename(ref_frame),
291 cpl_msg_error(__func__,
"Getting header from RAW file");
292 cpl_msg_indent_less() ;
295 dit_frame = crires_pfits_get_dit(plist) ;
296 cpl_propertylist_delete(plist) ;
297 if (cpl_error_get_code()) {
298 cpl_msg_error(__func__,
"Cannot get the DIT from RAW file") ;
299 cpl_msg_indent_less() ;
302 cpl_msg_info(__func__,
"DIT value: %g sec.", dit_frame) ;
306 cpl_msg_info(__func__,
"Verify the dark DIT") ;
307 cpl_msg_indent_more() ;
308 if ((plist=cpl_propertylist_load(dark, 0)) == NULL) {
309 cpl_msg_error(__func__,
"Getting header from DARK");
310 cpl_msg_indent_less() ;
313 dit_dark = crires_pfits_get_dit(plist) ;
314 cpl_propertylist_delete(plist) ;
315 if (cpl_error_get_code()) {
316 cpl_msg_error(__func__,
"Cannot get the DIT from DARK") ;
317 cpl_msg_indent_less() ;
320 if (fabs(dit_dark-dit_frame) > 1e-5) {
321 cpl_msg_error(__func__,
"Mismatch RAW DIT (%g) / DARK DIT (%g)",
322 dit_frame, dit_dark) ;
323 cpl_msg_indent_less() ;
326 cpl_msg_indent_less() ;
330 out = cpl_imagelist_new() ;
333 for (i=0 ; i<CRIRES_NB_DETECTORS ; i++) {
334 cpl_msg_info(__func__,
"Compute the MASTER FLAT for chip nb %d", i+1) ;
335 cpl_msg_indent_more() ;
338 in = crires_load_frameset(flatframes, crires_spec_flat_config.period,
339 i+1, CPL_TYPE_FLOAT) ;
340 nx = cpl_image_get_size_x(cpl_imagelist_get(in, 0)) ;
341 ny = cpl_image_get_size_y(cpl_imagelist_get(in, 0)) ;
344 if (crires_calib_chip_list(in, crires_spec_flat_config.period, i+1,
345 NULL, dark, NULL, detlin, dit_frame, -1, -1, -1)) {
346 cpl_msg_error(__func__,
"Cannot apply the calibrations") ;
347 cpl_imagelist_delete(in) ;
352 medians = cpl_vector_new(nframes) ;
356 ly = crires_get_detector_ly1(crires_spec_flat_config.period) ;
358 ly = crires_get_detector_ly2(crires_spec_flat_config.period) ;
360 ly = crires_get_detector_ly3(crires_spec_flat_config.period) ;
362 ly = crires_get_detector_ly4(crires_spec_flat_config.period) ;
366 starty = crires_spec_flat_config.starty ;
372 stopy = crires_spec_flat_config.stopy ;
380 cpl_msg_info(__func__,
"Normalise with the median") ;
381 for (j=0 ; j<nframes ; j++) {
382 median = cpl_image_get_median_window(cpl_imagelist_get(in, j),
383 1, starty, nx, stopy) ;
384 if (cpl_error_get_code()) {
385 cpl_msg_error(__func__,
"Invalid bounds Y : %d-%d", ly,
387 cpl_imagelist_delete(in) ;
388 cpl_vector_delete(medians) ;
391 cpl_vector_set(medians, j, median) ;
392 cpl_image_divide_scalar(cpl_imagelist_get(in, j), median) ;
396 crires_spec_flat_config.flat_mean[i] = cpl_vector_get_mean(medians) ;
397 if (cpl_vector_get_size(medians) > 1)
398 crires_spec_flat_config.flat_stdev[i]=cpl_vector_get_stdev(medians);
400 crires_spec_flat_config.flat_stdev[i] = -1.0 ;
401 crires_spec_flat_config.flat_flux[i] =
402 crires_spec_flat_config.flat_mean[i] / dit_frame ;
405 if ((ima = crires_combine_collapse_imagelist(in, medians,
406 crires_spec_flat_config.kappa_sigclip,
408 crires_spec_flat_config.coll_meth)) == NULL) {
409 cpl_msg_error(__func__,
"Cannot average the flats") ;
410 cpl_imagelist_delete(in) ;
411 cpl_vector_delete(medians) ;
414 cpl_vector_delete(medians) ;
415 cpl_imagelist_delete(in) ;
418 crires_spec_flat_config.flat_master_rms[i] =
419 cpl_image_get_stdev(ima) ;
422 cpl_imagelist_set(out, ima, i) ;
423 cpl_msg_indent_less() ;
439 static cpl_imagelist * crires_spec_flat_bpm(
440 cpl_imagelist * flat,
443 double bad_per_line_limit)
445 cpl_imagelist * bpm ;
447 cpl_image * bpm_cur ;
448 cpl_mask * mask_cur ;
449 cpl_binary * pmask_cur ;
450 int nx, ny, cur_bp_nb ;
454 if (flat == NULL)
return NULL ;
457 nima = cpl_imagelist_get_size(flat) ;
460 bpm = cpl_imagelist_new() ;
463 for (i=0 ; i<nima ; i++) {
465 if ((mask_cur = cpl_mask_threshold_image_create(
466 cpl_imagelist_get(flat, i), low, high)) == NULL) {
467 cpl_msg_error(__func__,
"Cannot create bad pixels map") ;
468 cpl_imagelist_delete(bpm) ;
471 cpl_mask_not(mask_cur) ;
479 nx = cpl_mask_get_size_x(mask_cur) ;
480 ny = cpl_mask_get_size_y(mask_cur) ;
481 pmask_cur = cpl_mask_get_data(mask_cur) ;
482 for (j=0 ; j<ny ; j++) {
483 cur_bp_nb = cpl_mask_count_window(mask_cur, 1, j+1, nx, j+1) ;
485 if (cur_bp_nb > bad_per_line_limit * nx) {
487 for (k=0 ; k<nx ; k++) {
488 pmask_cur[k+j*nx] = CPL_BINARY_0 ;
494 bpm_cur = cpl_image_new_from_mask(mask_cur) ;
495 crires_spec_flat_config.bpm_nb[i] = cpl_mask_count(mask_cur) ;
496 cpl_imagelist_set(bpm, bpm_cur, i) ;
499 if (crires_spec_flat_config.clean_flat_flag) {
500 cpl_image_reject_from_mask(cpl_imagelist_get(flat, i), mask_cur) ;
501 cpl_detector_interpolate_rejected(cpl_imagelist_get(flat, i)) ;
503 cpl_mask_delete(mask_cur) ;
506 if (crires_spec_flat_config.replace_flag) {
507 cpl_image_threshold(cpl_imagelist_get(flat, i),
508 low, high, 1.0, 1.0) ;
525 static int crires_spec_flat_save(
526 const cpl_imagelist * flat,
527 const cpl_imagelist * bpm,
530 const cpl_parameterlist * parlist,
531 cpl_frameset * set_tot)
533 cpl_propertylist ** qclists ;
534 const cpl_frame * ref_frame ;
536 cpl_propertylist * inputlist ;
537 const char * recipe_name =
"crires_spec_flat" ;
541 ref_frame = irplib_frameset_get_first_from_group(set, CPL_FRAME_GROUP_RAW) ;
544 qclists = cpl_malloc(CRIRES_NB_DETECTORS *
sizeof(cpl_propertylist*)) ;
545 for (i=0 ; i<CRIRES_NB_DETECTORS ; i++) {
546 qclists[i] = cpl_propertylist_new() ;
547 cpl_propertylist_append_int(qclists[i],
"ESO QC NBBAD",
548 crires_spec_flat_config.bpm_nb[i]) ;
549 cpl_propertylist_append_double(qclists[i],
"ESO QC FLAT MEAN",
550 crires_spec_flat_config.flat_mean[i]) ;
551 cpl_propertylist_append_double(qclists[i],
"ESO QC FLAT STDEV",
552 crires_spec_flat_config.flat_stdev[i]) ;
553 cpl_propertylist_append_double(qclists[i],
"ESO QC FLAT FLUX",
554 crires_spec_flat_config.flat_flux[i]) ;
555 cpl_propertylist_append_double(qclists[i],
"ESO QC FLAT MASTER RMS",
556 crires_spec_flat_config.flat_master_rms[i]) ;
559 inputlist = cpl_propertylist_load_regexp(
560 cpl_frame_get_filename(ref_frame), i+1,
561 CRIRES_HEADER_EXT_FORWARD, 0) ;
562 cpl_propertylist_copy_property_regexp(qclists[i], inputlist,
563 CRIRES_HEADER_EXT_FORWARD, 0) ;
564 cpl_propertylist_delete(inputlist) ;
568 filename = cpl_sprintf(
"%s_set%02d.fits", recipe_name, set_nb) ;
569 crires_image_save(set_tot,
576 crires_spec_flat_config.period,
578 (
const cpl_propertylist**)qclists,
579 PACKAGE
"/" PACKAGE_VERSION,
585 filename = cpl_sprintf(
"%s_set%02d_bpm.fits", recipe_name, set_nb) ;
586 crires_image_save(set_tot,
593 crires_spec_flat_config.period,
595 (
const cpl_propertylist**)qclists,
596 PACKAGE
"/" PACKAGE_VERSION,
602 for (i=0 ; i<CRIRES_NB_DETECTORS ; i++) {
603 cpl_propertylist_delete(qclists[i]) ;
617 static int crires_spec_flat_compare(
618 const cpl_frame * frame1,
619 const cpl_frame * frame2)
622 cpl_propertylist * plist1 ;
623 cpl_propertylist * plist2 ;
624 double dval1, dval2 ;
627 if (frame1==NULL || frame2==NULL)
return -1 ;
630 if ((plist1=cpl_propertylist_load(cpl_frame_get_filename(frame1),0))==NULL){
631 cpl_msg_error(__func__,
"getting header from reference frame");
634 if ((plist2=cpl_propertylist_load(cpl_frame_get_filename(frame2),0))==NULL){
635 cpl_msg_error(__func__,
"getting header from reference frame");
636 cpl_propertylist_delete(plist1) ;
641 if (cpl_error_get_code()) {
642 cpl_propertylist_delete(plist1) ;
643 cpl_propertylist_delete(plist2) ;
650 dval1 = crires_pfits_get_dit(plist1) ;
651 dval2 = crires_pfits_get_dit(plist2) ;
652 if (cpl_error_get_code()) {
653 cpl_msg_error(__func__,
"Cannot get the DIT");
654 cpl_propertylist_delete(plist1) ;
655 cpl_propertylist_delete(plist2) ;
658 if (fabs(dval1-dval2) > 1e-3) comparison = 0 ;
661 dval1 = crires_pfits_get_refwlen(plist1) ;
662 dval2 = crires_pfits_get_refwlen(plist2) ;
663 if (cpl_error_get_code()) {
664 cpl_msg_error(__func__,
"Cannot get the reference wavelength");
665 cpl_propertylist_delete(plist1) ;
666 cpl_propertylist_delete(plist2) ;
669 if (fabs(dval1-dval2) > 1e-3) comparison = 0 ;
672 dval1 = crires_pfits_get_bafflepos(plist1) ;
673 dval2 = crires_pfits_get_bafflepos(plist2) ;
674 if (cpl_error_get_code()) {
675 cpl_msg_error(__func__,
"Cannot get the baffle position");
676 cpl_propertylist_delete(plist1) ;
677 cpl_propertylist_delete(plist2) ;
680 if (fabs(dval1-dval2) > 1e-3) comparison = 0 ;
682 cpl_propertylist_delete(plist1) ;
683 cpl_propertylist_delete(plist2) ;