CR2RE Pipeline Reference Manual 1.6.10
cr2res_calib.c
1/*
2 * This file is part of the CR2RES Pipeline
3 * Copyright (C) 2002,2003 European Southern Observatory
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
18 */
19
20#ifdef HAVE_CONFIG_H
21#include <config.h>
22#endif
23
24/*-----------------------------------------------------------------------------
25 Includes
26 -----------------------------------------------------------------------------*/
27
28#include <math.h>
29#include <cpl.h>
30#include "cr2res_calib.h"
31#include "cr2res_bpm.h"
32#include "cr2res_pfits.h"
33#include "cr2res_io.h"
34#include "cr2res_detlin.h"
35#include "cr2res_utils.h"
36
37/*-----------------------------------------------------------------------------
38 Functions prototypes
39 -----------------------------------------------------------------------------*/
40
41int cr2res_add_shotnoise(hdrl_image * in, double dit, int ndit, int chip);
43 const hdrl_image * flat, cpl_size degree);
44
45/*----------------------------------------------------------------------------*/
49/*----------------------------------------------------------------------------*/
50
53/*----------------------------------------------------------------------------*/
70/*----------------------------------------------------------------------------*/
71hdrl_imagelist * cr2res_calib_imagelist(
72 const hdrl_imagelist * in,
73 int chip,
74 int clean_bad,
75 int subtract_nolight_rows,
76 int subtract_interorder_column,
77 int cosmics_corr,
78 const cpl_frame * flat,
79 const cpl_frame * dark,
80 const cpl_frame * bpm,
81 const cpl_frame * detlin,
82 const cpl_vector * dits,
83 const cpl_vector * ndits)
84{
85 hdrl_imagelist * out ;
86 double dit ;
87 int ndit ;
88 cpl_size i ;
89
90 /* Check Inputs */
91 if (in == NULL) return NULL ;
92
93 /* Initialise */
94 dit = 0.0 ;
95 ndit = 1 ;
96
97 /* Create calibrated image list */
98 out = hdrl_imagelist_new() ;
99
100 /* Loop on the images */
101 for (i = 0; i < hdrl_imagelist_get_size(in); i++) {
102 const hdrl_image *cur_ima;
103 hdrl_image *cur_ima_calib;
104 cur_ima = hdrl_imagelist_get(in, i) ;
105 //if (dark != NULL) dit = cpl_vector_get(dits, i) ;
106 if (dits != NULL) dit = cpl_vector_get(dits, i) ;
107 if (ndits != NULL) ndit = (int)cpl_vector_get(ndits, i) ;
108
109 /* Calibrate */
110 cpl_msg_info(__func__,
111 "Apply calibrations for image #%"CPL_SIZE_FORMAT, i+1) ;
112 cpl_msg_indent_more() ;
113 if ((cur_ima_calib = cr2res_calib_image(cur_ima, chip, clean_bad,
114 subtract_nolight_rows, subtract_interorder_column,
115 cosmics_corr, flat, dark, bpm,
116 detlin, dit, ndit)) == NULL) {
117 cpl_msg_error(__func__, "Failed to Calibrate the Data") ;
119 cpl_msg_indent_less() ;
120 return NULL ;
121 } else {
122 /* All the calibrated image in the list */
123 hdrl_imagelist_set(out, cur_ima_calib, i);
124 }
125 cpl_msg_indent_less() ;
126 }
127 return out ;
128}
129
130/*----------------------------------------------------------------------------*/
147/*----------------------------------------------------------------------------*/
149 const hdrl_image * in,
150 int chip,
151 int clean_bad,
152 int subtract_nolight_rows,
153 int subtract_interorder_column,
154 int cosmics_corr,
155 const cpl_frame * flat,
156 const cpl_frame * dark,
157 const cpl_frame * bpm,
158 const cpl_frame * detlin,
159 double dit,
160 int ndit)
161{
162 hdrl_image * out ;
163 hdrl_image * calib ;
164 hdrl_imagelist * calib_list ;
165
166 /* Test entries */
167 if (in == NULL) return NULL ;
168 if (chip < 1 || chip > CR2RES_NB_DETECTORS) return NULL ;
169
170 /* Create out image */
171 out = hdrl_image_duplicate(in) ;
172
173 /* Clean the bad pixels */
174 if (bpm != NULL) {
175 cpl_msg_info(__func__, "Correct the bad pixels") ;
177 cpl_frame_get_filename(bpm), chip, clean_bad) != 0) {
178 cpl_msg_error(__func__, "Cannot clean the bad pixels");
180 return NULL ;
181 }
182 }
183
184 /* Apply the non linearity correction */
185 if (detlin != NULL) {
186 cpl_msg_info(__func__, "Correct for the Non-Linearity") ;
187 if ((calib_list = cr2res_io_load_DETLIN_COEFFS(
188 cpl_frame_get_filename(detlin), chip)) == NULL) {
189 cpl_msg_error(__func__, "Cannot load the detlin") ;
191 return NULL ;
192 }
193 /* Detlin correction */
194 if (cr2res_detlin_correct(out, calib_list)) {
195 hdrl_imagelist_delete(calib_list) ;
197 cpl_msg_error(__func__, "Cannot correct for the Non-Linearity") ;
198 return NULL ;
199 }
200 }
201
202 /* Add shot-noise */
203 cpl_msg_info(__func__, "Add shot-noise") ;
204 if (cr2res_add_shotnoise(out, dit, ndit, chip)){
205 cpl_msg_error(__func__, "Cannot add shot-noise") ;
206 if(detlin != NULL)
207 hdrl_imagelist_delete(calib_list) ;
209 return NULL ;
210 }
211
212 /* Apply the dark */
213 if (dark != NULL) {
214 cpl_propertylist *plist;
215 double dark_dit;
216 cpl_msg_info(__func__, "Correct for the dark") ;
217
218 /* Load the dark */
219 if ((calib = cr2res_io_load_MASTER_DARK(cpl_frame_get_filename(dark),
220 chip)) == NULL) {
221 cpl_msg_error(__func__, "Cannot load the dark") ;
223 return NULL ;
224 }
225 if (detlin != NULL) {
226 cpl_msg_indent_more() ;
227 cpl_msg_info(__func__, "Correct DARK for Non-Linearity") ;
228 if (cr2res_detlin_correct(calib, calib_list)) {
229 hdrl_imagelist_delete(calib_list) ;
230 hdrl_image_delete(calib) ;
232 cpl_msg_error(__func__,"Cannot correct DARK for Non-Linearity");
233 cpl_msg_indent_less() ;
234 return NULL ;
235 }
236 cpl_msg_indent_less() ;
237 }
238
239 /* Get the dark DIT */
240 plist = cpl_propertylist_load(cpl_frame_get_filename(dark), 0);
241 dark_dit = cr2res_pfits_get_dit(plist) ;
242 cpl_propertylist_delete(plist) ;
243 /* Multiply the dark by dit/dark_dit */
244 hdrl_value hdrl_dit_corr = {dit/dark_dit, 0.0};
245 hdrl_image_mul_scalar(calib, hdrl_dit_corr) ;
246
247 /* Subtract the dark */
248 if (hdrl_image_sub_image(out, calib) != CPL_ERROR_NONE) {
249 cpl_msg_error(__func__, "Cannot apply the dark") ;
250 hdrl_image_delete(calib) ;
252 return NULL ;
253 }
254 hdrl_image_delete(calib) ;
255 }
256 if (detlin != NULL) {
257 hdrl_imagelist_delete(calib_list) ;
258 }
259
260 /* Subtract residual bias/dark from vignetted rows at bottom */
261 if (subtract_nolight_rows) {
262 cpl_size i;
263 cpl_image *img_tmp;
264 cpl_msg_info(__func__, "Subtract median of bottom rows");
265
266 img_tmp = cpl_image_collapse_median_create(
267 hdrl_image_get_image(out), 0, CR2RES_NB_BPM_EDGEPIX,
268 CR2RES_DETECTOR_SIZE-CR2RES_NB_BPM_VIGN_BOTTOM);
269 calib = hdrl_image_new(CR2RES_DETECTOR_SIZE,CR2RES_DETECTOR_SIZE);
270 for (i=1; i<=CR2RES_DETECTOR_SIZE; i++) {
271 hdrl_image_insert(calib, img_tmp, NULL, 1, i);
272 }
273 hdrl_image_sub_image(out, calib);
274 hdrl_image_delete(calib) ;
275 cpl_image_delete(img_tmp);
276 }
277
278 /* Apply the interorder correction */
279 if (subtract_interorder_column) {
280 if (flat == NULL) {
281 cpl_msg_info(__func__,
282 "Skip subtracting fit to inter-order pixels (no flat-field)");
283 } else {
284 cpl_msg_info(__func__, "Subtract fit to inter-order pixels");
285 if ((calib = cr2res_io_load_MASTER_FLAT(
286 cpl_frame_get_filename(flat), chip)) == NULL) {
287 cpl_msg_error(__func__, "Cannot load the flat field") ;
289 return NULL ;
290 }
291 if (cr2res_calib_subtract_interorder_column(out, calib, 0)){
292 cpl_msg_error(__func__, "Could not subtract inter-order fit");
293 }
294 hdrl_image_delete(calib);
295 }
296 }
297
298 /* Apply the flatfield */
299 if (flat != NULL) {
300 cpl_msg_info(__func__, "Correct for the flat field") ;
301 /* Load the flat */
302 if ((calib = cr2res_io_load_MASTER_FLAT(
303 cpl_frame_get_filename(flat), chip)) == NULL) {
304 cpl_msg_error(__func__, "Cannot load the flat field") ;
306 return NULL ;
307 }
308
309 /* Divide */
310 if (hdrl_image_div_image(out, calib) != CPL_ERROR_NONE) {
311 cpl_msg_error(__func__, "Cannot apply the flat field") ;
312 hdrl_image_delete(calib) ;
314 return NULL ;
315 }
316 hdrl_image_delete(calib) ;
317 }
318
319
320 /* Comics correction */
321 if (cosmics_corr) {
322 cpl_msg_info(__func__, "Apply the cosmics corrections") ;
323 hdrl_parameter * cospar = hdrl_lacosmic_parameter_create(12.0,2.5,1);
324 if (hdrl_lacosmic_parameter_verify(cospar)!=CPL_ERROR_NONE)
325 cpl_msg_warning(__func__,"invalid cospar");
326 cpl_mask* cosmask = hdrl_lacosmic_edgedetect(out,cospar);
327 cpl_mask_or(hdrl_image_get_mask(out), cosmask);
328 cpl_mask_delete(cosmask);
329 hdrl_parameter_delete(cospar);
330 }
331
332 return out ;
333}
334
335/*----------------------------------------------------------------------------*/
341/*----------------------------------------------------------------------------*/
342int cr2res_add_shotnoise(hdrl_image * in, double dit, int ndit, int chip){
343 double gain_sqrt;
344 cpl_image * error = hdrl_image_get_error(in);
345 cpl_image * adu = hdrl_image_get_image(in);
346 cpl_image * tmp_im;
347
348 if (adu==NULL){
349 cpl_msg_error(__func__,"Broken input image");
350 return -1;
351 }
352 if (error==NULL){
353 cpl_msg_error(__func__,"Error input null not supported");
354 }
355
356 if (chip == 1) gain_sqrt = sqrt(CR2RES_GAIN_CHIP1);
357 else if (chip == 2) gain_sqrt = sqrt(CR2RES_GAIN_CHIP2);
358 else if (chip == 3) gain_sqrt = sqrt(CR2RES_GAIN_CHIP3);
359 else {
360 cpl_msg_error(__func__,"Unknown detector");
361 return -1;
362 }
363
364 cpl_msg_debug(__func__, "chip:%d, sqrtgain:%g, ndit:%d",
365 chip, gain_sqrt, ndit);
366
367 if ( (tmp_im=cpl_image_abs_create(adu)) == NULL){
368 cpl_msg_error(__func__,"Abs failed");
369 return -1;
370 }
371 if ( (cpl_image_power(tmp_im, 0.5)) != CPL_ERROR_NONE){
372 cpl_msg_error(__func__,"Sqrt failed");
373 return -1;
374 }
375 cpl_image_divide_scalar(tmp_im, gain_sqrt);
376 cpl_image_divide_scalar(tmp_im, sqrt((float)ndit));
377 /* Add read noise following definition given in CRIRES
378 user manual.
379 read_noise_adu = read_noise_e- /(sqrt(ndit)*gain))*/
380 double min_dit = 1.427;
381 double lim_dit = 50.0;
382 double min_rn;
383 if (chip ==1){
384 min_rn = 11.0;
385 } else {
386 min_rn = 12.0;
387 }
388 double lim_rn = 6.0;
389 if(dit <= min_dit){
390 cpl_image_add_scalar(tmp_im, min_rn/(sqrt(ndit)*pow(gain_sqrt, 2)));
391 } else if(dit >= lim_dit) {
392 cpl_image_add_scalar(tmp_im, lim_rn/(sqrt(ndit)*pow(gain_sqrt, 2)));
393 } else {
394 double read_noise = min_rn + (lim_rn-min_rn)*(dit-min_dit)/(lim_dit-min_dit);
395 cpl_image_add_scalar(tmp_im, read_noise/(sqrt(ndit)*pow(gain_sqrt, 2)));
396 }
397
398 /* The BPM should not be stored in the error image */
399 /* Therefore it is removed from tmp_im before the addition to error */
400 cpl_image_accept_all(tmp_im) ;
401
402 cpl_image_add(error, tmp_im);
403 cpl_image_delete(tmp_im);
404 return 0;
405}
406
407/*----------------------------------------------------------------------------*/
419/*----------------------------------------------------------------------------*/
421 const hdrl_image * flat, cpl_size degree)
422{
423 // Define Parameters
424 cpl_image * img;
425 const cpl_image * flat_img;
426 cpl_polynomial * poly;
427 cpl_vector * tmp;
428 cpl_size ncolumns, nrow;
429 cpl_size i, j;
430 int badpix;
431 int badcols = 0;
432 double value, median, mad;
433
434 // Check Input
435 if (in == NULL || flat == NULL) return -1;
436 if (hdrl_image_get_size_x(in) != hdrl_image_get_size_x(flat)) return -1;
437 if (hdrl_image_get_size_y(in) != hdrl_image_get_size_y(flat)) return -1;
438
439 // Loop over the columns
440 img = hdrl_image_get_image(in);
441 flat_img = hdrl_image_get_image_const(flat);
442 ncolumns = cpl_image_get_size_x(img);
443 nrow = cpl_image_get_size_y(img);
444
445
446 for (i = CR2RES_NB_BPM_EDGEPIX + 1; i < ncolumns - CR2RES_NB_BPM_EDGEPIX;
447 i++) {
448 cpl_size npixel;
449 cpl_matrix *px;
450 cpl_vector *py;
451 // Fill vectors and matrix
452 px = cpl_matrix_new(1, nrow);
453 py = cpl_vector_new(nrow);
454 npixel = 0;
455 for ( j = CR2RES_NB_BPM_VIGN_BOTTOM + 5; j < nrow -20 ; j++)
456 {
457 // Filter pixels that are rejected in the flat (i.e. between orders)
458 // but not rejected by the image (i.e. not bad pixels)
459 if (cpl_image_is_rejected(flat_img, i, j)
460 && !cpl_image_is_rejected(img, i, j) &&
461 isnan(cpl_image_get(img, i, j, &badpix)) == 0){
462 cpl_matrix_set(px, 0, npixel, j);
463 cpl_vector_set(py, npixel, cpl_image_get(img, i, j, &badpix));
464 npixel++;
465 }
466 }
467 cpl_msg_debug(__func__, "Column %lld has %lld pixels for background",
468 i,npixel);
469 if (npixel < 10){
470 badcols += 1;
471 cpl_matrix_delete(px);
472 cpl_vector_delete(py);
473 continue;
474 }
475 cpl_matrix_set_size(px, 1, npixel);
476 cpl_vector_set_size(py, npixel);
477
478 // Reject outliers and filter
479 median = cr2res_vector_get_mad(py, &mad);
480 for (j=0;j<npixel;j++){
481 // Filtering for the range: median - 1*median to median + 1*median
482 if ((cpl_vector_get(py,j) > median*2) || (cpl_vector_get(py,j) < 0.0) )
483 cpl_vector_set(py,j,median);
484 }
485 //tmp = cpl_vector_filter_median_create(py,50);
486 tmp = cpl_vector_filter_lowpass_create(py,CPL_LOWPASS_LINEAR,100);
487 cpl_vector_delete(py);
488 py=tmp;
489 if (cpl_msg_get_level() == CPL_MSG_DEBUG){
490 tmp = cpl_vector_wrap(npixel, cpl_matrix_get_data(px));
491 cpl_vector_save(tmp, "debug_background.fits",
492 CPL_TYPE_DOUBLE, NULL, CPL_IO_CREATE);
493 cpl_vector_save(py, "debug_background.fits",
494 CPL_TYPE_DOUBLE, NULL, CPL_IO_EXTEND);
495 cpl_vector_unwrap(tmp);
496 }
497
498 // Fit polynomial
499 poly = cpl_polynomial_new(1);
500 if (cpl_polynomial_fit(poly, px, NULL, py, NULL, CPL_FALSE, NULL,
501 &degree)
502 != CPL_ERROR_NONE) {
503 badcols += 1;
504 cpl_msg_warning(__func__,
505 "Could not fit the background of column %"CPL_SIZE_FORMAT, i);
506 cpl_error_reset();
507 cpl_matrix_delete(px);
508 cpl_vector_delete(py);
509 cpl_polynomial_delete(poly);
510 continue;
511 }
512
513 // Subtract column
514 for ( j = 1; j < nrow + 1; j++)
515 {
516 value = cpl_image_get(img, i, j, &badpix);
517 // Just skip this pixel if it is bad
518 if (badpix) continue;
519 value -= cpl_polynomial_eval_1d(poly, j, NULL);
520 cpl_image_set(img, i, j, value);
521 }
522
523 // Clean up
524 cpl_matrix_delete(px);
525 cpl_vector_delete(py);
526 cpl_polynomial_delete(poly);
527 }
528
529 if (badcols){
530 cpl_msg_warning(__func__,
531 "%d columns could not be background-subtracted from inter-order gaps",
532 badcols);
533 }
534 return 0;
535}
536
int cr2res_bpm_set_and_correct_image(cpl_image *in, const char *bpm, int chip, int correct)
Set the BPM and optionally apply the correction to an image.
Definition: cr2res_bpm.c:199
hdrl_image * cr2res_calib_image(const hdrl_image *in, int chip, int clean_bad, int subtract_nolight_rows, int subtract_interorder_column, int cosmics_corr, const cpl_frame *flat, const cpl_frame *dark, const cpl_frame *bpm, const cpl_frame *detlin, double dit, int ndit)
The images calibration routine for a given chip.
Definition: cr2res_calib.c:148
int cr2res_add_shotnoise(hdrl_image *in, double dit, int ndit, int chip)
Add shot-noise to errors in HDRL-image.
Definition: cr2res_calib.c:342
hdrl_imagelist * cr2res_calib_imagelist(const hdrl_imagelist *in, int chip, int clean_bad, int subtract_nolight_rows, int subtract_interorder_column, int cosmics_corr, const cpl_frame *flat, const cpl_frame *dark, const cpl_frame *bpm, const cpl_frame *detlin, const cpl_vector *dits, const cpl_vector *ndits)
The images calibration routine for a given chip on a list.
Definition: cr2res_calib.c:71
int cr2res_calib_subtract_interorder_column(hdrl_image *in, const hdrl_image *flat, cpl_size degree)
Remove background column by column.
Definition: cr2res_calib.c:420
int cr2res_detlin_correct(hdrl_image *in, const hdrl_imagelist *detlin)
Apply the detector linearity correction.
Definition: cr2res_detlin.c:71
hdrl_imagelist * cr2res_io_load_DETLIN_COEFFS(const char *filename, int detector)
Load the detlin coefficients.
Definition: cr2res_io.c:1034
hdrl_image * cr2res_io_load_MASTER_DARK(const char *filename, int detector)
Load an image from a MASTER_DARK.
Definition: cr2res_io.c:990
hdrl_image * cr2res_io_load_MASTER_FLAT(const char *filename, int detector)
Load an hdrl image from a MASTER_FLAT.
Definition: cr2res_io.c:1066
double cr2res_pfits_get_dit(const cpl_propertylist *plist)
find out the DIT value
Definition: cr2res_pfits.c:199
double cr2res_vector_get_mad(cpl_vector *invec, double *mad)
Get MAD from a vector.
Definition: cr2res_utils.c:212
cpl_error_code hdrl_image_sub_image(hdrl_image *self, const hdrl_image *other)
Subtract two images, store the result in the first image.
cpl_error_code hdrl_image_div_image(hdrl_image *self, const hdrl_image *other)
Divide two images, store the result in the first image.
cpl_error_code hdrl_image_mul_scalar(hdrl_image *self, hdrl_value value)
Elementwise multiplication of an image with a scalar.
hdrl_image * hdrl_image_duplicate(const hdrl_image *himg)
copy hdrl_image
Definition: hdrl_image.c:391
cpl_mask * hdrl_image_get_mask(hdrl_image *himg)
get cpl bad pixel mask from image
Definition: hdrl_image.c:157
cpl_image * hdrl_image_get_error(hdrl_image *himg)
get error as cpl image
Definition: hdrl_image.c:131
cpl_size hdrl_image_get_size_y(const hdrl_image *self)
return size of Y dimension of image
Definition: hdrl_image.c:540
cpl_size hdrl_image_get_size_x(const hdrl_image *self)
return size of X dimension of image
Definition: hdrl_image.c:525
cpl_error_code hdrl_image_insert(hdrl_image *self, const cpl_image *image, const cpl_image *error, cpl_size xpos, cpl_size ypos)
Copy cpl images into an hdrl image.
Definition: hdrl_image.c:715
cpl_image * hdrl_image_get_image(hdrl_image *himg)
get data as cpl image
Definition: hdrl_image.c:105
const cpl_image * hdrl_image_get_image_const(const hdrl_image *himg)
get data as cpl image
Definition: hdrl_image.c:118
hdrl_image * hdrl_image_new(cpl_size nx, cpl_size ny)
create new zero filled hdrl image
Definition: hdrl_image.c:311
void hdrl_image_delete(hdrl_image *himg)
delete hdrl_image
Definition: hdrl_image.c:379
cpl_error_code hdrl_imagelist_set(hdrl_imagelist *himlist, hdrl_image *himg, cpl_size pos)
Insert an image into an imagelist.
void hdrl_imagelist_delete(hdrl_imagelist *himlist)
Free all memory used by a hdrl_imagelist object including the images.
cpl_size hdrl_imagelist_get_size(const hdrl_imagelist *himlist)
Get the number of images in the imagelist.
hdrl_imagelist * hdrl_imagelist_new(void)
Create an empty imagelist.
hdrl_image * hdrl_imagelist_get(const hdrl_imagelist *himlist, cpl_size inum)
Get an image from a list of images.
cpl_mask * hdrl_lacosmic_edgedetect(const hdrl_image *ima_in, const hdrl_parameter *params)
Detect bad-pixels / cosmic-rays on a single image.
cpl_error_code hdrl_lacosmic_parameter_verify(const hdrl_parameter *param)
Verify basic correctness of the LaCosmic parameters.
hdrl_parameter * hdrl_lacosmic_parameter_create(double sigma_lim, double f_lim, int max_iter)
Creates LaCosmic parameters object.
void hdrl_parameter_delete(hdrl_parameter *obj)
shallow delete of a parameter