CR2RE Pipeline Reference Manual 1.6.2
cr2res_calib.c
1/*
2 * This file is part of the CR2RES Pipeline
3 * Copyright (C) 2002,2003 European Southern Observatory
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
18 */
19
20#ifdef HAVE_CONFIG_H
21#include <config.h>
22#endif
23
24/*-----------------------------------------------------------------------------
25 Includes
26 -----------------------------------------------------------------------------*/
27
28#include <math.h>
29#include <cpl.h>
30#include "cr2res_calib.h"
31#include "cr2res_bpm.h"
32#include "cr2res_pfits.h"
33#include "cr2res_io.h"
34#include "cr2res_detlin.h"
35#include "cr2res_utils.h"
36
37/*-----------------------------------------------------------------------------
38 Functions prototypes
39 -----------------------------------------------------------------------------*/
40
41int cr2res_add_shotnoise(hdrl_image * in, int ndit, int chip);
43 const hdrl_image * flat, cpl_size degree);
44
45/*----------------------------------------------------------------------------*/
49/*----------------------------------------------------------------------------*/
50
53/*----------------------------------------------------------------------------*/
70/*----------------------------------------------------------------------------*/
71hdrl_imagelist * cr2res_calib_imagelist(
72 const hdrl_imagelist * in,
73 int chip,
74 int clean_bad,
75 int subtract_nolight_rows,
76 int subtract_interorder_column,
77 int cosmics_corr,
78 const cpl_frame * flat,
79 const cpl_frame * dark,
80 const cpl_frame * bpm,
81 const cpl_frame * detlin,
82 const cpl_vector * dits,
83 const cpl_vector * ndits)
84{
85 hdrl_imagelist * out ;
86 double dit ;
87 int ndit ;
88 cpl_size i ;
89
90 /* Check Inputs */
91 if (in == NULL) return NULL ;
92
93 /* Initialise */
94 dit = 0.0 ;
95 ndit = 1 ;
96
97 /* Create calibrated image list */
98 out = hdrl_imagelist_new() ;
99
100 /* Loop on the images */
101 for (i = 0; i < hdrl_imagelist_get_size(in); i++) {
102 const hdrl_image *cur_ima;
103 hdrl_image *cur_ima_calib;
104 cur_ima = hdrl_imagelist_get(in, i) ;
105 if (dark != NULL) dit = cpl_vector_get(dits, i) ;
106 if (ndits != NULL) ndit = (int)cpl_vector_get(ndits, i) ;
107
108 /* Calibrate */
109 cpl_msg_info(__func__,
110 "Apply calibrations for image #%"CPL_SIZE_FORMAT, i+1) ;
111 cpl_msg_indent_more() ;
112 if ((cur_ima_calib = cr2res_calib_image(cur_ima, chip, clean_bad,
113 subtract_nolight_rows, subtract_interorder_column,
114 cosmics_corr, flat, dark, bpm,
115 detlin, dit, ndit)) == NULL) {
116 cpl_msg_error(__func__, "Failed to Calibrate the Data") ;
118 cpl_msg_indent_less() ;
119 return NULL ;
120 } else {
121 /* All the calibrated image in the list */
122 hdrl_imagelist_set(out, cur_ima_calib, i);
123 }
124 cpl_msg_indent_less() ;
125 }
126 return out ;
127}
128
129/*----------------------------------------------------------------------------*/
146/*----------------------------------------------------------------------------*/
148 const hdrl_image * in,
149 int chip,
150 int clean_bad,
151 int subtract_nolight_rows,
152 int subtract_interorder_column,
153 int cosmics_corr,
154 const cpl_frame * flat,
155 const cpl_frame * dark,
156 const cpl_frame * bpm,
157 const cpl_frame * detlin,
158 double dit,
159 int ndit)
160{
161 hdrl_image * out ;
162 hdrl_image * calib ;
163 hdrl_imagelist * calib_list ;
164
165 /* Test entries */
166 if (in == NULL) return NULL ;
167 if (chip < 1 || chip > CR2RES_NB_DETECTORS) return NULL ;
168
169 /* Create out image */
170 out = hdrl_image_duplicate(in) ;
171
172 /* Clean the bad pixels */
173 if (bpm != NULL) {
174 cpl_msg_info(__func__, "Correct the bad pixels") ;
176 cpl_frame_get_filename(bpm), chip, clean_bad) != 0) {
177 cpl_msg_error(__func__, "Cannot clean the bad pixels");
179 return NULL ;
180 }
181 }
182
183 /* Apply the non linearity correction */
184 if (detlin != NULL) {
185 cpl_msg_info(__func__, "Correct for the Non-Linearity") ;
186 if ((calib_list = cr2res_io_load_DETLIN_COEFFS(
187 cpl_frame_get_filename(detlin), chip)) == NULL) {
188 cpl_msg_error(__func__, "Cannot load the detlin") ;
190 return NULL ;
191 }
192 /* Detlin correction */
193 if (cr2res_detlin_correct(out, calib_list)) {
194 hdrl_imagelist_delete(calib_list) ;
196 cpl_msg_error(__func__, "Cannot correct for the Non-Linearity") ;
197 return NULL ;
198 }
199 }
200
201 /* Add shot-noise */
202 cpl_msg_info(__func__, "Add shot-noise") ;
203 if (cr2res_add_shotnoise(out, ndit, chip)){
204 cpl_msg_error(__func__, "Cannot add shot-noise") ;
205 if(detlin != NULL)
206 hdrl_imagelist_delete(calib_list) ;
208 return NULL ;
209 }
210
211 /* Apply the dark */
212 if (dark != NULL) {
213 cpl_propertylist *plist;
214 double dark_dit;
215 cpl_msg_info(__func__, "Correct for the dark") ;
216
217 /* Load the dark */
218 if ((calib = cr2res_io_load_MASTER_DARK(cpl_frame_get_filename(dark),
219 chip)) == NULL) {
220 cpl_msg_error(__func__, "Cannot load the dark") ;
222 return NULL ;
223 }
224 if (detlin != NULL) {
225 cpl_msg_indent_more() ;
226 cpl_msg_info(__func__, "Correct DARK for Non-Linearity") ;
227 if (cr2res_detlin_correct(calib, calib_list)) {
228 hdrl_imagelist_delete(calib_list) ;
229 hdrl_image_delete(calib) ;
231 cpl_msg_error(__func__,"Cannot correct DARK for Non-Linearity");
232 cpl_msg_indent_less() ;
233 return NULL ;
234 }
235 cpl_msg_indent_less() ;
236 }
237
238 /* Get the dark DIT */
239 plist = cpl_propertylist_load(cpl_frame_get_filename(dark), 0);
240 dark_dit = cr2res_pfits_get_dit(plist) ;
241 cpl_propertylist_delete(plist) ;
242 /* Multiply the dark by dit/dark_dit */
243 hdrl_value hdrl_dit_corr = {dit/dark_dit, 0.0};
244 hdrl_image_mul_scalar(calib, hdrl_dit_corr) ;
245
246 /* Subtract the dark */
247 if (hdrl_image_sub_image(out, calib) != CPL_ERROR_NONE) {
248 cpl_msg_error(__func__, "Cannot apply the dark") ;
249 hdrl_image_delete(calib) ;
251 return NULL ;
252 }
253 hdrl_image_delete(calib) ;
254 }
255 if (detlin != NULL) {
256 hdrl_imagelist_delete(calib_list) ;
257 }
258
259 /* Subtract residual bias/dark from vignetted rows at bottom */
260 if (subtract_nolight_rows) {
261 cpl_size i;
262 cpl_image *img_tmp;
263 cpl_msg_info(__func__, "Subtract median of bottom rows");
264
265 img_tmp = cpl_image_collapse_median_create(
266 hdrl_image_get_image(out), 0, CR2RES_NB_BPM_EDGEPIX,
267 CR2RES_DETECTOR_SIZE-CR2RES_NB_BPM_VIGN_BOTTOM);
268 calib = hdrl_image_new(CR2RES_DETECTOR_SIZE,CR2RES_DETECTOR_SIZE);
269 for (i=1; i<=CR2RES_DETECTOR_SIZE; i++) {
270 hdrl_image_insert(calib, img_tmp, NULL, 1, i);
271 }
272 hdrl_image_sub_image(out, calib);
273 hdrl_image_delete(calib) ;
274 cpl_image_delete(img_tmp);
275 }
276
277 /* Apply the interorder correction */
278 if (subtract_interorder_column) {
279 if (flat == NULL) {
280 cpl_msg_info(__func__,
281 "Skip subtracting fit to inter-order pixels (no flat-field)");
282 } else {
283 cpl_msg_info(__func__, "Subtract fit to inter-order pixels");
284 if ((calib = cr2res_io_load_MASTER_FLAT(
285 cpl_frame_get_filename(flat), chip)) == NULL) {
286 cpl_msg_error(__func__, "Cannot load the flat field") ;
288 return NULL ;
289 }
290 if (cr2res_calib_subtract_interorder_column(out, calib, 0)){
291 cpl_msg_error(__func__, "Could not subtract inter-order fit");
292 }
293 hdrl_image_delete(calib);
294 }
295 }
296
297 /* Apply the flatfield */
298 if (flat != NULL) {
299 cpl_msg_info(__func__, "Correct for the flat field") ;
300 /* Load the flat */
301 if ((calib = cr2res_io_load_MASTER_FLAT(
302 cpl_frame_get_filename(flat), chip)) == NULL) {
303 cpl_msg_error(__func__, "Cannot load the flat field") ;
305 return NULL ;
306 }
307
308 /* Divide */
309 if (hdrl_image_div_image(out, calib) != CPL_ERROR_NONE) {
310 cpl_msg_error(__func__, "Cannot apply the flat field") ;
311 hdrl_image_delete(calib) ;
313 return NULL ;
314 }
315 hdrl_image_delete(calib) ;
316 }
317
318
319 /* Comics correction */
320 if (cosmics_corr) {
321 cpl_msg_info(__func__, "Apply the cosmics corrections") ;
322 hdrl_parameter * cospar = hdrl_lacosmic_parameter_create(12.0,2.5,1);
323 if (hdrl_lacosmic_parameter_verify(cospar)!=CPL_ERROR_NONE)
324 cpl_msg_warning(__func__,"invalid cospar");
325 cpl_mask* cosmask = hdrl_lacosmic_edgedetect(out,cospar);
326 cpl_mask_or(hdrl_image_get_mask(out), cosmask);
327 cpl_mask_delete(cosmask);
328 hdrl_parameter_delete(cospar);
329 }
330
331 return out ;
332}
333
334/*----------------------------------------------------------------------------*/
340/*----------------------------------------------------------------------------*/
341int cr2res_add_shotnoise(hdrl_image * in, int ndit, int chip){
342 double gain_sqrt;
343 cpl_image * error = hdrl_image_get_error(in);
344 cpl_image * adu = hdrl_image_get_image(in);
345 cpl_image * tmp_im;
346
347 if (adu==NULL){
348 cpl_msg_error(__func__,"Broken input image");
349 return -1;
350 }
351 if (error==NULL){
352 cpl_msg_error(__func__,"Error input null not supported");
353 }
354
355 if (chip == 1) gain_sqrt = sqrt(CR2RES_GAIN_CHIP1);
356 else if (chip == 2) gain_sqrt = sqrt(CR2RES_GAIN_CHIP2);
357 else if (chip == 3) gain_sqrt = sqrt(CR2RES_GAIN_CHIP3);
358 else {
359 cpl_msg_error(__func__,"Unknown detector");
360 return -1;
361 }
362
363 cpl_msg_debug(__func__, "chip:%d, sqrtgain:%g, ndit:%d",
364 chip, gain_sqrt, ndit);
365
366 if ( (tmp_im=cpl_image_abs_create(adu)) == NULL){
367 cpl_msg_error(__func__,"Abs failed");
368 return -1;
369 }
370 if ( (cpl_image_power(tmp_im, 0.5)) != CPL_ERROR_NONE){
371 cpl_msg_error(__func__,"Sqrt failed");
372 return -1;
373 }
374 cpl_image_divide_scalar(tmp_im, gain_sqrt);
375 cpl_image_divide_scalar(tmp_im, sqrt((float)ndit));
376
377 /* The BPM should not be stored in the error image */
378 /* Therefore it is removed from tmp_im before the addition to error */
379 cpl_image_accept_all(tmp_im) ;
380
381 cpl_image_add(error, tmp_im);
382 cpl_image_delete(tmp_im);
383 return 0;
384}
385
386/*----------------------------------------------------------------------------*/
398/*----------------------------------------------------------------------------*/
400 const hdrl_image * flat, cpl_size degree)
401{
402 // Define Parameters
403 cpl_image * img;
404 const cpl_image * flat_img;
405 cpl_polynomial * poly;
406 cpl_vector * tmp;
407 cpl_size ncolumns, nrow;
408 cpl_size i, j;
409 int badpix;
410 int badcols = 0;
411 double value, median, mad;
412
413 // Check Input
414 if (in == NULL || flat == NULL) return -1;
415 if (hdrl_image_get_size_x(in) != hdrl_image_get_size_x(flat)) return -1;
416 if (hdrl_image_get_size_y(in) != hdrl_image_get_size_y(flat)) return -1;
417
418 // Loop over the columns
419 img = hdrl_image_get_image(in);
420 flat_img = hdrl_image_get_image_const(flat);
421 ncolumns = cpl_image_get_size_x(img);
422 nrow = cpl_image_get_size_y(img);
423
424
425 for (i = CR2RES_NB_BPM_EDGEPIX + 1; i < ncolumns - CR2RES_NB_BPM_EDGEPIX;
426 i++) {
427 cpl_size npixel;
428 cpl_matrix *px;
429 cpl_vector *py;
430 // Fill vectors and matrix
431 px = cpl_matrix_new(1, nrow);
432 py = cpl_vector_new(nrow);
433 npixel = 0;
434 for ( j = CR2RES_NB_BPM_VIGN_BOTTOM + 5; j < nrow -20 ; j++)
435 {
436 // Filter pixels that are rejected in the flat (i.e. between orders)
437 // but not rejected by the image (i.e. not bad pixels)
438 if (cpl_image_is_rejected(flat_img, i, j)
439 && !cpl_image_is_rejected(img, i, j)){
440 cpl_matrix_set(px, 0, npixel, j);
441 cpl_vector_set(py, npixel, cpl_image_get(img, i, j, &badpix));
442 npixel++;
443 }
444 }
445 cpl_msg_debug(__func__, "Column %lld has %lld pixels for background",
446 i,npixel);
447 if (npixel < 10){
448 badcols += 1;
449 cpl_matrix_delete(px);
450 cpl_vector_delete(py);
451 continue;
452 }
453 cpl_matrix_set_size(px, 1, npixel);
454 cpl_vector_set_size(py, npixel);
455
456 // Reject outliers and filter
457 median = cr2res_vector_get_mad(py, &mad);
458 for (j=0;j<npixel;j++){
459 if (cpl_vector_get(py,j) > median*2)
460 cpl_vector_set(py,j,median);
461 }
462 //tmp = cpl_vector_filter_median_create(py,50);
463 tmp = cpl_vector_filter_lowpass_create(py,CPL_LOWPASS_LINEAR,100);
464 cpl_vector_delete(py);
465 py=tmp;
466 if (cpl_msg_get_level() == CPL_MSG_DEBUG){
467 tmp = cpl_vector_wrap(npixel, cpl_matrix_get_data(px));
468 cpl_vector_save(tmp, "debug_background.fits",
469 CPL_TYPE_DOUBLE, NULL, CPL_IO_CREATE);
470 cpl_vector_save(py, "debug_background.fits",
471 CPL_TYPE_DOUBLE, NULL, CPL_IO_EXTEND);
472 cpl_vector_unwrap(tmp);
473 }
474
475 // Fit polynomial
476 poly = cpl_polynomial_new(1);
477 if (cpl_polynomial_fit(poly, px, NULL, py, NULL, CPL_FALSE, NULL,
478 &degree)
479 != CPL_ERROR_NONE) {
480 badcols += 1;
481 cpl_msg_warning(__func__,
482 "Could not fit the background of column %"CPL_SIZE_FORMAT, i);
483 cpl_error_reset();
484 cpl_matrix_delete(px);
485 cpl_vector_delete(py);
486 cpl_polynomial_delete(poly);
487 continue;
488 }
489
490 // Subtract column
491 for ( j = 1; j < nrow + 1; j++)
492 {
493 value = cpl_image_get(img, i, j, &badpix);
494 // Just skip this pixel if it is bad
495 if (badpix) continue;
496 value -= cpl_polynomial_eval_1d(poly, j, NULL);
497 cpl_image_set(img, i, j, value);
498 }
499
500 // Clean up
501 cpl_matrix_delete(px);
502 cpl_vector_delete(py);
503 cpl_polynomial_delete(poly);
504 }
505
506 if (badcols){
507 cpl_msg_warning(__func__,
508 "%d columns could not be background-subtracted from inter-order gaps",
509 badcols);
510 }
511 return 0;
512}
513
int cr2res_bpm_set_and_correct_image(cpl_image *in, const char *bpm, int chip, int correct)
Set the BPM and optionally apply the correction to an image.
Definition: cr2res_bpm.c:199
hdrl_image * cr2res_calib_image(const hdrl_image *in, int chip, int clean_bad, int subtract_nolight_rows, int subtract_interorder_column, int cosmics_corr, const cpl_frame *flat, const cpl_frame *dark, const cpl_frame *bpm, const cpl_frame *detlin, double dit, int ndit)
The images calibration routine for a given chip.
Definition: cr2res_calib.c:147
hdrl_imagelist * cr2res_calib_imagelist(const hdrl_imagelist *in, int chip, int clean_bad, int subtract_nolight_rows, int subtract_interorder_column, int cosmics_corr, const cpl_frame *flat, const cpl_frame *dark, const cpl_frame *bpm, const cpl_frame *detlin, const cpl_vector *dits, const cpl_vector *ndits)
The images calibration routine for a given chip on a list.
Definition: cr2res_calib.c:71
int cr2res_add_shotnoise(hdrl_image *in, int ndit, int chip)
Add shot-noise to errors in HDRL-image.
Definition: cr2res_calib.c:341
int cr2res_calib_subtract_interorder_column(hdrl_image *in, const hdrl_image *flat, cpl_size degree)
Remove background column by column.
Definition: cr2res_calib.c:399
int cr2res_detlin_correct(hdrl_image *in, const hdrl_imagelist *detlin)
Apply the detector linearity correction.
Definition: cr2res_detlin.c:71
hdrl_imagelist * cr2res_io_load_DETLIN_COEFFS(const char *filename, int detector)
Load the detlin coefficients.
Definition: cr2res_io.c:1034
hdrl_image * cr2res_io_load_MASTER_DARK(const char *filename, int detector)
Load an image from a MASTER_DARK.
Definition: cr2res_io.c:990
hdrl_image * cr2res_io_load_MASTER_FLAT(const char *filename, int detector)
Load an hdrl image from a MASTER_FLAT.
Definition: cr2res_io.c:1066
double cr2res_pfits_get_dit(const cpl_propertylist *plist)
find out the DIT value
Definition: cr2res_pfits.c:199
double cr2res_vector_get_mad(cpl_vector *invec, double *mad)
Get MAD from a vector.
Definition: cr2res_utils.c:212
cpl_error_code hdrl_image_sub_image(hdrl_image *self, const hdrl_image *other)
Subtract two images, store the result in the first image.
cpl_error_code hdrl_image_div_image(hdrl_image *self, const hdrl_image *other)
Divide two images, store the result in the first image.
cpl_error_code hdrl_image_mul_scalar(hdrl_image *self, hdrl_value value)
Elementwise multiplication of an image with a scalar.
hdrl_image * hdrl_image_duplicate(const hdrl_image *himg)
copy hdrl_image
Definition: hdrl_image.c:391
cpl_mask * hdrl_image_get_mask(hdrl_image *himg)
get cpl bad pixel mask from image
Definition: hdrl_image.c:157
cpl_image * hdrl_image_get_error(hdrl_image *himg)
get error as cpl image
Definition: hdrl_image.c:131
cpl_size hdrl_image_get_size_y(const hdrl_image *self)
return size of Y dimension of image
Definition: hdrl_image.c:540
cpl_size hdrl_image_get_size_x(const hdrl_image *self)
return size of X dimension of image
Definition: hdrl_image.c:525
cpl_error_code hdrl_image_insert(hdrl_image *self, const cpl_image *image, const cpl_image *error, cpl_size xpos, cpl_size ypos)
Copy cpl images into an hdrl image.
Definition: hdrl_image.c:715
cpl_image * hdrl_image_get_image(hdrl_image *himg)
get data as cpl image
Definition: hdrl_image.c:105
const cpl_image * hdrl_image_get_image_const(const hdrl_image *himg)
get data as cpl image
Definition: hdrl_image.c:118
hdrl_image * hdrl_image_new(cpl_size nx, cpl_size ny)
create new zero filled hdrl image
Definition: hdrl_image.c:311
void hdrl_image_delete(hdrl_image *himg)
delete hdrl_image
Definition: hdrl_image.c:379
cpl_error_code hdrl_imagelist_set(hdrl_imagelist *himlist, hdrl_image *himg, cpl_size pos)
Insert an image into an imagelist.
void hdrl_imagelist_delete(hdrl_imagelist *himlist)
Free all memory used by a hdrl_imagelist object including the images.
cpl_size hdrl_imagelist_get_size(const hdrl_imagelist *himlist)
Get the number of images in the imagelist.
hdrl_imagelist * hdrl_imagelist_new(void)
Create an empty imagelist.
hdrl_image * hdrl_imagelist_get(const hdrl_imagelist *himlist, cpl_size inum)
Get an image from a list of images.
cpl_mask * hdrl_lacosmic_edgedetect(const hdrl_image *ima_in, const hdrl_parameter *params)
Detect bad-pixels / cosmic-rays on a single image.
cpl_error_code hdrl_lacosmic_parameter_verify(const hdrl_parameter *param)
Verify basic correctness of the LaCosmic parameters.
hdrl_parameter * hdrl_lacosmic_parameter_create(double sigma_lim, double f_lim, int max_iter)
Creates LaCosmic parameters object.
void hdrl_parameter_delete(hdrl_parameter *obj)
shallow delete of a parameter