VIRCAM Pipeline 2.3.15
casu_pawsky_minus.c
1/* $Id: casu_pawsky_minus.c,v 1.4 2015/11/25 10:26:31 jim Exp $
2 *
3 * This file is part of the CASU Pipeline utilities
4 * Copyright (C) 2015 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20
21/*
22 * $Author: jim $
23 * $Date: 2015/11/25 10:26:31 $
24 * $Revision: 1.4 $
25 * $Name: $
26 */
27
28/* Includes */
29
30#ifdef HAVE_CONFIG_H
31#include <config.h>
32#endif
33
34#include <cpl.h>
35#include <math.h>
36#include <string.h>
37
38#include "casu_sky.h"
39#include "casu_mask.h"
40#include "catalogue/casu_fits.h"
41#include "casu_mods.h"
42#include "catalogue/casu_utils.h"
43#include "casu_stats.h"
44#include "catalogue/casu_wcsutils.h"
45
46#define DATAMIN -1000.0
47#define DATAMAX 65535.0
48
49static void masksky_zeros(float **datas, unsigned char **masks, int nfiles,
50 int npts, float **zeros);
51static void combine(int nfiles, int stat, int npts, float **datas, float **vars,
52 unsigned char **masks, float **skyout, float **skyv,
53 unsigned char **skyout_bpm);
54static void combine_mult(int nfiles, int stat, int npts, float **datas,
55 float **vars, unsigned char **masks, float *skyout,
56 float *skyvar);
57static void domed(float *buf, int n, float *val);
58
61/*---------------------------------------------------------------------------*/
110/*---------------------------------------------------------------------------*/
111
112extern int casu_pawsky_minus(casu_fits **infiles, casu_fits **invar,
113 casu_fits *conf, casu_fits *objmaskfits,
114 int nfiles, casu_fits **skyout, casu_fits **skyvar,
115 int *status) {
116 cpl_wcs *wcsmask,*wcsimg;
117 unsigned char **masks,*sky_bpm;
118 cpl_image *im,*skyim,*skyimv;
119 double *xin = NULL, *yin = NULL, *xout = NULL, *yout = NULL, *dx = NULL, *dy = NULL, ddx, ddy;
120 int nx,ny,npts,ind,i,j,kind,ix,iy,jx,jy,*opm = NULL;
121 int nx_mask = -1; /* return value for cpl_image_get_size_x on error */
122 int nfiles2, *confdata = NULL;
123 float **datas,*sky,*skyv,*zeros,val,sig,**vars;
124 cpl_propertylist *plist;
125 casu_fits **infiles2,**invar2;
126 const char *fctid = "casu_pawsky_minus";
127
128 /* Inherited status */
129
130 *skyout = NULL;
131 *skyvar = NULL;
132 if (*status != CASU_OK)
133 return(*status);
134
135 /* If there aren't any images, then get out of here */
136
137 if (nfiles == 0) {
138 cpl_msg_error(fctid,"Sky correction impossible. No science frames");
139 return(CASU_FATAL);
140 }
141
142 /* Check to see which files are good */
143
144 infiles2 = cpl_malloc(nfiles*sizeof(casu_fits *));
145 if (invar != NULL)
146 invar2 = cpl_malloc(nfiles*sizeof(casu_fits *));
147 else
148 invar2 = NULL;
149 nfiles2 = 0;
150 for (i = 0; i < nfiles; i++) {
151 if (casu_fits_get_status(infiles[i]) == CASU_OK)
152 infiles2[nfiles2] = infiles[i];
153 if (invar != NULL)
154 invar2[nfiles2] = invar[i];
155 nfiles2++;
156 }
157
158 /* If no good images are present then wrap a dummy image and return it */
159
160 if (nfiles2 == 0) {
161 skyim = casu_dummy_image(infiles[0]);
162 *skyout = casu_fits_wrap(skyim,infiles[0],NULL,NULL);
164 casu_fits_set_status(*skyout,CASU_FATAL);
165 if (invar != NULL) {
166 skyimv = casu_dummy_image(infiles[0]);
167 *skyvar = casu_fits_wrap(skyimv,invar[0],NULL,NULL);
169 } else {
170 *skyvar = NULL;
171 }
172 cpl_msg_warning(fctid,"No good images in input list");
173 freespace(infiles2);
174 *status = CASU_WARN;
175 return(CASU_WARN);
176 }
177
178 /* Get space for arrays */
179
180 datas = cpl_malloc(nfiles2*sizeof(float *));
181 if (invar != NULL)
182 vars = cpl_malloc(nfiles2*sizeof(float *));
183 else
184 vars = NULL;
185 masks = cpl_malloc(nfiles2*sizeof(unsigned char *));
186 im = casu_fits_get_image(infiles2[0]);
187 nx = cpl_image_get_size_x(im);
188 ny = cpl_image_get_size_y(im);
189 npts = nx*ny;
190
191 /* Info about object mask if it exists */
192
193 if (objmaskfits != NULL) {
194 nx_mask = cpl_image_get_size_x(casu_fits_get_image(objmaskfits));
195 wcsmask = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(objmaskfits));
196 opm = cpl_image_get_data_int(casu_fits_get_image(objmaskfits));
197
198 /* We need this space so that we can work out which part of the
199 input mask is relevant for each input image */
200
201 xin = cpl_malloc(npts*sizeof(double));
202 yin = cpl_malloc(npts*sizeof(double));
203 xout = cpl_malloc(npts*sizeof(double));
204 yout = cpl_malloc(npts*sizeof(double));
205 dx = cpl_malloc(nfiles2*sizeof(double));
206 dy = cpl_malloc(nfiles2*sizeof(double));
207
208 /* Initialise the input xy arrays */
209
210 ind = 0;
211 for (j = 0; j < ny; j++) {
212 for (i = 0; i < nx; i++) {
213 xin[ind] = (double)(i+1);
214 yin[ind++] = (double)(j+1);
215 }
216 }
217
218 /* Define an output grid for the first image and then offsets for
219 the subsequent images */
220
221 ddx = 1.0;
222 ddy = 1.0;
223 for (i = 0; i < nfiles2; i++) {
224 wcsimg = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(infiles2[i]));
225 if (i == 0)
226 casu_xytoxy_list(wcsimg,wcsmask,npts,xin,yin,xout,yout);
227 casu_xytoxy_list(wcsimg,wcsmask,1,&ddx,&ddy,dx+i,dy+i);
228 dx[i] -= xout[0];
229 dy[i] -= yout[0];
230 cpl_wcs_delete(wcsimg);
231 }
232 cpl_wcs_delete(wcsmask);
233 }
234
235 /* Loop for each file and get a reference for the data array. Get
236 some space for a mask that will cover both the zeroed pixels in the
237 confidence maps and any object mask that might be used */
238
239 for (i = 0; i < nfiles2; i++) {
240 im = casu_fits_get_image(infiles2[i]);
241 datas[i] = cpl_image_get_data_float(im);
242 masks[i] = cpl_calloc(npts,sizeof(unsigned char));
243 if (i == 0)
244 confdata = cpl_image_get_data_int(casu_fits_get_image(conf));
245 if (invar != NULL)
246 vars[i] = cpl_image_get_data_float(casu_fits_get_image(invar2[i]));
247
248 /* Now loop for each pixel. If the confidence map is zero here then
249 mark this as a bad pixel. If not, then look at the master mask
250 and see which is its closest pixel to the current one. NB: the
251 -0.5 in the index is a combination of +0.5 to do a nint and a -1
252 to take account of the fact that arrays start from index zero. */
253
254 for (jy = 0; jy < ny; jy++) {
255 for (jx = 0; jx < nx; jx++) {
256 ind = jy*nx + jx;
257 if (confdata != NULL && confdata[ind] == 0) {
258 masks[i][ind] = 1;
259 } else if (objmaskfits != NULL) {
260 kind = (int)(yout[ind] + dy[i] - 0.5)*nx_mask +
261 (int)(xout[ind] + dx[i] - 0.5);
262 if(opm != NULL) {
263 masks[i][ind] = opm[kind];
264 }
265 }
266 }
267 }
268 }
269
270 /* Do an intermediate tidy */
271
272 if (objmaskfits != NULL) {
273 freespace(xin);
274 freespace(yin);
275 freespace(xout);
276 freespace(yout);
277 freespace(dx);
278 freespace(dy);
279 }
280
281 /* Offset the DC level */
282
283 masksky_zeros(datas,masks,nfiles2,npts,&zeros);
284
285 /* Right, combine these now */
286
287 combine(nfiles2,0,npts,datas,vars,masks,&sky,&skyv,&sky_bpm);
288
289 /* Wrap the sky and reject the pixels that had no sky contribution */
290
291 skyim = cpl_image_wrap_float(nx,ny,sky);
292 for (i = 0; i < npts; i++) {
293 iy = i/nx + 1;
294 ix = npts - (iy-1)*nx;
295 if (sky_bpm[i])
296 cpl_image_reject(skyim,(cpl_size)ix,(cpl_size)iy);
297 }
298 freespace(sky_bpm);
299 *skyout = casu_fits_wrap(skyim,infiles2[0],NULL,NULL);
300 plist = casu_fits_get_ehu(*skyout);
301 if (objmaskfits != NULL) {
302 cpl_propertylist_update_string(plist,"ESO DRS MASKUSED",
303 casu_fits_get_filename(objmaskfits));
304 cpl_propertylist_set_comment(plist,"ESO DRS MASKUSED",
305 "Object masked used to make sky");
306 }
307 cpl_propertylist_update_string(plist,"ESO DRS SKYALGO","pawsky_minus");
308 cpl_propertylist_set_comment(plist,"ESO DRS SKYALGO",
309 "Sky estimation algorithm");
310 casu_prov(plist,infiles2,nfiles2,1);
311 if (vars != NULL) {
312 skyimv = cpl_image_wrap_float(nx,ny,skyv);
313 *skyvar = casu_fits_wrap(skyimv,invar2[0],NULL,NULL);
314 } else {
315 *skyvar = NULL;
316 }
317
318 /* Fill in the bits with no sky contribution */
319
320 casu_inpaint(*skyout,64,status);
321
322 /* Do the multiple combine and subtraction */
323
324 combine_mult(nfiles2,0,npts,datas,vars,masks,sky,skyv);
325
326 /* Now add the DC level back onto the data */
327
328 for (i = 0; i < nfiles2; i++) {
329 casu_qmedsig(datas[i],masks[i],(long)npts,3.0,1,DATAMIN,DATAMAX,&val,
330 &sig);
331 val = zeros[i] - val;
332 for (j = 0; j < npts; j++)
333 datas[i][j] += val;
334 }
335 freespace(zeros);
336
337 /* Add a little something to the header and start cleaning up */
338
339 for (i = 0; i < nfiles2; i++) {
340 plist = casu_fits_get_ehu(infiles2[i]);
341 cpl_propertylist_update_string(plist,"ESO DRS SKYSUB",
342 "Done with pawsky_minus");
343 freespace(masks[i]);
344 }
345 freespace(masks);
346 freespace(datas);
347 freespace(vars);
348 freespace(infiles2);
349 freespace(invar2);
350 return(CASU_OK);
351}
352
353/*---------------------------------------------------------------------------*/
379/*---------------------------------------------------------------------------*/
380
381static void masksky_zeros(float **datas, unsigned char **masks, int nfiles,
382 int npts, float **zeros) {
383 int i,j;
384 float sig,off,medval;
385
386 /* Get some space for the zeros */
387
388 *zeros = cpl_malloc(nfiles*sizeof(float));
389
390 /* Loop for each input image and get the background median */
391
392 for (i = 0; i < nfiles; i++)
393 casu_qmedsig(datas[i],masks[i],(long)npts,3.0,1,DATAMIN,
394 DATAMAX,*zeros+i,&sig);
395
396 /* Get the median value of the array */
397
398 medval = casu_med(*zeros,NULL,nfiles);
399
400 /* Now work out the offset for each image and subtract it */
401
402 for (i = 0; i < nfiles; i++) {
403 off = medval - (*zeros)[i];
404 for (j = 0; j < npts; j++)
405 datas[i][j] += off;
406 }
407}
408
409/*---------------------------------------------------------------------------*/
444/*---------------------------------------------------------------------------*/
445
446static void combine(int nfiles, int stat, int npts, float **datas, float **vars,
447 unsigned char **masks, float **skyout, float **skyv,
448 unsigned char **skyout_bpm) {
449 int i,j,n;
450 float *buf,sum;
451
452 /* Get workspace for output arrays */
453
454 *skyout = cpl_malloc(npts*sizeof(float));
455 if (vars != NULL)
456 *skyv = cpl_malloc(npts*sizeof(float));
457 else
458 *skyv = NULL;
459 *skyout_bpm = cpl_malloc(npts*sizeof(unsigned char));
460 buf = cpl_malloc(nfiles*sizeof(float));
461
462 /* Loop now for each input pixel and form the median */
463
464 for (j = 0; j < npts; j++) {
465 n = 0;
466 sum = 0.0;
467 for (i = 0; i < nfiles; i++) {
468 if (masks[i][j] == 0) {
469 buf[n] = datas[i][j];
470 if (vars != NULL)
471 sum += vars[i][j];
472 n++;
473 }
474 }
475 if (n == 0) {
476 (*skyout)[j] = 0.0;
477 if (vars != NULL)
478 (*skyv)[j] = 0.0;
479 (*skyout_bpm)[j] = 1;
480 } else {
481 if (vars != NULL)
482 sum /= powf((float)n,2.0);
483 if (stat == 0) {
484 domed(buf,n,(*skyout)+j);
485 if (vars != NULL)
486 (*skyv)[j] = sum*CPL_MATH_PI_2;
487 } else {
488 (*skyout)[j] = casu_mean(buf,NULL,n);
489 if (vars != NULL)
490 (*skyv)[j] = sum;
491 }
492 (*skyout_bpm)[j] = 0;
493 }
494 }
495
496 /* Free some workspace and get out of here */
497
498 freespace(buf);
499 return;
500}
501
502/*---------------------------------------------------------------------------*/
536/*---------------------------------------------------------------------------*/
537
538static void combine_mult(int nfiles, int stat, int npts, float **datas,
539 float **vars, unsigned char **masks, float *skyout,
540 float *skyvar) {
541 int i,j,k,n;
542 float *buf,*bufv = NULL,*val,*valv = NULL;
543
544 /* Get some memory */
545
546 buf = cpl_malloc(nfiles*sizeof(float));
547 val = cpl_malloc(nfiles*sizeof(float));
548 if (vars != NULL) {
549 bufv = cpl_malloc(nfiles*sizeof(float));
550 valv = cpl_malloc(nfiles*sizeof(float));
551 }
552
553 /* Loop now for each input pixel */
554
555 for (j = 0; j < npts; j++) {
556
557 /* Loop for each file. If the current pixel in the current file
558 was flagged, then the sky for that pixel will be the same as
559 the general sky we did before. If not, then remove that file's
560 pixel and recalculate the sky */
561
562 for (i = 0; i < nfiles; i++) {
563 if (masks[i][j] != 0) {
564 val[i] = skyout[j];
565 if (vars != NULL)
566 valv[i] = skyvar[j];
567 } else {
568 n = 0;
569 for (k = 0; k < nfiles; k++) {
570 if (k == i || masks[k][j] != 0)
571 continue;
572 buf[n] = datas[k][j];
573 if (vars != NULL && vars[i] != NULL)
574 bufv[n++] = vars[k][j];
575 }
576 if (n == 0) {
577 val[i] = skyout[j];
578 if (vars != NULL)
579 valv[i] = skyvar[j];
580 } else {
581 if (stat == 0) {
582 domed(buf,n,val+i);
583 if (vars != NULL)
584 valv[i] = CPL_MATH_PI_2*casu_mean(bufv,NULL,n);
585 } else {
586 val[i] = casu_mean(buf,NULL,n);
587 if (vars != NULL)
588 valv[i] = casu_mean(bufv,NULL,n);
589 }
590 }
591 }
592 }
593
594 /* Now correct these pixels in the input images */
595
596 for (i = 0; i < nfiles; i++) {
597 datas[i][j] -= val[i];
598 if (vars != NULL)
599 vars[i][j] += valv[i];
600 }
601 }
602
603 /* Free some workspace and get out of here */
604
605 freespace(buf);
606 freespace(val);
607 if (vars != NULL) {
608 freespace(bufv);
609 freespace(valv);
610 }
611 return;
612}
613
614/*---------------------------------------------------------------------------*/
636/*---------------------------------------------------------------------------*/
637
638static void domed(float *buf, int n, float *val) {
639 int is_even,n2;
640
641 is_even = ! (n & 1);
642 casu_sort(&buf,n,1);
643 n2 = n/2;
644 if (is_even) {
645 *val = 0.5*(buf[n2-1] + buf[n2]);
646 } else {
647 if (n <= 5)
648 *val = buf[n2];
649 else
650 *val = 0.5*buf[n2] + 0.25*(buf[n2-1] + buf[n2+1]);
651 }
652}
653
656/*
657
658$Log: casu_pawsky_minus.c,v $
659Revision 1.4 2015/11/25 10:26:31 jim
660replaced some hardcoded numbers with defines
661
662Revision 1.3 2015/09/11 09:27:53 jim
663Fixed some problems with the docs
664
665Revision 1.2 2015/08/07 13:06:54 jim
666Fixed copyright to ESO
667
668Revision 1.1.1.1 2015/06/12 10:44:32 jim
669Initial import
670
671Revision 1.9 2015/03/12 09:16:51 jim
672Modified to remove some compiler moans
673
674Revision 1.8 2015/01/29 11:52:36 jim
675Modified comments + fixed error in variance propagation
676
677Revision 1.7 2015/01/09 12:13:15 jim
678*** empty log message ***
679
680Revision 1.6 2014/12/11 12:23:33 jim
681new version
682
683Revision 1.5 2014/04/09 11:08:21 jim
684Get rid of a couple of compiler moans
685
686Revision 1.4 2014/03/26 15:55:40 jim
687Uses floating point confidence
688
689Revision 1.3 2013/11/21 09:38:14 jim
690detabbed
691
692Revision 1.2 2013-10-24 09:25:21 jim
693traps for dummy input files and tries to remove them from the analysis
694
695Revision 1.1.1.1 2013-08-27 12:07:48 jim
696Imported
697
698
699*/
700
casu_fits * casu_fits_wrap(cpl_image *im, casu_fits *model, cpl_propertylist *phu, cpl_propertylist *ehu)
Definition: casu_fits.c:883
int casu_fits_get_status(casu_fits *p)
Definition: casu_fits.c:711
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
char * casu_fits_get_filename(casu_fits *p)
Definition: casu_fits.c:646
void casu_fits_set_status(casu_fits *p, int status)
Definition: casu_fits.c:801
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
int casu_inpaint(casu_fits *in, int nbsize, int *status)
Inpaint pixels or patches in a map.
Definition: casu_inpaint.c:83
int casu_pawsky_minus(casu_fits **infiles, casu_fits **invar, casu_fits *conf, casu_fits *objmaskfits, int nfiles, casu_fits **skyout, casu_fits **skyvar, int *status)
Background correct images using pawsky_minus algorithm.
float casu_med(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:89
void casu_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: casu_stats.c:258
float casu_mean(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:479
void casu_prov(cpl_propertylist *p, casu_fits **inlist, int n, int isextn)
Write provenance keywords.
Definition: casu_utils.c:287
void casu_dummy_property(cpl_propertylist *p)
Set dummy property keyword.
Definition: casu_utils.c:445
cpl_image * casu_dummy_image(casu_fits *model)
Create a dummy image of zeros based on a model.
Definition: casu_utils.c:533
void casu_sort(float **a, int n, int m)
Sort a 2d array by the first column and co-sort the rest.
Definition: casu_utils.c:349
void casu_xytoxy_list(cpl_wcs *wcs1, cpl_wcs *wcs2, int nc, double *x_1, double *y_1, double *x_2, double *y_2)