VIRCAM Pipeline  2.3.10
casu_pawsky_minus.c
1 /* $Id: casu_pawsky_minus.c,v 1.4 2015/11/25 10:26:31 jim Exp $
2  *
3  * This file is part of the CASU Pipeline utilities
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2015/11/25 10:26:31 $
24  * $Revision: 1.4 $
25  * $Name: $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <cpl.h>
35 #include <math.h>
36 #include <string.h>
37 
38 #include "casu_sky.h"
39 #include "casu_mask.h"
40 #include "catalogue/casu_fits.h"
41 #include "casu_mods.h"
42 #include "catalogue/casu_utils.h"
43 #include "casu_stats.h"
44 #include "catalogue/casu_wcsutils.h"
45 
46 #define DATAMIN -1000.0
47 #define DATAMAX 65535.0
48 
49 static void masksky_zeros(float **datas, unsigned char **masks, int nfiles,
50  int npts, float **zeros);
51 static void combine(int nfiles, int stat, int npts, float **datas, float **vars,
52  unsigned char **masks, float **skyout, float **skyv,
53  unsigned char **skyout_bpm);
54 static void combine_mult(int nfiles, int stat, int npts, float **datas,
55  float **vars, unsigned char **masks, float *skyout,
56  float *skyvar);
57 static void domed(float *buf, int n, float *val);
58 
61 /*---------------------------------------------------------------------------*/
110 /*---------------------------------------------------------------------------*/
111 
112 extern int casu_pawsky_minus(casu_fits **infiles, casu_fits **invar,
113  casu_fits *conf, casu_fits *objmaskfits,
114  int nfiles, casu_fits **skyout, casu_fits **skyvar,
115  int *status) {
116  cpl_wcs *wcsmask,*wcsimg;
117  unsigned char **masks,*sky_bpm;
118  cpl_image *im,*skyim,*skyimv;
119  double *xin = NULL, *yin = NULL, *xout = NULL, *yout = NULL, *dx = NULL, *dy = NULL, ddx, ddy;
120  int nx,ny,npts,ind,i,j,kind,ix,iy,jx,jy,*opm = NULL;
121  int nx_mask = -1; /* return value for cpl_image_get_size_x on error */
122  int nfiles2, *confdata = NULL;
123  float **datas,*sky,*skyv,*zeros,val,sig,**vars;
124  cpl_propertylist *plist;
125  casu_fits **infiles2,**invar2;
126  const char *fctid = "casu_pawsky_minus";
127 
128  /* Inherited status */
129 
130  *skyout = NULL;
131  *skyvar = NULL;
132  if (*status != CASU_OK)
133  return(*status);
134 
135  /* If there aren't any images, then get out of here */
136 
137  if (nfiles == 0) {
138  cpl_msg_error(fctid,"Sky correction impossible. No science frames");
139  return(CASU_FATAL);
140  }
141 
142  /* Check to see which files are good */
143 
144  infiles2 = cpl_malloc(nfiles*sizeof(casu_fits *));
145  if (invar != NULL)
146  invar2 = cpl_malloc(nfiles*sizeof(casu_fits *));
147  else
148  invar2 = NULL;
149  nfiles2 = 0;
150  for (i = 0; i < nfiles; i++) {
151  if (casu_fits_get_status(infiles[i]) == CASU_OK)
152  infiles2[nfiles2] = infiles[i];
153  if (invar != NULL)
154  invar2[nfiles2] = invar[i];
155  nfiles2++;
156  }
157 
158  /* If no good images are present then wrap a dummy image and return it */
159 
160  if (nfiles2 == 0) {
161  skyim = casu_dummy_image(infiles[0]);
162  *skyout = casu_fits_wrap(skyim,infiles[0],NULL,NULL);
164  casu_fits_set_status(*skyout,CASU_FATAL);
165  if (invar != NULL) {
166  skyimv = casu_dummy_image(infiles[0]);
167  *skyvar = casu_fits_wrap(skyimv,invar[0],NULL,NULL);
169  } else {
170  *skyvar = NULL;
171  }
172  cpl_msg_warning(fctid,"No good images in input list");
173  freespace(infiles2);
174  *status = CASU_WARN;
175  return(CASU_WARN);
176  }
177 
178  /* Get space for arrays */
179 
180  datas = cpl_malloc(nfiles2*sizeof(float *));
181  if (invar != NULL)
182  vars = cpl_malloc(nfiles2*sizeof(float *));
183  else
184  vars = NULL;
185  masks = cpl_malloc(nfiles2*sizeof(unsigned char *));
186  im = casu_fits_get_image(infiles2[0]);
187  nx = cpl_image_get_size_x(im);
188  ny = cpl_image_get_size_y(im);
189  npts = nx*ny;
190 
191  /* Info about object mask if it exists */
192 
193  if (objmaskfits != NULL) {
194  nx_mask = cpl_image_get_size_x(casu_fits_get_image(objmaskfits));
195  wcsmask = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(objmaskfits));
196  opm = cpl_image_get_data_int(casu_fits_get_image(objmaskfits));
197 
198  /* We need this space so that we can work out which part of the
199  input mask is relevant for each input image */
200 
201  xin = cpl_malloc(npts*sizeof(double));
202  yin = cpl_malloc(npts*sizeof(double));
203  xout = cpl_malloc(npts*sizeof(double));
204  yout = cpl_malloc(npts*sizeof(double));
205  dx = cpl_malloc(nfiles2*sizeof(double));
206  dy = cpl_malloc(nfiles2*sizeof(double));
207 
208  /* Initialise the input xy arrays */
209 
210  ind = 0;
211  for (j = 0; j < ny; j++) {
212  for (i = 0; i < nx; i++) {
213  xin[ind] = (double)(i+1);
214  yin[ind++] = (double)(j+1);
215  }
216  }
217 
218  /* Define an output grid for the first image and then offsets for
219  the subsequent images */
220 
221  ddx = 1.0;
222  ddy = 1.0;
223  for (i = 0; i < nfiles2; i++) {
224  wcsimg = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(infiles2[i]));
225  if (i == 0)
226  casu_xytoxy_list(wcsimg,wcsmask,npts,xin,yin,xout,yout);
227  casu_xytoxy_list(wcsimg,wcsmask,1,&ddx,&ddy,dx+i,dy+i);
228  dx[i] -= xout[0];
229  dy[i] -= yout[0];
230  cpl_wcs_delete(wcsimg);
231  }
232  cpl_wcs_delete(wcsmask);
233  }
234 
235  /* Loop for each file and get a reference for the data array. Get
236  some space for a mask that will cover both the zeroed pixels in the
237  confidence maps and any object mask that might be used */
238 
239  for (i = 0; i < nfiles2; i++) {
240  im = casu_fits_get_image(infiles2[i]);
241  datas[i] = cpl_image_get_data_float(im);
242  masks[i] = cpl_calloc(npts,sizeof(unsigned char));
243  if (i == 0)
244  confdata = cpl_image_get_data_int(casu_fits_get_image(conf));
245  if (invar != NULL)
246  vars[i] = cpl_image_get_data_float(casu_fits_get_image(invar2[i]));
247 
248  /* Now loop for each pixel. If the confidence map is zero here then
249  mark this as a bad pixel. If not, then look at the master mask
250  and see which is its closest pixel to the current one. NB: the
251  -0.5 in the index is a combination of +0.5 to do a nint and a -1
252  to take account of the fact that arrays start from index zero. */
253 
254  for (jy = 0; jy < ny; jy++) {
255  for (jx = 0; jx < nx; jx++) {
256  ind = jy*nx + jx;
257  if (confdata != NULL && confdata[ind] == 0) {
258  masks[i][ind] = 1;
259  } else if (objmaskfits != NULL) {
260  kind = (int)(yout[ind] + dy[i] - 0.5)*nx_mask +
261  (int)(xout[ind] + dx[i] - 0.5);
262  if(opm != NULL) {
263  masks[i][ind] = opm[kind];
264  }
265  }
266  }
267  }
268  }
269 
270  /* Do an intermediate tidy */
271 
272  if (objmaskfits != NULL) {
273  freespace(xin);
274  freespace(yin);
275  freespace(xout);
276  freespace(yout);
277  freespace(dx);
278  freespace(dy);
279  }
280 
281  /* Offset the DC level */
282 
283  masksky_zeros(datas,masks,nfiles2,npts,&zeros);
284 
285  /* Right, combine these now */
286 
287  combine(nfiles2,0,npts,datas,vars,masks,&sky,&skyv,&sky_bpm);
288 
289  /* Wrap the sky and reject the pixels that had no sky contribution */
290 
291  skyim = cpl_image_wrap_float(nx,ny,sky);
292  for (i = 0; i < npts; i++) {
293  iy = i/nx + 1;
294  ix = npts - (iy-1)*nx;
295  if (sky_bpm[i])
296  cpl_image_reject(skyim,(cpl_size)ix,(cpl_size)iy);
297  }
298  freespace(sky_bpm);
299  *skyout = casu_fits_wrap(skyim,infiles2[0],NULL,NULL);
300  plist = casu_fits_get_ehu(*skyout);
301  if (objmaskfits != NULL) {
302  cpl_propertylist_update_string(plist,"ESO DRS MASKUSED",
303  casu_fits_get_filename(objmaskfits));
304  cpl_propertylist_set_comment(plist,"ESO DRS MASKUSED",
305  "Object masked used to make sky");
306  }
307  cpl_propertylist_update_string(plist,"ESO DRS SKYALGO","pawsky_minus");
308  cpl_propertylist_set_comment(plist,"ESO DRS SKYALGO",
309  "Sky estimation algorithm");
310  casu_prov(plist,infiles2,nfiles2,1);
311  if (vars != NULL) {
312  skyimv = cpl_image_wrap_float(nx,ny,skyv);
313  *skyvar = casu_fits_wrap(skyimv,invar2[0],NULL,NULL);
314  } else {
315  *skyvar = NULL;
316  }
317 
318  /* Fill in the bits with no sky contribution */
319 
320  casu_inpaint(*skyout,64,status);
321 
322  /* Do the multiple combine and subtraction */
323 
324  combine_mult(nfiles2,0,npts,datas,vars,masks,sky,skyv);
325 
326  /* Now add the DC level back onto the data */
327 
328  for (i = 0; i < nfiles2; i++) {
329  casu_qmedsig(datas[i],masks[i],(long)npts,3.0,1,DATAMIN,DATAMAX,&val,
330  &sig);
331  val = zeros[i] - val;
332  for (j = 0; j < npts; j++)
333  datas[i][j] += val;
334  }
335  freespace(zeros);
336 
337  /* Add a little something to the header and start cleaning up */
338 
339  for (i = 0; i < nfiles2; i++) {
340  plist = casu_fits_get_ehu(infiles2[i]);
341  cpl_propertylist_update_string(plist,"ESO DRS SKYSUB",
342  "Done with pawsky_minus");
343  freespace(masks[i]);
344  }
345  freespace(masks);
346  freespace(datas);
347  freespace(vars);
348  freespace(infiles2);
349  freespace(invar2);
350  return(CASU_OK);
351 }
352 
353 /*---------------------------------------------------------------------------*/
379 /*---------------------------------------------------------------------------*/
380 
381 static void masksky_zeros(float **datas, unsigned char **masks, int nfiles,
382  int npts, float **zeros) {
383  int i,j;
384  float sig,off,medval;
385 
386  /* Get some space for the zeros */
387 
388  *zeros = cpl_malloc(nfiles*sizeof(float));
389 
390  /* Loop for each input image and get the background median */
391 
392  for (i = 0; i < nfiles; i++)
393  casu_qmedsig(datas[i],masks[i],(long)npts,3.0,1,DATAMIN,
394  DATAMAX,*zeros+i,&sig);
395 
396  /* Get the median value of the array */
397 
398  medval = casu_med(*zeros,NULL,nfiles);
399 
400  /* Now work out the offset for each image and subtract it */
401 
402  for (i = 0; i < nfiles; i++) {
403  off = medval - (*zeros)[i];
404  for (j = 0; j < npts; j++)
405  datas[i][j] += off;
406  }
407 }
408 
409 /*---------------------------------------------------------------------------*/
444 /*---------------------------------------------------------------------------*/
445 
446 static void combine(int nfiles, int stat, int npts, float **datas, float **vars,
447  unsigned char **masks, float **skyout, float **skyv,
448  unsigned char **skyout_bpm) {
449  int i,j,n;
450  float *buf,sum;
451 
452  /* Get workspace for output arrays */
453 
454  *skyout = cpl_malloc(npts*sizeof(float));
455  if (vars != NULL)
456  *skyv = cpl_malloc(npts*sizeof(float));
457  else
458  *skyv = NULL;
459  *skyout_bpm = cpl_malloc(npts*sizeof(unsigned char));
460  buf = cpl_malloc(nfiles*sizeof(float));
461 
462  /* Loop now for each input pixel and form the median */
463 
464  for (j = 0; j < npts; j++) {
465  n = 0;
466  sum = 0.0;
467  for (i = 0; i < nfiles; i++) {
468  if (masks[i][j] == 0) {
469  buf[n] = datas[i][j];
470  if (vars != NULL)
471  sum += vars[i][j];
472  n++;
473  }
474  }
475  if (n == 0) {
476  (*skyout)[j] = 0.0;
477  if (vars != NULL)
478  (*skyv)[j] = 0.0;
479  (*skyout_bpm)[j] = 1;
480  } else {
481  if (vars != NULL)
482  sum /= powf((float)n,2.0);
483  if (stat == 0) {
484  domed(buf,n,(*skyout)+j);
485  if (vars != NULL)
486  (*skyv)[j] = sum*CPL_MATH_PI_2;
487  } else {
488  (*skyout)[j] = casu_mean(buf,NULL,n);
489  if (vars != NULL)
490  (*skyv)[j] = sum;
491  }
492  (*skyout_bpm)[j] = 0;
493  }
494  }
495 
496  /* Free some workspace and get out of here */
497 
498  freespace(buf);
499  return;
500 }
501 
502 /*---------------------------------------------------------------------------*/
536 /*---------------------------------------------------------------------------*/
537 
538 static void combine_mult(int nfiles, int stat, int npts, float **datas,
539  float **vars, unsigned char **masks, float *skyout,
540  float *skyvar) {
541  int i,j,k,n;
542  float *buf,*bufv = NULL,*val,*valv = NULL;
543 
544  /* Get some memory */
545 
546  buf = cpl_malloc(nfiles*sizeof(float));
547  val = cpl_malloc(nfiles*sizeof(float));
548  if (vars != NULL) {
549  bufv = cpl_malloc(nfiles*sizeof(float));
550  valv = cpl_malloc(nfiles*sizeof(float));
551  }
552 
553  /* Loop now for each input pixel */
554 
555  for (j = 0; j < npts; j++) {
556 
557  /* Loop for each file. If the current pixel in the current file
558  was flagged, then the sky for that pixel will be the same as
559  the general sky we did before. If not, then remove that file's
560  pixel and recalculate the sky */
561 
562  for (i = 0; i < nfiles; i++) {
563  if (masks[i][j] != 0) {
564  val[i] = skyout[j];
565  if (vars != NULL)
566  valv[i] = skyvar[j];
567  } else {
568  n = 0;
569  for (k = 0; k < nfiles; k++) {
570  if (k == i || masks[k][j] != 0)
571  continue;
572  buf[n] = datas[k][j];
573  if (vars != NULL && vars[i] != NULL)
574  bufv[n++] = vars[k][j];
575  }
576  if (n == 0) {
577  val[i] = skyout[j];
578  if (vars != NULL)
579  valv[i] = skyvar[j];
580  } else {
581  if (stat == 0) {
582  domed(buf,n,val+i);
583  if (vars != NULL)
584  valv[i] = CPL_MATH_PI_2*casu_mean(bufv,NULL,n);
585  } else {
586  val[i] = casu_mean(buf,NULL,n);
587  if (vars != NULL)
588  valv[i] = casu_mean(bufv,NULL,n);
589  }
590  }
591  }
592  }
593 
594  /* Now correct these pixels in the input images */
595 
596  for (i = 0; i < nfiles; i++) {
597  datas[i][j] -= val[i];
598  if (vars != NULL)
599  vars[i][j] += valv[i];
600  }
601  }
602 
603  /* Free some workspace and get out of here */
604 
605  freespace(buf);
606  freespace(val);
607  if (vars != NULL) {
608  freespace(bufv);
609  freespace(valv);
610  }
611  return;
612 }
613 
614 /*---------------------------------------------------------------------------*/
636 /*---------------------------------------------------------------------------*/
637 
638 static void domed(float *buf, int n, float *val) {
639  int is_even,n2;
640 
641  is_even = ! (n & 1);
642  casu_sort(&buf,n,1);
643  n2 = n/2;
644  if (is_even) {
645  *val = 0.5*(buf[n2-1] + buf[n2]);
646  } else {
647  if (n <= 5)
648  *val = buf[n2];
649  else
650  *val = 0.5*buf[n2] + 0.25*(buf[n2-1] + buf[n2+1]);
651  }
652 }
653 
656 /*
657 
658 $Log: casu_pawsky_minus.c,v $
659 Revision 1.4 2015/11/25 10:26:31 jim
660 replaced some hardcoded numbers with defines
661 
662 Revision 1.3 2015/09/11 09:27:53 jim
663 Fixed some problems with the docs
664 
665 Revision 1.2 2015/08/07 13:06:54 jim
666 Fixed copyright to ESO
667 
668 Revision 1.1.1.1 2015/06/12 10:44:32 jim
669 Initial import
670 
671 Revision 1.9 2015/03/12 09:16:51 jim
672 Modified to remove some compiler moans
673 
674 Revision 1.8 2015/01/29 11:52:36 jim
675 Modified comments + fixed error in variance propagation
676 
677 Revision 1.7 2015/01/09 12:13:15 jim
678 *** empty log message ***
679 
680 Revision 1.6 2014/12/11 12:23:33 jim
681 new version
682 
683 Revision 1.5 2014/04/09 11:08:21 jim
684 Get rid of a couple of compiler moans
685 
686 Revision 1.4 2014/03/26 15:55:40 jim
687 Uses floating point confidence
688 
689 Revision 1.3 2013/11/21 09:38:14 jim
690 detabbed
691 
692 Revision 1.2 2013-10-24 09:25:21 jim
693 traps for dummy input files and tries to remove them from the analysis
694 
695 Revision 1.1.1.1 2013-08-27 12:07:48 jim
696 Imported
697 
698 
699 */
700 
int casu_fits_get_status(casu_fits *p)
Definition: casu_fits.c:711
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
char * casu_fits_get_filename(casu_fits *p)
Definition: casu_fits.c:646
casu_fits * casu_fits_wrap(cpl_image *im, casu_fits *model, cpl_propertylist *phu, cpl_propertylist *ehu)
Definition: casu_fits.c:883
void casu_fits_set_status(casu_fits *p, int status)
Definition: casu_fits.c:801
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
int casu_inpaint(casu_fits *in, int nbsize, int *status)
Inpaint pixels or patches in a map.
Definition: casu_inpaint.c:83
int casu_pawsky_minus(casu_fits **infiles, casu_fits **invar, casu_fits *conf, casu_fits *objmaskfits, int nfiles, casu_fits **skyout, casu_fits **skyvar, int *status)
Background correct images using pawsky_minus algorithm.
float casu_med(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:89
void casu_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: casu_stats.c:258
float casu_mean(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:479
cpl_image * casu_dummy_image(casu_fits *model)
Create a dummy image of zeros based on a model.
Definition: casu_utils.c:533
void casu_prov(cpl_propertylist *p, casu_fits **inlist, int n, int isextn)
Write provenance keywords.
Definition: casu_utils.c:287
void casu_dummy_property(cpl_propertylist *p)
Set dummy property keyword.
Definition: casu_utils.c:445
void casu_sort(float **a, int n, int m)
Sort a 2d array by the first column and co-sort the rest.
Definition: casu_utils.c:349
void casu_xytoxy_list(cpl_wcs *wcs1, cpl_wcs *wcs2, int nc, double *x_1, double *y_1, double *x_2, double *y_2)