VIRCAM Pipeline  2.3.12
vircam_tilesky_minus.c
1 /* $Id: vircam_tilesky_minus.c,v 1.1 2013-10-15 16:30:07 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2013-10-15 16:30:07 $
24  * $Revision: 1.1 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <cpl.h>
35 #include <math.h>
36 #include <string.h>
37 
38 #include <casu_utils.h>
39 #include <casu_mods.h>
40 #include <casu_mask.h>
41 #include <casu_stats.h>
42 #include <casu_wcsutils.h>
43 
44 #include "vircam_sky.h"
45 #include "vircam_mods.h"
46 #include "vircam_dfs.h"
47 #include "vircam_pfits.h"
48 
49 typedef struct {
50  int pawposition;
51  int nimages;
52  casu_fits **infiles;
53  float **datas;
54  unsigned char **masks;
55  float *master_xsky;
56  unsigned char *master_xsky_bpm;
57  float **xsky_variant;
58  unsigned char **xsky_variant_bpm;
59  float *zeros;
60  float *dx;
61  float *dy;
62 } jpos;
63 static void combine_master(int nfiles, int stat, int npts, float **datas,
64  unsigned char **masks, float **skyout,
65  unsigned char **skyout_bpm);
66 static void combine_variant(int stat, int npts, jpos *jp, int skipme);
67 static void combine_final(int nxskys, int stat, int npts, float **xskys,
68  unsigned char **xsky_masks, float *data,
69  float *master_sky, unsigned char *master_bpm);
70 static void masksky_zeros(jpos *jp, int njit, int npts);
71 static void domed(float *buf, int n, float *val);
72 static jpos *jpos_init(int njit);
73 static void jpos_alloc(casu_fits **infiles, int nfiles, int pawpos, jpos *jp);
74 static void jpos_free(jpos *jp, int njit);
75 
76 extern int vircam_tilesky_minus(casu_fits **infiles, casu_fits *conf,
77  casu_fits *objmaskfits, int nfiles,
78  casu_fits **skyout, int *status) {
79  int i, nx_mask = 0,ind,j,nx,ny,kind,njit,*confdata = NULL,jx,jy,ix,iy,k;
80  int npts,*opm = NULL;
81  jpos *jp=NULL;
82  cpl_wcs *wcsmask,*wcsimg;
83  cpl_image *im,*skyim;
84  cpl_propertylist *plist;
85  unsigned char **xsky_masks,*master_sky_lev2_bpm;
86  double *xin = NULL,*yin = NULL,*xout = NULL,*yout = NULL,*dx = NULL,*dy = NULL,ddx,ddy;
87  float **xskys,*master_sky_lev2,val,sig;
88  const char *fctid = "vircam_tilesky_minus";
89 
90  /* Inherited status */
91 
92  *skyout = NULL;
93  if (*status != CASU_OK)
94  return(*status);
95 
96  /* If there aren't any images, then get out of here */
97 
98  if (nfiles == 0) {
99  cpl_msg_error(fctid,"Sky correction impossible. No science frames");
100  return(CASU_FATAL);
101  }
102 
103  /* Initialise the jpos structure. There is one for each jitter position.
104  Each jitter position has a maximum number of observations which is
105  the number of jitter sequences. This is nominally 6, but will
106  sometimes be 3 if we are observing in stripes or something else
107  if the tile is incomplete. */
108 
109  (void)vircam_pfits_get_njsteps(casu_fits_get_phu(infiles[0]),&njit);
110  jp = jpos_init(njit);
111  for (i = 1; i <= njit; i++)
112  jpos_alloc(infiles,nfiles,i,jp+i-1);
113  nx = cpl_image_get_size_x(casu_fits_get_image(infiles[0]));
114  ny = cpl_image_get_size_y(casu_fits_get_image(infiles[0]));
115  npts = nx*ny;
116 
117  /* Get the wcs descriptor for the mask (if it exists) */
118 
119  if (objmaskfits != NULL) {
120  nx_mask = cpl_image_get_size_x(casu_fits_get_image(objmaskfits));
121  wcsmask = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(objmaskfits));
122  opm = cpl_image_get_data_int(casu_fits_get_image(objmaskfits));
123 
124  /* We need this space so that we can work out which part of the
125  input mask is relevant for each input image */
126 
127  xin = cpl_malloc(npts*sizeof(double));
128  yin = cpl_malloc(npts*sizeof(double));
129  xout = cpl_malloc(npts*sizeof(double));
130  yout = cpl_malloc(npts*sizeof(double));
131  dx = cpl_malloc(nfiles*sizeof(double));
132  dy = cpl_malloc(nfiles*sizeof(double));
133 
134  /* Initialise the input xy arrays */
135 
136  ind = 0;
137  for (j = 0; j < ny; j++) {
138  for (i = 0; i < nx; i++) {
139  xin[ind] = (double)(i+1);
140  yin[ind++] = (double)(j+1);
141  }
142  }
143 
144  /* Define an output grid for the first image and then offsets for
145  the subsequent images */
146 
147  ddx = 1.0;
148  ddy = 1.0;
149  for (i = 0; i < nfiles; i++) {
150  wcsimg = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(infiles[i]));
151  if (i == 0)
152  casu_xytoxy_list(wcsimg,wcsmask,npts,xin,yin,xout,yout);
153  casu_xytoxy_list(wcsimg,wcsmask,1,&ddx,&ddy,dx+i,dy+i);
154  dx[i] -= xout[0];
155  dy[i] -= yout[0];
156  cpl_wcs_delete(wcsimg);
157  }
158  cpl_wcs_delete(wcsmask);
159  }
160 
161  /* Loop for each file and get a reference for the data array. Get
162  some space for a mask that will cover both the zeroed pixels in the
163  confidence maps and any object mask that might be used */
164 
165  for (j = 0; j < njit; j++) {
166  for (i = 0; i < jp[j].nimages; i++) {
167  im = casu_fits_get_image(jp[j].infiles[i]);
168  jp[j].datas[i] = cpl_image_get_data_float(im);
169  jp[j].masks[i] = cpl_calloc(npts,sizeof(unsigned char));
170  if (i == 0 && j == 0)
171  confdata = cpl_image_get_data_int(casu_fits_get_image(conf));
172 
173  /* Now loop for each pixel. If the confidence map is zero here then
174  mark this as a bad pixel. If not, then look at the master mask
175  and see which is its closest pixel to the current one. NB: the
176  -0.5 in the index is a combination of +0.5 to do a nint and a -1
177  to take account of the fact that arrays start from index zero. */
178 
179  for (jy = 0; jy < ny; jy++) {
180  for (jx = 0; jx < nx; jx++) {
181  ind = jy*nx + jx;
182  if (confdata[ind] == 0) {
183  jp[j].masks[i][ind] = 1;
184  } else if (objmaskfits != NULL) {
185  kind = (int)(yout[ind] + dy[i] - 0.5)*nx_mask +
186  (int)(xout[ind] + dx[i] - 0.5);
187  jp[j].masks[i][ind] = opm[kind];
188  }
189  }
190  }
191  }
192  }
193 
194  /* Do an intermediate tidy */
195 
196  if (objmaskfits != NULL) {
197  freespace(xin);
198  freespace(yin);
199  freespace(xout);
200  freespace(yout);
201  freespace(dx);
202  freespace(dy);
203  }
204 
205  /* Offset the DC level for all images together */
206 
207  masksky_zeros(jp,njit,npts);
208 
209  /* Now loop through all the positions again and form the xskys and
210  all their variants. The 'master' is an xsky that is formed from
211  all the images. The variant xskys are formed from all but one
212  of the input images (removing one image from the list each time). */
213 
214  for (i = 0; i < njit; i++) {
215 
216  /* Form the master xsky for this position */
217 
218  combine_master(jp[i].nimages,0,npts,jp[i].datas,jp[i].masks,
219  &(jp[i].master_xsky),&(jp[i].master_xsky_bpm));
220 
221  /* Form the variant xskys for this position */
222 
223  for (j = 0; j < jp[i].nimages; j++)
224  combine_variant(0,npts,jp+i,j);
225  }
226 
227  /* Form master level 2 sky */
228 
229  xskys = cpl_malloc(njit*sizeof(float *));
230  xsky_masks = cpl_malloc(njit*sizeof(unsigned char *));
231  for (i = 0; i < njit; i++) {
232  xskys[i] = jp[i].master_xsky;
233  xsky_masks[i] = jp[i].master_xsky_bpm;
234  }
235  combine_master(njit,0,npts,xskys,xsky_masks,&master_sky_lev2,
236  &master_sky_lev2_bpm);
237 
238  /* Wrap the sky and reject the pixels that had no sky contribution */
239 
240  skyim = cpl_image_wrap_float(nx,ny,master_sky_lev2);
241  for (i = 0; i < npts; i++) {
242  iy = i/nx + 1;
243  ix = i - (iy-1)*nx + 1;
244  if (master_sky_lev2_bpm[i])
245  cpl_image_reject(skyim,(cpl_size)ix,(cpl_size)iy);
246  }
247  *skyout = casu_fits_wrap(skyim,infiles[0],NULL,NULL);
248  plist = casu_fits_get_ehu(*skyout);
249  if (objmaskfits != NULL) {
250  cpl_propertylist_update_string(plist,"ESO DRS MASKUSED",
251  casu_fits_get_filename(objmaskfits));
252  cpl_propertylist_set_comment(plist,"ESO DRS MASKUSED",
253  "Object masked used to make sky");
254  }
255  cpl_propertylist_update_string(plist,"ESO DRS SKYALGO","tilesky_minus");
256  cpl_propertylist_set_comment(plist,"ESO DRS SKYALGO",
257  "Sky estimation algorithm");
258  casu_prov(plist,infiles,nfiles,1);
259 
260  /* Fill in the bits with no sky contribution */
261 
262  casu_inpaint(*skyout,64,status);
263 
264  /* OK, now loop through each of the jitter positions again and
265  form the variant level 2 sky and subtract it off the data frames */
266 
267  for (j = 0; j < njit; j++) {
268  for (i = 0; i < njit; i++) {
269  xskys[i] = jp[i].master_xsky;
270  xsky_masks[i] = jp[i].master_xsky_bpm;
271  }
272  for (i = 0; i < jp[j].nimages; i++) {
273  xskys[j] = jp[j].xsky_variant[i];
274  xsky_masks[j] = jp[j].xsky_variant_bpm[i];
275  combine_final(njit,0,npts,xskys,xsky_masks,jp[j].datas[i],
276  master_sky_lev2,master_sky_lev2_bpm);
277 
278  /* Readjust the DC to be what it was before sky subtraction */
279 
280  casu_qmedsig(jp[j].datas[i],jp[j].masks[i],(long)npts,3.0,1,
281  -1000.0,65535.0,&val,&sig);
282  val = jp[j].zeros[i] - val;
283  for (k = 0; k < npts; k++)
284  jp[j].datas[i][k] += val;
285  }
286  }
287 
288  /* Tidy up */
289 
290  freespace(xskys);
291  freespace(xsky_masks);
292  freespace(master_sky_lev2_bpm);
293  jpos_free(jp,njit);
294  return(CASU_OK);
295 }
296 
297 static void combine_master(int nfiles, int stat, int npts, float **datas,
298  unsigned char **masks, float **skyout,
299  unsigned char **skyout_bpm) {
300  int i,j,n;
301  float *buf;
302 
303  /* Get workspace for output arrays */
304 
305  *skyout = cpl_malloc(npts*sizeof(float));
306  *skyout_bpm = cpl_malloc(npts*sizeof(unsigned char));
307  buf = cpl_malloc(nfiles*sizeof(float));
308 
309  /* Loop now for each input pixel and form the median */
310 
311  for (j = 0; j < npts; j++) {
312  n = 0;
313  for (i = 0; i < nfiles; i++) {
314  if (masks[i][j] == 0)
315  buf[n++] = datas[i][j];
316  }
317  if (n == 0) {
318  (*skyout)[j] = 0.0;
319  (*skyout_bpm)[j] = 1;
320  } else {
321  if (stat == 0)
322  domed(buf,n,(*skyout)+j);
323  else
324  (*skyout)[j] = casu_mean(buf,NULL,n);
325  (*skyout_bpm)[j] = 0;
326  }
327  }
328 
329  /* Free some workspace and get out of here */
330 
331  freespace(buf);
332  return;
333 }
334 
335 static void combine_variant(int stat, int npts, jpos *jp, int skipme) {
336  int i,j,n,nfiles;
337  float *buf,**datas,*skyout,*master_sky;
338  unsigned char **masks,*skyout_bpm,*master_sky_bpm;
339 
340  /* Create some shortcuts */
341 
342  datas = jp->datas;
343  masks = jp->masks;
344  master_sky = jp->master_xsky;
345  master_sky_bpm = jp->master_xsky_bpm;
346  nfiles = jp->nimages;
347 
348  /* Get workspace for output arrays */
349 
350  skyout = cpl_malloc(npts*sizeof(float));
351  jp->xsky_variant[skipme] = skyout;
352  skyout_bpm = cpl_malloc(npts*sizeof(unsigned char));
353  jp->xsky_variant_bpm[skipme] = skyout_bpm;
354  buf = cpl_malloc(nfiles*sizeof(float));
355 
356  /* If there is only one file in this group then just replace the
357  variant with the master and get out of here */
358 
359  if (nfiles == 1) {
360  memmove(skyout,jp->master_xsky,npts*sizeof(float));
361  memmove(skyout_bpm,jp->master_xsky_bpm,npts*sizeof(unsigned char));
362  freespace(buf);
363  return;
364  }
365 
366  /* Loop now for each input pixel and form the median */
367 
368  for (j = 0; j < npts; j++) {
369  n = 0;
370 
371  /* If the current pixel was masked out in the skipped image, then
372  there isn't any point in recalculating the mean value again */
373 
374  if (masks[skipme][j]) {
375  skyout[j] = master_sky[j];
376  skyout_bpm[j] = master_sky_bpm[j];
377  continue;
378  }
379 
380  /* Recalculate */
381 
382  for (i = 0; i < nfiles; i++) {
383  if (masks[i][j] == 0 && i != skipme)
384  buf[n++] = datas[i][j];
385  }
386  if (n == 0) {
387  skyout[j] = 0.0;
388  skyout_bpm[j] = 1;
389  } else {
390  if (stat == 0)
391  domed(buf,n,skyout+j);
392  else
393  skyout[j] = casu_mean(buf,NULL,n);
394  skyout_bpm[j] = 0;
395  }
396  }
397 
398  /* Free some workspace and get out of here */
399 
400  freespace(buf);
401  return;
402 }
403 
404 static void combine_final(int nxskys, int stat, int npts, float **xskys,
405  unsigned char **xsky_masks, float *data,
406  float *master_sky, unsigned char *master_bpm) {
407  int i,j,n;
408  float *buf,val;
409 
410  /* Get some memory */
411 
412  buf = cpl_malloc(nxskys*sizeof(float));
413 
414  /* Loop now for each input pixel */
415 
416  for (j = 0; j < npts; j++) {
417 
418  /* If the master bpm is flagged, then this is an interpolated point
419  and there's no reason to recalculate */
420 
421  if (master_bpm[j]) {
422  val = master_sky[j];
423  } else {
424 
425  /* Loop for each file. If the current pixel in the current file
426  was flagged, then the sky for that pixel will be the same as
427  the general sky we did before. If not, then remove that file's
428  pixel and recalculate the sky */
429 
430  n = 0;
431  for (i = 0; i < nxskys; i++) {
432  if (xsky_masks[i][j] != 0)
433  continue;
434  buf[n++] = xskys[i][j];
435  }
436  if (n == 0) {
437  val = master_sky[j];
438  } else {
439  if (stat == 0)
440  domed(buf,n,&val);
441  else
442  val = casu_mean(buf,NULL,n);
443  }
444  }
445 
446  /* Now correct this pixel in the input image */
447 
448  data[j] -= val;
449  }
450 
451  /* Free some workspace and get out of here */
452 
453  freespace(buf);
454  return;
455 }
456 
457 
458 static jpos *jpos_init(int njit) {
459  jpos *jp;
460  int i;
461 
462  jp = cpl_malloc(njit*sizeof(jpos));
463  for (i = 0; i < njit; i++) {
464  jp[i].pawposition = i + 1;
465  jp[i].nimages = 0;
466  jp[i].infiles = NULL;
467  jp[i].datas = NULL;
468  jp[i].masks = NULL;
469  jp[i].master_xsky = NULL;
470  jp[i].master_xsky_bpm = NULL;
471  jp[i].xsky_variant = NULL;
472  jp[i].xsky_variant_bpm = NULL;
473  jp[i].zeros = NULL;
474  jp[i].dx = NULL;
475  jp[i].dy = NULL;
476  }
477  return(jp);
478 }
479 
480 static void jpos_alloc(casu_fits **infiles, int nfiles, int pawpos, jpos *jp) {
481  int i,ji;
482 
483  /* First the paw position */
484 
485  jp->pawposition = pawpos;
486 
487  /* Now loop through the files and find which ones are at this position */
488 
489  jp->nimages = 0;
490  jp->infiles = cpl_malloc(6*sizeof(casu_fits *));
491  for (i = 0; i < nfiles; i++) {
492  (void)vircam_pfits_get_jitteri(casu_fits_get_phu(infiles[i]),&ji);
493  if (ji != pawpos)
494  continue;
495  jp->infiles[jp->nimages] = infiles[i];
496  jp->nimages += 1;
497  }
498  jp->infiles = cpl_realloc(jp->infiles,jp->nimages*sizeof(casu_fits *));
499 
500  /* Right, safely through all the hard bits. Finish allocating memory */
501 
502  jp->datas = cpl_calloc(jp->nimages,sizeof(float *));
503  jp->masks = cpl_calloc(jp->nimages,sizeof(unsigned char *));
504  jp->master_xsky = NULL;
505  jp->master_xsky_bpm = NULL;
506  jp->xsky_variant = cpl_calloc(jp->nimages,sizeof(float *));
507  jp->xsky_variant_bpm = cpl_calloc(jp->nimages,sizeof(unsigned char *));
508  jp->zeros = cpl_calloc(jp->nimages,sizeof(float));
509  jp->dx = cpl_calloc(jp->nimages,sizeof(float));
510  jp->dy = cpl_calloc(jp->nimages,sizeof(float));
511 }
512 
513 static void jpos_free(jpos *jp, int njit) {
514  int i,j;
515 
516  for (i = 0; i < njit; i++) {
517  freespace(jp[i].infiles);
518  freespace(jp[i].master_xsky);
519  freespace(jp[i].master_xsky_bpm);
520  freespace(jp[i].zeros);
521  for (j = 0; j < jp[i].nimages; j++) {
522  freespace(jp[i].masks[j]);
523  freespace(jp[i].xsky_variant[j]);
524  freespace(jp[i].xsky_variant_bpm[j]);
525  }
526  freespace(jp[i].datas);
527  freespace(jp[i].masks);
528  freespace(jp[i].xsky_variant);
529  freespace(jp[i].xsky_variant_bpm);
530  freespace(jp[i].dx);
531  freespace(jp[i].dy);
532  }
533  freespace(jp);
534 }
535 
536 static void masksky_zeros(jpos *jp, int njit, int npts) {
537  int i,j,k,nf;
538  float sig,medval,off,*zeros;
539 
540  /* How many files total are there? */
541 
542  nf = 0;
543  for (j = 0; j < njit; j++)
544  nf += jp[j].nimages;
545 
546  /* Get some workspace */
547 
548  zeros = cpl_malloc(nf*sizeof(float));
549 
550  /* Loop for each input image and get the background median */
551 
552  nf = 0;
553  for (j = 0; j < njit; j++) {
554  for (i = 0; i < jp[j].nimages; i++) {
555  casu_qmedsig(jp[j].datas[i],jp[j].masks[i],(long)npts,3.0,1,
556  -1000.0,65535.0,zeros+nf,&sig);
557  jp[j].zeros[i] = zeros[nf];
558  nf++;
559  }
560  }
561 
562  /* Get the median value of the array */
563 
564  medval = casu_med(zeros,NULL,nf);
565 
566  /* Now work out the offset for each image */
567 
568  for (j = 0; j < njit; j++) {
569  for (i = 0; i < jp[j].nimages; i++) {
570  off = medval - jp[j].zeros[i];
571  for (k = 0; k < npts; k++)
572  jp[j].datas[i][k] += off;
573  }
574  }
575  freespace(zeros);
576 }
577 
578 static void domed(float *buf, int n, float *val) {
579  int is_even,n2;
580 
581  is_even = ! (n & 1);
582  casu_sort(&buf,n,1);
583  n2 = n/2;
584  if (is_even) {
585  *val = 0.5*(buf[n2-1] + buf[n2]);
586  } else {
587  if (n <= 5)
588  *val = buf[n2];
589  else
590  *val = 0.5*buf[n2] + 0.25*(buf[n2-1] + buf[n2+1]);
591  }
592 }
593 
594 /*
595 
596 $Log: not supported by cvs2svn $
597 
598 */
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
char * casu_fits_get_filename(casu_fits *p)
Definition: casu_fits.c:646
casu_fits * casu_fits_wrap(cpl_image *im, casu_fits *model, cpl_propertylist *phu, cpl_propertylist *ehu)
Definition: casu_fits.c:883
cpl_propertylist * casu_fits_get_phu(casu_fits *p)
Definition: casu_fits.c:531
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
int casu_inpaint(casu_fits *in, int nbsize, int *status)
Inpaint pixels or patches in a map.
Definition: casu_inpaint.c:83
float casu_med(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:89
void casu_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: casu_stats.c:258
float casu_mean(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:479
void casu_prov(cpl_propertylist *p, casu_fits **inlist, int n, int isextn)
Write provenance keywords.
Definition: casu_utils.c:287
void casu_sort(float **a, int n, int m)
Sort a 2d array by the first column and co-sort the rest.
Definition: casu_utils.c:349
void casu_xytoxy_list(cpl_wcs *wcs1, cpl_wcs *wcs2, int nc, double *x_1, double *y_1, double *x_2, double *y_2)
int vircam_pfits_get_jitteri(const cpl_propertylist *plist, int *jitteri)
Get the position number of an observations in a jitter sequence.
Definition: vircam_pfits.c:530
int vircam_pfits_get_njsteps(const cpl_propertylist *plist, int *njsteps)
Get the value of the number of observations in a jitter sequence.
Definition: vircam_pfits.c:476