VIRCAM Pipeline 2.3.15
vircam_tilesky_minus.c
1/* $Id: vircam_tilesky_minus.c,v 1.1 2013-10-15 16:30:07 jim Exp $
2 *
3 * This file is part of the VIRCAM Pipeline
4 * Copyright (C) 2015 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20
21/*
22 * $Author: jim $
23 * $Date: 2013-10-15 16:30:07 $
24 * $Revision: 1.1 $
25 * $Name: not supported by cvs2svn $
26 */
27
28/* Includes */
29
30#ifdef HAVE_CONFIG_H
31#include <config.h>
32#endif
33
34#include <cpl.h>
35#include <math.h>
36#include <string.h>
37
38#include <casu_utils.h>
39#include <casu_mods.h>
40#include <casu_mask.h>
41#include <casu_stats.h>
42#include <casu_wcsutils.h>
43
44#include "vircam_sky.h"
45#include "vircam_mods.h"
46#include "vircam_dfs.h"
47#include "vircam_pfits.h"
48
49typedef struct {
50 int pawposition;
51 int nimages;
52 casu_fits **infiles;
53 float **datas;
54 unsigned char **masks;
55 float *master_xsky;
56 unsigned char *master_xsky_bpm;
57 float **xsky_variant;
58 unsigned char **xsky_variant_bpm;
59 float *zeros;
60 float *dx;
61 float *dy;
62} jpos;
63static void combine_master(int nfiles, int stat, int npts, float **datas,
64 unsigned char **masks, float **skyout,
65 unsigned char **skyout_bpm);
66static void combine_variant(int stat, int npts, jpos *jp, int skipme);
67static void combine_final(int nxskys, int stat, int npts, float **xskys,
68 unsigned char **xsky_masks, float *data,
69 float *master_sky, unsigned char *master_bpm);
70static void masksky_zeros(jpos *jp, int njit, int npts);
71static void domed(float *buf, int n, float *val);
72static jpos *jpos_init(int njit);
73static void jpos_alloc(casu_fits **infiles, int nfiles, int pawpos, jpos *jp);
74static void jpos_free(jpos *jp, int njit);
75
76extern int vircam_tilesky_minus(casu_fits **infiles, casu_fits *conf,
77 casu_fits *objmaskfits, int nfiles,
78 casu_fits **skyout, int *status) {
79 int i, nx_mask = 0,ind,j,nx,ny,kind,njit,*confdata = NULL,jx,jy,ix,iy,k;
80 int npts,*opm = NULL;
81 jpos *jp=NULL;
82 cpl_wcs *wcsmask,*wcsimg;
83 cpl_image *im,*skyim;
84 cpl_propertylist *plist;
85 unsigned char **xsky_masks,*master_sky_lev2_bpm;
86 double *xin = NULL,*yin = NULL,*xout = NULL,*yout = NULL,*dx = NULL,*dy = NULL,ddx,ddy;
87 float **xskys,*master_sky_lev2,val,sig;
88 const char *fctid = "vircam_tilesky_minus";
89
90 /* Inherited status */
91
92 *skyout = NULL;
93 if (*status != CASU_OK)
94 return(*status);
95
96 /* If there aren't any images, then get out of here */
97
98 if (nfiles == 0) {
99 cpl_msg_error(fctid,"Sky correction impossible. No science frames");
100 return(CASU_FATAL);
101 }
102
103 /* Initialise the jpos structure. There is one for each jitter position.
104 Each jitter position has a maximum number of observations which is
105 the number of jitter sequences. This is nominally 6, but will
106 sometimes be 3 if we are observing in stripes or something else
107 if the tile is incomplete. */
108
109 (void)vircam_pfits_get_njsteps(casu_fits_get_phu(infiles[0]),&njit);
110 jp = jpos_init(njit);
111 for (i = 1; i <= njit; i++)
112 jpos_alloc(infiles,nfiles,i,jp+i-1);
113 nx = cpl_image_get_size_x(casu_fits_get_image(infiles[0]));
114 ny = cpl_image_get_size_y(casu_fits_get_image(infiles[0]));
115 npts = nx*ny;
116
117 /* Get the wcs descriptor for the mask (if it exists) */
118
119 if (objmaskfits != NULL) {
120 nx_mask = cpl_image_get_size_x(casu_fits_get_image(objmaskfits));
121 wcsmask = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(objmaskfits));
122 opm = cpl_image_get_data_int(casu_fits_get_image(objmaskfits));
123
124 /* We need this space so that we can work out which part of the
125 input mask is relevant for each input image */
126
127 xin = cpl_malloc(npts*sizeof(double));
128 yin = cpl_malloc(npts*sizeof(double));
129 xout = cpl_malloc(npts*sizeof(double));
130 yout = cpl_malloc(npts*sizeof(double));
131 dx = cpl_malloc(nfiles*sizeof(double));
132 dy = cpl_malloc(nfiles*sizeof(double));
133
134 /* Initialise the input xy arrays */
135
136 ind = 0;
137 for (j = 0; j < ny; j++) {
138 for (i = 0; i < nx; i++) {
139 xin[ind] = (double)(i+1);
140 yin[ind++] = (double)(j+1);
141 }
142 }
143
144 /* Define an output grid for the first image and then offsets for
145 the subsequent images */
146
147 ddx = 1.0;
148 ddy = 1.0;
149 for (i = 0; i < nfiles; i++) {
150 wcsimg = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(infiles[i]));
151 if (i == 0)
152 casu_xytoxy_list(wcsimg,wcsmask,npts,xin,yin,xout,yout);
153 casu_xytoxy_list(wcsimg,wcsmask,1,&ddx,&ddy,dx+i,dy+i);
154 dx[i] -= xout[0];
155 dy[i] -= yout[0];
156 cpl_wcs_delete(wcsimg);
157 }
158 cpl_wcs_delete(wcsmask);
159 }
160
161 /* Loop for each file and get a reference for the data array. Get
162 some space for a mask that will cover both the zeroed pixels in the
163 confidence maps and any object mask that might be used */
164
165 for (j = 0; j < njit; j++) {
166 for (i = 0; i < jp[j].nimages; i++) {
167 im = casu_fits_get_image(jp[j].infiles[i]);
168 jp[j].datas[i] = cpl_image_get_data_float(im);
169 jp[j].masks[i] = cpl_calloc(npts,sizeof(unsigned char));
170 if (i == 0 && j == 0)
171 confdata = cpl_image_get_data_int(casu_fits_get_image(conf));
172
173 /* Now loop for each pixel. If the confidence map is zero here then
174 mark this as a bad pixel. If not, then look at the master mask
175 and see which is its closest pixel to the current one. NB: the
176 -0.5 in the index is a combination of +0.5 to do a nint and a -1
177 to take account of the fact that arrays start from index zero. */
178
179 for (jy = 0; jy < ny; jy++) {
180 for (jx = 0; jx < nx; jx++) {
181 ind = jy*nx + jx;
182 if (confdata[ind] == 0) {
183 jp[j].masks[i][ind] = 1;
184 } else if (objmaskfits != NULL) {
185 kind = (int)(yout[ind] + dy[i] - 0.5)*nx_mask +
186 (int)(xout[ind] + dx[i] - 0.5);
187 jp[j].masks[i][ind] = opm[kind];
188 }
189 }
190 }
191 }
192 }
193
194 /* Do an intermediate tidy */
195
196 if (objmaskfits != NULL) {
197 freespace(xin);
198 freespace(yin);
199 freespace(xout);
200 freespace(yout);
201 freespace(dx);
202 freespace(dy);
203 }
204
205 /* Offset the DC level for all images together */
206
207 masksky_zeros(jp,njit,npts);
208
209 /* Now loop through all the positions again and form the xskys and
210 all their variants. The 'master' is an xsky that is formed from
211 all the images. The variant xskys are formed from all but one
212 of the input images (removing one image from the list each time). */
213
214 for (i = 0; i < njit; i++) {
215
216 /* Form the master xsky for this position */
217
218 combine_master(jp[i].nimages,0,npts,jp[i].datas,jp[i].masks,
219 &(jp[i].master_xsky),&(jp[i].master_xsky_bpm));
220
221 /* Form the variant xskys for this position */
222
223 for (j = 0; j < jp[i].nimages; j++)
224 combine_variant(0,npts,jp+i,j);
225 }
226
227 /* Form master level 2 sky */
228
229 xskys = cpl_malloc(njit*sizeof(float *));
230 xsky_masks = cpl_malloc(njit*sizeof(unsigned char *));
231 for (i = 0; i < njit; i++) {
232 xskys[i] = jp[i].master_xsky;
233 xsky_masks[i] = jp[i].master_xsky_bpm;
234 }
235 combine_master(njit,0,npts,xskys,xsky_masks,&master_sky_lev2,
236 &master_sky_lev2_bpm);
237
238 /* Wrap the sky and reject the pixels that had no sky contribution */
239
240 skyim = cpl_image_wrap_float(nx,ny,master_sky_lev2);
241 for (i = 0; i < npts; i++) {
242 iy = i/nx + 1;
243 ix = i - (iy-1)*nx + 1;
244 if (master_sky_lev2_bpm[i])
245 cpl_image_reject(skyim,(cpl_size)ix,(cpl_size)iy);
246 }
247 *skyout = casu_fits_wrap(skyim,infiles[0],NULL,NULL);
248 plist = casu_fits_get_ehu(*skyout);
249 if (objmaskfits != NULL) {
250 cpl_propertylist_update_string(plist,"ESO DRS MASKUSED",
251 casu_fits_get_filename(objmaskfits));
252 cpl_propertylist_set_comment(plist,"ESO DRS MASKUSED",
253 "Object masked used to make sky");
254 }
255 cpl_propertylist_update_string(plist,"ESO DRS SKYALGO","tilesky_minus");
256 cpl_propertylist_set_comment(plist,"ESO DRS SKYALGO",
257 "Sky estimation algorithm");
258 casu_prov(plist,infiles,nfiles,1);
259
260 /* Fill in the bits with no sky contribution */
261
262 casu_inpaint(*skyout,64,status);
263
264 /* OK, now loop through each of the jitter positions again and
265 form the variant level 2 sky and subtract it off the data frames */
266
267 for (j = 0; j < njit; j++) {
268 for (i = 0; i < njit; i++) {
269 xskys[i] = jp[i].master_xsky;
270 xsky_masks[i] = jp[i].master_xsky_bpm;
271 }
272 for (i = 0; i < jp[j].nimages; i++) {
273 xskys[j] = jp[j].xsky_variant[i];
274 xsky_masks[j] = jp[j].xsky_variant_bpm[i];
275 combine_final(njit,0,npts,xskys,xsky_masks,jp[j].datas[i],
276 master_sky_lev2,master_sky_lev2_bpm);
277
278 /* Readjust the DC to be what it was before sky subtraction */
279
280 casu_qmedsig(jp[j].datas[i],jp[j].masks[i],(long)npts,3.0,1,
281 -1000.0,65535.0,&val,&sig);
282 val = jp[j].zeros[i] - val;
283 for (k = 0; k < npts; k++)
284 jp[j].datas[i][k] += val;
285 }
286 }
287
288 /* Tidy up */
289
290 freespace(xskys);
291 freespace(xsky_masks);
292 freespace(master_sky_lev2_bpm);
293 jpos_free(jp,njit);
294 return(CASU_OK);
295}
296
297static void combine_master(int nfiles, int stat, int npts, float **datas,
298 unsigned char **masks, float **skyout,
299 unsigned char **skyout_bpm) {
300 int i,j,n;
301 float *buf;
302
303 /* Get workspace for output arrays */
304
305 *skyout = cpl_malloc(npts*sizeof(float));
306 *skyout_bpm = cpl_malloc(npts*sizeof(unsigned char));
307 buf = cpl_malloc(nfiles*sizeof(float));
308
309 /* Loop now for each input pixel and form the median */
310
311 for (j = 0; j < npts; j++) {
312 n = 0;
313 for (i = 0; i < nfiles; i++) {
314 if (masks[i][j] == 0)
315 buf[n++] = datas[i][j];
316 }
317 if (n == 0) {
318 (*skyout)[j] = 0.0;
319 (*skyout_bpm)[j] = 1;
320 } else {
321 if (stat == 0)
322 domed(buf,n,(*skyout)+j);
323 else
324 (*skyout)[j] = casu_mean(buf,NULL,n);
325 (*skyout_bpm)[j] = 0;
326 }
327 }
328
329 /* Free some workspace and get out of here */
330
331 freespace(buf);
332 return;
333}
334
335static void combine_variant(int stat, int npts, jpos *jp, int skipme) {
336 int i,j,n,nfiles;
337 float *buf,**datas,*skyout,*master_sky;
338 unsigned char **masks,*skyout_bpm,*master_sky_bpm;
339
340 /* Create some shortcuts */
341
342 datas = jp->datas;
343 masks = jp->masks;
344 master_sky = jp->master_xsky;
345 master_sky_bpm = jp->master_xsky_bpm;
346 nfiles = jp->nimages;
347
348 /* Get workspace for output arrays */
349
350 skyout = cpl_malloc(npts*sizeof(float));
351 jp->xsky_variant[skipme] = skyout;
352 skyout_bpm = cpl_malloc(npts*sizeof(unsigned char));
353 jp->xsky_variant_bpm[skipme] = skyout_bpm;
354 buf = cpl_malloc(nfiles*sizeof(float));
355
356 /* If there is only one file in this group then just replace the
357 variant with the master and get out of here */
358
359 if (nfiles == 1) {
360 memmove(skyout,jp->master_xsky,npts*sizeof(float));
361 memmove(skyout_bpm,jp->master_xsky_bpm,npts*sizeof(unsigned char));
362 freespace(buf);
363 return;
364 }
365
366 /* Loop now for each input pixel and form the median */
367
368 for (j = 0; j < npts; j++) {
369 n = 0;
370
371 /* If the current pixel was masked out in the skipped image, then
372 there isn't any point in recalculating the mean value again */
373
374 if (masks[skipme][j]) {
375 skyout[j] = master_sky[j];
376 skyout_bpm[j] = master_sky_bpm[j];
377 continue;
378 }
379
380 /* Recalculate */
381
382 for (i = 0; i < nfiles; i++) {
383 if (masks[i][j] == 0 && i != skipme)
384 buf[n++] = datas[i][j];
385 }
386 if (n == 0) {
387 skyout[j] = 0.0;
388 skyout_bpm[j] = 1;
389 } else {
390 if (stat == 0)
391 domed(buf,n,skyout+j);
392 else
393 skyout[j] = casu_mean(buf,NULL,n);
394 skyout_bpm[j] = 0;
395 }
396 }
397
398 /* Free some workspace and get out of here */
399
400 freespace(buf);
401 return;
402}
403
404static void combine_final(int nxskys, int stat, int npts, float **xskys,
405 unsigned char **xsky_masks, float *data,
406 float *master_sky, unsigned char *master_bpm) {
407 int i,j,n;
408 float *buf,val;
409
410 /* Get some memory */
411
412 buf = cpl_malloc(nxskys*sizeof(float));
413
414 /* Loop now for each input pixel */
415
416 for (j = 0; j < npts; j++) {
417
418 /* If the master bpm is flagged, then this is an interpolated point
419 and there's no reason to recalculate */
420
421 if (master_bpm[j]) {
422 val = master_sky[j];
423 } else {
424
425 /* Loop for each file. If the current pixel in the current file
426 was flagged, then the sky for that pixel will be the same as
427 the general sky we did before. If not, then remove that file's
428 pixel and recalculate the sky */
429
430 n = 0;
431 for (i = 0; i < nxskys; i++) {
432 if (xsky_masks[i][j] != 0)
433 continue;
434 buf[n++] = xskys[i][j];
435 }
436 if (n == 0) {
437 val = master_sky[j];
438 } else {
439 if (stat == 0)
440 domed(buf,n,&val);
441 else
442 val = casu_mean(buf,NULL,n);
443 }
444 }
445
446 /* Now correct this pixel in the input image */
447
448 data[j] -= val;
449 }
450
451 /* Free some workspace and get out of here */
452
453 freespace(buf);
454 return;
455}
456
457
458static jpos *jpos_init(int njit) {
459 jpos *jp;
460 int i;
461
462 jp = cpl_malloc(njit*sizeof(jpos));
463 for (i = 0; i < njit; i++) {
464 jp[i].pawposition = i + 1;
465 jp[i].nimages = 0;
466 jp[i].infiles = NULL;
467 jp[i].datas = NULL;
468 jp[i].masks = NULL;
469 jp[i].master_xsky = NULL;
470 jp[i].master_xsky_bpm = NULL;
471 jp[i].xsky_variant = NULL;
472 jp[i].xsky_variant_bpm = NULL;
473 jp[i].zeros = NULL;
474 jp[i].dx = NULL;
475 jp[i].dy = NULL;
476 }
477 return(jp);
478}
479
480static void jpos_alloc(casu_fits **infiles, int nfiles, int pawpos, jpos *jp) {
481 int i,ji;
482
483 /* First the paw position */
484
485 jp->pawposition = pawpos;
486
487 /* Now loop through the files and find which ones are at this position */
488
489 jp->nimages = 0;
490 jp->infiles = cpl_malloc(6*sizeof(casu_fits *));
491 for (i = 0; i < nfiles; i++) {
492 (void)vircam_pfits_get_jitteri(casu_fits_get_phu(infiles[i]),&ji);
493 if (ji != pawpos)
494 continue;
495 jp->infiles[jp->nimages] = infiles[i];
496 jp->nimages += 1;
497 }
498 jp->infiles = cpl_realloc(jp->infiles,jp->nimages*sizeof(casu_fits *));
499
500 /* Right, safely through all the hard bits. Finish allocating memory */
501
502 jp->datas = cpl_calloc(jp->nimages,sizeof(float *));
503 jp->masks = cpl_calloc(jp->nimages,sizeof(unsigned char *));
504 jp->master_xsky = NULL;
505 jp->master_xsky_bpm = NULL;
506 jp->xsky_variant = cpl_calloc(jp->nimages,sizeof(float *));
507 jp->xsky_variant_bpm = cpl_calloc(jp->nimages,sizeof(unsigned char *));
508 jp->zeros = cpl_calloc(jp->nimages,sizeof(float));
509 jp->dx = cpl_calloc(jp->nimages,sizeof(float));
510 jp->dy = cpl_calloc(jp->nimages,sizeof(float));
511}
512
513static void jpos_free(jpos *jp, int njit) {
514 int i,j;
515
516 for (i = 0; i < njit; i++) {
517 freespace(jp[i].infiles);
518 freespace(jp[i].master_xsky);
519 freespace(jp[i].master_xsky_bpm);
520 freespace(jp[i].zeros);
521 for (j = 0; j < jp[i].nimages; j++) {
522 freespace(jp[i].masks[j]);
523 freespace(jp[i].xsky_variant[j]);
524 freespace(jp[i].xsky_variant_bpm[j]);
525 }
526 freespace(jp[i].datas);
527 freespace(jp[i].masks);
528 freespace(jp[i].xsky_variant);
529 freespace(jp[i].xsky_variant_bpm);
530 freespace(jp[i].dx);
531 freespace(jp[i].dy);
532 }
533 freespace(jp);
534}
535
536static void masksky_zeros(jpos *jp, int njit, int npts) {
537 int i,j,k,nf;
538 float sig,medval,off,*zeros;
539
540 /* How many files total are there? */
541
542 nf = 0;
543 for (j = 0; j < njit; j++)
544 nf += jp[j].nimages;
545
546 /* Get some workspace */
547
548 zeros = cpl_malloc(nf*sizeof(float));
549
550 /* Loop for each input image and get the background median */
551
552 nf = 0;
553 for (j = 0; j < njit; j++) {
554 for (i = 0; i < jp[j].nimages; i++) {
555 casu_qmedsig(jp[j].datas[i],jp[j].masks[i],(long)npts,3.0,1,
556 -1000.0,65535.0,zeros+nf,&sig);
557 jp[j].zeros[i] = zeros[nf];
558 nf++;
559 }
560 }
561
562 /* Get the median value of the array */
563
564 medval = casu_med(zeros,NULL,nf);
565
566 /* Now work out the offset for each image */
567
568 for (j = 0; j < njit; j++) {
569 for (i = 0; i < jp[j].nimages; i++) {
570 off = medval - jp[j].zeros[i];
571 for (k = 0; k < npts; k++)
572 jp[j].datas[i][k] += off;
573 }
574 }
575 freespace(zeros);
576}
577
578static void domed(float *buf, int n, float *val) {
579 int is_even,n2;
580
581 is_even = ! (n & 1);
582 casu_sort(&buf,n,1);
583 n2 = n/2;
584 if (is_even) {
585 *val = 0.5*(buf[n2-1] + buf[n2]);
586 } else {
587 if (n <= 5)
588 *val = buf[n2];
589 else
590 *val = 0.5*buf[n2] + 0.25*(buf[n2-1] + buf[n2+1]);
591 }
592}
593
594/*
595
596$Log: not supported by cvs2svn $
597
598*/
casu_fits * casu_fits_wrap(cpl_image *im, casu_fits *model, cpl_propertylist *phu, cpl_propertylist *ehu)
Definition: casu_fits.c:883
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
cpl_propertylist * casu_fits_get_phu(casu_fits *p)
Definition: casu_fits.c:531
char * casu_fits_get_filename(casu_fits *p)
Definition: casu_fits.c:646
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
int casu_inpaint(casu_fits *in, int nbsize, int *status)
Inpaint pixels or patches in a map.
Definition: casu_inpaint.c:83
float casu_med(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:89
void casu_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: casu_stats.c:258
float casu_mean(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:479
void casu_prov(cpl_propertylist *p, casu_fits **inlist, int n, int isextn)
Write provenance keywords.
Definition: casu_utils.c:287
void casu_sort(float **a, int n, int m)
Sort a 2d array by the first column and co-sort the rest.
Definition: casu_utils.c:349
void casu_xytoxy_list(cpl_wcs *wcs1, cpl_wcs *wcs2, int nc, double *x_1, double *y_1, double *x_2, double *y_2)
int vircam_pfits_get_jitteri(const cpl_propertylist *plist, int *jitteri)
Get the position number of an observations in a jitter sequence.
Definition: vircam_pfits.c:530
int vircam_pfits_get_njsteps(const cpl_propertylist *plist, int *njsteps)
Get the value of the number of observations in a jitter sequence.
Definition: vircam_pfits.c:476