VIRCAM Pipeline 2.3.12
casu_imdither.c
1/* $Id: casu_imdither.c,v 1.5 2015/11/25 10:26:31 jim Exp $
2 *
3 * This file is part of the CASU Pipeline utilities
4 * Copyright (C) 2015 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20
21/*
22 * $Author: jim $
23 * $Date: 2015/11/25 10:26:31 $
24 * $Revision: 1.5 $
25 * $Name: $
26 */
27
28/* Includes */
29
30#ifdef HAVE_CONFIG_H
31#include <config.h>
32#endif
33
34#include <math.h>
35#include <cpl.h>
36#include "casu_mods.h"
37#include "catalogue/casu_utils.h"
38#include "catalogue/casu_fits.h"
39#include "casu_stats.h"
40
41typedef struct {
42 casu_fits *fname;
43 casu_fits *conf;
44 float xoff;
45 float yoff;
46 int ixoff;
47 int iyoff;
48 int nx;
49 int ny;
50 float sky;
51 float skydiff;
52 float noise;
53 float expscale;
54 float weight;
55 float *data;
56 int *cdata;
57 int ndata;
58} dstrct;
59
60typedef struct {
61 float *values;
62 float *confs;
63 float *weights;
64 short int *iff;
65 int n;
66 long outindex;
67 unsigned char clipped;
68} keeptabs;
69
70#define NROWSBUF 512
71#define MINDATA -1000.0
72#define MAXDATA 65535.0
73
74static void average(dstrct *fileptrs, keeptabs *c, float *outdata,
75 float *outconf, float cliplow, float cliphigh, float lthr,
76 float hthr, float extra, float sumweight);
77static keeptabs *clip_open(int nimages, int nxo);
78static void clip_close(keeptabs **c, int nxo);
79static void skyest(float *data, int *cdata, long npts, float thresh,
80 float dmin, float dmax, float *skymed, float *skynoise);
81static void tidy(int nxo, dstrct *fileptrs, keeptabs *clipmon, float *owdata);
82
85/*---------------------------------------------------------------------------*/
149/*---------------------------------------------------------------------------*/
150
151extern int casu_imdither(casu_fits **inf, casu_fits **inconf, int nimages,
152 int nconfs, float lthr, float hthr,
153 cpl_propertylist **p, const char *expkey,
154 cpl_image **out, cpl_image **outc, int *status) {
155
156 int i,itx,iy,ccur,clast,ix,n,iline,icol,jy,jx,j,nxo,nyo,*ocdata;
157 long npts,ielm,iloc,index_y,index;
158 dstrct *fileptrs=NULL,*dd;
159 keeptabs *clipmon=NULL,*c;
160 float minxoff,minyoff,expref,sky,skynoise,clip1,clip2,outdata;
161 float outconf,avlev,avvar,renorm,exposure,sumweight,*odata,*owdata=NULL;
162 double crpix1,crpix2;
163 cpl_propertylist *ehu,*p2,*phu;
164 const char *fctid = "casu_imdither";
165 char timestamp[25];
166
167 /* Inherited status */
168
169 *out = NULL;
170 *outc = NULL;
171 *p = NULL;
172 if (*status != CASU_OK)
173 return(*status);
174
175 /* Is there any point in being here? */
176
177 if (nimages == 0) {
178 cpl_msg_error(fctid,"No input files to combine");
179 tidy(0,fileptrs,clipmon,owdata);
180 FATAL_ERROR
181 }
182
183 /* Allocate file struct array and fill in some values */
184
185 fileptrs = cpl_malloc(nimages*sizeof(dstrct));
186 if (cpl_propertylist_has(casu_fits_get_phu(inf[0]),expkey)) {
187 exposure = (float)cpl_propertylist_get_double(casu_fits_get_phu(inf[0]),
188 expkey);
189 } else {
190 exposure = 0.5;
191 }
192 expref = max(0.5,exposure);
193 minxoff = 1.0e10;
194 minyoff = 1.0e10;
195 for (i = 0; i < nimages; i++) {
196 dd = fileptrs + i;
197 dd->fname = inf[i];
198 dd->data = cpl_image_get_data_float(casu_fits_get_image(inf[i]));
199 if (nconfs == 0) {
200 dd->conf = NULL;
201 } else if (nconfs == 1) {
202 dd->conf = inconf[0];
203 dd->cdata = cpl_image_get_data_int(casu_fits_get_image(inconf[0]));
204 } else {
205 dd->conf = inconf[i];
206 dd->cdata = cpl_image_get_data_int(casu_fits_get_image(inconf[i]));
207 }
208 phu = casu_fits_get_phu(dd->fname);
209 ehu = casu_fits_get_ehu(dd->fname);
210 dd->xoff = cpl_propertylist_get_float(ehu,"ESO DRS XOFFDITHER");
211 dd->yoff = cpl_propertylist_get_float(ehu,"ESO DRS YOFFDITHER");
212 minxoff = min(dd->xoff,minxoff);
213 minyoff = min(dd->yoff,minyoff);
214 if (cpl_propertylist_has(phu,expkey)) {
215 exposure = (float)cpl_propertylist_get_double(phu,expkey);
216 exposure = max(0.5,exposure);
217 } else {
218 exposure = 0.5;
219 }
220 dd->expscale = exposure/expref;
221
222 /* Now work out a background and background noise estimate */
223
224 dd->nx = (int)cpl_image_get_size_x(casu_fits_get_image(dd->fname));
225 dd->ny = (int)cpl_image_get_size_y(casu_fits_get_image(dd->fname));
226 npts = dd->nx*dd->ny;
227 skyest(dd->data,dd->cdata,npts,3.0,MINDATA,MAXDATA,&sky,&skynoise);
228 dd->sky = sky;
229 dd->noise = skynoise;
230
231 /* Double check to make sure the confidence maps and images have the
232 same dimensions */
233
234 if ((int)cpl_image_get_size_x(casu_fits_get_image(dd->conf)) != dd->nx ||
235 (int)cpl_image_get_size_y(casu_fits_get_image(dd->conf)) != dd->ny) {
236 cpl_msg_error(fctid,"Image %s and Confidence map %s don't match",
237 casu_fits_get_fullname(dd->fname),
238 casu_fits_get_fullname(dd->conf));
239 tidy(0,fileptrs,clipmon,owdata);
240 FATAL_ERROR
241 }
242 }
243
244 /* Redo the offsets so that they are all positive. */
245
246 for (i = 0; i < nimages; i++) {
247 dd = fileptrs + i;
248 dd->xoff -= minxoff;
249 dd->yoff -= minyoff;
250 dd->ixoff = (int)(dd->xoff + 0.5);
251 dd->iyoff = (int)(dd->yoff + 0.5);
252 }
253
254 /* Redo the zero point offsets so that they are all relative to
255 the first image in the list. Make sure to divide by the relative
256 exposure time first! Set up weights*/
257
258 fileptrs->sky /= fileptrs->expscale;
259 fileptrs->skydiff = 0.0;
260 fileptrs->weight = 1.0;
261 sumweight = 1.0;
262 for (i = 1; i < nimages; i++) {
263 dd = fileptrs + i;
264 dd->sky /= dd->expscale;
265 dd->skydiff = fileptrs->sky - dd->sky;
266 dd->noise /= (float)sqrt((double)dd->expscale);
267 dd->weight = (float)(pow((double)fileptrs->noise,2.0)/pow((double)dd->noise,2.0));
268 sumweight += dd->weight;
269 }
270
271 /* Scale data (don't do image 0 since that has 1 scale and 0 offset) */
272
273 for (i = 1; i < nimages; i++) {
274 dd = fileptrs + i;
275 npts = dd->nx*dd->ny;
276 for (j = 0; j < npts; j++)
277 dd->data[j] = dd->data[j]/dd->expscale + dd->skydiff;
278 }
279
280 /* Set up clipping levels */
281
282 clip1 = fileptrs->sky - lthr*fileptrs->noise;
283 clip2 = fileptrs->sky + hthr*fileptrs->noise;
284
285 /* Open the output file. First of all work out how big the output map
286 needs to be. Then create it based on the first image in the list */
287
288 nxo = 0;
289 nyo = 0;
290 for (i = 0; i < nimages; i++) {
291 dd = fileptrs + i;
292 itx = dd->nx + dd->ixoff;
293 nxo = max(nxo,itx);
294 itx = dd->ny + dd->iyoff;
295 nyo = max(nyo,itx);
296 }
297
298 /* Create the output image */
299
300 *out = cpl_image_new((cpl_size)nxo,(cpl_size)nyo,CPL_TYPE_FLOAT);
301
302 /* If an output confidence map has been specified, then create it now. */
303
304 if (nconfs != 0)
305 *outc = cpl_image_new((cpl_size)nxo,(cpl_size)nyo,CPL_TYPE_INT);
306 else
307 *outc = NULL;
308
309 /* Get the data arrays for the output images */
310
311 npts = nxo*nyo;
312 odata = cpl_image_get_data_float(*out);
313 if (*outc != NULL)
314 owdata = cpl_malloc(npts*sizeof(float));
315 clipmon = clip_open(nimages,nxo);
316
317 /* Right, now try and do the work. Start by deciding whether for a given
318 output line an input line is able to contribute */
319
320 for (iy = 0; iy < nyo; iy++) {
321 ccur = (iy % 2)*nxo;
322 clast = nxo - ccur;
323 for (ix = 0; ix < nxo; ix++) {
324 c = clipmon + ccur + ix;
325 c->n = 0;
326 c->clipped = 0;
327 n = 0;
328 for (i = 0; i < nimages; i++) {
329 dd = fileptrs + i;
330 iline = iy - dd->iyoff;
331 if (iline < 0 || iline >= dd->ny)
332 continue;
333 icol = ix - dd->ixoff;
334 if (icol < 0 || icol >= dd->nx)
335 continue;
336
337 /* Load up any input data for this pixel from the current
338 image */
339
340 ielm = dd->nx*iline + icol;
341 c->values[n] = dd->data[ielm];
342 c->confs[n] = (float)dd->cdata[ielm];
343 c->weights[n] = dd->weight;
344 c->iff[n] = (short int)i;
345 n++;
346 }
347 c->outindex = nxo*iy + ix;
348 c->n = n;
349 average(fileptrs,c,&outdata,&outconf,clip1,clip2,lthr,hthr,0.0,
350 sumweight);
351 odata[c->outindex] = outdata;
352 if (owdata != NULL)
353 owdata[c->outindex] = outconf;
354 }
355
356 /* If we're away from the edges, have a look and see which pixels in
357 the previous row had clipping. Evaluate whether that clipping was
358 really justified or not */
359
360 if (iy < 2)
361 continue;
362 for (ix = 1; ix < nxo-1; ix++) {
363 c = clipmon + clast + ix;
364 if (! c->clipped)
365 continue;
366
367 /* If it was clipped, then evaluate the amount of 'noise' there
368 is spatially */
369
370 iloc = c->outindex;
371 avlev = 0.0;
372 for (jy = -1; jy <= 1; jy++) {
373 index_y = iloc + jy*nxo;
374 for (jx = -1; jx <= 1; jx++) {
375 index = index_y + jx;
376 avlev += odata[index];
377 }
378 }
379 avlev /= 9.0;
380 avvar = 0.0;
381 for (jy = -1; jy <= 1; jy++) {
382 index_y = iloc + jy*nxo;
383 for (jx = -1; jx <= 1; jx++) {
384 index = index_y + jx;
385 avvar += fabs(odata[index] - avlev);
386 }
387 }
388 avvar /= 9.0;
389
390 /* If the average level in this cell is below the upper clip level
391 or the mean absolute deviation is smaller than the poisson
392 noise in the cell, then the clip was probably justified. */
393
394 if (avlev < clip2 || avvar <= 2.0*fileptrs->noise)
395 continue;
396
397 /* Otherwise, create new clip levels and redo the average */
398
399 average(fileptrs,c,&outdata,&outconf,clip1,clip2,lthr,hthr,
400 3.0*avvar,sumweight);
401 odata[c->outindex] = outdata;
402 if (owdata != NULL)
403 owdata[c->outindex] = outconf;
404 }
405 }
406
407 /* Normalise the output confidence map */
408
409 if (owdata != NULL) {
410 skyest(owdata,NULL,npts,3.0,1.0,MAXDATA,&sky,&skynoise);
411 renorm = 100.0/sky;
412 ocdata = cpl_image_get_data_int(*outc);
413 for (i = 0; i < npts; i++)
414 ocdata[i] = max(0,min(1000,(int)(owdata[i]*renorm + 0.5)));
415 }
416
417 /* Create the output propertylist with some provenance info */
418
419 *p = cpl_propertylist_duplicate(casu_fits_get_ehu(inf[0]));
420 casu_prov(*p,inf,nimages,1);
421
422 /* Add a timestamp to the propertylist */
423
424 casu_timestamp(timestamp,25);
425 p2 = casu_fits_get_phu(inf[0]);
426 cpl_propertylist_update_string(p2,"ESO CASU_TIME",timestamp);
427 cpl_propertylist_set_comment(p2,"ESO CASU_TIME",
428 "Timestamp for matching to conf map");
429
430 /* Update the WCS in the header to reflect the new offset */
431
432 if (cpl_propertylist_has(*p,"CRPIX1") &&
433 cpl_propertylist_has(*p,"CRPIX2")) {
434 crpix1 = cpl_propertylist_get_double(*p,"CRPIX1");
435 crpix2 = cpl_propertylist_get_double(*p,"CRPIX2");
436 crpix1 += fileptrs->xoff;
437 crpix2 += fileptrs->yoff;
438 cpl_propertylist_update_double(*p,"CRPIX1",crpix1);
439 cpl_propertylist_update_double(*p,"CRPIX2",crpix2);
440 }
441
442 /* Get out of here */
443
444 tidy(nxo,fileptrs,clipmon,owdata);
445 GOOD_STATUS
446}
447
448/*---------------------------------------------------------------------------*/
492/*---------------------------------------------------------------------------*/
493
494static void average(dstrct *fileptrs, keeptabs *c, float *outdata,
495 float *outconf, float cliplow, float cliphigh, float lsig,
496 float hsig, float extra, float sumweight) {
497 int i,imin,imax;
498 float valuemax,valuemin,cwmin,cwmax,sum,cnumb,numb,cw,cv,reflev,noise;
499 float sky,clipval;
500
501 /* If there aren't any pixels defined for this (kind of a funny state
502 to be in, but never mind), give it some nominal value, which is the
503 sky background of the first input image. Flag it with zero confidence */
504
505 if (c->n <= 0) {
506 *outdata = fileptrs->sky;
507 *outconf = 0.0;
508 return;
509 }
510
511 /* Initialise a few things (avoid boring compiler errors about
512 uninitialised variables */
513
514 valuemin = 1.0e10;
515 valuemax = -1.0e10;
516 cwmin = 1.0e10;
517 cwmax = -1.0e10;
518 imin = 0;
519 imax = 0;
520 sum = 0.0;
521 cnumb = 0.0;
522 numb = 0.0;
523
524 /* Now loop through all the data for this point, keeping track of the
525 min and max */
526
527 for (i = 0; i < c->n; i++) {
528 cw = c->weights[i]*c->confs[i];
529 cv = c->values[i];
530 sum += cv*cw;
531 cnumb +=cw;
532 numb += c->confs[i];
533 if (cv < valuemin) {
534 valuemin = cv;
535 cwmin = cw;
536 imin = i;
537 }
538 if (cv > valuemax) {
539 valuemax = cv;
540 cwmax = cw;
541 imax = i;
542 }
543 }
544 if (cnumb > 0.0)
545 *outdata = sum/cnumb;
546 else
547 *outdata = fileptrs->sky;
548
549 /* See if we need to clip. Look at bright one first */
550
551 if (valuemax > cliphigh && numb > 150.0 && cnumb > 150.0) {
552 reflev = (sum - valuemax*cwmax)/(cnumb - cwmax);
553 noise = (fileptrs+c->iff[imax])->noise;
554 sky = (fileptrs+c->iff[imax])->sky;
555 clipval = reflev + hsig*noise*max(1.0,reflev)/max(1.0,sky) + extra;
556 if (valuemax > clipval) {
557 sum -= valuemax*cwmax;
558 cnumb -= cwmax;
559 *outdata = reflev;
560 c->clipped = 1;
561 }
562 }
563
564 /* Now look at the lowest value */
565
566 if (valuemin < cliplow && numb > 150.0 && cnumb > 150.0) {
567 reflev = (sum - valuemin*cwmin)/(cnumb - cwmin);
568 noise = (fileptrs+c->iff[imin])->noise;
569 clipval = reflev - lsig*noise;
570 if (valuemin < clipval) {
571 cnumb -= cwmin;
572 *outdata = reflev;
573 }
574 }
575
576 /* Do the output confidence */
577
578 *outconf = cnumb/sumweight;
579}
580
581/*---------------------------------------------------------------------------*/
603/*---------------------------------------------------------------------------*/
604
605static keeptabs *clip_open(int nimages, int nxo) {
606 keeptabs *c;
607 int i;
608 short int *iff;
609 float *dptr;
610
611 c = cpl_malloc(2*nxo*sizeof(keeptabs));
612 for (i = 0; i < 2*nxo; i++) {
613 dptr = cpl_malloc(3*nimages*sizeof(*dptr));
614 iff = cpl_malloc(nimages*sizeof(*iff));
615 (c+i)->values = dptr;
616 (c+i)->confs = dptr + nimages;
617 (c+i)->weights = dptr + 2*nimages;
618 (c+i)->iff = iff;
619 (c+i)->n = 0;
620 (c+i)->outindex = -1;
621 (c+i)->clipped = 0;
622 }
623 return(c);
624}
625
626/*---------------------------------------------------------------------------*/
643/*---------------------------------------------------------------------------*/
644
645static void clip_close(keeptabs **c, int nxo) {
646 int i;
647
648 for (i = 0; i < 2*nxo; i++) {
649 freespace((*c+i)->values);
650 freespace((*c+i)->iff);
651 }
652 freespace(*c);
653}
654
655/*---------------------------------------------------------------------------*/
685/*---------------------------------------------------------------------------*/
686
687static void skyest(float *data, int *cdata, long npts, float thresh,
688 float dmin, float dmax, float *skymed, float *skynoise) {
689 unsigned char *bpm;
690 int i;
691
692 /* Get a dummy bad pixel mask */
693
694 bpm = cpl_calloc(npts,sizeof(*bpm));
695 if (cdata != NULL) {
696 for (i = 0; i < npts; i++)
697 bpm[i] = (cdata[i] == 0);
698 }
699
700 /* Get the stats */
701
702 casu_qmedsig(data,bpm,npts,thresh,2,dmin,dmax,skymed,skynoise);
703
704 /* Clean up */
705
706 freespace(bpm);
707}
708
709/*---------------------------------------------------------------------------*/
713/*---------------------------------------------------------------------------*/
714
715static void tidy(int nxo, dstrct *fileptrs, keeptabs *clipmon, float *owdata) {
716
717 if(owdata != NULL) {cpl_free(owdata);}
718 //freespace(owdata);
719 clip_close(&clipmon,nxo);
720 if(fileptrs != NULL) {cpl_free(fileptrs);}
721 //freespace(fileptrs);
722}
723
726/*
727
728$Log: casu_imdither.c,v $
729Revision 1.5 2015/11/25 10:26:31 jim
730replaced some hardcoded numbers with defines
731
732Revision 1.4 2015/11/18 20:06:12 jim
733Fixed CASU_TIME keyword
734
735Revision 1.3 2015/09/11 09:27:53 jim
736Fixed some problems with the docs
737
738Revision 1.2 2015/08/07 13:06:54 jim
739Fixed copyright to ESO
740
741Revision 1.1.1.1 2015/06/12 10:44:32 jim
742Initial import
743
744Revision 1.7 2015/01/29 11:51:56 jim
745modified comments
746
747Revision 1.6 2014/12/11 12:23:33 jim
748new version
749
750Revision 1.5 2014/04/09 11:08:21 jim
751Get rid of a couple of compiler moans
752
753Revision 1.4 2014/04/09 09:09:51 jim
754Detabbed
755
756Revision 1.3 2014/03/26 15:45:45 jim
757Modified to remove globals and to allow for floating point confidence maps
758
759Revision 1.2 2013/11/21 09:38:14 jim
760detabbed
761
762Revision 1.1.1.1 2013-08-27 12:07:48 jim
763Imported
764
765
766*/
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
char * casu_fits_get_fullname(casu_fits *p)
Definition: casu_fits.c:680
cpl_propertylist * casu_fits_get_phu(casu_fits *p)
Definition: casu_fits.c:531
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
int casu_imdither(casu_fits **inf, casu_fits **inconf, int nimages, int nconfs, float lthr, float hthr, cpl_propertylist **p, const char *expkey, cpl_image **out, cpl_image **outc, int *status)
Dither a set of jittered observations.
void casu_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: casu_stats.c:258
void casu_prov(cpl_propertylist *p, casu_fits **inlist, int n, int isextn)
Write provenance keywords.
Definition: casu_utils.c:287
void casu_timestamp(char *out, int n)
Create a timestamp string.
Definition: casu_utils.c:849