VIRCAM Pipeline  2.3.12
casu_imdither.c
1 /* $Id: casu_imdither.c,v 1.5 2015/11/25 10:26:31 jim Exp $
2  *
3  * This file is part of the CASU Pipeline utilities
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2015/11/25 10:26:31 $
24  * $Revision: 1.5 $
25  * $Name: $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <math.h>
35 #include <cpl.h>
36 #include "casu_mods.h"
37 #include "catalogue/casu_utils.h"
38 #include "catalogue/casu_fits.h"
39 #include "casu_stats.h"
40 
41 typedef struct {
42  casu_fits *fname;
43  casu_fits *conf;
44  float xoff;
45  float yoff;
46  int ixoff;
47  int iyoff;
48  int nx;
49  int ny;
50  float sky;
51  float skydiff;
52  float noise;
53  float expscale;
54  float weight;
55  float *data;
56  int *cdata;
57  int ndata;
58 } dstrct;
59 
60 typedef struct {
61  float *values;
62  float *confs;
63  float *weights;
64  short int *iff;
65  int n;
66  long outindex;
67  unsigned char clipped;
68 } keeptabs;
69 
70 #define NROWSBUF 512
71 #define MINDATA -1000.0
72 #define MAXDATA 65535.0
73 
74 static void average(dstrct *fileptrs, keeptabs *c, float *outdata,
75  float *outconf, float cliplow, float cliphigh, float lthr,
76  float hthr, float extra, float sumweight);
77 static keeptabs *clip_open(int nimages, int nxo);
78 static void clip_close(keeptabs **c, int nxo);
79 static void skyest(float *data, int *cdata, long npts, float thresh,
80  float dmin, float dmax, float *skymed, float *skynoise);
81 static void tidy(int nxo, dstrct *fileptrs, keeptabs *clipmon, float *owdata);
82 
85 /*---------------------------------------------------------------------------*/
149 /*---------------------------------------------------------------------------*/
150 
151 extern int casu_imdither(casu_fits **inf, casu_fits **inconf, int nimages,
152  int nconfs, float lthr, float hthr,
153  cpl_propertylist **p, const char *expkey,
154  cpl_image **out, cpl_image **outc, int *status) {
155 
156  int i,itx,iy,ccur,clast,ix,n,iline,icol,jy,jx,j,nxo,nyo,*ocdata;
157  long npts,ielm,iloc,index_y,index;
158  dstrct *fileptrs=NULL,*dd;
159  keeptabs *clipmon=NULL,*c;
160  float minxoff,minyoff,expref,sky,skynoise,clip1,clip2,outdata;
161  float outconf,avlev,avvar,renorm,exposure,sumweight,*odata,*owdata=NULL;
162  double crpix1,crpix2;
163  cpl_propertylist *ehu,*p2,*phu;
164  const char *fctid = "casu_imdither";
165  char timestamp[25];
166 
167  /* Inherited status */
168 
169  *out = NULL;
170  *outc = NULL;
171  *p = NULL;
172  if (*status != CASU_OK)
173  return(*status);
174 
175  /* Is there any point in being here? */
176 
177  if (nimages == 0) {
178  cpl_msg_error(fctid,"No input files to combine");
179  tidy(0,fileptrs,clipmon,owdata);
180  FATAL_ERROR
181  }
182 
183  /* Allocate file struct array and fill in some values */
184 
185  fileptrs = cpl_malloc(nimages*sizeof(dstrct));
186  if (cpl_propertylist_has(casu_fits_get_phu(inf[0]),expkey)) {
187  exposure = (float)cpl_propertylist_get_double(casu_fits_get_phu(inf[0]),
188  expkey);
189  } else {
190  exposure = 0.5;
191  }
192  expref = max(0.5,exposure);
193  minxoff = 1.0e10;
194  minyoff = 1.0e10;
195  for (i = 0; i < nimages; i++) {
196  dd = fileptrs + i;
197  dd->fname = inf[i];
198  dd->data = cpl_image_get_data_float(casu_fits_get_image(inf[i]));
199  if (nconfs == 0) {
200  dd->conf = NULL;
201  } else if (nconfs == 1) {
202  dd->conf = inconf[0];
203  dd->cdata = cpl_image_get_data_int(casu_fits_get_image(inconf[0]));
204  } else {
205  dd->conf = inconf[i];
206  dd->cdata = cpl_image_get_data_int(casu_fits_get_image(inconf[i]));
207  }
208  phu = casu_fits_get_phu(dd->fname);
209  ehu = casu_fits_get_ehu(dd->fname);
210  dd->xoff = cpl_propertylist_get_float(ehu,"ESO DRS XOFFDITHER");
211  dd->yoff = cpl_propertylist_get_float(ehu,"ESO DRS YOFFDITHER");
212  minxoff = min(dd->xoff,minxoff);
213  minyoff = min(dd->yoff,minyoff);
214  if (cpl_propertylist_has(phu,expkey)) {
215  exposure = (float)cpl_propertylist_get_double(phu,expkey);
216  exposure = max(0.5,exposure);
217  } else {
218  exposure = 0.5;
219  }
220  dd->expscale = exposure/expref;
221 
222  /* Now work out a background and background noise estimate */
223 
224  dd->nx = (int)cpl_image_get_size_x(casu_fits_get_image(dd->fname));
225  dd->ny = (int)cpl_image_get_size_y(casu_fits_get_image(dd->fname));
226  npts = dd->nx*dd->ny;
227  skyest(dd->data,dd->cdata,npts,3.0,MINDATA,MAXDATA,&sky,&skynoise);
228  dd->sky = sky;
229  dd->noise = skynoise;
230 
231  /* Double check to make sure the confidence maps and images have the
232  same dimensions */
233 
234  if ((int)cpl_image_get_size_x(casu_fits_get_image(dd->conf)) != dd->nx ||
235  (int)cpl_image_get_size_y(casu_fits_get_image(dd->conf)) != dd->ny) {
236  cpl_msg_error(fctid,"Image %s and Confidence map %s don't match",
237  casu_fits_get_fullname(dd->fname),
238  casu_fits_get_fullname(dd->conf));
239  tidy(0,fileptrs,clipmon,owdata);
240  FATAL_ERROR
241  }
242  }
243 
244  /* Redo the offsets so that they are all positive. */
245 
246  for (i = 0; i < nimages; i++) {
247  dd = fileptrs + i;
248  dd->xoff -= minxoff;
249  dd->yoff -= minyoff;
250  dd->ixoff = (int)(dd->xoff + 0.5);
251  dd->iyoff = (int)(dd->yoff + 0.5);
252  }
253 
254  /* Redo the zero point offsets so that they are all relative to
255  the first image in the list. Make sure to divide by the relative
256  exposure time first! Set up weights*/
257 
258  fileptrs->sky /= fileptrs->expscale;
259  fileptrs->skydiff = 0.0;
260  fileptrs->weight = 1.0;
261  sumweight = 1.0;
262  for (i = 1; i < nimages; i++) {
263  dd = fileptrs + i;
264  dd->sky /= dd->expscale;
265  dd->skydiff = fileptrs->sky - dd->sky;
266  dd->noise /= (float)sqrt((double)dd->expscale);
267  dd->weight = (float)(pow((double)fileptrs->noise,2.0)/pow((double)dd->noise,2.0));
268  sumweight += dd->weight;
269  }
270 
271  /* Scale data (don't do image 0 since that has 1 scale and 0 offset) */
272 
273  for (i = 1; i < nimages; i++) {
274  dd = fileptrs + i;
275  npts = dd->nx*dd->ny;
276  for (j = 0; j < npts; j++)
277  dd->data[j] = dd->data[j]/dd->expscale + dd->skydiff;
278  }
279 
280  /* Set up clipping levels */
281 
282  clip1 = fileptrs->sky - lthr*fileptrs->noise;
283  clip2 = fileptrs->sky + hthr*fileptrs->noise;
284 
285  /* Open the output file. First of all work out how big the output map
286  needs to be. Then create it based on the first image in the list */
287 
288  nxo = 0;
289  nyo = 0;
290  for (i = 0; i < nimages; i++) {
291  dd = fileptrs + i;
292  itx = dd->nx + dd->ixoff;
293  nxo = max(nxo,itx);
294  itx = dd->ny + dd->iyoff;
295  nyo = max(nyo,itx);
296  }
297 
298  /* Create the output image */
299 
300  *out = cpl_image_new((cpl_size)nxo,(cpl_size)nyo,CPL_TYPE_FLOAT);
301 
302  /* If an output confidence map has been specified, then create it now. */
303 
304  if (nconfs != 0)
305  *outc = cpl_image_new((cpl_size)nxo,(cpl_size)nyo,CPL_TYPE_INT);
306  else
307  *outc = NULL;
308 
309  /* Get the data arrays for the output images */
310 
311  npts = nxo*nyo;
312  odata = cpl_image_get_data_float(*out);
313  if (*outc != NULL)
314  owdata = cpl_malloc(npts*sizeof(float));
315  clipmon = clip_open(nimages,nxo);
316 
317  /* Right, now try and do the work. Start by deciding whether for a given
318  output line an input line is able to contribute */
319 
320  for (iy = 0; iy < nyo; iy++) {
321  ccur = (iy % 2)*nxo;
322  clast = nxo - ccur;
323  for (ix = 0; ix < nxo; ix++) {
324  c = clipmon + ccur + ix;
325  c->n = 0;
326  c->clipped = 0;
327  n = 0;
328  for (i = 0; i < nimages; i++) {
329  dd = fileptrs + i;
330  iline = iy - dd->iyoff;
331  if (iline < 0 || iline >= dd->ny)
332  continue;
333  icol = ix - dd->ixoff;
334  if (icol < 0 || icol >= dd->nx)
335  continue;
336 
337  /* Load up any input data for this pixel from the current
338  image */
339 
340  ielm = dd->nx*iline + icol;
341  c->values[n] = dd->data[ielm];
342  c->confs[n] = (float)dd->cdata[ielm];
343  c->weights[n] = dd->weight;
344  c->iff[n] = (short int)i;
345  n++;
346  }
347  c->outindex = nxo*iy + ix;
348  c->n = n;
349  average(fileptrs,c,&outdata,&outconf,clip1,clip2,lthr,hthr,0.0,
350  sumweight);
351  odata[c->outindex] = outdata;
352  if (owdata != NULL)
353  owdata[c->outindex] = outconf;
354  }
355 
356  /* If we're away from the edges, have a look and see which pixels in
357  the previous row had clipping. Evaluate whether that clipping was
358  really justified or not */
359 
360  if (iy < 2)
361  continue;
362  for (ix = 1; ix < nxo-1; ix++) {
363  c = clipmon + clast + ix;
364  if (! c->clipped)
365  continue;
366 
367  /* If it was clipped, then evaluate the amount of 'noise' there
368  is spatially */
369 
370  iloc = c->outindex;
371  avlev = 0.0;
372  for (jy = -1; jy <= 1; jy++) {
373  index_y = iloc + jy*nxo;
374  for (jx = -1; jx <= 1; jx++) {
375  index = index_y + jx;
376  avlev += odata[index];
377  }
378  }
379  avlev /= 9.0;
380  avvar = 0.0;
381  for (jy = -1; jy <= 1; jy++) {
382  index_y = iloc + jy*nxo;
383  for (jx = -1; jx <= 1; jx++) {
384  index = index_y + jx;
385  avvar += fabs(odata[index] - avlev);
386  }
387  }
388  avvar /= 9.0;
389 
390  /* If the average level in this cell is below the upper clip level
391  or the mean absolute deviation is smaller than the poisson
392  noise in the cell, then the clip was probably justified. */
393 
394  if (avlev < clip2 || avvar <= 2.0*fileptrs->noise)
395  continue;
396 
397  /* Otherwise, create new clip levels and redo the average */
398 
399  average(fileptrs,c,&outdata,&outconf,clip1,clip2,lthr,hthr,
400  3.0*avvar,sumweight);
401  odata[c->outindex] = outdata;
402  if (owdata != NULL)
403  owdata[c->outindex] = outconf;
404  }
405  }
406 
407  /* Normalise the output confidence map */
408 
409  if (owdata != NULL) {
410  skyest(owdata,NULL,npts,3.0,1.0,MAXDATA,&sky,&skynoise);
411  renorm = 100.0/sky;
412  ocdata = cpl_image_get_data_int(*outc);
413  for (i = 0; i < npts; i++)
414  ocdata[i] = max(0,min(1000,(int)(owdata[i]*renorm + 0.5)));
415  }
416 
417  /* Create the output propertylist with some provenance info */
418 
419  *p = cpl_propertylist_duplicate(casu_fits_get_ehu(inf[0]));
420  casu_prov(*p,inf,nimages,1);
421 
422  /* Add a timestamp to the propertylist */
423 
424  casu_timestamp(timestamp,25);
425  p2 = casu_fits_get_phu(inf[0]);
426  cpl_propertylist_update_string(p2,"ESO CASU_TIME",timestamp);
427  cpl_propertylist_set_comment(p2,"ESO CASU_TIME",
428  "Timestamp for matching to conf map");
429 
430  /* Update the WCS in the header to reflect the new offset */
431 
432  if (cpl_propertylist_has(*p,"CRPIX1") &&
433  cpl_propertylist_has(*p,"CRPIX2")) {
434  crpix1 = cpl_propertylist_get_double(*p,"CRPIX1");
435  crpix2 = cpl_propertylist_get_double(*p,"CRPIX2");
436  crpix1 += fileptrs->xoff;
437  crpix2 += fileptrs->yoff;
438  cpl_propertylist_update_double(*p,"CRPIX1",crpix1);
439  cpl_propertylist_update_double(*p,"CRPIX2",crpix2);
440  }
441 
442  /* Get out of here */
443 
444  tidy(nxo,fileptrs,clipmon,owdata);
445  GOOD_STATUS
446 }
447 
448 /*---------------------------------------------------------------------------*/
492 /*---------------------------------------------------------------------------*/
493 
494 static void average(dstrct *fileptrs, keeptabs *c, float *outdata,
495  float *outconf, float cliplow, float cliphigh, float lsig,
496  float hsig, float extra, float sumweight) {
497  int i,imin,imax;
498  float valuemax,valuemin,cwmin,cwmax,sum,cnumb,numb,cw,cv,reflev,noise;
499  float sky,clipval;
500 
501  /* If there aren't any pixels defined for this (kind of a funny state
502  to be in, but never mind), give it some nominal value, which is the
503  sky background of the first input image. Flag it with zero confidence */
504 
505  if (c->n <= 0) {
506  *outdata = fileptrs->sky;
507  *outconf = 0.0;
508  return;
509  }
510 
511  /* Initialise a few things (avoid boring compiler errors about
512  uninitialised variables */
513 
514  valuemin = 1.0e10;
515  valuemax = -1.0e10;
516  cwmin = 1.0e10;
517  cwmax = -1.0e10;
518  imin = 0;
519  imax = 0;
520  sum = 0.0;
521  cnumb = 0.0;
522  numb = 0.0;
523 
524  /* Now loop through all the data for this point, keeping track of the
525  min and max */
526 
527  for (i = 0; i < c->n; i++) {
528  cw = c->weights[i]*c->confs[i];
529  cv = c->values[i];
530  sum += cv*cw;
531  cnumb +=cw;
532  numb += c->confs[i];
533  if (cv < valuemin) {
534  valuemin = cv;
535  cwmin = cw;
536  imin = i;
537  }
538  if (cv > valuemax) {
539  valuemax = cv;
540  cwmax = cw;
541  imax = i;
542  }
543  }
544  if (cnumb > 0.0)
545  *outdata = sum/cnumb;
546  else
547  *outdata = fileptrs->sky;
548 
549  /* See if we need to clip. Look at bright one first */
550 
551  if (valuemax > cliphigh && numb > 150.0 && cnumb > 150.0) {
552  reflev = (sum - valuemax*cwmax)/(cnumb - cwmax);
553  noise = (fileptrs+c->iff[imax])->noise;
554  sky = (fileptrs+c->iff[imax])->sky;
555  clipval = reflev + hsig*noise*max(1.0,reflev)/max(1.0,sky) + extra;
556  if (valuemax > clipval) {
557  sum -= valuemax*cwmax;
558  cnumb -= cwmax;
559  *outdata = reflev;
560  c->clipped = 1;
561  }
562  }
563 
564  /* Now look at the lowest value */
565 
566  if (valuemin < cliplow && numb > 150.0 && cnumb > 150.0) {
567  reflev = (sum - valuemin*cwmin)/(cnumb - cwmin);
568  noise = (fileptrs+c->iff[imin])->noise;
569  clipval = reflev - lsig*noise;
570  if (valuemin < clipval) {
571  cnumb -= cwmin;
572  *outdata = reflev;
573  }
574  }
575 
576  /* Do the output confidence */
577 
578  *outconf = cnumb/sumweight;
579 }
580 
581 /*---------------------------------------------------------------------------*/
603 /*---------------------------------------------------------------------------*/
604 
605 static keeptabs *clip_open(int nimages, int nxo) {
606  keeptabs *c;
607  int i;
608  short int *iff;
609  float *dptr;
610 
611  c = cpl_malloc(2*nxo*sizeof(keeptabs));
612  for (i = 0; i < 2*nxo; i++) {
613  dptr = cpl_malloc(3*nimages*sizeof(*dptr));
614  iff = cpl_malloc(nimages*sizeof(*iff));
615  (c+i)->values = dptr;
616  (c+i)->confs = dptr + nimages;
617  (c+i)->weights = dptr + 2*nimages;
618  (c+i)->iff = iff;
619  (c+i)->n = 0;
620  (c+i)->outindex = -1;
621  (c+i)->clipped = 0;
622  }
623  return(c);
624 }
625 
626 /*---------------------------------------------------------------------------*/
643 /*---------------------------------------------------------------------------*/
644 
645 static void clip_close(keeptabs **c, int nxo) {
646  int i;
647 
648  for (i = 0; i < 2*nxo; i++) {
649  freespace((*c+i)->values);
650  freespace((*c+i)->iff);
651  }
652  freespace(*c);
653 }
654 
655 /*---------------------------------------------------------------------------*/
685 /*---------------------------------------------------------------------------*/
686 
687 static void skyest(float *data, int *cdata, long npts, float thresh,
688  float dmin, float dmax, float *skymed, float *skynoise) {
689  unsigned char *bpm;
690  int i;
691 
692  /* Get a dummy bad pixel mask */
693 
694  bpm = cpl_calloc(npts,sizeof(*bpm));
695  if (cdata != NULL) {
696  for (i = 0; i < npts; i++)
697  bpm[i] = (cdata[i] == 0);
698  }
699 
700  /* Get the stats */
701 
702  casu_qmedsig(data,bpm,npts,thresh,2,dmin,dmax,skymed,skynoise);
703 
704  /* Clean up */
705 
706  freespace(bpm);
707 }
708 
709 /*---------------------------------------------------------------------------*/
713 /*---------------------------------------------------------------------------*/
714 
715 static void tidy(int nxo, dstrct *fileptrs, keeptabs *clipmon, float *owdata) {
716 
717  if(owdata != NULL) {cpl_free(owdata);}
718  //freespace(owdata);
719  clip_close(&clipmon,nxo);
720  if(fileptrs != NULL) {cpl_free(fileptrs);}
721  //freespace(fileptrs);
722 }
723 
726 /*
727 
728 $Log: casu_imdither.c,v $
729 Revision 1.5 2015/11/25 10:26:31 jim
730 replaced some hardcoded numbers with defines
731 
732 Revision 1.4 2015/11/18 20:06:12 jim
733 Fixed CASU_TIME keyword
734 
735 Revision 1.3 2015/09/11 09:27:53 jim
736 Fixed some problems with the docs
737 
738 Revision 1.2 2015/08/07 13:06:54 jim
739 Fixed copyright to ESO
740 
741 Revision 1.1.1.1 2015/06/12 10:44:32 jim
742 Initial import
743 
744 Revision 1.7 2015/01/29 11:51:56 jim
745 modified comments
746 
747 Revision 1.6 2014/12/11 12:23:33 jim
748 new version
749 
750 Revision 1.5 2014/04/09 11:08:21 jim
751 Get rid of a couple of compiler moans
752 
753 Revision 1.4 2014/04/09 09:09:51 jim
754 Detabbed
755 
756 Revision 1.3 2014/03/26 15:45:45 jim
757 Modified to remove globals and to allow for floating point confidence maps
758 
759 Revision 1.2 2013/11/21 09:38:14 jim
760 detabbed
761 
762 Revision 1.1.1.1 2013-08-27 12:07:48 jim
763 Imported
764 
765 
766 */
cpl_image * casu_fits_get_image(casu_fits *p)
Definition: casu_fits.c:436
char * casu_fits_get_fullname(casu_fits *p)
Definition: casu_fits.c:680
cpl_propertylist * casu_fits_get_phu(casu_fits *p)
Definition: casu_fits.c:531
cpl_propertylist * casu_fits_get_ehu(casu_fits *p)
Definition: casu_fits.c:576
int casu_imdither(casu_fits **inf, casu_fits **inconf, int nimages, int nconfs, float lthr, float hthr, cpl_propertylist **p, const char *expkey, cpl_image **out, cpl_image **outc, int *status)
Dither a set of jittered observations.
void casu_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: casu_stats.c:258
void casu_prov(cpl_propertylist *p, casu_fits **inlist, int n, int isextn)
Write provenance keywords.
Definition: casu_utils.c:287
void casu_timestamp(char *out, int n)
Create a timestamp string.
Definition: casu_utils.c:849