VIRCAM Pipeline  2.3.12
imcore_background.c
1 /* $Id: imcore_background.c,v 1.4 2015/08/12 11:16:55 jim Exp $
2  *
3  * This file is part of the CASU Pipeline utilities
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2015/08/12 11:16:55 $
24  * $Revision: 1.4 $
25  * $Name: $
26  */
27 
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <math.h>
32 
33 #include <cpl.h>
34 
35 #include "floatmath.h"
36 #include "util.h"
37 #include "imcore.h"
38 
39 static int **hist = NULL;
40 static int *nnp = NULL;
41 static int npvx;
42 static int npvy;
43 static void tidy(void);
44 static void sortit (float [], int);
45 
48 /*---------------------------------------------------------------------------*/
79 /*---------------------------------------------------------------------------*/
80 
81 extern int imcore_background(ap_t *ap, int nbsize, float nullval) {
82  float fracx,fracy,skymed,sigma,skymedc,sigmac,avsky,fnbsize,dely,delx;
83  float t1,t2,dsky,*map,**bvals,*work;
84  int ifracx,ifracy,nbsizx,nbsizy,nbx,nby,npixstripe,l,i,ll;
85  int isquare,ilev,j,iclip,mcpix,iloop,irej,nbsizo2,kk,k,iby,ibyp1,ibx,ibxp1;
86  int *shist;
87  unsigned char *mflag,*bbad;
88  long nx,ny;
89 
90  /* Set up some variables */
91 
92  map = ap->indata;
93  mflag = ap->mflag;
94  nx = ap->lsiz;
95  ny = ap->csiz;
96 
97  /* check to see if nbsize is close to exact divisor */
98 
99  fracx = ((float)nx)/((float)nbsize);
100  fracy = ((float)ny)/((float)nbsize);
101  ifracx = (int)(fracx + 0.1);
102  ifracy = (int)(fracy + 0.1);
103  nbsizx = nx/ifracx;
104  nbsizy = ny/ifracy;
105  nbsize = MAX(NINT(0.9*nbsize), MIN(nbsize, MIN(nbsizx,nbsizy)));
106  nbsize = MIN(nx,MIN(ny,nbsize)); /* trap for small maps */
107 
108  /* Divide the map into partitions */
109 
110  nbx = nx/nbsize;
111  nby = ny/nbsize;
112  npixstripe = nbsize*nx;
113  npvx = nbx;
114  npvy = nby;
115 
116  /* Get histogram workspace if you can */
117 
118  hist = cpl_malloc(nbx*sizeof(int *));
119  for (l = 0; l < nbx; l++)
120  hist[l] = cpl_malloc(MAXHIST*sizeof(int));
121 
122  /* Same for background values array */
123 
124  bvals = cpl_malloc(nby*sizeof(float *));
125  for (l = 0; l < nby; l++)
126  bvals[l] = cpl_malloc(nbx*sizeof(float));
127  bbad = cpl_calloc(nbx*nby,sizeof(unsigned char));
128 
129  /* Store some of this away for use later */
130 
131  ap->backmap.nbx = nbx;
132  ap->backmap.nby = nby;
133  ap->backmap.nbsize = nbsize;
134  ap->backmap.bvals = bvals;
135 
136  /* Finally a counter array */
137 
138  nnp = cpl_malloc(nbx*sizeof(int));
139 
140  /* Loop for each row of background squares. Start by initialising
141  the accumulators and histograms */
142 
143  for (l = 0; l < nby; l++) {
144  memset((char *)nnp,0,nbx*sizeof(*nnp));
145  for (i = 0; i < nbx; i++)
146  memset((char *)hist[i],0,MAXHIST*sizeof(int));
147 
148  /* Skim through the data in this stripe. Find out which square each
149  belongs to and add it it to the relevant histogram */
150 
151  ll = l*npixstripe;
152  for (i = 0; i < npixstripe; i++) {
153  if (map[ll+i] != nullval && mflag[ll+i] != MF_ZEROCONF &&
154  mflag[ll+i] != MF_STUPID_VALUE) {
155  isquare = (int)((float)(i % nx)/(float)nbsize);
156  isquare = MIN(nbx-1,MAX(0,isquare));
157  ilev = MIN(MAXHISTVAL,MAX(MINHISTVAL,NINT(map[i+ll])));
158  hist[isquare][ilev-MINHISTVAL] += 1;
159  nnp[isquare] += 1;
160  }
161  }
162 
163  /* but only do background estimation if enough pixels ----------- */
164 
165  for (j = 0; j < nbx; j++) {
166  if (nnp[j] > 0.25*nbsize*nbsize){
167  shist = hist[j];
168  imcore_medsig(shist,MAXHIST,MINHISTVAL-1,nnp[j],&skymed,
169  &sigma);
170 
171  /* do an iterative 3-sigma upper clip to give a more robust
172  estimator */
173 
174  iclip = MAXHISTVAL;
175  mcpix = nnp[j];
176  skymedc = skymed;
177  sigmac = sigma;
178  for (iloop = 0; iloop < 3; iloop++) {
179  irej = 0;
180  for(i = NINT(skymedc+3.0*sigmac); i <= iclip; i++)
181  irej += shist[i-MINHISTVAL];
182  if (irej == 0)
183  break;
184  iclip = NINT(skymedc+3.0*sigmac) - 1;
185  mcpix = mcpix - irej;
186  imcore_medsig(shist,MAXHIST,MINHISTVAL-1,mcpix,
187  &skymedc,&sigmac);
188  }
189  bvals[l][j] = skymedc;
190  } else {
191  bvals[l][j] = -1000.0;
192  bbad[j+nbx*l] = 1;
193  }
194  }
195  }
196 
197  /* filter raw background values */
198 
199  imcore_bfilt(bvals,nbx,nby);
200 
201  /* compute average sky level */
202 
203  work = cpl_malloc(nbx*nby*sizeof(*work));
204  k = 0;
205  for(l = 0; l < nby; l++)
206  for(j = 0; j < nbx; j++)
207  if (! bbad[l*nbx+j])
208  work[k++] = bvals[l][j];
209  sortit(work,k);
210  avsky = work[(k)/2];
211  freespace(work);
212  freespace(bbad);
213 
214  /* ok now correct map for background variations and put avsky back on */
215 
216  nbsizo2 = nbsize/2;
217  fnbsize = 1.0/((float)nbsize);
218  for (k = 0; k < ny; k++) {
219  kk = k*nx;
220 
221  /* Nearest background pixel vertically */
222 
223  iby = (k + 1 + nbsizo2)/nbsize;
224  ibyp1 = iby + 1;
225  iby = MIN(nby,MAX(1,iby));
226  ibyp1 = MIN(nby,ibyp1);
227  dely = (k + 1 - nbsize*iby + nbsizo2)*fnbsize;
228 
229  for (j = 0; j < nx; j++) {
230  if (map[kk+j] == nullval)
231  continue;
232 
233  /* nearest background pixel across */
234 
235  ibx = (j + 1 + nbsizo2)/nbsize;
236  ibxp1 = ibx + 1;
237  ibx = MIN(nbx,MAX(1,ibx));
238  ibxp1 = MIN(nbx,ibxp1);
239  delx = (j + 1 - nbsize*ibx + nbsizo2)*fnbsize;
240 
241  /* bilinear interpolation to find background */
242 
243  t1 = (1.0 - dely)*bvals[iby-1][ibx-1] + dely*bvals[ibyp1-1][ibx-1];
244  t2 = (1.0 - dely)*bvals[iby-1][ibxp1-1] + dely*bvals[ibyp1-1][ibxp1-1];
245  dsky = avsky - (1.0 - delx)*t1 - delx*t2;
246  map[kk+j] += dsky;
247  }
248  }
249 
250  /* Free some workspace */
251 
252  tidy();
253  return(CASU_OK);
254 }
255 
256 /*---------------------------------------------------------------------------*/
293 /*---------------------------------------------------------------------------*/
294 
295 extern int imcore_backstats(ap_t *ap, float nullval, int satonly,
296  float *skymed, float *skysig, float *sat) {
297  int ilev,iclip,iloop,i,*ihist,isat,iter;
298  long mpix,npix,k,mcpix,irej,lpix,nx,ny;
299  float skymedc,sigmac,*map,sata,fac,skyref;
300  unsigned char *mflag;
301 
302  /* Get some info from the ap structure */
303 
304  map = ap->indata;
305  nx = ap->lsiz;
306  ny = ap->csiz;
307  mflag = ap->mflag;
308 
309  /* Check to make sure there are some non-zero values here */
310 
311  ilev = 1;
312  for (i = 0; i < nx*ny; i++) {
313  if (map[i] != nullval && mflag[i] != MF_ZEROCONF &&
314  mflag[i] != MF_STUPID_VALUE) {
315  ilev = 0;
316  break;
317  }
318  }
319  if (ilev == 1) {
320  *skymed = 0.0;
321  *skysig = 0.0;
322  *sat = 0.0;
323  return(CASU_WARN);
324  }
325 
326  /* First, get some workspace for the background histogram */
327 
328  ihist = cpl_calloc(MAXHIST,sizeof(*ihist));
329 
330  /* Loop for up to 10 iterations. For each iteration we multiply the
331  input data by a successively higher power of 2 in order to
332  try and deal with data that has very small noise estimates */
333 
334  fac = 0.5;
335  skyref = 0.0;
336  for (iter = 0; iter <= 9; iter++) {
337  fac *= 2.0;
338  if (iter == 1)
339  skyref = skymedc;
340  for (k = 0; k < MAXHIST; k++)
341  ihist[k] = 0;
342 
343  /* Now form the histogram of all pixel intensities */
344 
345  mpix = 0;
346  isat = 0;
347  npix = nx*ny;
348  for (k = 0; k < npix; k++) {
349  if (map[k] != nullval && mflag[k] != MF_ZEROCONF &&
350  mflag[k] != MF_STUPID_VALUE) {
351  ilev = MIN(MAXHISTVAL,MAX(MINHISTVAL,NINT(fac*(map[k]-skyref))));
352  ihist[ilev - MINHISTVAL] += 1;
353  isat = MAX(isat,ilev);
354  mpix++;
355  }
356  }
357  sata = MIN(MAXHISTVAL,MAX(MINSATURATE,0.9*((float)isat))/fac);
358  lpix = ihist[isat - MINHISTVAL];
359  while (lpix < mpix/1000 && isat > MINHISTVAL) {
360  isat--;
361  lpix += ihist[isat - MINHISTVAL];
362  }
363  *sat = ((float)isat)/fac + skyref;
364  *sat = MIN(MAXHISTVAL,MAX(MINSATURATE,MAX(0.95*(*sat),sata)));
365 
366  /* If all you want is the saturation level, then get out of here...*/
367 
368  if (satonly) {
369  freespace(ihist);
370  return(CASU_OK);
371  }
372 
373  /* Now find the median and sigma */
374 
375  imcore_medsig(ihist,MAXHIST,MINHISTVAL-1,mpix,skymed,skysig);
376 
377  /* Do an iterative 3-sigma upper clip to give a more robust
378  estimator */
379 
380  iclip = MAXHISTVAL;
381  mcpix = mpix;
382  skymedc = *skymed;
383  sigmac = *skysig;
384  for (iloop = 0; iloop < 3; iloop++) {
385  irej = 0;
386  for (i = NINT(skymedc+3.0*sigmac); i <= iclip; i++)
387  irej += ihist[i - MINHISTVAL];
388  if (irej == 0)
389  break;
390  iclip = NINT(skymedc+3.0*sigmac)-1;
391  mcpix = mcpix-irej;
392  imcore_medsig(ihist,MAXHIST,MINHISTVAL-1,mcpix,&skymedc,
393  &sigmac);
394  }
395  if (sigmac > 2.5)
396  break;
397  }
398 
399  /* Set the final answer */
400 
401  *skymed = skymedc/fac + skyref;
402  *skysig = sigmac/fac;
403  freespace(ihist);
404  return(CASU_OK);
405 }
406 
407 /*---------------------------------------------------------------------------*/
441 /*---------------------------------------------------------------------------*/
442 
443 extern void imcore_backest(ap_t *ap, float x, float y, float *skylev,
444  float *skyrms) {
445  int i,j,nbx,nby,nbsize,nbsizo2,iby,ibyp1,ibx,ibxp1;
446  float **bvals,fnbsize,dely,delx,t1,t2;
447 
448  /* Define some local variables */
449 
450  nbx = ap->backmap.nbx;
451  nby = ap->backmap.nby;
452  nbsize = ap->backmap.nbsize;
453  bvals = ap->backmap.bvals;
454 
455  /* Get closest pixel to the input location */
456 
457  i = NINT(x);
458  j = NINT(y);
459 
460  /* Now, work out where in the map to do the interpolation */
461 
462  nbsizo2 = nbsize/2;
463  fnbsize = 1.0/((float)nbsize);
464  iby = (j + nbsizo2)/nbsize;
465  ibyp1 = iby + 1;
466  iby = MIN(nby,MAX(1,iby));
467  ibyp1 = MIN(nby,ibyp1);
468  dely = (j - nbsize*iby + nbsizo2)*fnbsize;
469  ibx = (i + nbsizo2)/nbsize;
470  ibxp1 = ibx + 1;
471  ibx = MIN(nbx,MAX(1,ibx));
472  ibxp1 = MIN(nbx,ibxp1);
473  delx = (i - nbsize*ibx + nbsizo2)*fnbsize;
474 
475  /* Now do a linear interpolation to find the background. Calculate MAD of
476  the four adjacent background cells as an estimate of the RMS */
477 
478  t1 = (1.0 - dely)*bvals[iby-1][ibx-1] + dely*bvals[ibyp1-1][ibx-1];
479  t2 = (1.0 - dely)*bvals[iby-1][ibxp1-1] + dely*bvals[ibyp1-1][ibxp1-1];
480  *skylev = (1.0 - delx)*t1 + delx*t2;
481  *skyrms = 0.25*(fabsf(bvals[iby-1][ibx-1] - *skylev) +
482  fabsf(bvals[ibyp1-1][ibx-1] - *skylev) +
483  fabsf(bvals[iby-1][ibxp1-1] - *skylev) +
484  fabsf(bvals[ibyp1-1][ibxp1-1] - *skylev));
485 }
486 
487 /*---------------------------------------------------------------------------*/
524 /*---------------------------------------------------------------------------*/
525 
526 extern void imcore_medsig(int *shist, int nh, int ist, int itarg,
527  float *med, float *sig) {
528  int isum, medata;
529  float ffrac,sigmed;
530 
531  /* median */
532 
533  isum = 0;
534  medata = ist;
535  while (isum <= (itarg+1)/2 && (medata-MINHISTVAL) < nh) {
536  medata++;
537  isum += shist[medata-MINHISTVAL];
538  }
539  if (shist[medata-MINHISTVAL] == 0) {
540  ffrac = 0.0;
541  } else {
542  ffrac = (float)(isum - (itarg+1)/2)/(float)shist[medata-MINHISTVAL];
543  }
544  *med = (float)medata - ffrac + 0.5;
545 
546  /* sigma */
547 
548  isum = 0;
549  medata = ist;
550  while (isum <= (itarg+3)/4 && (medata-MINHISTVAL) < nh) {
551  medata++;
552  isum += shist[medata-MINHISTVAL];
553  }
554  if (shist[medata-MINHISTVAL] == 0) {
555  ffrac = 0.0;
556  } else {
557  ffrac = (float)(isum - (itarg+3)/4)/(float)shist[medata-MINHISTVAL];
558  }
559  sigmed = (float)medata - ffrac + 0.5;
560  *sig = 1.48*(*med - sigmed);
561  *sig = MAX(0.5,*sig);
562 }
563 
564 /*---------------------------------------------------------------------------*/
583 /*---------------------------------------------------------------------------*/
584 
585 static void sortit (float ia[], int n) {
586  int i, j, ii, jj, ifin;
587  float it;
588 
589  jj = 4;
590  while (jj < n)
591  jj = 2 * jj;
592  jj = MIN(n,(3 * jj)/4 - 1);
593  while (jj > 1) {
594  jj = jj/2;
595  ifin = n - jj;
596  for (ii = 0; ii < ifin; ii++) {
597  i = ii;
598  j = i + jj;
599  if (ia[i] <= ia[j])
600  continue;
601  it = ia[j];
602  do {
603  ia[j] = ia[i];
604  j = i;
605  i = i - jj;
606  if (i < 0)
607  break;
608  } while (ia[i] > it);
609  ia[j] = it;
610  }
611  }
612  return;
613 }
614 
615 
616 static void tidy(void) {
617  int i;
618 
619  freespace(nnp);
620  if (hist != NULL) {
621  for (i = 0; i < npvx; i++)
622  freespace(hist[i]);
623  }
624  freespace(hist);
625  return;
626 }
627 
630 /*
631 
632 $Log: imcore_background.c,v $
633 Revision 1.4 2015/08/12 11:16:55 jim
634 Modified procedure names to protect namespace
635 
636 Revision 1.3 2015/08/07 13:06:54 jim
637 Fixed copyright to ESO
638 
639 Revision 1.2 2015/08/06 05:33:31 jim
640 Small modification to help with backgrounds where the vast majority of
641 the pixels are bad
642 
643 Revision 1.1.1.1 2015/06/12 10:44:32 jim
644 Initial import
645 
646 Revision 1.4 2014/04/09 11:08:21 jim
647 Get rid of a couple of compiler moans
648 
649 Revision 1.3 2014/04/09 09:09:51 jim
650 Detabbed
651 
652 Revision 1.2 2014/03/26 15:25:19 jim
653 Modified for floating point confidence maps
654 
655 Revision 1.1.1.1 2013/08/27 12:07:48 jim
656 Imported
657 
658 
659 */
660 
int imcore_background(ap_t *ap, int nbsize, float nullval)
Model and create background map.
int imcore_backstats(ap_t *ap, float nullval, int satonly, float *skymed, float *skysig, float *sat)
Work out robust background estimate over a whole input image.
void imcore_medsig(int *shist, int nh, int ist, int itarg, float *med, float *sig)
Analyse histogram to work out median and sigma.
void imcore_backest(ap_t *ap, float x, float y, float *skylev, float *skyrms)
Work out estimated sky for a pixel position.
void imcore_bfilt(float **xbuf, int nx, int ny)
Do bilinear median and linear filtering on background values.
Definition: imcore_filter.c:73