VIRCAM Pipeline  2.3.12
casu_stats.c
1 /* $Id: casu_stats.c,v 1.2 2015/08/07 13:06:54 jim Exp $
2  *
3  * This file is part of the CASU Pipeline utilities
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2015/08/07 13:06:54 $
24  * $Revision: 1.2 $
25  * $Name: $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <math.h>
35 #include <string.h>
36 #include <cpl.h>
37 #include <cxtypes.h>
38 
39 #include "casu_stats.h"
40 #include "catalogue/casu_utils.h"
41 
42 /* Static subroutine prototypes */
43 
44 static float kselect(float *a, long n, long k);
45 static double dkselect(double *a, long n, long k);
46 static float histexam(int *histo, int nhist, int level);
47 
61 /*---------------------------------------------------------------------------*/
87 /*---------------------------------------------------------------------------*/
88 
89 extern float casu_med(float *data, unsigned char *bpm, long npts) {
90  long i,j,is_even,ilevel;
91  float *buf,value;
92 
93  /* Is there any point being here? */
94 
95  if (npts == 0)
96  return(CX_MAXFLOAT);
97 
98  /* If there is not BPM, then just do a straight forward median */
99 
100  buf = cpl_malloc(npts*sizeof(*buf));
101  if (bpm == NULL) {
102  is_even = !(npts & 1);
103  memmove((char *)buf,(char *)data,npts*sizeof(float));
104  if (is_even) {
105  ilevel = npts/2 - 1;
106  value = kselect(buf,npts,ilevel);
107  ilevel = npts/2;
108  value = 0.5*(value + kselect(buf,npts,ilevel));
109  } else {
110  ilevel = npts/2;
111  value = kselect(buf,npts,ilevel);
112  }
113 
114  /* Otherwise get rid of the dodgy values and then do the median */
115 
116  } else {
117  j = 0;
118  for (i = 0; i < npts; i++) {
119  if (bpm[i] == 0)
120  buf[j++] = data[i];
121  }
122  if (j == 0) {
123  cpl_free(buf);
124  value = CX_MAXFLOAT;
125  return(value);
126  }
127  is_even = !(j & 1);
128  if (is_even) {
129  ilevel = j/2 - 1;
130  value = kselect(buf,j,ilevel);
131  ilevel = j/2;
132  value = 0.5*(value + kselect(buf,j,ilevel));
133  } else {
134  ilevel = j/2;
135  value = kselect(buf,j,ilevel);
136  }
137  }
138  cpl_free(buf);
139  return(value);
140 }
141 
142 /*---------------------------------------------------------------------------*/
168 /*---------------------------------------------------------------------------*/
169 
170 extern double casu_dmed(double *data, unsigned char *bpm, long npts) {
171  long i,j,is_even,ilevel;
172  double *buf,value;
173 
174  /* If there is not BPM, then just do a straight forward median */
175 
176  buf = cpl_malloc(npts*sizeof(*buf));
177  if (bpm == NULL) {
178  is_even = !(npts & 1);
179  memmove((char *)buf,(char *)data,npts*sizeof(double));
180  if (is_even) {
181  ilevel = npts/2 - 1;
182  value = dkselect(buf,npts,ilevel);
183  ilevel = npts/2;
184  value = 0.5*(value + dkselect(buf,npts,ilevel));
185  } else {
186  ilevel = npts/2;
187  value = dkselect(buf,npts,ilevel);
188  }
189 
190  /* Otherwise get rid of the dodgy values and then do the median */
191 
192  } else {
193  j = 0;
194  for (i = 0; i < npts; i++) {
195  if (bpm[i] == 0)
196  buf[j++] = data[i];
197  }
198  if (j == 0) {
199  cpl_free(buf);
200  value = CX_MAXDOUBLE;
201  return(value);
202  }
203  is_even = !(j & 1);
204  if (is_even) {
205  ilevel = j/2 - 1;
206  value = dkselect(buf,j,ilevel);
207  ilevel = j/2;
208  value = 0.5*(value + dkselect(buf,j,ilevel));
209  } else {
210  ilevel = j/2;
211  value = dkselect(buf,j,ilevel);
212  }
213  }
214  cpl_free(buf);
215  return(value);
216 }
217 
218 /*---------------------------------------------------------------------------*/
256 /*---------------------------------------------------------------------------*/
257 
258 extern void casu_qmedsig(float *data, unsigned char *bpm, long npts,
259  float thresh, int niter, float lowv, float highv,
260  float *median, float *sigma) {
261  int *histo,nbins,nhist,ilev,iclip,nhist2,halflev,quartlev;
262  int irej,jst,j;
263  long i;
264  float mlev,qlev;
265  unsigned char *b;
266 
267  /* Right, first thing is to histogram the data. Cut values below
268  and above the 'ceiling' values */
269 
270  if (bpm == NULL)
271  b = cpl_calloc(npts,sizeof(unsigned char));
272  else
273  b = bpm;
274  nbins = casu_nint(highv - lowv + 1.0);
275  histo = cpl_calloc(nbins,sizeof(*histo));
276  nhist = 0;
277  for (i = 0; i < npts; i++) {
278  if (b[i] || data[i] < lowv || data[i] > highv)
279  continue;
280  ilev = casu_nint(data[i] - lowv);
281  ilev = max(0,min(nbins-1,ilev));
282  histo[ilev] += 1;
283  nhist += 1;
284  }
285  if (bpm == NULL)
286  freespace(b);
287  if (nhist == 0) {
288  *median = CX_MAXFLOAT;
289  *sigma = CX_MAXFLOAT;
290  freespace(histo);
291  return;
292  }
293 
294  /* Right, find the median value and the first quartile. */
295 
296  iclip = nbins - 1;
297  nhist2 = nhist;
298  for (i = 0; i <= niter; i++) {
299  halflev = (nhist2 + 1)/2;
300  quartlev = (nhist2 + 3)/4;
301  mlev = histexam(histo,nbins,halflev);
302  *median = mlev + lowv;
303  qlev = histexam(histo,nbins,quartlev);
304  *sigma = (mlev - qlev)*1.48;
305  if (i == niter)
306  break;
307  irej = 0;
308  jst = casu_nint(mlev + thresh*(*sigma));
309  for (j = jst; j <= iclip; j++)
310  irej += histo[j];
311  if (irej == 0)
312  break;
313  iclip = jst - 1;
314  nhist2 -= irej;
315  }
316  cpl_free(histo);
317 }
318 
319 /*---------------------------------------------------------------------------*/
345 /*---------------------------------------------------------------------------*/
346 
347 extern void casu_medmad(float *data, unsigned char *bpm, long np, float *med,
348  float *mad) {
349  long i;
350  float *work;
351 
352  /* First find the median value */
353 
354  *med = casu_med(data,bpm,np);
355 
356  /* Now work out the MAD. Start by getting a bit of workspace and filling
357  it with absolute residuals */
358 
359  work = cpl_malloc(np*sizeof(*work));
360  for (i = 0; i < np; i++)
361  work[i] = (float)fabs((double)(data[i] - *med));
362 
363  /* Now get the median value of the absolute deviations */
364 
365  *mad = casu_med(work,bpm,np);
366 
367  /* Tidy and exit */
368 
369  cpl_free(work);
370 }
371 
372 /*---------------------------------------------------------------------------*/
404 /*---------------------------------------------------------------------------*/
405 
406 extern void casu_medmadcut(float *data, unsigned char *bpm, long np, float lcut,
407  float hcut, float *med, float *mad) {
408  long i;
409  float *work;
410  unsigned char *bad;
411 
412  /* Get a workspace for the pseudo-pabd pixel mask... */
413 
414  bad = cpl_calloc(np,sizeof(*bad));
415  if (bpm != NULL) {
416  for (i = 0; i < np; i++)
417  if (bpm[i] != 0 || data[i] < lcut || data[i] > hcut)
418  bad[i] = 1;
419  } else {
420  for (i = 0; i < np; i++)
421  if (data[i] < lcut || data[i] > hcut)
422  bad[i] = 1;
423  }
424 
425  /* First find the median value */
426 
427  *med = casu_med(data,bad,np);
428  if (*med == CX_MAXFLOAT) {
429  *mad = 0.0;
430  cpl_free(bad);
431  return;
432  }
433 
434  /* Now work out the MAD. Start by getting a bit of workspace and filling
435  it with absolute residuals */
436 
437  work = cpl_malloc(np*sizeof(*work));
438  for (i = 0; i < np; i++)
439  work[i] = (float)fabs((double)(data[i] - *med));
440 
441  /* Now get the median value of the absolute deviations */
442 
443  *mad = casu_med(work,bad,np);
444 
445  /* Tidy and exit */
446 
447  cpl_free(work);
448  cpl_free(bad);
449 }
450 
451 /*---------------------------------------------------------------------------*/
477 /*---------------------------------------------------------------------------*/
478 
479 extern float casu_mean(float *data, unsigned char *bpm, long npts) {
480  long i,n;
481  float sum,value;
482 
483  /* Separate sections depending on whether there is a BPM or not */
484 
485  sum = 0.0;
486  if (bpm == NULL) {
487  n = npts;
488  for (i = 0; i < npts; i++)
489  sum += data[i];
490  } else {
491  n = 0;
492  for (i = 0; i < npts; i++) {
493  if (bpm[i] == 0) {
494  sum += data[i];
495  n++;
496  }
497  }
498  }
499  if (n > 0)
500  value = sum/(float)n;
501  else
502  value = CX_MAXFLOAT;
503  return(value);
504 }
505 
506 /*---------------------------------------------------------------------------*/
532 /*---------------------------------------------------------------------------*/
533 
534 extern double casu_dmean(double *data, unsigned char *bpm, long npts) {
535  long i,n;
536  double sum,value;
537 
538  /* Separate sections depending on whether there is a BPM or not */
539 
540  sum = 0.0;
541  if (bpm == NULL) {
542  n = npts;
543  for (i = 0; i < npts; i++)
544  sum += data[i];
545  } else {
546  n = 0;
547  for (i = 0; i < npts; i++) {
548  if (bpm[i] == 0) {
549  sum += data[i];
550  n++;
551  }
552  }
553  }
554  if (n > 0)
555  value = sum/(float)n;
556  else
557  value = CX_MAXDOUBLE;
558  return(value);
559 }
560 
561 /*---------------------------------------------------------------------------*/
593 /*---------------------------------------------------------------------------*/
594 
595 extern int casu_meansig(float *data, unsigned char *bpm, long npts,
596  float *mean, float *sig) {
597  long i,n;
598  double sum,sum2,d;
599  const char *fctid = "casu_meansig";
600 
601  /* Separate sections depending on whether there is a BPM or not */
602 
603  sum = 0.0;
604  sum2 = 0.0;
605  if (bpm == NULL) {
606  n = npts;
607  for (i = 0; i < npts; i++) {
608  d = (double)(data[i]);
609  sum += d;
610  sum2 += d*d;
611  }
612  } else {
613  n = 0;
614  for (i = 0; i < npts; i++) {
615  if (bpm[i] == 0) {
616  d = (double)(data[i]);
617  sum += d;
618  sum2 += d*d;
619  n++;
620  }
621  }
622  }
623 
624  /* Check whether we can do the mean and sigma calculations */
625 
626  switch (n) {
627  case 0:
628  *mean = CX_MAXFLOAT;
629  *sig = CX_MAXFLOAT;
630  cpl_msg_warning(fctid,"All values flagged as bad");
631  return(CASU_WARN);
632  case 1:
633  *mean = (float)sum;
634  *sig = 0.0;
635  return(CASU_OK);
636  default:
637  sum /= (double)n;
638  *mean = (float)sum;
639  sum2 = sum2/(double)n - sum*sum;
640  *sig = (float)sqrt(max(1.0e-12,sum2));
641  return(CASU_OK);
642  }
643 }
644 
645 /*---------------------------------------------------------------------------*/
670 /*---------------------------------------------------------------------------*/
671 
672 extern void casu_medsig(float *data, unsigned char *bpm, long np, float *med,
673  float *sig) {
674  long i,n;
675  float sum,resid;
676 
677  /* First find the median value */
678 
679  *med = casu_med(data,bpm,np);
680  if (*med == CX_MAXFLOAT) {
681  *sig = 0.0;
682  return;
683  }
684 
685  /* If no bpm is present the just use them all */
686 
687  if (bpm == NULL) {
688  sum = 0.0;
689  for (i = 0; i < np; i++) {
690  resid = data[i] - *med;
691  sum += resid*resid;
692  }
693  *sig = sqrt(sum/(float)np);
694 
695  /* Otherwise test the bpm */
696 
697  } else {
698  sum = 0.0;
699  n = 0;
700  for (i = 0; i < np; i++) {
701  if (bpm[i] == 0) {
702  n++;
703  resid = data[i] - *med;
704  sum += resid*resid;
705  }
706  }
707  if (n > 0)
708  *sig = sqrt(sum/(float)n);
709  else
710  *sig = 0.0;
711  }
712 }
713 
714 /*---------------------------------------------------------------------------*/
736 /*---------------------------------------------------------------------------*/
737 
738 extern int casu_sumbpm(unsigned char *bpm, long npts, int *sumb) {
739  long j;
740 
741  *sumb = 0;
742  for (j = 0; j < npts; j++)
743  *sumb += bpm[j];
744  return(CASU_OK);
745 }
746 
749 static float histexam(int *histo, int nhist, int level) {
750  int ilev,ii;
751  float value;
752 
753  ii = 0;
754  ilev = -1;
755  while (ii < level && ilev < nhist-1)
756  ii += histo[++ilev];
757  value = (float)ilev - (float)(ii - level)/(float)histo[ilev] + 0.5;
758  return(value);
759 }
760 
761 /*
762  * given an array a of n elements, return the element that would be at
763  * position k, (0 <= k < n), if the array were sorted. from Algorithms
764  * and Data Structures in C++ by Leendert Ammeraal, pg. 82. O(n).
765  *
766  * NB: partially reorders data in array
767  */
768 
769 /* Stolen from C. Sabbey */
770 
771 static float kselect(float *a, long n, long k) {
772  while (n > 1) {
773  long i = 0, j = n - 1;
774  float x = a[j/2], w;
775 
776  do {
777  while (a[i] < x) i++;
778  while (a[j] > x) j--;
779  if (i < j) {
780  w = a[i]; a[i] = a[j]; a[j] = w;
781  } else {
782  if (i == j) i++;
783  break;
784  }
785  } while (++i <= --j);
786 
787  if (k < i)
788  n = i;
789  else {
790  a += i; n -= i; k -= i;
791  }
792  }
793 
794  return a[0];
795 }
796 
797 static double dkselect(double *a, long n, long k) {
798  while (n > 1) {
799  long i = 0, j = n - 1;
800  double x = a[j/2], w;
801 
802  do {
803  while (a[i] < x) i++;
804  while (a[j] > x) j--;
805  if (i < j) {
806  w = a[i]; a[i] = a[j]; a[j] = w;
807  } else {
808  if (i == j) i++;
809  break;
810  }
811  } while (++i <= --j);
812 
813  if (k < i)
814  n = i;
815  else {
816  a += i; n -= i; k -= i;
817  }
818  }
819 
820  return a[0];
821 }
822 
823 /*
824 
825 $Log: casu_stats.c,v $
826 Revision 1.2 2015/08/07 13:06:54 jim
827 Fixed copyright to ESO
828 
829 Revision 1.1.1.1 2015/06/12 10:44:32 jim
830 Initial import
831 
832 Revision 1.4 2015/01/29 11:54:33 jim
833 Added medmadcut routine. Some modified comments
834 
835 Revision 1.3 2014/04/26 18:38:53 jim
836 changed integer to long in some routines
837 
838 Revision 1.2 2013/11/21 09:38:14 jim
839 detabbed
840 
841 Revision 1.1.1.1 2013-08-27 12:07:48 jim
842 Imported
843 
844 
845 */
846 
double casu_dmean(double *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:534
void casu_medmad(float *data, unsigned char *bpm, long np, float *med, float *mad)
Definition: casu_stats.c:347
float casu_med(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:89
void casu_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: casu_stats.c:258
void casu_medsig(float *data, unsigned char *bpm, long np, float *med, float *sig)
Definition: casu_stats.c:672
float casu_mean(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:479
double casu_dmed(double *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:170
int casu_sumbpm(unsigned char *bpm, long npts, int *sumb)
Definition: casu_stats.c:738
void casu_medmadcut(float *data, unsigned char *bpm, long np, float lcut, float hcut, float *med, float *mad)
Definition: casu_stats.c:406
int casu_meansig(float *data, unsigned char *bpm, long npts, float *mean, float *sig)
Definition: casu_stats.c:595