VIRCAM Pipeline 2.3.12
casu_stats.c
1/* $Id: casu_stats.c,v 1.2 2015/08/07 13:06:54 jim Exp $
2 *
3 * This file is part of the CASU Pipeline utilities
4 * Copyright (C) 2015 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20
21/*
22 * $Author: jim $
23 * $Date: 2015/08/07 13:06:54 $
24 * $Revision: 1.2 $
25 * $Name: $
26 */
27
28/* Includes */
29
30#ifdef HAVE_CONFIG_H
31#include <config.h>
32#endif
33
34#include <math.h>
35#include <string.h>
36#include <cpl.h>
37#include <cxtypes.h>
38
39#include "casu_stats.h"
40#include "catalogue/casu_utils.h"
41
42/* Static subroutine prototypes */
43
44static float kselect(float *a, long n, long k);
45static double dkselect(double *a, long n, long k);
46static float histexam(int *histo, int nhist, int level);
47
61/*---------------------------------------------------------------------------*/
87/*---------------------------------------------------------------------------*/
88
89extern float casu_med(float *data, unsigned char *bpm, long npts) {
90 long i,j,is_even,ilevel;
91 float *buf,value;
92
93 /* Is there any point being here? */
94
95 if (npts == 0)
96 return(CX_MAXFLOAT);
97
98 /* If there is not BPM, then just do a straight forward median */
99
100 buf = cpl_malloc(npts*sizeof(*buf));
101 if (bpm == NULL) {
102 is_even = !(npts & 1);
103 memmove((char *)buf,(char *)data,npts*sizeof(float));
104 if (is_even) {
105 ilevel = npts/2 - 1;
106 value = kselect(buf,npts,ilevel);
107 ilevel = npts/2;
108 value = 0.5*(value + kselect(buf,npts,ilevel));
109 } else {
110 ilevel = npts/2;
111 value = kselect(buf,npts,ilevel);
112 }
113
114 /* Otherwise get rid of the dodgy values and then do the median */
115
116 } else {
117 j = 0;
118 for (i = 0; i < npts; i++) {
119 if (bpm[i] == 0)
120 buf[j++] = data[i];
121 }
122 if (j == 0) {
123 cpl_free(buf);
124 value = CX_MAXFLOAT;
125 return(value);
126 }
127 is_even = !(j & 1);
128 if (is_even) {
129 ilevel = j/2 - 1;
130 value = kselect(buf,j,ilevel);
131 ilevel = j/2;
132 value = 0.5*(value + kselect(buf,j,ilevel));
133 } else {
134 ilevel = j/2;
135 value = kselect(buf,j,ilevel);
136 }
137 }
138 cpl_free(buf);
139 return(value);
140}
141
142/*---------------------------------------------------------------------------*/
168/*---------------------------------------------------------------------------*/
169
170extern double casu_dmed(double *data, unsigned char *bpm, long npts) {
171 long i,j,is_even,ilevel;
172 double *buf,value;
173
174 /* If there is not BPM, then just do a straight forward median */
175
176 buf = cpl_malloc(npts*sizeof(*buf));
177 if (bpm == NULL) {
178 is_even = !(npts & 1);
179 memmove((char *)buf,(char *)data,npts*sizeof(double));
180 if (is_even) {
181 ilevel = npts/2 - 1;
182 value = dkselect(buf,npts,ilevel);
183 ilevel = npts/2;
184 value = 0.5*(value + dkselect(buf,npts,ilevel));
185 } else {
186 ilevel = npts/2;
187 value = dkselect(buf,npts,ilevel);
188 }
189
190 /* Otherwise get rid of the dodgy values and then do the median */
191
192 } else {
193 j = 0;
194 for (i = 0; i < npts; i++) {
195 if (bpm[i] == 0)
196 buf[j++] = data[i];
197 }
198 if (j == 0) {
199 cpl_free(buf);
200 value = CX_MAXDOUBLE;
201 return(value);
202 }
203 is_even = !(j & 1);
204 if (is_even) {
205 ilevel = j/2 - 1;
206 value = dkselect(buf,j,ilevel);
207 ilevel = j/2;
208 value = 0.5*(value + dkselect(buf,j,ilevel));
209 } else {
210 ilevel = j/2;
211 value = dkselect(buf,j,ilevel);
212 }
213 }
214 cpl_free(buf);
215 return(value);
216}
217
218/*---------------------------------------------------------------------------*/
256/*---------------------------------------------------------------------------*/
257
258extern void casu_qmedsig(float *data, unsigned char *bpm, long npts,
259 float thresh, int niter, float lowv, float highv,
260 float *median, float *sigma) {
261 int *histo,nbins,nhist,ilev,iclip,nhist2,halflev,quartlev;
262 int irej,jst,j;
263 long i;
264 float mlev,qlev;
265 unsigned char *b;
266
267 /* Right, first thing is to histogram the data. Cut values below
268 and above the 'ceiling' values */
269
270 if (bpm == NULL)
271 b = cpl_calloc(npts,sizeof(unsigned char));
272 else
273 b = bpm;
274 nbins = casu_nint(highv - lowv + 1.0);
275 histo = cpl_calloc(nbins,sizeof(*histo));
276 nhist = 0;
277 for (i = 0; i < npts; i++) {
278 if (b[i] || data[i] < lowv || data[i] > highv)
279 continue;
280 ilev = casu_nint(data[i] - lowv);
281 ilev = max(0,min(nbins-1,ilev));
282 histo[ilev] += 1;
283 nhist += 1;
284 }
285 if (bpm == NULL)
286 freespace(b);
287 if (nhist == 0) {
288 *median = CX_MAXFLOAT;
289 *sigma = CX_MAXFLOAT;
290 freespace(histo);
291 return;
292 }
293
294 /* Right, find the median value and the first quartile. */
295
296 iclip = nbins - 1;
297 nhist2 = nhist;
298 for (i = 0; i <= niter; i++) {
299 halflev = (nhist2 + 1)/2;
300 quartlev = (nhist2 + 3)/4;
301 mlev = histexam(histo,nbins,halflev);
302 *median = mlev + lowv;
303 qlev = histexam(histo,nbins,quartlev);
304 *sigma = (mlev - qlev)*1.48;
305 if (i == niter)
306 break;
307 irej = 0;
308 jst = casu_nint(mlev + thresh*(*sigma));
309 for (j = jst; j <= iclip; j++)
310 irej += histo[j];
311 if (irej == 0)
312 break;
313 iclip = jst - 1;
314 nhist2 -= irej;
315 }
316 cpl_free(histo);
317}
318
319/*---------------------------------------------------------------------------*/
345/*---------------------------------------------------------------------------*/
346
347extern void casu_medmad(float *data, unsigned char *bpm, long np, float *med,
348 float *mad) {
349 long i;
350 float *work;
351
352 /* First find the median value */
353
354 *med = casu_med(data,bpm,np);
355
356 /* Now work out the MAD. Start by getting a bit of workspace and filling
357 it with absolute residuals */
358
359 work = cpl_malloc(np*sizeof(*work));
360 for (i = 0; i < np; i++)
361 work[i] = (float)fabs((double)(data[i] - *med));
362
363 /* Now get the median value of the absolute deviations */
364
365 *mad = casu_med(work,bpm,np);
366
367 /* Tidy and exit */
368
369 cpl_free(work);
370}
371
372/*---------------------------------------------------------------------------*/
404/*---------------------------------------------------------------------------*/
405
406extern void casu_medmadcut(float *data, unsigned char *bpm, long np, float lcut,
407 float hcut, float *med, float *mad) {
408 long i;
409 float *work;
410 unsigned char *bad;
411
412 /* Get a workspace for the pseudo-pabd pixel mask... */
413
414 bad = cpl_calloc(np,sizeof(*bad));
415 if (bpm != NULL) {
416 for (i = 0; i < np; i++)
417 if (bpm[i] != 0 || data[i] < lcut || data[i] > hcut)
418 bad[i] = 1;
419 } else {
420 for (i = 0; i < np; i++)
421 if (data[i] < lcut || data[i] > hcut)
422 bad[i] = 1;
423 }
424
425 /* First find the median value */
426
427 *med = casu_med(data,bad,np);
428 if (*med == CX_MAXFLOAT) {
429 *mad = 0.0;
430 cpl_free(bad);
431 return;
432 }
433
434 /* Now work out the MAD. Start by getting a bit of workspace and filling
435 it with absolute residuals */
436
437 work = cpl_malloc(np*sizeof(*work));
438 for (i = 0; i < np; i++)
439 work[i] = (float)fabs((double)(data[i] - *med));
440
441 /* Now get the median value of the absolute deviations */
442
443 *mad = casu_med(work,bad,np);
444
445 /* Tidy and exit */
446
447 cpl_free(work);
448 cpl_free(bad);
449}
450
451/*---------------------------------------------------------------------------*/
477/*---------------------------------------------------------------------------*/
478
479extern float casu_mean(float *data, unsigned char *bpm, long npts) {
480 long i,n;
481 float sum,value;
482
483 /* Separate sections depending on whether there is a BPM or not */
484
485 sum = 0.0;
486 if (bpm == NULL) {
487 n = npts;
488 for (i = 0; i < npts; i++)
489 sum += data[i];
490 } else {
491 n = 0;
492 for (i = 0; i < npts; i++) {
493 if (bpm[i] == 0) {
494 sum += data[i];
495 n++;
496 }
497 }
498 }
499 if (n > 0)
500 value = sum/(float)n;
501 else
502 value = CX_MAXFLOAT;
503 return(value);
504}
505
506/*---------------------------------------------------------------------------*/
532/*---------------------------------------------------------------------------*/
533
534extern double casu_dmean(double *data, unsigned char *bpm, long npts) {
535 long i,n;
536 double sum,value;
537
538 /* Separate sections depending on whether there is a BPM or not */
539
540 sum = 0.0;
541 if (bpm == NULL) {
542 n = npts;
543 for (i = 0; i < npts; i++)
544 sum += data[i];
545 } else {
546 n = 0;
547 for (i = 0; i < npts; i++) {
548 if (bpm[i] == 0) {
549 sum += data[i];
550 n++;
551 }
552 }
553 }
554 if (n > 0)
555 value = sum/(float)n;
556 else
557 value = CX_MAXDOUBLE;
558 return(value);
559}
560
561/*---------------------------------------------------------------------------*/
593/*---------------------------------------------------------------------------*/
594
595extern int casu_meansig(float *data, unsigned char *bpm, long npts,
596 float *mean, float *sig) {
597 long i,n;
598 double sum,sum2,d;
599 const char *fctid = "casu_meansig";
600
601 /* Separate sections depending on whether there is a BPM or not */
602
603 sum = 0.0;
604 sum2 = 0.0;
605 if (bpm == NULL) {
606 n = npts;
607 for (i = 0; i < npts; i++) {
608 d = (double)(data[i]);
609 sum += d;
610 sum2 += d*d;
611 }
612 } else {
613 n = 0;
614 for (i = 0; i < npts; i++) {
615 if (bpm[i] == 0) {
616 d = (double)(data[i]);
617 sum += d;
618 sum2 += d*d;
619 n++;
620 }
621 }
622 }
623
624 /* Check whether we can do the mean and sigma calculations */
625
626 switch (n) {
627 case 0:
628 *mean = CX_MAXFLOAT;
629 *sig = CX_MAXFLOAT;
630 cpl_msg_warning(fctid,"All values flagged as bad");
631 return(CASU_WARN);
632 case 1:
633 *mean = (float)sum;
634 *sig = 0.0;
635 return(CASU_OK);
636 default:
637 sum /= (double)n;
638 *mean = (float)sum;
639 sum2 = sum2/(double)n - sum*sum;
640 *sig = (float)sqrt(max(1.0e-12,sum2));
641 return(CASU_OK);
642 }
643}
644
645/*---------------------------------------------------------------------------*/
670/*---------------------------------------------------------------------------*/
671
672extern void casu_medsig(float *data, unsigned char *bpm, long np, float *med,
673 float *sig) {
674 long i,n;
675 float sum,resid;
676
677 /* First find the median value */
678
679 *med = casu_med(data,bpm,np);
680 if (*med == CX_MAXFLOAT) {
681 *sig = 0.0;
682 return;
683 }
684
685 /* If no bpm is present the just use them all */
686
687 if (bpm == NULL) {
688 sum = 0.0;
689 for (i = 0; i < np; i++) {
690 resid = data[i] - *med;
691 sum += resid*resid;
692 }
693 *sig = sqrt(sum/(float)np);
694
695 /* Otherwise test the bpm */
696
697 } else {
698 sum = 0.0;
699 n = 0;
700 for (i = 0; i < np; i++) {
701 if (bpm[i] == 0) {
702 n++;
703 resid = data[i] - *med;
704 sum += resid*resid;
705 }
706 }
707 if (n > 0)
708 *sig = sqrt(sum/(float)n);
709 else
710 *sig = 0.0;
711 }
712}
713
714/*---------------------------------------------------------------------------*/
736/*---------------------------------------------------------------------------*/
737
738extern int casu_sumbpm(unsigned char *bpm, long npts, int *sumb) {
739 long j;
740
741 *sumb = 0;
742 for (j = 0; j < npts; j++)
743 *sumb += bpm[j];
744 return(CASU_OK);
745}
746
749static float histexam(int *histo, int nhist, int level) {
750 int ilev,ii;
751 float value;
752
753 ii = 0;
754 ilev = -1;
755 while (ii < level && ilev < nhist-1)
756 ii += histo[++ilev];
757 value = (float)ilev - (float)(ii - level)/(float)histo[ilev] + 0.5;
758 return(value);
759}
760
761/*
762 * given an array a of n elements, return the element that would be at
763 * position k, (0 <= k < n), if the array were sorted. from Algorithms
764 * and Data Structures in C++ by Leendert Ammeraal, pg. 82. O(n).
765 *
766 * NB: partially reorders data in array
767 */
768
769/* Stolen from C. Sabbey */
770
771static float kselect(float *a, long n, long k) {
772 while (n > 1) {
773 long i = 0, j = n - 1;
774 float x = a[j/2], w;
775
776 do {
777 while (a[i] < x) i++;
778 while (a[j] > x) j--;
779 if (i < j) {
780 w = a[i]; a[i] = a[j]; a[j] = w;
781 } else {
782 if (i == j) i++;
783 break;
784 }
785 } while (++i <= --j);
786
787 if (k < i)
788 n = i;
789 else {
790 a += i; n -= i; k -= i;
791 }
792 }
793
794 return a[0];
795}
796
797static double dkselect(double *a, long n, long k) {
798 while (n > 1) {
799 long i = 0, j = n - 1;
800 double x = a[j/2], w;
801
802 do {
803 while (a[i] < x) i++;
804 while (a[j] > x) j--;
805 if (i < j) {
806 w = a[i]; a[i] = a[j]; a[j] = w;
807 } else {
808 if (i == j) i++;
809 break;
810 }
811 } while (++i <= --j);
812
813 if (k < i)
814 n = i;
815 else {
816 a += i; n -= i; k -= i;
817 }
818 }
819
820 return a[0];
821}
822
823/*
824
825$Log: casu_stats.c,v $
826Revision 1.2 2015/08/07 13:06:54 jim
827Fixed copyright to ESO
828
829Revision 1.1.1.1 2015/06/12 10:44:32 jim
830Initial import
831
832Revision 1.4 2015/01/29 11:54:33 jim
833Added medmadcut routine. Some modified comments
834
835Revision 1.3 2014/04/26 18:38:53 jim
836changed integer to long in some routines
837
838Revision 1.2 2013/11/21 09:38:14 jim
839detabbed
840
841Revision 1.1.1.1 2013-08-27 12:07:48 jim
842Imported
843
844
845*/
846
double casu_dmean(double *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:534
void casu_medmad(float *data, unsigned char *bpm, long np, float *med, float *mad)
Definition: casu_stats.c:347
float casu_med(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:89
void casu_qmedsig(float *data, unsigned char *bpm, long npts, float thresh, int niter, float lowv, float highv, float *median, float *sigma)
Definition: casu_stats.c:258
void casu_medsig(float *data, unsigned char *bpm, long np, float *med, float *sig)
Definition: casu_stats.c:672
float casu_mean(float *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:479
double casu_dmed(double *data, unsigned char *bpm, long npts)
Definition: casu_stats.c:170
int casu_sumbpm(unsigned char *bpm, long npts, int *sumb)
Definition: casu_stats.c:738
void casu_medmadcut(float *data, unsigned char *bpm, long np, float lcut, float hcut, float *med, float *mad)
Definition: casu_stats.c:406
int casu_meansig(float *data, unsigned char *bpm, long npts, float *mean, float *sig)
Definition: casu_stats.c:595