High-Level Data Reduction Library: hdrl_mode.c Source File

High-Level Data Reduction Library 1.6.0 High-Level data reduction routines for ESO pipelines

Loading...
Searching...
No Matches
hdrl_mode.c
Go to the documentation of this file.
1/*
2 * This file is part of the HDRL
3 * Copyright (C) 2021 European Southern Observatory
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19
20#ifdef HAVE_CONFIG_H
21#include <config.h>
22#endif
23
24/*-----------------------------------------------------------------------------
25 Includes
26-----------------------------------------------------------------------------*/
27#include "hdrl_mode.h"
28#include "hdrl_sigclip.h"
29#include "hdrl_utils.h"
30#include "hdrl_collapse.h"
31#include "hdrl_random.h"
32
33#ifndef _OPENMP
34#define omp_get_max_threads() 1
35#define omp_get_thread_num() 0
36#else
37#include <omp.h>
38#endif
39
40#include <cpl.h>
41//#include <cpl_vector.h>
42#include <string.h>
43#include <math.h>
44#include <time.h>
45#include <gsl/gsl_multifit.h>
46#include <gsl/gsl_vector.h>
47#include <gsl/gsl_histogram.h>
48#include <gsl/gsl_poly.h>
49
50 static const char * method_to_string(const hdrl_mode_type method)
51{
52 switch (method) {
53 case HDRL_MODE_MEDIAN:
54 return "MEDIAN";
55 break;
56 case HDRL_MODE_WEIGHTED:
57 return "WEIGHTED";
58 break;
59 case HDRL_MODE_FIT:
60 return "FIT";
61 break;
62 default :
63 cpl_error_set_message(cpl_func, CPL_ERROR_ILLEGAL_INPUT,
64 "mode method unknown");
65 return "";
66 break;
67 }
68}
69
70/* ---------------------------------------------------------------------------*/
88/* ---------------------------------------------------------------------------*/
89cpl_parameterlist * hdrl_mode_parameter_create_parlist(
90 const char *base_context,
91 const char *prefix,
92 const hdrl_parameter *defaults)
93{
94 cpl_ensure(base_context && prefix && defaults,
95 CPL_ERROR_NULL_INPUT, NULL);
96
97 cpl_ensure(hdrl_collapse_parameter_is_mode(defaults),
98 CPL_ERROR_INCOMPATIBLE_INPUT, NULL);
99
100 cpl_parameterlist * parlist = cpl_parameterlist_new();
101
102 /* --prefix.histo-min */
103 hdrl_setup_vparameter(parlist, prefix, ".", "",
104 "histo-min", base_context,
105 "Minimum pixel value to accept for mode computation",
106 CPL_TYPE_DOUBLE,
107 hdrl_collapse_mode_parameter_get_histo_min(defaults));
108
109 /* --prefix.histo-max */
110 hdrl_setup_vparameter(parlist, prefix, ".", "",
111 "histo-max", base_context,
112 "Maximum pixel value to accept for mode computation",
113 CPL_TYPE_DOUBLE,
114 hdrl_collapse_mode_parameter_get_histo_max(defaults));
115
116 /* --prefix.bin-size */
117 hdrl_setup_vparameter(parlist, prefix, ".", "",
118 "bin-size", base_context,
119 "Binsize of the histogram",
120 CPL_TYPE_DOUBLE,
121 hdrl_collapse_mode_parameter_get_bin_size(defaults));
122
123 /* --prefix.method */
124 char * name ;
125 char * context = hdrl_join_string(".", 2, base_context, prefix);
126
127 hdrl_mode_type method = hdrl_collapse_mode_parameter_get_method(defaults);
128 const char * method_def = method_to_string(method);
129 name = hdrl_join_string(".", 2, context, "method");
130 cpl_free(context);
131 cpl_parameter * par = cpl_parameter_new_enum(name, CPL_TYPE_STRING,
132 "Mode method (algorithm) to use",
133 base_context, method_def, 3, "MEDIAN", "WEIGHTED", "FIT");
134 cpl_free(name);
135 name = hdrl_join_string(".", 2, prefix, "method");
136 cpl_parameter_set_alias(par, CPL_PARAMETER_MODE_CLI, name);
137 cpl_parameter_disable(par, CPL_PARAMETER_MODE_ENV);
138 cpl_free(name);
139 cpl_parameterlist_append(parlist, par);
140
141/*
142 --prefix.method
143 hdrl_setup_vparameter(parlist, prefix, ".", "",
144 "method", base_context,
145 "Mode method",
146 CPL_TYPE_INT,
147 hdrl_collapse_mode_parameter_get_method(defaults));
148*/
149
150 /* --prefix.error-niter */
151 hdrl_setup_vparameter(parlist, prefix, ".", "",
152 "error-niter", base_context,
153 "Iterations to compute the mode error",
154 CPL_TYPE_INT,
155 hdrl_collapse_mode_parameter_get_error_niter(defaults));
156
157 if (cpl_error_get_code()) {
158 cpl_parameterlist_delete(parlist);
159 return NULL;
160 }
161
162 return parlist;
163}
164/* ---------------------------------------------------------------------------*/
185/* ---------------------------------------------------------------------------*/
186cpl_error_code hdrl_mode_parameter_parse_parlist(
187 const cpl_parameterlist * parlist,
188 const char * prefix,
189 double * histo_min,
190 double * histo_max,
191 double * bin_size,
192 hdrl_mode_type * method,
193 cpl_size * error_niter)
194{
195 cpl_ensure_code(prefix && parlist, CPL_ERROR_NULL_INPUT);
196 char * name;
197
198 if (histo_min) {
199 name = hdrl_join_string(".", 2, prefix, "mode.histo-min");
200 const cpl_parameter * par = cpl_parameterlist_find_const(parlist, name);
201 *histo_min = cpl_parameter_get_double(par);
202 cpl_free(name);
203 }
204 if (histo_max) {
205 name = hdrl_join_string(".", 2, prefix, "mode.histo-max");
206 const cpl_parameter * par = cpl_parameterlist_find_const(parlist, name);
207 *histo_max = cpl_parameter_get_double(par);
208 cpl_free(name);
209 }
210
211 if (bin_size) {
212 name = hdrl_join_string(".", 2, prefix, "mode.bin-size");
213 const cpl_parameter * par = cpl_parameterlist_find_const(parlist, name);
214 *bin_size = cpl_parameter_get_double(par);
215 cpl_free(name);
216 }
217
218
219 /* --method */
220 if (method) {
221 name = hdrl_join_string(".", 2, prefix, "mode.method");
222 const cpl_parameter * par = cpl_parameterlist_find_const(parlist, name) ;
223 const char * tmp_str = cpl_parameter_get_string(par);
224 if (tmp_str == NULL) {
225 cpl_free(name);
226 return cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND,
227 "Parameter mode.method not found");
228 }
229 if(!strcmp(tmp_str, "MEDIAN")) {
230 *method = HDRL_MODE_MEDIAN;
231 } else if(!strcmp(tmp_str, "WEIGHTED")) {
232 *method = HDRL_MODE_WEIGHTED;
233 } else if(!strcmp(tmp_str, "FIT")) {
234 *method = HDRL_MODE_FIT;
235 }
236 cpl_free(name) ;
237
238 }
239
240/*
241 if (method) {
242 name = hdrl_join_string(".", 2, prefix, "mode.method");
243 const cpl_parameter * par = cpl_parameterlist_find_const(parlist, name);
244 *method = cpl_parameter_get_int(par);
245 cpl_free(name);
246 }
247*/
248
249 if (error_niter) {
250 name = hdrl_join_string(".", 2, prefix, "mode.error-niter");
251 const cpl_parameter * par = cpl_parameterlist_find_const(parlist, name);
252 *error_niter = cpl_parameter_get_int(par);
253 cpl_free(name);
254 }
255
256 if (cpl_error_get_code()) {
257 return cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND,
258 "Error while parsing parameterlist "
259 "with prefix %s", prefix);
260 }
261
262 return CPL_ERROR_NONE;
263}
264
277 static double
278hdrl_mode_compute_binsize(cpl_vector* vec)
279{
280
281 cpl_size size = cpl_vector_get_size(vec);
282 double mad = 0;
283 hcpl_vector_get_mad_window(vec, 1, size, &mad);
284 double std_deviation_from_mad = mad * CPL_MATH_STD_MAD;
285 /*
286 cpl_msg_debug(cpl_func, "mad: %g standard deviation from mad: %16.10f",
287 mad, std_deviation_from_mad);
288 */
289
290 double binsize = 2. * 3.49 * std_deviation_from_mad / pow(size, 1. / 3. );
291
292 if(binsize <= 0){
293 binsize = nextafter(0,1.0);
294 }
295
296 return binsize ;
297
298}
306 static cpl_size
307hdrl_mode_get_nbin(const double min, const double max, const double size)
308{
309
310 return (cpl_size)floor( (max - min) / size ) + 1;
311
312}
313
326 static gsl_histogram *
327hdrl_mode_histogram(const cpl_vector* vec, const double histogram_min,
328 const double histogram_max, const cpl_size nbin)
329{
330
331 cpl_error_ensure(nbin > 0, CPL_ERROR_ILLEGAL_INPUT,
332 return NULL, "Number of bins must be > 0");
333
334 cpl_error_ensure(histogram_max > histogram_min, CPL_ERROR_ILLEGAL_INPUT,
335 return NULL, "histo_max must be larger than histo_min");
336
337 gsl_histogram * histogram = gsl_histogram_alloc(nbin);
338
339 gsl_histogram_set_ranges_uniform (histogram, histogram_min, histogram_max);
340
341 const cpl_size size = cpl_vector_get_size(vec);
342 const double* data = cpl_vector_get_data_const(vec);
343 for(cpl_size i = 0; i < size; i++) {
344
345 gsl_histogram_increment(histogram, data[i]);
346
347 }
348
349 return histogram;
350}
351/*-------------------------------------------------------------------------*/
372 static cpl_table *
373hdrl_mode_histogram_to_table(gsl_histogram* histogram,
374 const double histogram_min,
375 const double histogram_step, const cpl_size nbin){
376
377 cpl_table* table = cpl_table_new(nbin);
378
379 cpl_table_new_column(table, "BIN", CPL_TYPE_DOUBLE);
380 cpl_table_new_column(table, "INTERVAL_LOWER", CPL_TYPE_DOUBLE);
381 cpl_table_new_column(table, "INTERVAL_UPPER", CPL_TYPE_DOUBLE);
382 cpl_table_new_column(table, "COUNTS", CPL_TYPE_DOUBLE);
383
384 cpl_table_fill_column_window(table, "BIN", 0, nbin, 0);
385 cpl_table_fill_column_window(table, "INTERVAL_LOWER", 0, nbin, 0);
386 cpl_table_fill_column_window(table, "INTERVAL_UPPER", 0, nbin, 0);
387 cpl_table_fill_column_window(table, "COUNTS", 0, nbin, 0);
388
389 double* phb = cpl_table_get_data_double(table, "BIN");
390 double* phl = cpl_table_get_data_double(table, "INTERVAL_LOWER");
391 double* phu = cpl_table_get_data_double(table, "INTERVAL_UPPER");
392 double* phc = cpl_table_get_data_double(table, "COUNTS");
393
394 //cpl_msg_debug(cpl_func,"histogram bin step: %16.10f", histo_step);
395 for(cpl_size i = 0; i < nbin; i++) {
396 phb[i] = (double)i;
397 phl[i] = histogram_min + phb[i] * histogram_step;
398 phu[i] = phl[i] + histogram_step;
399 phc[i] = histogram->bin[i];
400 }
401 return table;
402}
403/*----------------------------------------------------------------------------*/
422/*----------------------------------------------------------------------------*/
423 static gsl_matrix *
424hdrl_gsl_fit_poly (const double* data_x,
425 const double* data_y,
426 const double* errs_y,
427 const size_t n_sampling_points,
428 const cpl_size poly_degree,
429 double *fit,
430 double *coeffs_val,
431 double *coeffs_err,
432 double *chisq) {
433
434 cpl_size i, j;
435 double err, chisq_loc;
436 gsl_vector *x = NULL, *y = NULL, *w = NULL, *p = NULL;
437 gsl_matrix *X = NULL, *covar = NULL;
438
439 //double *fit;
440 gsl_multifit_linear_workspace *work = NULL;
441
442 x = gsl_vector_alloc (n_sampling_points);
443 y = gsl_vector_alloc (n_sampling_points);
444 w = gsl_vector_alloc (n_sampling_points);
445 p = gsl_vector_alloc (poly_degree);
446 X = gsl_matrix_alloc (n_sampling_points, poly_degree);
447 covar = gsl_matrix_alloc (poly_degree, poly_degree);
448
449 for (i = 0; i < (cpl_size) n_sampling_points; i++) {
450 gsl_vector_set (x, i, data_x[i]);
451 gsl_vector_set (y, i, data_y[i]);
452 err = errs_y[i];
453
454 gsl_vector_set (w, i, 1.0 / err / err);
455
456 for (j = 0; j < poly_degree; j++)
457 gsl_matrix_set (X, i, j, gsl_pow_int(gsl_vector_get(x, i), j));
458 }
459
460 work = gsl_multifit_linear_alloc (n_sampling_points, poly_degree);
461 gsl_multifit_wlinear (X, w, y, p, covar, &chisq_loc, work);
462 gsl_multifit_linear_free (work);
463
464 for (i = 0; i < (cpl_size) n_sampling_points; i++) {
465
466 fit[i] = 0.;
467 for (j = 0; j < poly_degree; j++)
468 fit[i] += gsl_matrix_get (X, i, j) * gsl_vector_get (p, j);
469 }
470
471 *chisq = chisq_loc/(n_sampling_points - poly_degree);
472 /*
473 * NOTE we scale covar arbitrarily by chisq_loc to get results as
474 * python code by Lodo
475 */
476 for (j = 0; j < poly_degree; j++) {
477 gsl_matrix_set (covar, j, j, gsl_matrix_get (covar, j, j)*chisq_loc);
478 coeffs_val[j] = gsl_vector_get (p, j);
479 coeffs_err[j] = sqrt(gsl_matrix_get (covar, j, j));
480 }
481
482 /*
483 cpl_msg_debug(cpl_func,"GSL poly fit c0: %16.8f c1: %16.8f, c2: %16.8f",
484 coeffs_val[0], coeffs_val[1], coeffs_val[2]);
485 cpl_msg_debug(cpl_func,"GSL poly fit e0: %16.8f e1: %16.8f, e2: %16.8f",
486 coeffs_err[0], coeffs_err[1], coeffs_err[2]);
487 cpl_msg_debug(cpl_func,"GSL poly fit chi2: %16.8f", *chisq);
488 */
489
490 gsl_vector_free(x);
491 gsl_vector_free(y);
492 gsl_vector_free(w);
493 gsl_vector_free(p);
494 gsl_matrix_free(X);
495
496
497 return covar;
498}
499
500/*----------------------------------------------------------------------------*/
513 static cpl_vector*
514hdrl_mode_vector_trim(const cpl_vector* vec, const double min,
515 const double max) {
516 cpl_size size = cpl_vector_get_size(vec);
517 cpl_error_ensure(size > 0, CPL_ERROR_ILLEGAL_INPUT,
518 return NULL, "vector size must be > 0");
519 cpl_vector * vec_trim = cpl_vector_new(size);
520 const double * pvec = cpl_vector_get_data_const(vec);
521 double * pvec_trim = cpl_vector_get_data(vec_trim);
522
523 cpl_size index = 0;
524
525 for (cpl_size i = 0; i < size; i++) {
526 if (pvec[i] >= min && pvec[i] <= max) {
527 pvec_trim[index] = pvec[i];
528 index++;
529 } else {
530 continue;
531 }
532 }
533
534 if (index > 0) {
535 /* return resized vector */
536 cpl_vector_set_size(vec_trim, index);
537 return vec_trim;
538 } else {
539 /* return NULL */
540 cpl_vector_delete(vec_trim);
541 return NULL;
542 }
543}
544
545/*----------------------------------------------------------------------------*/
563/*----------------------------------------------------------------------------*/
564 static cpl_error_code hdrl_mode_median(
565 const cpl_vector * vec,
566 const double histo_min,
567 const double histo_max,
568 const cpl_size nbin,
569 const cpl_size error_niter,
570 double * mode_median,
571 double * mode_median_err)
572{
573
574
575 cpl_error_ensure(vec != NULL, CPL_ERROR_NULL_INPUT,
576 return CPL_ERROR_NULL_INPUT, "Null input vector data");
577
578 gsl_histogram * histogram = hdrl_mode_histogram(vec, histo_min, histo_max,
579 nbin);
580
581 cpl_error_ensure(histogram != NULL, CPL_ERROR_NULL_INPUT,
582 return CPL_ERROR_NULL_INPUT, "Histogram can not be created");
583
584 /* for debug purposes
585 cpl_table* t = hdrl_mode_histogram_to_table(h, histo_min, nbin);
586 cpl_table_save(t,NULL,NULL,"debug.fits",CPL_IO_CREATE);
587 */
588
589 cpl_size histogram_bin_max = gsl_histogram_max_bin(histogram);
590 //cpl_msg_debug(cpl_func, "histogram bin max: %lld", histogram_bin_max);
591
592 /* get median and error associated to the values in the histogram max bin */
593
594 /* to compute the mode we need to determine the median of the values in the
595 * histogram bin max. The error is the standard deviation of these values.
596 * */
597 double lower = 0.;
598 double upper = 0.;
599 gsl_histogram_get_range(histogram, histogram_bin_max, &lower, &upper);
600
601 cpl_vector * vec_final = hdrl_mode_vector_trim(vec, lower, upper);
602 *mode_median = cpl_vector_get_median(vec_final);
603
604 if(error_niter == 0) {
605 *mode_median_err = cpl_vector_get_stdev(vec_final);
606 cpl_msg_debug(cpl_func, "(method median) computed mode: %g, associated error: %g",
607 *mode_median, *mode_median_err);
608 } else {
609 /* Derive only the analytical error, if requested */
610 *mode_median_err = 0;
611 }
612
613 /* free memory */
614 gsl_histogram_free(histogram);
615 cpl_vector_delete(vec_final);
616
617 return cpl_error_get_code();
618}
619/*----------------------------------------------------------------------------*/
639/*----------------------------------------------------------------------------*/
640 static cpl_error_code hdrl_mode_weight(
641 const cpl_vector * vec,
642 const double histo_min,
643 const double histo_max,
644 const double bin_size,
645 const cpl_size nbin,
646 const cpl_size error_niter,
647 double * mode_weight,
648 double * mode_weight_err)
649{
650
651 cpl_error_ensure(vec != NULL, CPL_ERROR_NULL_INPUT,
652 return CPL_ERROR_NULL_INPUT, "Null input vector data");
653
654 gsl_histogram * histogram = hdrl_mode_histogram(vec, histo_min, histo_max,
655 nbin);
656
657 cpl_error_ensure(histogram != NULL, CPL_ERROR_NULL_INPUT,
658 return CPL_ERROR_NULL_INPUT, "Histogram can not be created");
659
660 cpl_table* table = hdrl_mode_histogram_to_table(histogram, histo_min,
661 bin_size, nbin);
662 //cpl_table_save(table, NULL, NULL, "histo_tab.fits", CPL_IO_DEFAULT);
663
664 double histogram_max = gsl_histogram_max_val(histogram);
665 //cpl_msg_debug(cpl_func, "histogram value at max: %g", histogram_max);
666 cpl_size histogram_bin_max = gsl_histogram_max_bin(histogram);
667 //cpl_msg_debug(cpl_func, "histogram bin max: %lld", histogram_bin_max);
668 //cpl_msg_debug(cpl_func, "histogram bin size: %ld", gsl_histogram_bins(histogram));
669 if(histogram_bin_max > 0 &&
670 (histogram_bin_max < ((cpl_size)gsl_histogram_bins(histogram) - 1))) {
671 cpl_msg_debug(cpl_func, "histogram (bin_max-1) value: %16.8g",
672 gsl_histogram_get(histogram,histogram_bin_max-1));
673 cpl_msg_debug(cpl_func, "histogram (bin_max+1) value: %16.8g",
674 gsl_histogram_get(histogram,histogram_bin_max+1));
675 }
676
677 /* determine the mean of the values that lead to the histogram bin max */
678 double lower = 0;
679 double upper = 0;
680 gsl_histogram_get_range(histogram, histogram_bin_max, &lower, &upper);
681 //cpl_msg_debug(cpl_func, "histogram range lower: %16.8g upper: %16.8g", lower, upper);
682 cpl_table_and_selected_double(table,"COUNTS",CPL_EQUAL_TO, histogram_max);
683 cpl_table* extract = cpl_table_extract_selected(table);
684
685 //cpl_table_save(extract, NULL, NULL, "histo_tab_extr1.fits", CPL_IO_DEFAULT);
686
687 double mean = cpl_table_get_column_mean(extract, "INTERVAL_LOWER");
688
689 cpl_table_delete(extract);
690
691 cpl_size max_pos = 0;
692 cpl_table_get_column_maxpos(table, "INTERVAL_LOWER", &max_pos);
693
694 cpl_table_delete(table);
695
696 double freq1 = histogram_max;
697 double level1 = mean;
698
699 double freq0, freq2;
700 //cpl_msg_debug(cpl_func, "nbin: %lld histogram_bin_max: %lld", nbin,histogram_bin_max);
701 if(histogram_bin_max + 1 <= ((cpl_size)nbin - 1)) {
702 freq2 = gsl_histogram_get(histogram, histogram_bin_max + 1);
703 } else {
704 freq2 = 0;
705 }
706
707
708 if(histogram_bin_max > 0) {
709 freq0 = gsl_histogram_get(histogram, histogram_bin_max - 1);
710 } else {
711 freq0 = 0;
712 }
713
714 /*
715 cpl_msg_debug(cpl_func, "Frequencies: f0: %16.8g f1: %16.10g f2: %16.8g l1: %16.10g",
716 freq0, freq1, freq2, level1);
717 */
718
719 double diff1 = freq1 - freq0;
720 double diff2 = freq1 - freq2;
721 double factor = diff1 / (diff1 + diff2);
722 if( factor == 0 || isnan(factor) ) {
723 factor = 0.5;
724 }
725 /* compute mode */
726 *mode_weight = level1 + bin_size * factor;
727
728 /* compute error associated to mode */
729 if (error_niter == 0) {
730 double dd1 = sqrt(freq0 + freq1);
731 double dd2 = sqrt(freq1 + freq2);
732 double dw2=1;
733
734 double term1 = diff2 * dd1 / pow((diff1 + diff2), 2);
735 double term2 = diff1 * dd2 / pow((diff1 + diff2), 2);
736 dw2 = pow(term1, 2.) + pow(term2, 2.);
737 *mode_weight_err = bin_size * sqrt(dw2);
738 /*
739 cpl_msg_debug(cpl_func,"fct: %16.8g m: %16.8g, dd1: %16.8g dd2: %16.8g",
740 factor,*mode_weight,dd1,dd2);
741 cpl_msg_debug(cpl_func,"dw2: %16.8g e: %16.8g", dw2, *mode_weight_err);
742 */
743
744 } else {
745 /* Derive only the analytical error, if requested */
746 *mode_weight_err = 0;
747 }
748
749 cpl_msg_debug(cpl_func, "(method weight) computed mode: %16.10g error: %16.10g",
750 *mode_weight, *mode_weight_err);
751
752 /* free memory */
753 gsl_histogram_free(histogram);
754
755 return cpl_error_get_code();
756}
765 static double
766hdrl_mode_fit_analytical_error(const double* coeffs_val,
767 const double* coeffs_err, gsl_matrix* covar,
768 const double scale_factor){
769
770
771 double a2 = coeffs_val[2];
772 double a1 = coeffs_val[1];
773 //double a0 = coeffs_val[0];
774
775 /*NOTE we scale arbirtarily by scale_factor = chi2 / dof, where dof is the
776 * degree of freedom to get results as python code by Lodo
777 */
778 double cvar_matrix = gsl_matrix_get (covar, 2, 1) * scale_factor;
779
780 double da2 = coeffs_err[2];
781 double da1 = coeffs_err[1];
782 //double da0 = coeffs_err[0];
783
784 double cvr_term = 2 * (-1. / (2 * a2)) * (a1 / (2 * a2 * a2)) * cvar_matrix;
785
786 double add1 = (da1 / (2 * a2));
787 add1 *= add1;
788 double a_square = a2 * a2;
789 double add2 = (a1 * da2 / (2 * a_square ));
790 add2 *= add2;
791 double err2 = add1 + add2;
792
793 err2 += cvr_term;
794
795 return sqrt(err2);;
796}
797
798/*----------------------------------------------------------------------------*/
817/*----------------------------------------------------------------------------*/
818 static cpl_error_code hdrl_mode_fit(
819 const cpl_vector * vec,
820 const double histo_min,
821 const double histo_max,
822 const double bin_size,
823 const cpl_size nbin,
824 const cpl_size error_niter,
825 double * mode_fit,
826 double * mode_fit_err)
827{
828
829 cpl_error_ensure(vec != NULL, CPL_ERROR_NULL_INPUT,
830 return CPL_ERROR_NULL_INPUT, "Null input vector data");
831
832 /* We use semi_fit_window = 2 as the fit polynomial degree used is 2
833 * We cannot use a larger value to prevent asymmetries (for asymmetric
834 * distributions) that cannot be fit well by a parabola */
835 cpl_size semi_fit_window = 2;
836
837 double hmin = histo_min;
838 double hmax = histo_max;
839
840 gsl_histogram * h = hdrl_mode_histogram(vec, hmin, hmax, nbin);
841
842 cpl_error_ensure(h != NULL, CPL_ERROR_NULL_INPUT,
843 return CPL_ERROR_NULL_INPUT, "Histogram can not be created");
844
845 cpl_size histogram_bin_max = (cpl_size) gsl_histogram_max_bin(h);
846 cpl_size histo_nbins = (cpl_size) gsl_histogram_bins(h);
847
848 if(histogram_bin_max > 0) {
849 cpl_msg_debug(cpl_func, "histogram (bin_max-1) value: %16.8g",
850 gsl_histogram_get(h, histogram_bin_max - 1));
851 }
852 if(histogram_bin_max < (histo_nbins-1)) {
853 cpl_msg_debug(cpl_func, "histogram (bin_max+1) value: %16.8g",
854 gsl_histogram_get(h, histogram_bin_max + 1));
855 }
856
857 double lower = 0.;
858 double upper = 0.;
859 gsl_histogram_get_range(h, gsl_histogram_max_bin(h), &lower, &upper);
860 double value_at_max = lower;
861
862 /* check if one has enough points (at least 3) to be able to do a polynomial
863 * fit. If too less points, switch to weight mode
864 */
865 if( histo_nbins < 3 ) {
866 cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
867 cpl_msg_debug(cpl_func,"Cannot do polynomial fit with less than 3 points.");
868 gsl_histogram_free(h);
869 return cpl_error_get_code();
870 } else {
871 cpl_size mn = histogram_bin_max - semi_fit_window;
872 cpl_size mx = histogram_bin_max + semi_fit_window;
873 mn = histogram_bin_max - semi_fit_window;
874 mn = (mn > 0) ? mn : 0;
875 mx = histogram_bin_max + semi_fit_window;
876 mx = (mx < histo_nbins) ? mx : histo_nbins-1; // Cannot go till histo_nbins else after for loop seg fault
877
878 /* fit a parabola */
879 const cpl_size good_points = (mx - mn + 1);
880 const cpl_size win_size = 2 * semi_fit_window + 1;
881 const cpl_size sz = (good_points < win_size) ? good_points : win_size;
882
883 double* x_vals = cpl_calloc(sz, sizeof(double));
884 double* y_vals = cpl_calloc(sz, sizeof(double));
885 double* y_errs = cpl_calloc(sz, sizeof(double));
886 cpl_size j = 0;
887 /* we assume uniform error as we do not know how to compute the error
888 * associated to each bin size
889 */
890 for(cpl_size i = mn; i <= mx; i++) {
891 double lower_local = 0.;
892 double upper_local = 0.;
893 gsl_histogram_get_range(h, i, &lower_local, &upper_local);
894 x_vals[j] = lower_local;
895 y_vals[j] = gsl_histogram_get(h, i);
896 y_errs[j] = 1;
897 j++;
898 }
899
900 /* do the polynomial fit, using GSL */
901 cpl_size deg = 2;
902 double chi2;
903 double* coeffs_val = (double *) cpl_calloc (sz, sizeof(double));
904 double* coeffs_err = (double *) cpl_calloc (sz, sizeof(double));
905 double* fit = (double *) cpl_calloc (sz, sizeof(double));
906 gsl_matrix* covar =
907 hdrl_gsl_fit_poly (x_vals, y_vals, y_errs, sz, deg+1, fit, coeffs_val,
908 coeffs_err, &chi2);
909
910 double m = -coeffs_val[1] / 2. / coeffs_val[2];
911 double value_at_mode = gsl_poly_eval(coeffs_val, sz, m);
912
913 *mode_fit = m + bin_size / 2.;
914
915 /* check that we are not at an edge and that we found a true maximum
916 * (that means the point distribution had a proper shape)
917 */
918 double y_min_val = gsl_poly_eval(coeffs_val, sz,x_vals[0]);
919 double y_max_val = gsl_poly_eval(coeffs_val, sz,x_vals[sz - 1]);
920 double max_parab = (y_max_val > y_min_val ) ? y_max_val : y_min_val;
921 cpl_boolean is_not_supported_fit = CPL_FALSE;
922 if( (fabs(value_at_max - m) > bin_size / 2.) ||
923 (value_at_mode < max_parab ) ) {
924 /* if we are near an edge point or the point distribution is not
925 * proper it is safer to use the weight method to get the mode
926 */
927 if( fabs(value_at_max - m) > bin_size / 2. ) {
928 cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
929 cpl_msg_debug(cpl_func, "Max too close to point distribution edge: abs(value_at_max+bin_size/2-m) > bin_size");
930 is_not_supported_fit = CPL_TRUE;
931 }
932
933 if(value_at_mode < max_parab) {
934 cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
935 cpl_msg_debug(cpl_func, "Value at mode is not a valid maximum: value_at_mode < np.max(parab)");
936 is_not_supported_fit = CPL_TRUE;
937 }
938 if(is_not_supported_fit == CPL_TRUE) {
939 gsl_matrix_free(covar);
940 gsl_histogram_free(h);
941 cpl_free(fit);
942 cpl_free(coeffs_val);
943 cpl_free(coeffs_err);
944 cpl_free(y_errs);
945 cpl_free(x_vals);
946 cpl_free(y_vals);
947 return cpl_error_get_code();
948 }
949
950 } else {
951 if (error_niter == 0) {
952 /* As requested, compute the analytical error, */
953 double dof = sz - (deg +1);
954 double scale_factor = (chi2 / dof);
955
956 *mode_fit_err =
957 hdrl_mode_fit_analytical_error(coeffs_val, coeffs_err, covar,
958 scale_factor);
959 } else {
960 /* As requested, do not compute the analytical error */
961 *mode_fit_err = 0;
962 }
963
964 }
965 /* check if problem occurred during poly fit or in error computation*/
966 if(!isfinite(*mode_fit_err) || !isfinite(*mode_fit)) {
967 cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_OUTPUT);
968 *mode_fit_err = NAN;
969 *mode_fit = NAN;
970 }
971 cpl_msg_debug(cpl_func, "(method fit) computed mode: %16.10g err: %16.10g ",
972 *mode_fit, *mode_fit_err);
973
974 gsl_matrix_free(covar);
975 cpl_free(fit);
976 cpl_free(coeffs_val);
977 cpl_free(coeffs_err);
978 cpl_free(x_vals);
979 cpl_free(y_vals);
980 cpl_free(y_errs);
981
982 }
983
984 /* free memory */
985 gsl_histogram_free(h);
986
987 return cpl_error_get_code();
988}
989
990/*----------------------------------------------------------------------------*/
1064/*----------------------------------------------------------------------------*/
1065cpl_error_code hdrl_mode_clip_image(
1066 const cpl_image * source,
1067 const double histo_min,
1068 const double histo_max,
1069 const double bin_size,
1070 const hdrl_mode_type method,
1071 const cpl_size error_niter,
1072 double * mode,
1073 double * mode_error,
1074 cpl_size * naccepted)
1075{
1076 cpl_vector * vec_source = NULL;
1077
1078 /* Check Entries */
1079 cpl_error_ensure(source != NULL, CPL_ERROR_NULL_INPUT,
1080 return CPL_ERROR_NULL_INPUT, "Null input source image!");
1081
1082 cpl_image* data =(cpl_image*) source;
1083
1084 /* compress images to vectors excluding the bad pixels */
1085 vec_source = hdrl_image_to_vector(data, cpl_image_get_bpm_const(source));
1086
1087 if (vec_source != NULL) {
1088 /* compute mode */
1089 hdrl_mode_clip(vec_source, histo_min, histo_max, bin_size, method,
1090 error_niter, mode, mode_error, naccepted);
1091
1092 if(error_niter > 0) {
1093 /* Calculate the error using the bootstrap technique */
1094 hdrl_mode_bootstrap(vec_source, histo_min, histo_max, bin_size,
1095 method, error_niter, mode_error);
1096 }
1097
1098 if (CPL_ERROR_NONE != cpl_error_get_code()) {
1099 /* if error occurred exit after cleaning memory */
1100 cpl_vector_delete(vec_source);
1101 return cpl_error_get_code();
1102 }
1103
1104 } else {
1105 /* no good pixels */
1106 *mode = NAN;
1107 *mode_error = NAN;
1108 cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
1109 }
1110
1111 cpl_vector_delete(vec_source);
1112 return cpl_error_get_code();
1113}
1114
1115
1116/*----------------------------------------------------------------------------*/
1135/*----------------------------------------------------------------------------*/
1136
1137cpl_error_code hdrl_mode_clip(
1138 cpl_vector * vec,
1139 const double histo_min,
1140 const double histo_max,
1141 const double bin_size,
1142 const hdrl_mode_type method,
1143 const cpl_size error_niter,
1144 double * mode,
1145 double * mode_error,
1146 cpl_size * naccepted)
1147{
1148 cpl_error_ensure(vec != NULL, CPL_ERROR_NULL_INPUT,
1149 return CPL_ERROR_NULL_INPUT, "Null input source image!");
1150
1151 *naccepted = 0; /* In case an error happens - if not, re-set it later */
1152
1153 cpl_vector * loc_vec = NULL;
1154 double bsize = bin_size;
1155
1156 /* if binsize <=0 : the routine derives the value from a formula */
1157 if (bsize <= DBL_EPSILON) {
1158 bsize = hdrl_mode_compute_binsize(vec);
1159 }
1160
1161 double hmin = histo_min;
1162 double hmax = histo_max;
1163 cpl_size nbin = 0;
1164 cpl_vector* trim_vec = NULL;
1165 /* if histo_min >= histo_max: the routine derives the two values from the
1166 * data (and also the number of histogram bins, nbin)
1167 */
1168 if ( histo_min >= histo_max) {
1169
1170 trim_vec = cpl_vector_duplicate(vec);
1171 hmin = cpl_vector_get_min(vec) - bsize / 2;
1172 hmax = cpl_vector_get_max(vec) + bsize / 2;
1173 /* the following two are required to have same results as Lodo's code
1174 * as implemented in the unit tests
1175 */
1176 nbin = hdrl_mode_get_nbin(hmin, hmax, bsize);
1177 hmax = hmin + (nbin * bsize);
1178 /* Fallback solution for the case there is only one single value */
1179 if(hmin == hmax) {
1180 hmin = nextafter(hmin, hmin - FLT_EPSILON);
1181 hmax = nextafter(hmax, hmax + FLT_EPSILON);
1182 bsize = nextafter(0.0, 1.0);
1183 nbin = 1 ;
1184 }
1185
1186
1187 } else {
1188 nbin = hdrl_mode_get_nbin(hmin, hmax, bsize);
1189 trim_vec = hdrl_mode_vector_trim(vec, hmin, hmax);
1190 if( hmin + (nbin * bsize) >= hmax) {
1191 /* adjust hmax to use gsl_histogram without changing bsize */
1192 //cpl_msg_warning(cpl_func,"Strange");
1193 hmax = hmin + (nbin * bsize);
1194 }
1195 }
1196 cpl_msg_debug(cpl_func,"Histogram bin size: %g min: %g max: %g number of bins: %lld",
1197 bsize, hmin, hmax, nbin);
1198
1199 cpl_error_ensure(trim_vec != NULL, CPL_ERROR_NULL_INPUT,
1200 return CPL_ERROR_NULL_INPUT, "No data for mode "
1201 "computation. Try to change mode parameters ... ");
1202
1203 loc_vec = trim_vec;
1204
1205
1206 /* method == enum with mode_fit, mode_weight, mode_median */
1207 switch(method) {
1208
1209 case HDRL_MODE_FIT:
1210
1211 if (CPL_ERROR_NONE != hdrl_mode_fit(loc_vec, hmin, hmax, bsize, nbin, error_niter, mode,
1212 mode_error) ) {
1213 //cpl_error_set(cpl_func,CPL_ERROR_ILLEGAL_INPUT);
1214 cpl_msg_debug(cpl_func,"Mode computation failed using method fit. Try method weight or median.");
1215 }
1216 break;
1217
1218 case HDRL_MODE_WEIGHTED:
1219
1220 if (CPL_ERROR_NONE != hdrl_mode_weight(loc_vec, hmin, hmax, bsize, nbin,
1221 error_niter, mode, mode_error) ) {
1222 //cpl_error_set(cpl_func,CPL_ERROR_UNSUPPORTED_MODE);
1223 cpl_msg_debug(cpl_func,"Mode computation failed using method weight. Try method fit or median.");
1224
1225 }
1226 break;
1227
1228 case HDRL_MODE_MEDIAN:
1229
1230 if (CPL_ERROR_NONE != hdrl_mode_median(loc_vec, hmin, hmax, nbin,
1231 error_niter, mode, mode_error) ) {
1232 //cpl_error_set(cpl_func,CPL_ERROR_UNSUPPORTED_MODE);
1233 cpl_msg_debug(cpl_func,"Mode computation failed using method median. Try method fit or weight");
1234
1235 }
1236
1237 break;
1238
1239 default:
1240 cpl_msg_debug(cpl_func,"Unsupported mode method. Supported methods are: fit, weight, median");
1241 return CPL_ERROR_UNSUPPORTED_MODE;
1242
1243 }
1244
1245 *naccepted = cpl_vector_get_size(vec);
1246 cpl_vector_delete(trim_vec);
1247
1248 return cpl_error_get_code();
1249}
1250
1251/* ---------------------------------------------------------------------------*/
1270/* ---------------------------------------------------------------------------*/
1271
1272cpl_error_code
1273 hdrl_mode_bootstrap(
1274 const cpl_vector * vec,
1275 const double histo_min,
1276 const double histo_max,
1277 const double bin_size,
1278 const hdrl_mode_type method,
1279 const cpl_size error_niter,
1280 double * mode_error)
1281{
1282
1283
1284 /* To ensure new random numbers each time the compiled program is called,
1285 * call srand(), e.g. srand(time(NULL)) */
1286
1287 /* Predefine the state for the threads as they get manipulated in the thread*/
1288 hdrl_random_state ** pstate = cpl_calloc(omp_get_max_threads(),
1289 sizeof(hdrl_random_state *));
1290
1291 for(cpl_size i = 0 ; i < (cpl_size)omp_get_max_threads(); i++){
1292 uint64_t seed[2] = {(uint64_t)rand(), (uint64_t)rand()};
1293 pstate[i] = hdrl_random_state_new(1, seed);
1294 }
1295
1296 cpl_size vec_size = cpl_vector_get_size(vec);
1297 const double * pvec = cpl_vector_get_data_const(vec);
1298 cpl_image * ima_mode = cpl_image_new(1, error_niter, CPL_TYPE_DOUBLE);
1299 double * pima_mode = cpl_image_get_data_double(ima_mode);
1300
1301 /* In case the loop gets parallelized use cpl_image_get_bpm outside the loop
1302 * once to get a single bad pixel mask for the image */
1303 cpl_binary * pima_mode_bpm = cpl_mask_get_data(cpl_image_get_bpm(ima_mode));
1304
1305 HDRL_OMP(omp parallel for)
1306 for (cpl_size i = 0; i < error_niter; i++) {
1307 cpl_error_code err = CPL_ERROR_NONE;
1308 cpl_vector * vec_simul = cpl_vector_new(vec_size);
1309 double mode = 0.;
1310 double mode_err = 0.;
1311 cpl_size naccepted = 0;
1312 double * pvec_simul = cpl_vector_get_data(vec_simul);
1313
1314 /* simulate the new vector by bootstraping */
1315 for (cpl_size j = 0; j < vec_size; j++) {
1316 pvec_simul[j] = pvec[(cpl_size)hdrl_random_uniform_int64(pstate[omp_get_thread_num()], 0, vec_size - 1)];
1317 }
1318
1319 err = hdrl_mode_clip(vec_simul, histo_min, histo_max, bin_size, method,
1320 -1, &mode, &mode_err, &naccepted);
1321 cpl_vector_delete(vec_simul);
1322
1323 if (err == CPL_ERROR_NONE) {
1324 pima_mode[i] = mode;
1325 pima_mode_bpm[i] = CPL_BINARY_0; // may be overdoing ....
1326 } else {
1327 pima_mode[i] = NAN;
1328 pima_mode_bpm[i] = CPL_BINARY_1;
1329 cpl_error_reset();
1330 }
1331 }
1332
1333 /* Return the mad scaled standard deviation */
1334 //double mad = 0;
1335 //cpl_image_get_mad(ima_mode, &mad);
1336 //*mode_error = mad * CPL_MATH_STD_MAD;
1337
1338 /* Return the pure standard deviation as the mode on the various simulations
1339 * is not very smoothly distributed - the MAD based mode depends thus more on
1340 * the number of iterations */
1341 *mode_error = cpl_image_get_stdev(ima_mode);
1342
1343 /* clean memory */
1344 cpl_image_delete(ima_mode);
1345
1346 for(cpl_size i = 0 ; i < (cpl_size)omp_get_max_threads(); i++){
1347 hdrl_random_state_delete(pstate[i]);
1348 }
1349 cpl_free(pstate);
1350
1351 return cpl_error_get_code();
1352}
1353
hdrl_mode.h
HDRL_OMP
#define HDRL_OMP(x)
Definition hdrl_cat_def.h:35
hdrl_random.h
hdrl_mode_clip
cpl_error_code hdrl_mode_clip(cpl_vector *vec, const double histo_min, const double histo_max, const double bin_size, const hdrl_mode_type method, const cpl_size error_niter, double *mode, double *mode_error, cpl_size *naccepted)
Compute mode of data.
Definition hdrl_mode.c:1137
omp_get_max_threads
#define omp_get_max_threads()
Definition hdrl_mode.c:34
omp_get_thread_num
#define omp_get_thread_num()
Definition hdrl_mode.c:35
hdrl_mode_bootstrap
cpl_error_code hdrl_mode_bootstrap(const cpl_vector *vec, const double histo_min, const double histo_max, const double bin_size, const hdrl_mode_type method, const cpl_size error_niter, double *mode_error)
uses Montecarlo simulations based on the bootstrap technique to determine the error of the mode of th...
Definition hdrl_mode.c:1273
hdrl_mode_clip_image
cpl_error_code hdrl_mode_clip_image(const cpl_image *source, const double histo_min, const double histo_max, const double bin_size, const hdrl_mode_type method, const cpl_size error_niter, double *mode, double *mode_error, cpl_size *naccepted)
Compute mode of data.
Definition hdrl_mode.c:1065
hdrl_mode_parameter_create_parlist
cpl_parameterlist * hdrl_mode_parameter_create_parlist(const char *base_context, const char *prefix, const hdrl_parameter *defaults)
Create parameters for the mode collapse.
Definition hdrl_mode.c:89
hdrl_mode_parameter_parse_parlist
cpl_error_code hdrl_mode_parameter_parse_parlist(const cpl_parameterlist *parlist, const char *prefix, double *histo_min, double *histo_max, double *bin_size, hdrl_mode_type *method, cpl_size *error_niter)
parse parameterlist for mode parameters to init corresponding hdrl structure parameters
Definition hdrl_mode.c:186
hdrl_sigclip.h
hdrl_mode_type
hdrl_mode_type
Define the type of the mode that should be computed.
Definition hdrl_mode_defs.h:33
HDRL_MODE_MEDIAN
@ HDRL_MODE_MEDIAN
Definition hdrl_mode_defs.h:36
HDRL_MODE_WEIGHTED
@ HDRL_MODE_WEIGHTED
Definition hdrl_mode_defs.h:38
HDRL_MODE_FIT
@ HDRL_MODE_FIT
Definition hdrl_mode_defs.h:40
hdrl_collapse.h
hdrl_random_state_new
hdrl_random_state * hdrl_random_state_new(int type, uint64_t *seed)
create random number generator state
Definition hdrl_random.c:108
hdrl_random_state_delete
void hdrl_random_state_delete(hdrl_random_state *state)
delete random number generator state structure
Definition hdrl_random.c:130
hdrl_random_uniform_int64
int64_t hdrl_random_uniform_int64(hdrl_random_state *state, int64_t minval, int64_t maxval)
generatore uniformly distributed 64 bit integers within range
Definition hdrl_random.c:153
hdrl_utils.h
hdrl_parameter
struct _hdrl_parameter_ hdrl_parameter
Definition hdrl_parameter.h:27
hdrl_collapse_mode_parameter_get_bin_size
double hdrl_collapse_mode_parameter_get_bin_size(const hdrl_parameter *p)
get size of the histogram bins
Definition hdrl_collapse.c:690
hdrl_collapse_mode_parameter_get_histo_min
double hdrl_collapse_mode_parameter_get_histo_min(const hdrl_parameter *p)
get min value
Definition hdrl_collapse.c:658
hdrl_collapse_mode_parameter_get_histo_max
double hdrl_collapse_mode_parameter_get_histo_max(const hdrl_parameter *p)
get high value
Definition hdrl_collapse.c:674
hdrl_collapse_mode_parameter_get_error_niter
cpl_size hdrl_collapse_mode_parameter_get_error_niter(const hdrl_parameter *p)
get the error type of the mode
Definition hdrl_collapse.c:722
hdrl_collapse_parameter_is_mode
cpl_boolean hdrl_collapse_parameter_is_mode(const hdrl_parameter *self)
check if parameter is a mode parameter
Definition hdrl_collapse.c:594
hdrl_collapse_mode_parameter_get_method
hdrl_mode_type hdrl_collapse_mode_parameter_get_method(const hdrl_parameter *p)
get the mode determination method
Definition hdrl_collapse.c:706
hdrl_join_string
char * hdrl_join_string(const char *sep_, int n,...)
join strings together
Definition hdrl_utils.c:810