CR2RE Pipeline Reference Manual 1.6.2
hdrl_fit-test.c
1/*
2 * This file is part of the HDRL
3 * Copyright (C) 2014 European Southern Observatory
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19
20#ifdef HAVE_CONFIG_H
21#include <config.h>
22#endif
23
24/*-----------------------------------------------------------------------------
25 Includes
26 -----------------------------------------------------------------------------*/
27
28#include "hdrl_fit.c"
29#include "hdrl_imagelist.h"
30#include "hdrl_test.h"
31
32#include <cpl.h>
33
34#include <math.h>
35
36#ifndef ARRAY_LEN
37#define ARRAY_LEN(a) sizeof((a))/sizeof((a)[0])
38#endif
39
40/*----------------------------------------------------------------------------*/
45/*----------------------------------------------------------------------------*/
46
47#define matrix_eq(m, exp, eps) \
48 do { \
49 for (size_t i = 0; i < (size_t)cpl_matrix_get_nrow(m); i++) { \
50 for (size_t j = 0; j < (size_t)cpl_matrix_get_ncol(m); j++) { \
51 cpl_test_abs(cpl_matrix_get(m, i, j), exp[i * \
52 cpl_matrix_get_ncol(m) + j], eps); \
53 } \
54 } \
55 } while (0)
56
57static void test_vander1d(void)
58{
59 {
60 double p[] = {1, 2, 3, 4};
61 double exp[] = {1., 1., 1.,
62 1., 2., 4.,
63 1., 3., 9.,
64 1., 4., 16.};
65 cpl_vector * pv = cpl_vector_wrap(ARRAY_LEN(p), p);
66 cpl_matrix * v = polyvander1d(pv, 2);
67 cpl_test_error(CPL_ERROR_NONE);
68 matrix_eq(v, exp, DBL_EPSILON * 5);
69 cpl_matrix_delete(v);
70
71 v = polyvander1d(pv, 2);
72 cpl_test_error(CPL_ERROR_NONE);
73 matrix_eq(v, exp, DBL_EPSILON * 5);
74 cpl_matrix_delete(v);
75
76 cpl_vector_unwrap(pv);
77 }
78 /* could be used for weight_design function */
79 if (0) {
80 double p[] = {1, 2, 3, 4};
81 double e[] = {1, 1, 4, 1};
82 double exp[] = {1., 1., 1.,
83 1., 2., 4.,
84 0.25, 0.75, 2.25,
85 1., 4., 16.};
86 cpl_vector * pv = cpl_vector_wrap(ARRAY_LEN(p), p);
87 cpl_vector * ev = cpl_vector_wrap(ARRAY_LEN(e), e);
88 cpl_matrix * v = polyvander1d(pv, 2);
89 cpl_test_error(CPL_ERROR_NONE);
90
91 matrix_eq(v, exp, DBL_EPSILON * 5);
92 cpl_vector_unwrap(pv);
93 cpl_vector_unwrap(ev);
94 cpl_matrix_delete(v);
95 }
96}
97
98static void test_fit(void)
99{
100 {
101 double s[] = {1, 2, 3, 4};
102 double p[] = {2, 2.5, 3, 3.5};
103 double exp[] = {1.5, 0.5, 0.};
104 double eres[] = {0., 0., 0., 0.};
105 cpl_vector * sv = cpl_vector_wrap(ARRAY_LEN(s), s);
106 cpl_vector * pv = cpl_vector_wrap(ARRAY_LEN(p), p);
107 cpl_vector * vre = cpl_vector_wrap(ARRAY_LEN(eres), eres);
108 cpl_matrix * v = polyvander1d(sv, 2);
109 hdrl_ls_fit_result * r = fit(v, pv, NULL);
110 cpl_test_error(CPL_ERROR_NONE);
111 matrix_eq(r->coef, exp, DBL_EPSILON * 10);
112 cpl_vector * res = hdrl_ls_fit_result_get_residuals(r, pv);
113 cpl_test_vector_abs(res, vre, DBL_EPSILON * 5);
114 cpl_vector_unwrap(vre);
115 cpl_vector_delete(res);
116 cpl_matrix_delete(v);
117 cpl_vector_unwrap(pv);
118 cpl_vector_unwrap(sv);
119 hdrl_ls_fit_result_delete(r);
120 }
121 {
122 double s[] = {1, 2, 3, 4, 5};
123 double p[] = {1.1, 2.5, 3.4, 3.8, 7};
124 double e[] = {0.3, 0.2, 0.2, 0.1, 0.5};
125 double exp[] = {0.54529, 0.858981};
126 double cexp[] = {0.0756216, -0.0206541, -0.0206541, 0.00613226};
127 cpl_vector * vfit = cpl_vector_new(ARRAY_LEN(s));
128 cpl_vector * vres = cpl_vector_new(ARRAY_LEN(s));
129 for (size_t i = 0; i < ARRAY_LEN(s); i++) {
130 cpl_vector_set(vfit, i, exp[0] + exp[1] * s[i]);
131 cpl_vector_set(vres, i, p[i] - (exp[0] + exp[1] * s[i]));
132 }
133
134 cpl_vector * sv = cpl_vector_wrap(ARRAY_LEN(s), s);
135 cpl_vector * pv = cpl_vector_wrap(ARRAY_LEN(p), p);
136 cpl_vector * ev = cpl_vector_wrap(ARRAY_LEN(e), e);
137 cpl_matrix * v = polyvander1d(sv, 1);
138
139 hdrl_ls_fit_result * r = fit(v, pv, ev);
140 cpl_test_error(CPL_ERROR_NONE);
141 matrix_eq(r->coef, exp, DBL_EPSILON * 1e10);
142 matrix_eq(r->cov, cexp, DBL_EPSILON * 1e10);
143
144 cpl_vector * values = hdrl_ls_fit_result_get_fitted_values(r);
145 cpl_test_vector_abs(values, vfit, DBL_EPSILON * 1e10);
146 cpl_vector_delete(vfit);
147 cpl_vector_delete(values);
148 cpl_vector * resi = hdrl_ls_fit_result_get_residuals(r, pv);
149 cpl_test_vector_abs(resi, vres, DBL_EPSILON * 1e10);
150 cpl_vector_delete(vres);
151 cpl_vector_delete(resi);
152 cpl_matrix_delete(v);
153
154 hdrl_ls_fit_result_delete(r);
155
156 r = polyfit1d(sv, pv, ev, 1);
157 cpl_test_error(CPL_ERROR_NONE);
158 matrix_eq(r->coef, exp, DBL_EPSILON * 1e10);
159 hdrl_ls_fit_result_delete(r);
160
161 cpl_vector_unwrap(pv);
162 cpl_vector_unwrap(ev);
163 cpl_vector_unwrap(sv);
164 }
165}
166
167/* ---------------------------------------------------------------------------*/
181/* ---------------------------------------------------------------------------*/
182static inline cpl_vector * hdrl_ls_fit_result_get_fit_interval(
183 const hdrl_ls_fit_result * r,
184 const cpl_vector * data,
185 cpl_vector * errors)
186{
187 /* mse = sum(sqrt(weights) * residuals ** 2) / df */
188 double mse = hdrl_ls_fit_result_get_chi2(r, data, errors) /
189 hdrl_ls_fit_result_get_residual_dof(r);
190 /* var = mse / weights */
191 cpl_vector * serror = cpl_vector_duplicate(errors);
192 cpl_vector_multiply(serror, serror);
193 cpl_vector_multiply_scalar(serror, mse);
194 cpl_vector_power(serror, 0.5);
195
196 /* TODO: accounting for covariance missing
197 * + (exog * np.dot(covb, exog.T).T).sum(axis=1) */
198
199 return serror;
200}
201
202
203void test_poisson(void)
204{
205 double x[] = { 10. , 62.1 , 114.2, 166.3, 218.4, 270.5, 322.6,
206 374.7, 426.8, 478.9, 531.1, 583.2, 635.3, 687.4, 739.5,
207 791.6, 843.7, 895.8, 947.9, 1000. };
208 /* poisson data with lambda: x / 10 (== variance) and 100 offset */
209 double y[] = { 103, 107, 111, 112, 117, 127, 126, 125, 139, 150, 157, 162,
210 153, 158, 162, 184, 191, 195, 182, 196 };
211 cpl_vector * vx = cpl_vector_wrap(ARRAY_LEN(x), x);
212 cpl_vector * vy = cpl_vector_wrap(ARRAY_LEN(y), y);
213 /* relative model errors (poisson model ~ sqrt(x)) */
214 cpl_vector * ve_model = cpl_vector_duplicate(vx);
215 cpl_vector_power(ve_model, 0.5);
216 /* real absolute error of the population != relative model errors */
217 cpl_vector * ve_real = cpl_vector_duplicate(vx);
218 cpl_vector_divide_scalar(ve_real, 10);
219 cpl_vector_power(ve_real, 0.5);
220
221
222 double exp_c[] = {101.4164, 0.0919476};
223 cpl_matrix * design = polyvander1d(vx, 1);
224 hdrl_ls_fit_result * res = polyfit1d(vx, vy, ve_model, 1);
225 matrix_eq(res->coef, exp_c, DBL_EPSILON * 2e12);
226 /* get sample error */
227 cpl_vector * pred_e =
228 hdrl_ls_fit_result_get_fit_interval(res, vy, ve_model);
229 /* should be less than < 20% deviation to population error */
230 cpl_test_vector_abs(pred_e, ve_real, 0.7);
231 cpl_vector_delete(pred_e);
232
233 cpl_vector_unwrap(vx);
234 cpl_vector_unwrap(vy);
235 cpl_vector_delete(ve_model);
236 cpl_vector_delete(ve_real);
237 cpl_matrix_delete(design);
238 hdrl_ls_fit_result_delete(res);
239}
240
241void test_imglistfit(void)
242{
243 {
244 size_t n = 5;
245 hdrl_imagelist * input = hdrl_imagelist_new();
246 cpl_vector * samp = cpl_vector_new(n);
247 hdrl_imagelist * out_coef = NULL;
248 cpl_image * out_chi2 = NULL;
249 cpl_image * out_dof = NULL;
250 hdrl_fit_polynomial_imagelist(NULL, NULL, 0, NULL, NULL, NULL);
251 cpl_test_error(CPL_ERROR_NULL_INPUT);
252
253 hdrl_fit_polynomial_imagelist(input, NULL, 0, NULL, NULL, NULL);
254 cpl_test_error(CPL_ERROR_NULL_INPUT);
255
256 hdrl_fit_polynomial_imagelist(NULL, samp, 0, NULL, NULL, NULL);
257 cpl_test_error(CPL_ERROR_NULL_INPUT);
258
259 hdrl_fit_polynomial_imagelist(input, samp, 0, NULL, NULL, NULL);
260 cpl_test_error(CPL_ERROR_NULL_INPUT);
261
262 hdrl_fit_polynomial_imagelist(input, samp, -1, &out_coef, &out_chi2, &out_dof);
263 cpl_test_error(CPL_ERROR_INCOMPATIBLE_INPUT);
264
265 hdrl_fit_polynomial_imagelist(input, samp, n + 2, &out_coef, &out_chi2, &out_dof);
266 cpl_test_error(CPL_ERROR_INCOMPATIBLE_INPUT);
267
268 for (size_t i = 0; i < n; i++) {
269 double t = (i + 1) * 100;
270 hdrl_image * img = hdrl_image_new(10, 10);
271 /* slight deviation from linear to get nonzero scale */
273 (hdrl_value){0.5 * t + 100 - i, sqrt(0.5 * t)});
274 if (i == 3) {
275 hdrl_image_reject(img, 3, 4);
276 }
277 hdrl_imagelist_set(input, img, i);
278 cpl_vector_set(samp, i, t);
279 }
280
281 hdrl_fit_polynomial_imagelist(input, samp, 1, &out_coef, &out_chi2, &out_dof);
282 cpl_test_error(CPL_ERROR_NONE);
283 cpl_test_eq(cpl_image_get_type(out_dof), HDRL_TYPE_DATA);
284 cpl_test_eq(cpl_image_get_type(out_chi2), HDRL_TYPE_DATA);
285 cpl_test_eq(hdrl_imagelist_get_size(out_coef), 2);
286
287 hdrl_image * coef0 = hdrl_imagelist_get(out_coef, 0);
288 hdrl_image * expect = hdrl_image_new(10, 10);
289 hdrl_image_add_scalar(expect, (hdrl_value){101, 9.0045});
290 hdrl_image_set_pixel(expect, 3, 4, (hdrl_value){101, 9.29448});
291 hdrl_test_image_abs(coef0, expect, HDRL_EPS_DATA * 1e11);
292 hdrl_image_delete(expect);
293
294 hdrl_image * coef1 = hdrl_imagelist_get(out_coef, 1);
295 expect = hdrl_image_new(10, 10);
296 hdrl_image_add_scalar(expect, (hdrl_value){0.49, 0.0351317});
297 hdrl_image_set_pixel(expect, 3, 4, (hdrl_value){0.49, 0.0399607});
298 hdrl_test_image_abs(coef1, expect, HDRL_EPS_DATA * 1e11);
299 hdrl_image_delete(expect);
300
301 cpl_image * cexpect = cpl_image_new(10, 10, HDRL_TYPE_DATA);
302 /* dof*scaling between weights and measured error */
303 cpl_image_add_scalar(cexpect, 1.831e-29);
304 cpl_test_image_abs(out_chi2, cexpect, DBL_EPSILON * 1e9);
305 cpl_image_delete(cexpect);
306
307 cexpect = cpl_image_new(10, 10, HDRL_TYPE_DATA);
308 cpl_image_add_scalar(cexpect, 3);
309 cpl_image_set(cexpect, 3, 4, 2);
310 cpl_test_image_abs(out_dof, cexpect, 0);
311 cpl_image_delete(cexpect);
312
313 hdrl_imagelist_delete(out_coef);
314 cpl_image_delete(out_chi2);
315 cpl_image_delete(out_dof);
316
317
318 /* smoke test less good pixels than fit degree */
319 hdrl_image_reject(hdrl_imagelist_get(input, 0), 2, 2);
320 hdrl_image_reject(hdrl_imagelist_get(input, 1), 2, 2);
321 hdrl_image_reject(hdrl_imagelist_get(input, 2), 2, 2);
322 hdrl_image_reject(hdrl_imagelist_get(input, 3), 2, 2);
323 hdrl_fit_polynomial_imagelist(input, samp, 3, &out_coef, &out_chi2, &out_dof);
324
326 cpl_vector_delete(samp);
327 hdrl_imagelist_delete(out_coef);
328 cpl_image_delete(out_chi2);
329 cpl_image_delete(out_dof);
330 }
331}
332
333void test_imglistfit2(void)
334{
335 {
336 size_t n = 5;
337 hdrl_imagelist * input = hdrl_imagelist_new();
338 cpl_imagelist * samp = cpl_imagelist_new();
339 hdrl_imagelist * out_coef = NULL;
340 cpl_image * out_chi2 = NULL;
341 cpl_image * out_dof = NULL;
342 hdrl_fit_polynomial_imagelist2(NULL, NULL, 0, NULL, NULL, NULL);
343 cpl_test_error(CPL_ERROR_NULL_INPUT);
344
345 hdrl_fit_polynomial_imagelist2(input, NULL, 0, NULL, NULL, NULL);
346 cpl_test_error(CPL_ERROR_NULL_INPUT);
347
348 hdrl_fit_polynomial_imagelist2(NULL, samp, 0, NULL, NULL, NULL);
349 cpl_test_error(CPL_ERROR_NULL_INPUT);
350
351 hdrl_fit_polynomial_imagelist2(input, samp, 0, NULL, NULL, NULL);
352 cpl_test_error(CPL_ERROR_NULL_INPUT);
353
354 hdrl_fit_polynomial_imagelist2(input, samp, -1, &out_coef, &out_chi2, &out_dof);
355 cpl_test_error(CPL_ERROR_INCOMPATIBLE_INPUT);
356
357 hdrl_fit_polynomial_imagelist2(input, samp, n + 2, &out_coef, &out_chi2, &out_dof);
358 cpl_test_error(CPL_ERROR_INCOMPATIBLE_INPUT);
359
360 for (size_t i = 0; i < n; i++) {
361 double t = (i + 1) * 100;
362 hdrl_image * img = hdrl_image_new(10, 10);
363 /* slight deviation from linear to get nonzero scale */
365 (hdrl_value){0.5 * t + 100 - i, sqrt(0.5 * t)});
366 hdrl_imagelist_set(input, img, i);
367 cpl_image * sampi = cpl_image_new(10, 10, HDRL_TYPE_DATA);
368 cpl_image_add_scalar(sampi, t);
369 if (i == 3) {
370 cpl_image_reject(sampi, 3, 4);
371 }
372 cpl_imagelist_set(samp, sampi, i);
373 }
374
375 hdrl_fit_polynomial_imagelist2(input, samp, 1, &out_coef, &out_chi2, &out_dof);
376 cpl_test_error(CPL_ERROR_NONE);
377 cpl_test_eq(cpl_image_get_type(out_dof), HDRL_TYPE_DATA);
378 cpl_test_eq(cpl_image_get_type(out_chi2), HDRL_TYPE_DATA);
379 cpl_test_eq(hdrl_imagelist_get_size(out_coef), 2);
380
381 hdrl_image * coef0 = hdrl_imagelist_get(out_coef, 0);
382 hdrl_image * expect = hdrl_image_new(10, 10);
383 hdrl_image_add_scalar(expect, (hdrl_value){101, 9.0045});
384 hdrl_image_set_pixel(expect, 3, 4, (hdrl_value){101, 9.29448});
385 hdrl_test_image_abs(coef0, expect, HDRL_EPS_DATA * 1e11);
386 hdrl_image_delete(expect);
387
388 hdrl_image * coef1 = hdrl_imagelist_get(out_coef, 1);
389 expect = hdrl_image_new(10, 10);
390 hdrl_image_add_scalar(expect, (hdrl_value){0.49, 0.0351317});
391 hdrl_image_set_pixel(expect, 3, 4, (hdrl_value){0.49, 0.0399607});
392 hdrl_test_image_abs(coef1, expect, HDRL_EPS_DATA * 1e11);
393 hdrl_image_delete(expect);
394
395 cpl_image * cexpect = cpl_image_new(10, 10, HDRL_TYPE_DATA);
396 /* dof*scaling between weights and measured error */
397 cpl_image_add_scalar(cexpect, 1.831e-29);
398 cpl_test_image_abs(out_chi2, cexpect, DBL_EPSILON * 1e9);
399 cpl_image_delete(cexpect);
400
401 cexpect = cpl_image_new(10, 10, HDRL_TYPE_DATA);
402 cpl_image_add_scalar(cexpect, 3);
403 cpl_image_set(cexpect, 3, 4, 2);
404 cpl_test_image_abs(out_dof, cexpect, 0);
405 cpl_image_delete(cexpect);
406
407 hdrl_imagelist_delete(out_coef);
408 cpl_image_delete(out_chi2);
409 cpl_image_delete(out_dof);
410
411
412 /* smoke test less good pixels than fit degree */
413 hdrl_image_reject(hdrl_imagelist_get(input, 0), 2, 2);
414 hdrl_image_reject(hdrl_imagelist_get(input, 1), 2, 2);
415 hdrl_image_reject(hdrl_imagelist_get(input, 2), 2, 2);
416 hdrl_image_reject(hdrl_imagelist_get(input, 3), 2, 2);
417 hdrl_fit_polynomial_imagelist2(input, samp, 3, &out_coef, &out_chi2, &out_dof);
418
420 cpl_imagelist_delete(samp);
421 hdrl_imagelist_delete(out_coef);
422 cpl_image_delete(out_chi2);
423 cpl_image_delete(out_dof);
424 }
425}
426
427void test_real_data(void)
428{
429 /* pixel from vcam ramp data with poisson model error (gain 2.4, ron 10) */
430 double x[] = { 3., 3., 5., 5., 7., 7., 10., 10., 12., 12.,
431 15., 15., 20., 20. };
432 double y[] = { 3441, 3420, 5606, 5586, 7814, 7815, 11003, 10970,
433 13292, 13198, 16347, 16175, 21267, 21318 };
434 double e[] = { 39.16312027, 39.05124664, 49.35416031, 49.26966476,
435 57.92955399, 57.93315125, 68.4440155 , 68.34349823,
436 75.08883667, 74.82757568, 83.13392639, 82.7017746 ,
437 94.66387939, 94.77605438 };
438 cpl_vector * vx = cpl_vector_wrap(ARRAY_LEN(x), x);
439 cpl_vector * vy = cpl_vector_wrap(ARRAY_LEN(y), y);
440 cpl_vector * ve = cpl_vector_wrap(ARRAY_LEN(e), e);
441
442 double exp_c[] = {296.10245659, 1063.12005477};
443 cpl_matrix * design = polyvander1d(vx, 1);
444 hdrl_ls_fit_result * res = polyfit1d(vx, vy, ve, 1);
445 matrix_eq(res->coef, exp_c, DBL_EPSILON * 2e10);
446
447 cpl_vector_unwrap(vx);
448 cpl_vector_unwrap(vy);
449 cpl_vector_unwrap(ve);
450 cpl_matrix_delete(design);
451 hdrl_ls_fit_result_delete(res);
452}
453
454/*----------------------------------------------------------------------------*/
458/*----------------------------------------------------------------------------*/
459int main(void)
460{
461 cpl_test_init(PACKAGE_BUGREPORT, CPL_MSG_WARNING);
462
463 test_vander1d();
464 test_fit();
465 test_poisson();
466 test_imglistfit();
467 test_imglistfit2();
468 test_real_data();
469
470 return cpl_test_end(0);
471}
cpl_error_code hdrl_fit_polynomial_imagelist2(const hdrl_imagelist *list, const cpl_imagelist *samplepos, const int degree, hdrl_imagelist **coef, cpl_image **chi2, cpl_image **dof)
weighted least squares polynomial fit of each pixel of a imagelist
Definition: hdrl_fit.c:540
cpl_error_code hdrl_fit_polynomial_imagelist(const hdrl_imagelist *list, const cpl_vector *samplepos, const int degree, hdrl_imagelist **coef, cpl_image **chi2, cpl_image **dof)
weighted least squares polynomial fit of each pixel of a imagelist
Definition: hdrl_fit.c:367
cpl_error_code hdrl_image_set_pixel(hdrl_image *self, cpl_size xpos, cpl_size ypos, hdrl_value value)
set pixel values of hdrl_image
Definition: hdrl_image.c:594
cpl_error_code hdrl_image_add_scalar(hdrl_image *self, hdrl_value value)
Elementwise addition of a scalar to an image.
cpl_error_code hdrl_image_reject(hdrl_image *self, cpl_size xpos, cpl_size ypos)
mark pixel as bad
Definition: hdrl_image.c:427
hdrl_image * hdrl_image_new(cpl_size nx, cpl_size ny)
create new zero filled hdrl image
Definition: hdrl_image.c:311
void hdrl_image_delete(hdrl_image *himg)
delete hdrl_image
Definition: hdrl_image.c:379
cpl_error_code hdrl_imagelist_set(hdrl_imagelist *himlist, hdrl_image *himg, cpl_size pos)
Insert an image into an imagelist.
void hdrl_imagelist_delete(hdrl_imagelist *himlist)
Free all memory used by a hdrl_imagelist object including the images.
cpl_size hdrl_imagelist_get_size(const hdrl_imagelist *himlist)
Get the number of images in the imagelist.
hdrl_imagelist * hdrl_imagelist_new(void)
Create an empty imagelist.
hdrl_image * hdrl_imagelist_get(const hdrl_imagelist *himlist, cpl_size inum)
Get an image from a list of images.