CR2RE Pipeline Reference Manual 1.6.8
hdrl_cat_conf.c
1/*
2 * This file is part of the HDRL
3 * Copyright (C) 2017 European Southern Observatory
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <string.h>
21
22#include "hdrl_cat_conf.h"
23
24#include "hdrl_cat_apio.h"
25#include "hdrl_cat_apline.h"
26#include "hdrl_cat_table.h"
27#include "hdrl_cat_background.h"
28#include "hdrl_cat_terminate.h"
29
30
31/*** Defines ***/
32#define NW 5 /* */
33#define STUPID_VALUE -1000 /* Minimum value of a pixel */
34
35
36/*** Internal global variables **/
37
38static double *g_smoothed = NULL;
39static double *g_smoothedc = NULL;
40static unsigned char *g_mflag = NULL;
41static double *g_indata = NULL;
42static double *g_confdata = NULL;
43static double *g_confsqrt = NULL;
44
45static ap_t g_ap;
46static cpl_size g_freeconf = 0;
47
48static double g_weights[NW*NW];
49static cpl_size g_nx;
50static cpl_size g_ny;
51
52
53/*** Prototypes ***/
54
55static void crweights(double filtfwhm);
56static void convolve( cpl_size ir );
57static void clean_up( cpl_table *tab );
58
59/*---------------------------------------------------------------------------*/
66/*----------------------------------------------------------------------------*/
67
70/* ---------------------------------------------------------------------------*/
111/* ---------------------------------------------------------------------------*/
112cpl_error_code hdrl_catalogue_conf(
113 hdrl_casu_fits *infile, hdrl_casu_fits *conf, cpl_size ipix,
114 double threshold, cpl_size icrowd, double rcore,
115 cpl_size bkg_subtr, cpl_size nbsize,
116 cpl_size cattype, double filtfwhm, double gain,
117 double saturation, hdrl_casu_result *res)
118{
119 /* Initialize output */
120 res->catalogue = NULL;
121
122 /* Useful constants */
123 double fconst = CPL_MATH_LOG2E;
124 cpl_size nobjects = 0;
125
126 /* Open input image */
127 cpl_table *tab = NULL;
128 cpl_image *map = hdrl_casu_fits_get_image(infile);
129
130 if ((g_indata = cpl_image_get_data_double(map)) == NULL) {
131 clean_up(tab);
132 cpl_error_set_message(cpl_func, CPL_ERROR_NULL_INPUT,
133 "hdrl_cat_catalogue_conf - Error getting image data");
134 return CPL_ERROR_NULL_INPUT;
135 }
136
137 g_nx = cpl_image_get_size_x(map);
138 g_ny = cpl_image_get_size_y(map);
139
140 cpl_size npts = g_nx * g_ny;
141 cpl_size npix = g_nx * g_ny;
142
143 /* Open the associated confidence map, if it exists */
144 cpl_image *cmap;
145 if (conf != NULL) {
146
147 cmap = hdrl_casu_fits_get_image(conf);
148 if ((g_confdata = cpl_image_get_data(cmap)) == NULL) {
149 clean_up(tab);
150 cpl_error_set_message(cpl_func, CPL_ERROR_NULL_INPUT,
151 "hdrl_cat_catalogue_conf - Error getting confidence map data");
152 return CPL_ERROR_NULL_INPUT;
153 }
154
155 cpl_size nxc = cpl_image_get_size_x(cmap);
156 cpl_size nyc = cpl_image_get_size_y(cmap);
157 if ((g_nx != nxc) || (g_ny != nyc)) {
158 clean_up(tab);
159 cpl_error_set_message(cpl_func, CPL_ERROR_INCOMPATIBLE_INPUT,
160 "hdrl_cat_catalogue_conf - Input image and confidence dimensions don't match");
161 return CPL_ERROR_INCOMPATIBLE_INPUT;
162 }
163
164 g_freeconf = 0;
165 } else {
166 g_confdata = cpl_malloc(npts * sizeof(*g_confdata));
167 for (cpl_size i = 0; i < npts; i++) {
168 g_confdata[i] = 100;
169 }
170 g_freeconf = 1;
171 cmap = NULL;
172 }
173
174 /* Get g_mflag array for flagging saturated pixels */
175 g_mflag = cpl_calloc(npix, sizeof(*g_mflag));
176
177 /* Open the g_ap structure and define some stuff in it */
178 g_ap.lsiz = g_nx;
179 g_ap.csiz = g_ny;
180 g_ap.inframe = map;
181 g_ap.conframe = cmap;
182
183 hdrl_apinit(&g_ap);
184
185 g_ap.indata = g_indata;
186 g_ap.confdata = g_confdata;
187 g_ap.multiply = 1;
188 g_ap.ipnop = ipix;
189 g_ap.mflag = g_mflag;
190 g_ap.rcore = rcore;
191 g_ap.filtfwhm = filtfwhm;
192 g_ap.icrowd = icrowd;
193 g_ap.fconst = fconst;
194
195 /* Open the output catalogue FITS table */
196 cpl_size hdrl_xcol;
197 cpl_size hdrl_ycol;
198 hdrl_tabinit(&g_ap, &hdrl_xcol, &hdrl_ycol, cattype, &tab, res);
199
200 /* Set up the data flags */
201 for (cpl_size i = 0; i < npix; i++) {
202 if (g_confdata[i] == 0) {
203 g_mflag[i] = MF_ZEROCONF;
204 } else if (g_indata[i] < STUPID_VALUE) {
205 g_mflag[i] = MF_STUPID_VALUE;
206 } else {
207 g_mflag[i] = MF_CLEANPIX;
208 }
209 }
210
211 /* Flag up regions where the value is above the saturation level*/
212 for (cpl_size i = 0; i < npix ; i++) {
213 if (g_mflag[i] == MF_CLEANPIX && g_indata[i] > saturation) {
214 g_mflag[i] = MF_SATURATED;
215 }
216 }
217
218 /* Compute the background variation and remove it from the data*/
219 if (hdrl_background(&g_ap, nbsize, bkg_subtr, res) != CPL_ERROR_NONE) {
220 clean_up(tab);
221 return cpl_error_get_code();
222 }
223
224 /* Compute background statistics */
225 double skymed;
226 double skysig;
227 if (hdrl_backstats(&g_ap, &skymed, &skysig) != CPL_ERROR_NONE) {
228 clean_up(tab);
229 return cpl_error_get_code();
230 }
231
232 /* Take mean sky level out of data. */
233 if (bkg_subtr) {
234 for (cpl_size i = 0; i < g_nx * g_ny; i++) {
235 g_indata[i] -= skymed;
236 }
237 }
238
239 /* Work out isophotal detection threshold levels */
240 double thresh = threshold * skysig;
241 if (!bkg_subtr && thresh < skymed) {
242 clean_up(tab);
243 cpl_error_set_message(cpl_func, CPL_ERROR_INCOMPATIBLE_INPUT,
244 "Bad background corrected input. Background estimation disabled "
245 "but image median larger than threshold * sigma.");
246 return CPL_ERROR_INCOMPATIBLE_INPUT;
247 }
248
249 /* Minimum intensity for consideration */
250 double xintmin = 1.5 * thresh * (double)ipix;
251
252 /* Actual areal profile levels: T, 2T, 4T, 8T,...but written wrt T i.e. threshold as a power of 2 */
253 double offset = log(thresh)*fconst;
254
255 /* Get a bit of workspace for buffers */
256 g_smoothed = cpl_malloc(g_nx * sizeof(*g_smoothed));
257 g_smoothedc = cpl_malloc(g_nx * sizeof(*g_smoothedc));
258
259 /* Minimum size for considering multiple images */
260 cpl_size mulpix = CPL_MAX(8, 2 * ipix);
261
262 /* Define a few things more things in g_ap structure */
263 g_ap.mulpix = mulpix;
264 g_ap.areal_offset = offset;
265 g_ap.thresh = thresh;
266 g_ap.xintmin = xintmin;
267 g_ap.sigma = skysig;
268
269 if (bkg_subtr) {
270 g_ap.background = skymed;
271 g_ap.saturation = saturation - skymed;
272 } else {
273 g_ap.background = 0.;
274 g_ap.saturation = saturation;
275 }
276
277 /* Set the weights */
278 crweights(filtfwhm);
279 cpl_size nw2 = NW / 2;
280
281 /* sqrt confdata, need as many rows as the convolution is wide */
282 g_confsqrt = cpl_malloc(g_nx * NW * sizeof(*g_confsqrt));
283 for (cpl_size j = 0; j < NW; j++) {
284 for (cpl_size i = 0; i < g_nx; i++) {
285 g_confsqrt[j * g_nx + i] = sqrt(0.01 * (double)(g_confdata[j * g_nx + i]));
286 }
287 }
288
289 /* Right, now for the extraction loop. Begin by defining a group of three rows of data and confidence */
290 for (cpl_size j = nw2; j < g_ny-nw2; j++) {
291
292 double *current = g_indata + j * g_nx;
293
294 if (j != nw2) {
295
296 /* rotate buffer, could be a more efficient structure, but this sufficent for now */
297 memmove(g_confsqrt, g_confsqrt + g_nx, g_nx * (NW - 1) * sizeof(*g_confsqrt));
298
299 /* fill last row of buffer */
300 for (cpl_size i = 0; i < g_nx; i++) {
301 g_confsqrt[(NW - 1) * g_nx + i] = sqrt(0.01 * (double)(g_confdata[(j + nw2) * g_nx + i]));
302 }
303 }
304
305 /* current row is the center of the buffer */
306 double *currentc = g_confsqrt + nw2 * g_nx;
307 convolve(j);
308
309 /* Do the detection now */
310 hdrl_apline(&g_ap, current, currentc, g_smoothed, g_smoothedc, j, NULL);
311
312 /* Make sure we're not overruning the stacks */
313 if (g_ap.ibstack > g_ap.maxbl - g_ap.lsiz) hdrl_apfu(&g_ap);
314 if (g_ap.ipstack > g_ap.maxpa * 3 / 4 ) hdrl_apfu(&g_ap);
315
316 /* See if there are any images to terminate */
317 if (g_ap.ipstack > 1) {
318 hdrl_terminate(&g_ap, gain, &nobjects, tab, res);
319 }
320 }
321
322 /* Post process. First truncate the cpl_table to the correct size and then work out an estimate of the seeing */
323 cpl_table_set_size(tab, nobjects);
324
325 if (hdrl_do_seeing(&g_ap, nobjects, tab) != CPL_ERROR_NONE) {
326 clean_up(tab);
327 return cpl_error_get_code();
328 }
329
330 /* Create a property list with extra parameters. First QC parameters */
331 cpl_propertylist *extra = cpl_propertylist_duplicate(hdrl_casu_fits_get_ehu(infile));
332
333
334 /* QC parameters */
335 cpl_propertylist_update_double(extra, "ESO QC SATURATION", g_ap.saturation);
336 cpl_propertylist_update_double(extra, "ESO QC MEAN_SKY", g_ap.background);
337 cpl_propertylist_update_double(extra, "ESO QC SKY_NOISE", g_ap.sigma);
338
339 cpl_propertylist_set_comment( extra, "ESO QC SATURATION", "[adu] Saturation level");
340 cpl_propertylist_set_comment( extra, "ESO QC MEAN_SKY", "[adu] Median sky brightness");
341 cpl_propertylist_set_comment( extra, "ESO QC SKY_NOISE", "[adu] Pixel noise at sky level");
342
343
344 /* DRS parameters */
345 cpl_propertylist_update_double(extra, "ESO DRS THRESHOL", g_ap.thresh);
346 cpl_propertylist_update_int( extra, "ESO DRS MINPIX", g_ap.ipnop);
347 cpl_propertylist_update_int( extra, "ESO DRS CROWDED", g_ap.icrowd);
348 cpl_propertylist_update_double(extra, "ESO DRS RCORE", g_ap.rcore);
349 cpl_propertylist_update_double(extra, "ESO DRS SEEING", g_ap.fwhm);
350 cpl_propertylist_update_double(extra, "ESO DRS FILTFWHM", g_ap.filtfwhm);
351 cpl_propertylist_update_int( extra, "ESO DRS XCOL", hdrl_xcol);
352 cpl_propertylist_update_int( extra, "ESO DRS YCOL", hdrl_ycol);
353 cpl_propertylist_update_int( extra, "ESO DRS NXOUT", g_nx);
354 cpl_propertylist_update_int( extra, "ESO DRS NYOUT", g_ny);
355
356 cpl_propertylist_set_comment( extra, "ESO DRS THRESHOL", "[adu] Isophotal analysis threshold");
357 cpl_propertylist_set_comment( extra, "ESO DRS MINPIX", "[pixels] Minimum size for images");
358 cpl_propertylist_set_comment( extra, "ESO DRS CROWDED", "Crowded field analysis flag");
359 cpl_propertylist_set_comment( extra, "ESO DRS RCORE", "[pixels] Core radius for default profile fit");
360 cpl_propertylist_set_comment( extra, "ESO DRS SEEING", "[pixels] Average FWHM");
361 cpl_propertylist_set_comment( extra, "ESO DRS FILTFWHM", "[pixels] FWHM of smoothing kernel");
362 cpl_propertylist_set_comment( extra, "ESO DRS XCOL", "Column for X position");
363 cpl_propertylist_set_comment( extra, "ESO DRS YCOL", "Column for Y position");
364 cpl_propertylist_set_comment( extra, "ESO DRS NXOUT", "X Dimension of input image");
365 cpl_propertylist_set_comment( extra, "ESO DRS NYOUT", "Y Dimension of input image");
366
367 res->catalogue = hdrl_casu_tfits_wrap(tab, extra);
368
369 /* Clean up */
370 clean_up(NULL);
371
372 return CPL_ERROR_NONE;
373}
374
377/* ---------------------------------------------------------------------------*/
386/* ---------------------------------------------------------------------------*/
387static void crweights(double filtfwhm)
388{
389 /* Get the kernel size */
390 cpl_size nw2 = NW / 2;
391
392 /* Set the normalisation constants: 2, 2.35 one should use CPL_MATH_FWHM_SIG */
393 double gsigsq = 1. / (2. * pow(CPL_MAX(1., (double)filtfwhm) / 2.35, 2.));
394 double renorm = 0.;
395
396 /* Now work out the weights */
397 cpl_size n = -1;
398 for (cpl_size i = -nw2; i <= nw2; i++) {
399
400 double di = (double)i;
401 di *= gsigsq * di;
402
403 for (cpl_size j = -nw2; j <= nw2; j++) {
404
405 double dj = (double)j;
406 dj *= gsigsq * dj;
407
408 n++;
409
410 g_weights[n] = exp(-(di + dj));
411
412 renorm += g_weights[n];
413 }
414 }
415
416 /* Now normalise the weights */
417 cpl_size nn = -1;
418 for (cpl_size i = -nw2; i <= nw2; i++) {
419 for (cpl_size j = -nw2; j <= nw2; j++) {
420
421 nn++;
422
423 g_weights[nn] /= renorm;
424 }
425 }
426}
427
428/* ---------------------------------------------------------------------------*/
437/* ---------------------------------------------------------------------------*/
438static void convolve(cpl_size ir)
439{
440 /* Zero the summations */
441 for (cpl_size i = 0; i < g_nx; i++) {
442
443 g_smoothed[i] = 0.;
444 g_smoothedc[i] = 0.;
445 }
446
447 /* Now big is the smoothing kernel? */
448 cpl_size nw2 = NW / 2;
449
450 /* Now loop for each column */
451 for (cpl_size ix = nw2; ix < g_nx - nw2; ix++) {
452
453
454 double sum = 0.;
455 double sumc = 0.;
456
457 cpl_size n = -1;
458 for (cpl_size jy = ir - nw2; jy <= ir + nw2; jy++) {
459
460 /* g_confsqrt [0..NW] rows */
461 double *cdata = g_confsqrt + (jy - ir + nw2) * g_nx;
462 double *idata = g_indata + jy * g_nx;
463
464 for (cpl_size jx = ix - nw2; jx <= ix + nw2; jx++) {
465
466 n++;
467
468 sum += g_weights[n] * idata[jx];
469 sumc += g_weights[n] * idata[jx] * cdata[jx];
470 }
471 }
472
473 g_smoothed[ix] = sum;
474 g_smoothedc[ix] = sumc;
475 }
476}
477
478/* ---------------------------------------------------------------------------*/
485/* ---------------------------------------------------------------------------*/
486static void clean_up(cpl_table *tab) {
487
488 if (tab) cpl_table_delete(tab);
489
490 if (g_freeconf) {
491 if (g_confdata) {cpl_free(g_confdata); g_confdata = NULL;}
492 }
493
494 if (g_confsqrt ) {cpl_free(g_confsqrt); g_confsqrt = NULL;}
495 if (g_smoothed ) {cpl_free(g_smoothed); g_smoothed = NULL;}
496 if (g_smoothedc) {cpl_free(g_smoothedc); g_smoothedc = NULL;}
497 if (g_mflag ) {cpl_free(g_mflag); g_mflag = NULL;}
498
499 hdrl_apclose(&g_ap);
500}
cpl_error_code hdrl_catalogue_conf(hdrl_casu_fits *infile, hdrl_casu_fits *conf, cpl_size ipix, double threshold, cpl_size icrowd, double rcore, cpl_size bkg_subtr, cpl_size nbsize, cpl_size cattype, double filtfwhm, double gain, double saturation, hdrl_casu_result *res)
Do source extraction.
void hdrl_apinit(ap_t *ap)
Initialize the ap structure.
Definition: hdrl_cat_apio.c:54
void hdrl_apclose(ap_t *ap)
Close ap structure.
void hdrl_apline(ap_t *ap, double dat[], double conf[], double smoothed[], double smoothedc[], cpl_size j, unsigned char *bpm)
Detect objects on a line of data.
cpl_error_code hdrl_background(ap_t *ap, cpl_size nbsize, cpl_size bkg_subtr, hdrl_casu_result *res)
Model and create background map.
cpl_error_code hdrl_backstats(ap_t *ap, double *skymed, double *skysig)
Work out robust background estimate over a whole input image.
cpl_image * hdrl_casu_fits_get_image(hdrl_casu_fits *p)
Get the CPL image from the hdrl_casu_fits object.
hdrl_casu_tfits * hdrl_casu_tfits_wrap(cpl_table *tab, cpl_propertylist *ehu)
Wrap an table in a hdrl_casu_tfits wrapper.
Definition: hdrl_cat_casu.c:59
cpl_propertylist * hdrl_casu_fits_get_ehu(hdrl_casu_fits *p)
Get the propertylist for the extension header for a given hdrl_casu_fits image.
cpl_error_code hdrl_do_seeing(ap_t *ap, cpl_size nobjects, cpl_table *tab)
Do seeing estimate.
cpl_error_code hdrl_tabinit(ap_t *ap, cpl_size *xcol, cpl_size *ycol, hdrl_catalogue_options cattype, cpl_table **tab, hdrl_casu_result *res)
Initialize catalogues.
void hdrl_apfu(ap_t *ap)
Get rid of the largest contributor in an ap structure.
void hdrl_terminate(ap_t *ap, double gain, cpl_size *nobjects, cpl_table *tab, hdrl_casu_result *res)
Check for objects that have terminated.