MUSE Pipeline Reference Manual  0.18.5
muse_sky_rowbyrow.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  *
5  * This file is part of the MUSE Instrument Pipeline
6  * Copyright (C) 2005-2011 European Southern Observatory
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21  */
22 
23 #ifdef HAVE_CONFIG_H
24 #include <config.h>
25 #endif
26 
27 /*----------------------------------------------------------------------------*
28  * Includes *
29  *----------------------------------------------------------------------------*/
30 #include <math.h>
31 #include <string.h>
32 
33 #include <cpl.h>
34 
35 #include "muse_sky.h"
36 #include "muse_instrument.h"
37 #include "muse_quality.h"
38 #include "muse_tracing.h"
39 #include "muse_utils.h"
40 
44 /*----------------------------------------------------------------------------*
45  * Debugging Macros *
46  * Set these to 1 or higher for (lots of) debugging output *
47  *----------------------------------------------------------------------------*/
48 #define SKY_ROWBYROW_DEBUG 0 /* debug output in muse_sky_subtract_rowbyrow() */
49 
50 /*----------------------------------------------------------------------------*/
73 /*----------------------------------------------------------------------------*/
74 cpl_error_code
75 muse_sky_subtract_rowbyrow_mask(muse_image *aImage, cpl_table *aTrace)
76 {
77  cpl_ensure_code(aImage, CPL_ERROR_NULL_INPUT);
78  unsigned short nslices = 0;
79  if (aTrace) {
80  nslices = cpl_table_get_nrow(aTrace);
81  } else if (!aTrace) { /* check for the slice center headers */
82  unsigned short slice = 0;
83  cpl_boolean exists = CPL_FALSE;
84  do {
85  char *keyword = cpl_sprintf("ESO DRS MUSE SLICE%hu CENTER", ++slice);
86  exists = cpl_propertylist_has(aImage->header, keyword);
87 #if 0
88  cpl_msg_debug(__func__, "%s %s", keyword, exists ? "exists" : "doesn't exist");
89 #endif
90  cpl_free(keyword);
91  } while (exists);
92  nslices = slice - 1;
93  }
94  cpl_msg_debug(__func__, "Found %hu slices", nslices);
95  cpl_ensure_code(nslices, CPL_ERROR_ILLEGAL_INPUT);
96  const unsigned int kOrdObj = 1; /* a linear fit to reject objects */
97  const float kRSigObj = 3.; /* use 3 sigma rejection to find objects */
98 
99  /* vertical median as object mask */
100  cpl_image *objmask = cpl_image_collapse_median_create(aImage->data, 0, 0, 0);
101 #if SKY_ROWBYROW_DEBUG
102  cpl_msg_debug(__func__, "saving \"sky_removed_objmask.fits\"");
103  cpl_image_save(objmask, "sky_removed_objmask.fits", CPL_TYPE_UNSPECIFIED,
104  NULL, CPL_IO_CREATE);
105 #endif
106  /* get image properties and data */
107  int nx = cpl_image_get_size_x(aImage->data),
108  ny = cpl_image_get_size_y(aImage->data);
109  int *dq = cpl_image_get_data_int(aImage->dq);
110 
111  /* loop through all slices to fit and mask the objects */
112  unsigned short islice;
113  for (islice = 0; islice < nslices; islice++) {
114  cpl_msg_debug(__func__, "Processing slice %hu", islice + 1);
115 
116  /* find slice edge, either through trace info or *
117  * through centers plus "missing data" flags */
118  cpl_polynomial **ptrace = NULL;
119  int i1 = 0, i2 = nx, istart = 0;
120  if (aTrace) {
121  /* get the tracing polynomials for this slice */
122  ptrace = muse_trace_table_get_polys_for_slice(aTrace, islice + 1);
123  if (!ptrace) {
124  cpl_msg_warning(__func__, "slice %2d: tracing polynomials missing!",
125  islice + 1);
126  continue;
127  }
128  /* use trace of vertical center, be generous with the edges */
129  i1 = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], ny/2, NULL));
130  i2 = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], ny/2, NULL));
131  if (i1 < 1 || i2 > nx || i1 > i2) {
132  cpl_msg_warning(__func__, "slice %2d: faulty polynomial detected at "
133  "y=%d (borders: %d ... %d)", islice + 1, ny/2, i1, i2);
134  muse_trace_polys_delete(ptrace);
135  continue; /* next slice */
136  }
137  } else { /* find slice edge without tracing info */
138  char *keyword = cpl_sprintf("ESO DRS MUSE SLICE%hu CENTER", islice + 1);
139  istart = cpl_propertylist_get_float(aImage->header, keyword);
140  cpl_free(keyword);
141  /* move outwards */
142  int ix = istart;
143  while (!(dq[(--ix-1) + ny/2*nx] & EURO3D_MISSDATA)) {
144  i1 = ix;
145  }
146  ix = istart;
147  while (!(dq[(++ix-1) + ny/2*nx] & EURO3D_MISSDATA)) {
148  i2 = ix;
149  }
150  } /* else, no aTrace */
151 
152  /* fit and apply object mask */
153  cpl_matrix *p = cpl_matrix_new(1, i2 - i1 + 1);
154  cpl_vector *v = cpl_vector_new(i2 - i1 + 1);
155  int i;
156  for (i = i1; i <= i2; i++) { /* x pos start at 1 */
157  cpl_matrix_set(p, 0, i - i1, i);
158  int err;
159  double value = cpl_image_get(objmask, i, 1, &err);
160  if (err) {
161  cpl_vector_set(v, i - i1, NAN); /* set to bad */
162  } else {
163  cpl_vector_set(v, i - i1, value);
164  }
165  } /* for i (relevant columns) */
166  /* do iterative low-order polynomial fit (originally 1st order) */
167  cpl_polynomial *p1 = muse_utils_iterate_fit_polynomial(p, v, NULL, NULL,
168  kOrdObj, kRSigObj,
169  NULL, NULL);
170  int idxold = 0;
171  for (i = i1; i <= i2; i++) { /* x pos start at 1 */
172  int idx = idxold,
173  ncol = cpl_matrix_get_ncol(p);
174  while (idx < ncol && (int)cpl_matrix_get(p, 0, idx) < i) { /* object pixels */
175  idx++;
176  }
177  /* check if we found the entry, if so skip the masking loop */
178  if (idx < ncol && (int)cpl_matrix_get(p, 0, idx) == i) {
179  idxold = idx; /* matrix is sorted, so keep track of position */
180  continue;
181  }
182  int j;
183  for (j = 0; j < ny; j++) { /* set all column pixels to "object" in DQ */
184  dq[(i-1) + j*nx] |= EURO3D_OBJECT;
185  } /* for j (vertical pixels) */
186  } /* for i (relevant columns) */
187  cpl_vector_delete(v);
188  cpl_matrix_delete(p);
189  cpl_polynomial_delete(p1);
190  muse_trace_polys_delete(ptrace); /* NULL check in there... */
191  } /* for islice */
192  cpl_image_delete(objmask);
193 
194  return CPL_ERROR_NONE;
195 } /* muse_sky_subtract_rowbyrow_mask() */
196 
197 /*----------------------------------------------------------------------------*/
223 /*----------------------------------------------------------------------------*/
224 cpl_error_code
225 muse_sky_subtract_rowbyrow(muse_image *aImage, cpl_table *aTrace, float aRSigma,
226  unsigned int aOrder)
227 {
228  cpl_ensure_code(aImage, CPL_ERROR_NULL_INPUT);
229  unsigned short nslices = 0;
230  if (aTrace) {
231  nslices = cpl_table_get_nrow(aTrace);
232  } else if (!aTrace) { /* check for the slice center headers */
233  unsigned short slice = 0;
234  cpl_boolean exists = CPL_FALSE;
235  do {
236  char *keyword = cpl_sprintf("ESO DRS MUSE SLICE%hu CENTER", ++slice);
237  exists = cpl_propertylist_has(aImage->header, keyword);
238 #if 0
239  cpl_msg_debug(__func__, "%s %s", keyword, exists ? "exists" : "doesn't exist");
240 #endif
241  cpl_free(keyword);
242  } while (exists);
243  nslices = slice - 1;
244  }
245  cpl_msg_debug(__func__, "Found %hu slices", nslices);
246  cpl_ensure_code(nslices, CPL_ERROR_ILLEGAL_INPUT);
247 
248  /* get image properties and data */
249  int nx = cpl_image_get_size_x(aImage->data),
250  ny = cpl_image_get_size_y(aImage->data);
251  float *data = cpl_image_get_data_float(aImage->data),
252  *stat = cpl_image_get_data_float(aImage->stat);
253  int *dq = cpl_image_get_data_int(aImage->dq);
254 
255  /* loop through all slices to fit and remove the sky */
256  unsigned short islice;
257  for (islice = 0; islice < nslices; islice++) {
258  cpl_msg_debug(__func__, "Processing slice %hu", islice + 1);
259 
260  /* find slice edge, either through trace info or *
261  * through centers plus "missing data" flags */
262  cpl_polynomial **ptrace = NULL;
263  int i1 = 0, i2 = nx, istart = 0;
264  if (aTrace) {
265  /* get the tracing polynomials for this slice */
266  ptrace = muse_trace_table_get_polys_for_slice(aTrace, islice + 1);
267  if (!ptrace) {
268  cpl_msg_warning(__func__, "slice %2d: tracing polynomials missing!",
269  islice + 1);
270  continue;
271  }
272  /* use trace of vertical center, be generous with the edges */
273  i1 = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], ny/2, NULL));
274  i2 = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], ny/2, NULL));
275  if (i1 < 1 || i2 > nx || i1 > i2) {
276  cpl_msg_warning(__func__, "slice %2d: faulty polynomial detected at "
277  "y=%d (borders: %d ... %d)", islice + 1, ny/2, i1, i2);
278  muse_trace_polys_delete(ptrace);
279  continue; /* next slice */
280  }
281  } else { /* find slice edge without tracing info */
282  char *keyword = cpl_sprintf("ESO DRS MUSE SLICE%hu CENTER", islice + 1);
283  istart = cpl_propertylist_get_float(aImage->header, keyword);
284  cpl_free(keyword);
285  /* move outwards */
286  int ix = istart;
287  while (!(dq[(--ix-1) + ny/2*nx] & EURO3D_MISSDATA)) {
288  i1 = ix;
289  }
290  ix = istart;
291  while (!(dq[(++ix-1) + ny/2*nx] & EURO3D_MISSDATA)) {
292  i2 = ix;
293  }
294  } /* else, no aTrace */
295 #if SKY_ROWBYROW_DEBUG
296  cpl_msg_debug(__func__, "1 slice %d row %d edges: %d %d", islice+1, ny/2, i1, i2);
297 #endif
298  /* loop through all image rows */
299  int j;
300  for (j = 0; j < ny; j++) {
301  /* include pixels that have more than half of them inside the slice, *
302  * use the same method to compute the cutoff as muse_pixtable_create() */
303  int ileft = istart, icenter = istart, iright = istart;
304  if (ptrace) { /* slice edges with tracing info */
305  ileft = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], j+1, NULL)),
306  icenter = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER], j+1, NULL),
307  iright = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], j+1, NULL));
308  } else { /* find slice edges here without tracing info */
309  /* move outwards */
310  int ix = istart;
311  while (!(dq[(--ix-1) + j*nx] & EURO3D_MISSDATA)) {
312  ileft = ix;
313  }
314  ix = istart;
315  while (!(dq[(++ix-1) + j*nx] & EURO3D_MISSDATA)) {
316  iright = ix;
317  }
318  icenter = (iright + ileft) / 2.; /* re-"compute" center at this pos */
319  } /* else, no aTrace */
320 #if SKY_ROWBYROW_DEBUG
321  cpl_msg_debug(__func__, "2 slice %d row %d edges: %d %d", islice+1, j+1,
322  ileft, iright);
323 #endif
324 
325  /* fill data into vectors, copy the matrix */
326  cpl_matrix *pos = cpl_matrix_new(1, iright - ileft + 1);
327  cpl_vector *values = cpl_vector_new(iright - ileft + 1);
328  /* unused: *errors = cpl_vector_new(iright - ileft + 1); */
329  unsigned int nval = 0;
330  int i;
331  for (i = ileft; i <= iright; i++) { /* x pos start at 1 */
332  cpl_matrix_set(pos, 0, i - ileft, i - icenter); /* relative coordinates */
333  if (dq[(i-1) + j*nx] != EURO3D_GOODPIXEL) {
334  cpl_vector_set(values, i - ileft, NAN); /* mark as bad entry */
335  } else {
336  cpl_vector_set(values, i - ileft, data[(i-1) + j*nx]);
337  nval++;
338  }
339  /* XXX sigmas are not supported by cpl_polynomial_fit() yet *
340  * cpl_vector_set(errors, i - ileft, sqrt(data[i + j*nx])); */
341  } /* for i (relevant columns) */
342  if (nval < 1) { /* skip image rows with only NANs */
343  cpl_vector_delete(values);
344  /* unused: cpl_vector_delete(errors); */
345  cpl_matrix_delete(pos);
346  continue;
347  }
348  /* iterate the horizontal polynomial fit (originally 4th order) */
349  unsigned int order = aOrder > nval + 1 ? nval - 1 : aOrder;
350  double mse;
351  cpl_polynomial *poly = muse_utils_iterate_fit_polynomial(pos, values,
352  NULL, NULL, order,
353  aRSigma, &mse,
354  NULL);
355  int npix = cpl_vector_get_size(values); /* keep number of final pixels */
356  cpl_vector_delete(values);
357  /* unused: cpl_vector_delete(errors); */
358  cpl_matrix_delete(pos);
359 
360  /* subtract the fit, add the mean-squared error onto the variance */
361  for (i = ileft - 1; i < iright; i++) { /* x pos start at 0! */
362  /* evaluate polynomial at the same relative position within slice */
363  double sky = cpl_polynomial_eval_1d(poly, i+1 - icenter, NULL);
364 #if SKY_ROWBYROW_DEBUG > 1
365  if (islice+1 == SKY_ROWBYROW_DEBUG_SLICE &&
366  j+1 >= SKY_ROWBYROW_DEBUG_ROW1 && j+1 <= SKY_ROWBYROW_DEBUG_ROW2) {
367  printf("subtracting slice %d row %d %d %f ", islice+1, j+1,
368  i+1 - icenter, data[i + j*nx]);
369  }
370 #endif
371  data[i + j*nx] -= sky;
372  /* to correct the variance, add the mean-squared error divided *
373  * by number of pixels minus the degrees of freedom */
374  stat[i + j*nx] += mse / (npix - order-1);
375 #if SKY_ROWBYROW_DEBUG > 1
376  if (islice+1 == SKY_ROWBYROW_DEBUG_SLICE &&
377  j+1 >= SKY_ROWBYROW_DEBUG_ROW1 && j+1 <= SKY_ROWBYROW_DEBUG_ROW2) {
378  printf("%f\n", data[i + j*nx]);
379  fflush(stdout);
380  }
381 #endif
382  } /* for i (relevant columns) */
383  cpl_polynomial_delete(poly);
384  } /* for j (all image rows) */
385 
386  muse_trace_polys_delete(ptrace); /* NULL check in there... */
387  } /* for islice */
388 
389 #if SKY_ROWBYROW_DEBUG
390  cpl_msg_debug(__func__, "saving \"sky_removed_iterated_edge2.fits\"");
391  muse_image_save(aImage, "sky_removed_iterated_edge2.fits");
392 #endif
393 
394  return CPL_ERROR_NONE;
395 } /* muse_sky_subtract_rowbyrow() */
396 
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
cpl_image * data
the data extension
Definition: muse_image.h:47
cpl_error_code muse_sky_subtract_rowbyrow(muse_image *aImage, cpl_table *aTrace, float aRSigma, unsigned int aOrder)
Subtract the sky row-by-row from a CCD-based image.
cpl_image * stat
the statistics extension
Definition: muse_image.h:65
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:41
cpl_propertylist * header
the FITS header
Definition: muse_image.h:73
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
cpl_image * dq
the data quality extension
Definition: muse_image.h:57
cpl_polynomial * muse_utils_iterate_fit_polynomial(cpl_matrix *aPos, cpl_vector *aVal, cpl_vector *aErr, cpl_table *aExtra, const unsigned int aOrder, const double aRSigma, double *aMSE, double *aChiSq)
Iterate a polynomial fit.
Definition: muse_utils.c:1829
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
Definition: muse_image.c:396
cpl_error_code muse_sky_subtract_rowbyrow_mask(muse_image *aImage, cpl_table *aTrace)
Prepare an (object) mask for the sky row-by-row fitting.