MUSE Pipeline Reference Manual  0.18.5
muse_flat.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  *
5  * This file is part of the MUSE Instrument Pipeline
6  * Copyright (C) 2005-2011 European Southern Observatory
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21  */
22 
23 #ifdef HAVE_CONFIG_H
24 #include <config.h>
25 #endif
26 
27 /*----------------------------------------------------------------------------*
28  * Includes *
29  *----------------------------------------------------------------------------*/
30 #include <stdio.h>
31 #include <float.h>
32 #include <math.h>
33 #include <string.h>
34 #include <cpl.h>
35 #include <muse.h>
36 #include "muse_flat_z.h"
37 
38 /*---------------------------------------------------------------------------*
39  * Functions code *
40  *---------------------------------------------------------------------------*/
41 
42 /*---------------------------------------------------------------------------*/
48 /*---------------------------------------------------------------------------*/
49 static void
50 muse_flat_qc_header(muse_image *aImage, muse_imagelist *aList)
51 {
52  cpl_msg_debug(__func__, "Adding QC keywords");
53 
54  unsigned stats = CPL_STATS_MEDIAN | CPL_STATS_MEAN | CPL_STATS_STDEV
55  | CPL_STATS_MIN | CPL_STATS_MAX;
56 
57  /* write the image statistics for input flat field exposures */
58  unsigned int i;
59  for (i = 0; i < muse_imagelist_get_size(aList); i++) {
60  char *keyword = cpl_sprintf(QC_FLAT_PREFIXi, i+1);
62  aImage->header, keyword, stats);
63  cpl_free(keyword);
64  keyword = cpl_sprintf(QC_FLAT_PREFIXi" "QC_BASIC_NSATURATED, i+1);
65  int nsaturated = cpl_propertylist_get_int(muse_imagelist_get(aList, i)->header,
66  MUSE_HDR_TMP_NSAT);
67  cpl_propertylist_update_int(aImage->header, keyword, nsaturated);
68  cpl_free(keyword);
69  } /* for i (all images in list) */
70 
71  /* properties of the resulting master frame */
72  stats |= CPL_STATS_FLUX;
74  QC_FLAT_MASTER_PREFIX, stats);
75 } /* muse_flat_qc_header() */
76 
77 /*---------------------------------------------------------------------------*/
83 /*---------------------------------------------------------------------------*/
84 static void
85 muse_flat_qc_trace_header(cpl_propertylist *aHeader, const cpl_table *aTrace)
86 {
87  if (!aHeader || !aTrace) {
88  return;
89  }
90  unsigned char ifu = muse_utils_get_ifu(aHeader);
91  cpl_msg_debug(__func__, "Adding tracing QC keywords for IFU %hhu", ifu);
92 
93  /* create vector for gap statistics */
94  cpl_array *gaps = cpl_array_new(kMuseSlicesPerCCD - 1, CPL_TYPE_DOUBLE);
95  double redge = -1; /* keep right edge of slice */
96  int nslice;
97  for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
98  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
99  nslice);
100  if (!ptrace) {
101  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
102  "missing!", nslice, ifu);
103  continue;
104  }
105 
106  /* ask for the center of the slice at the vertical center of the CCD */
107  float xpos = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
108  kMuseOutputYTop/2, NULL);
109  if (xpos < 1 || xpos > kMuseOutputXRight || !isnormal(xpos)) {
110  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty polynomial "
111  "detected at y=%d (center: %f)", nslice, ifu,
112  kMuseOutputYTop/2, xpos);
113  muse_trace_polys_delete(ptrace);
114  continue; /* next slice */
115  }
116  /* also may need the centers of the trace *
117  * at bottom and top for tilt computation: */
118  double x1 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER], 1, NULL),
119  x2 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
120  kMuseOutputYTop, NULL);
121  float tilt = atan((x2 - x1) / (kMuseOutputYTop - 1)) * CPL_MATH_DEG_RAD;
122  if (nslice == 1) {
123  cpl_propertylist_append_float(aHeader, QC_TRACE_L_XPOS, xpos);
124  cpl_propertylist_append_float(aHeader, QC_TRACE_L_TILT, tilt);
125  } else if (nslice == kMuseSlicesPerCCD) {
126  cpl_propertylist_append_float(aHeader, QC_TRACE_R_XPOS, xpos);
127  cpl_propertylist_append_float(aHeader, QC_TRACE_R_TILT, tilt);
128  }
129  if (redge > 0) {
130  /* the gap is the difference between the left edge of the *
131  * current slice and the right edge of the previous slice */
132  double gap = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
133  kMuseOutputYTop/2, NULL)
134  - redge;
135  cpl_array_set_double(gaps, nslice - 2, gap);
136  }
137  /* compute right edge for gap computation */
138  redge = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], kMuseOutputYTop/2,
139  NULL);
140 
141  /* Compute derivatives of all three tracing polynominals to *
142  * derive the maximum slope of the solutions within the CCD. */
143  double maxslope = 0.;
144  int ipoly, j, jmax = -1, ipolymax = -1;
145  for (ipoly = MUSE_TRACE_CENTER; ipoly <= MUSE_TRACE_RIGHT; ipoly++) {
146  cpl_polynomial_derivative(ptrace[ipoly], 0);
147  /* have to evaluate the polynomial at all positions to *
148  * get the maximum within the typical CCD boundaries... */
149  for (j = 1; j <= kMuseOutputYTop; j++) {
150  double slope = cpl_polynomial_eval_1d(ptrace[ipoly], j, NULL);
151  if (fabs(slope) > fabs(maxslope)) {
152  maxslope = slope;
153  jmax = j;
154  ipolymax = ipoly;
155  } /* if */
156  } /* for j (all positions in vertical direction) */
157  } /* for ipoly (all tracing polynomials) */
158 #define MAX_NORMAL_SLOPE 0.015
159  if (fabs(maxslope) > MAX_NORMAL_SLOPE) {
160  cpl_msg_warning(__func__, "Slope in slice %2d of IFU %hhu is unusually "
161  "high (maximum in polynomial %d at y=%d is |%f| > %f)",
162  nslice, ifu, ipolymax, jmax, maxslope, MAX_NORMAL_SLOPE);
163  }
164  char *kw = cpl_sprintf(QC_TRACE_SLICE_MAXSLOPE, nslice);
165  cpl_propertylist_append_float(aHeader, kw, maxslope);
166  cpl_free(kw);
167  muse_trace_polys_delete(ptrace);
168  } /* for nslice */
169 
170  float median, mean, stdev, min, max;
171  /* now add the gap statistics as QC parameters */
172  median = cpl_array_get_median(gaps);
173  mean = cpl_array_get_mean(gaps);
174  stdev = cpl_array_get_stdev(gaps);
175  min = cpl_array_get_min(gaps);
176  max = cpl_array_get_max(gaps);
177  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MEDIAN, median);
178  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MEAN, mean);
179  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_STDEV, stdev);
180  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MIN, min);
181  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MAX, max);
182  cpl_array_delete(gaps);
183 
184  /* add the slice widths also as QC paramters */
185  median = cpl_table_get_column_median(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
186  mean = cpl_table_get_column_mean(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
187  stdev = cpl_table_get_column_stdev(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
188  min = cpl_table_get_column_min(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
189  max = cpl_table_get_column_max(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
190  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MEDIAN, median);
191  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MEAN, mean);
192  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_STDEV, stdev);
193  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MIN, min);
194  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MAX, max);
195 } /* muse_flat_qc_trace_header() */
196 
197 /*---------------------------------------------------------------------------*/
203 /*---------------------------------------------------------------------------*/
204 static void
205 muse_flat_qc_per_slice(muse_image *aImage, const cpl_table *aTrace)
206 {
207  if (!aImage->header || !aTrace) {
208  return;
209  }
210  unsigned char ifu = muse_utils_get_ifu(aImage->header);
211  cpl_msg_debug(__func__, "Adding per-slice QC statistics for IFU %hhu", ifu);
212 
213  /* common vertical region in each slice, around the slice center */
214  const int y1 = kMuseOutputYTop/2 - 100,
215  y2 = kMuseOutputYTop/2 + 100;
216  int nslice;
217  for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
218  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
219  nslice);
220  if (!ptrace) {
221  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
222  "missing!", nslice, ifu);
223  continue;
224  }
225  char *kwmean = cpl_sprintf(QC_FLAT_MASTER_SLICEj_MEAN, nslice),
226  *kwstdev = cpl_sprintf(QC_FLAT_MASTER_SLICEj_STDEV, nslice);
227  /* horizontal region for this slice */
228  int x1 = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
229  kMuseOutputYTop/2, NULL)) + 1,
230  x2 = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
231  kMuseOutputYTop/2, NULL)) - 1;
232  if (x1 < 1 || x2 > kMuseOutputXRight || x1 > x2) {
233  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty polynomial "
234  "detected at y=%d (borders: %d ... %d)", nslice, ifu,
235  kMuseOutputYTop/2, x1, x2);
236  muse_trace_polys_delete(ptrace);
237  continue; /* next slice */
238  }
239  double mean = cpl_image_get_mean_window(aImage->data, x1, y1, x2, y2),
240  stdev = cpl_image_get_stdev_window(aImage->data, x1, y1, x2, y2);
241  cpl_propertylist_update_float(aImage->header, kwmean, mean);
242  cpl_propertylist_update_float(aImage->header, kwstdev, stdev);
243 
244  muse_trace_polys_delete(ptrace);
245  cpl_free(kwmean);
246  cpl_free(kwstdev);
247  } /* for nslice */
248 } /* muse_flat_qc_per_slice() */
249 
250 /*---------------------------------------------------------------------------*/
258 /*---------------------------------------------------------------------------*/
259 static int
260 muse_flat_trace_badpix(muse_processing *aProcessing,
261  muse_flat_params_t *aParams, muse_image *aImage)
262 {
263  cpl_table *samples = NULL;
264  cpl_table *tracetable = muse_trace(aImage, aParams->nsum, aParams->edgefrac,
265  aParams->order,
266  aParams->samples ? &samples : NULL);
267  if (!tracetable) {
268  cpl_table_delete(samples); /* just in case... */
269  return cpl_error_get_code();
270  }
271 
272  /* create table header by copying most of the image header */
273  cpl_propertylist *header = cpl_propertylist_duplicate(aImage->header);
274  cpl_propertylist_erase_regexp(header,
275  "^SIMPLE$|^BITPIX$|^NAXIS|^EXTEND$|^XTENSION$|"
276  "^DATASUM$|^DATAMIN$|^DATAMAX$|^DATAMD5$|"
277  "^PCOUNT$|^GCOUNT$|^HDUVERS$|^BLANK$|"
278  "^BZERO$|^BSCALE$|^BUNIT$|^CHECKSUM$|^INHERIT$|"
279  "^PIPEFILE$|^ESO QC |^ESO PRO ", 0);
280 
281  muse_flat_qc_trace_header(header, tracetable);
282  int rc = muse_processing_save_table(aProcessing, aParams->nifu, tracetable,
283  header, MUSE_TAG_TRACE_TABLE,
285  if (samples) {
286  /* don't want to save QC headers with the samples table: */
287  cpl_propertylist_erase_regexp(header, QC_TRACE_PREFIX, 0);
288  muse_processing_save_table(aProcessing, aParams->nifu, samples, header,
289  MUSE_TAG_TRACE_SAMPLES, MUSE_TABLE_TYPE_CPL);
290  cpl_table_delete(samples);
291  }
292  cpl_propertylist_delete(header);
293 
294  if (rc != CPL_ERROR_NONE) {
295  cpl_table_delete(tracetable);
296  return rc;
297  }
298 
299  /* if we have successfully traced, we can also search for dark pixels */
300  int nbad = muse_quality_flat_badpix(aImage, tracetable,
301  aParams->losigmabadpix,
302  aParams->hisigmabadpix);
303  cpl_msg_info(__func__, "found %d bad pixels in the flat-field image", nbad);
304 
305  /* add image statistics as QC now that we know the slice locations */
306  muse_flat_qc_per_slice(aImage, tracetable);
307 
308  cpl_table_delete(tracetable);
309  return rc;
310 } /* muse_flat_trace_badpix() */
311 
312 /*---------------------------------------------------------------------------*/
319 /*---------------------------------------------------------------------------*/
320 int
321 muse_flat_compute(muse_processing *aProcessing, muse_flat_params_t *aParams)
322 {
324  "muse.muse_flat");
325  muse_imagelist *images = muse_basicproc_load(aProcessing, aParams->nifu, bpars);
327  cpl_ensure(images, cpl_error_get_code(), -1);
328 
329  muse_combinepar *cpars = muse_combinepar_new(aProcessing->parameters,
330  "muse.muse_flat");
331  muse_image *masterimage = muse_combine_images(cpars, images);
332  muse_combinepar_delete(cpars);
333  if (!masterimage) {
334  cpl_msg_error(__func__, "Combining input frames failed!");
335  muse_imagelist_delete(images);
336  return -1;
337  }
338  cpl_propertylist_erase_regexp(masterimage->header, MUSE_WCS_KEYS, 0);
339 
340  muse_flat_qc_header(masterimage, images);
341  muse_imagelist_delete(images);
342 
343  int rc = CPL_ERROR_NONE;
344  if (aParams->trace) {
345  /* trace and search for bad pixels; do this before saving the *
346  * masterimage, so that we can add bad pixels to its DQ extension */
347  rc = muse_flat_trace_badpix(aProcessing, aParams, masterimage);
348  if (rc != CPL_ERROR_NONE) {
349  /* warn but don't return yet; tracing is important but the user should *
350  * still be able to get the masterimage saved to take a look at it */
351  cpl_msg_warning(__func__, "Tracing/bad pixel search failed");
352  }
353  }
354 
355  /* if requested, normalize the output flat-field to 1 using the average value */
356  if (aParams->normalize) {
357  double mean = cpl_propertylist_get_float(masterimage->header,
358  QC_FLAT_MASTER_PREFIX" MEAN");
359  muse_image_scale(masterimage, 1. / mean);
360  }
361 
362  muse_basicproc_qc_saturated(masterimage, QC_FLAT_MASTER_PREFIX);
363  rc = muse_processing_save_image(aProcessing, aParams->nifu, masterimage,
364  MUSE_TAG_MASTER_FLAT);
365  muse_image_delete(masterimage);
366  return rc == CPL_ERROR_NONE ? 0 : -1;
367 } /* muse_flat_compute() */
muse_imagelist * muse_basicproc_load(muse_processing *aProcessing, unsigned char aIFU, muse_basicproc_params *aBPars)
Load the raw input files from disk and do basic processing.
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
int nifu
IFU to handle. If set to 0, all IFUs are processed serially. If set to -1, all IFUs are processed in ...
Definition: muse_flat_z.h:50
int order
Order of polynomial fit to the trace.
Definition: muse_flat_z.h:97
Structure definition for a collection of muse_images.
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:84
int muse_quality_flat_badpix(muse_image *aFlat, cpl_table *aTrace, double aSigmaLo, double aSigmaHi)
Find bad (especially dark) pixels (in a master flat).
Definition: muse_quality.c:315
int muse_image_scale(muse_image *aImage, double aScale)
Scale a muse_image with correct treatment of variance.
Definition: muse_image.c:687
cpl_table * muse_trace(const muse_image *aImage, int aNSum, double aEdgeFrac, int aFitorder, cpl_table **aSamples)
carry out the tracing of the slices on CCD, save parameters in table
Definition: muse_tracing.c:918
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
Definition: muse_utils.c:95
cpl_image * data
the data extension
Definition: muse_image.h:47
int normalize
Normalize the master flat to the average flux.
Definition: muse_flat_z.h:88
Structure to hold the parameters of the muse_flat recipe.
Definition: muse_flat_z.h:48
void muse_imagelist_delete(muse_imagelist *aList)
Free the memory of the MUSE image list.
muse_basicproc_params * muse_basicproc_params_new(cpl_parameterlist *aParameters, const char *aPrefix)
Create a new structure of basic processing parameters.
muse_image * muse_combine_images(muse_combinepar *aCPars, muse_imagelist *aImages)
Combine several images into one.
Definition: muse_combine.c:742
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:41
void muse_basicproc_params_delete(muse_basicproc_params *aBPars)
Free a structure of basic processing parameters.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:73
unsigned int muse_imagelist_get_size(muse_imagelist *aList)
Return the number of stored images.
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
void muse_combinepar_delete(muse_combinepar *aCPars)
Clear the combination parameters.
Definition: muse_combine.c:716
muse_image * muse_imagelist_get(muse_imagelist *aList, unsigned int aIdx)
Get the muse_image of given list index.
int nsum
Number of lines over which to average when tracing.
Definition: muse_flat_z.h:94
int samples
Create a table containing all tracing sample points.
Definition: muse_flat_z.h:109
muse_combinepar * muse_combinepar_new(cpl_parameterlist *aParameters, const char *aPrefix)
Create a new set of combination parameters.
Definition: muse_combine.c:673
cpl_error_code muse_basicproc_stats_append_header(cpl_image *aImage, cpl_propertylist *aHeader, const char *aPrefix, unsigned aStats)
Compute image statistics of an image and add them to a header.
int muse_processing_save_image(muse_processing *aProcessing, int aIFU, muse_image *aImage, const char *aTag)
Save a computed MUSE image to disk.
double edgefrac
Fractional change required to identify edge when tracing.
Definition: muse_flat_z.h:100
double losigmabadpix
Low sigma to find dark pixels in the master flat.
Definition: muse_flat_z.h:103
cpl_error_code muse_processing_save_table(muse_processing *aProcessing, int aIFU, void *aTable, cpl_propertylist *aHeader, const char *aTag, muse_table_type aType)
Save a computed table to disk.
int trace
Trace the position of the slices on the master flat.
Definition: muse_flat_z.h:91
Structure of basic processing parameters.
cpl_parameterlist * parameters
double hisigmabadpix
High sigma to find bright pixels in the master flat.
Definition: muse_flat_z.h:106
cpl_error_code muse_basicproc_qc_saturated(muse_image *aImage, const char *aPrefix)
Add QC parameter about saturated pixels to a muse_image.