MUSE Pipeline Reference Manual  0.18.5
muse_wavecalib.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*----------------------------------------------------------------------------*
27  * Includes *
28  *----------------------------------------------------------------------------*/
29 #if HAVE_POPEN && HAVE_PCLOSE
30 #define _BSD_SOURCE /* force popen/pclose, mkdtemp definitions from stdio/stdlib */
31 #endif
32 #include <cpl.h>
33 #include <math.h>
34 #include <string.h>
35 
36 #include "muse_wavecalib.h"
37 #include "muse_instrument.h"
38 
39 #include "muse_combine.h"
40 #include "muse_cplwrappers.h"
41 #include "muse_dfs.h"
42 #include "muse_pfits.h"
43 #include "muse_tracing.h"
44 #include "muse_utils.h"
45 
46 /*----------------------------------------------------------------------------*
47  * Debugging Macros *
48  * Set these to 1 or higher for (lots of) debugging output *
49  *----------------------------------------------------------------------------*/
50 #define SEARCH_DEBUG 0 /* debugging in muse_wave_lines_search(), 1 or higher */
51 #define SEARCH_DEBUG_FILES 0 /* save different versions of the columns into *
52  * FITS files in muse_wave_lines_search() */
53 #define DEBUG_GAUSSFIT 0 /* debugging the Gaussian fit in muse_wave_lines_search() */
54 #define MUSE_WAVE_LINES_SEARCH_SHIFT_WARN 3.0 /* [pix] output debug message if *
55  * a shift of more than this occurs */
56 #define MUSE_WAVE_LINE_FIT_MAXSHIFT 2.0 /* [pix] don't use the fit when a shift *
57  * of more than this is detected */
58 #define MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT 0.25 /* [pix] maximum shift between *
59  * adjacent CCD columns */
60 
61 /*----------------------------------------------------------------------------*/
65 /*----------------------------------------------------------------------------*/
66 
69 /*----------------------------------------------------------------------------*/
94 /*----------------------------------------------------------------------------*/
96  { "lampno", CPL_TYPE_INT, "", "%d", "Number of the lamp", CPL_TRUE },
97  { "lampname", CPL_TYPE_STRING, "", "%s", "Name of the lamp", CPL_TRUE },
98  { "x", CPL_TYPE_DOUBLE, "pix", "%.2f", "x-position on CCD", CPL_TRUE },
99  { "y", CPL_TYPE_DOUBLE, "pix", "%.2f", "first-guess y-position on CCD", CPL_TRUE },
100  { "peak", CPL_TYPE_DOUBLE, "pix", "%g", "Peak of line", CPL_TRUE },
101  { "center", CPL_TYPE_DOUBLE, "pix", "%.4f", "Gaussian line center", CPL_TRUE },
102  { "cerr", CPL_TYPE_DOUBLE, "pix", "%.4f", "error estimate of line center", CPL_TRUE },
103  { "sigma", CPL_TYPE_DOUBLE, "pix", "%.3f", "Gaussian sigma", CPL_TRUE },
104  { "fwhm", CPL_TYPE_DOUBLE, "pix", "%.3f", "Gaussian FWHM", CPL_TRUE },
105  { "flux", CPL_TYPE_DOUBLE, "count", "%g", "Gaussian area (flux)", CPL_TRUE },
106  { "bg", CPL_TYPE_DOUBLE, "count", "%.2f", "background level", CPL_TRUE },
107  { "mse", CPL_TYPE_DOUBLE, "", "%e", "mean squared error", CPL_TRUE },
108  { "lambda", CPL_TYPE_DOUBLE, "Angstrom", "%9.3f",
109  "identified wavelength of the line", CPL_TRUE },
110  { NULL, 0, NULL, NULL, NULL, CPL_FALSE }
111 };
112 
113 /*----------------------------------------------------------------------------*/
117 /*----------------------------------------------------------------------------*/
119  { "slice", CPL_TYPE_INT, "", "%02d", "slice number", CPL_TRUE},
120  { "iteration", CPL_TYPE_INT, "", "%d", "iteration", CPL_TRUE},
121  { "x", CPL_TYPE_INT, "pix", "%04d", "x-position on CCD", CPL_TRUE},
122  { "y", CPL_TYPE_FLOAT, "pix", "%8.3f", "y-position on CCD", CPL_TRUE},
123  { "lambda", CPL_TYPE_FLOAT, "Angstrom", "%8.3f", "wavelength", CPL_TRUE},
124  { "residual", CPL_TYPE_DOUBLE, "Angstrom", "%.4e", "residual at this point", CPL_TRUE},
125  { "rejlimit", CPL_TYPE_DOUBLE, "Angstrom", "%.4e",
126  "rejection limit for this iteration", CPL_TRUE},
127  { NULL, 0, NULL, NULL, NULL, CPL_FALSE }
128 };
129 
130 static void muse_wave_lines_add_flux_for_lsf(cpl_table *, cpl_table *);
131 
132 /* corresponds to muse_wave_weighting_type, *
133  * keep in sync with those values! */
134 const char *muse_wave_weighting_string[] = {
135  "uniform",
136  "centroid error",
137  "per-line RMS scatter",
138  "centroid error plus per-line RMS scatter"
139 };
140 
141 /*----------------------------------------------------------------------------*/
154 /*----------------------------------------------------------------------------*/
157 {
158  muse_wave_params *p = cpl_malloc(sizeof(muse_wave_params));
159  /* set defaults */
160  p->xorder = 2;
161  p->yorder = 6;
162  p->detsigma = 1.;
163  p->ddisp = 0.05; /* from 1.20...1.30 Angstrom/pixel */
164  p->tolerance = 0.1;
165  p->linesigma = -1.;
166  p->rflag = CPL_FALSE; /* do not create a residuals table by default */
167  p->residuals = NULL;
168  p->fitsigma = -1.;
170  p->targetrms = 0.03;
171  return p;
172 }
173 
174 /*----------------------------------------------------------------------------*/
184 /*----------------------------------------------------------------------------*/
185 void
187 {
188  if (!aParams) {
189  return;
190  }
191  cpl_table_delete(aParams->residuals);
192  aParams->residuals = NULL;
193  memset(aParams, 0, sizeof(muse_wave_params)); /* null out everything */
194  cpl_free(aParams);
195 }
196 
197 /*----------------------------------------------------------------------------*/
209 /*----------------------------------------------------------------------------*/
210 static void
211 muse_wave_calib_qc_peaks(cpl_table *aLines, cpl_propertylist *aHeader,
212  const unsigned short aSlice)
213 {
214  char keyword[KEYWORD_LENGTH];
215  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MEAN, aSlice);
216  cpl_propertylist_append_float(aHeader, keyword,
217  cpl_table_get_column_mean(aLines, "peak"));
218  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_STDEV, aSlice);
219  cpl_propertylist_append_float(aHeader, keyword,
220  cpl_table_get_column_stdev(aLines, "peak"));
221  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MIN, aSlice);
222  cpl_propertylist_append_float(aHeader, keyword,
223  cpl_table_get_column_min(aLines, "peak"));
224  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MAX, aSlice);
225  cpl_propertylist_append_float(aHeader, keyword,
226  cpl_table_get_column_max(aLines, "peak"));
227  if (aSlice != 20) {
228  return;
229  }
230  /* if we are in slice 20 (a slice that is near the middle of a stack, so
231  * should never be vignetted loop through all possible lamps */
232  int n, nlamps = muse_pfits_get_lampnum(aHeader);
233  for (n = 1; n < nlamps; n++) {
234  /* select and extract all table entries matching the lamp */
235  cpl_table_unselect_all(aLines);
236  cpl_table_or_selected_int(aLines, "lampno", CPL_EQUAL_TO, n);
237  cpl_table *lamplines = cpl_table_extract_selected(aLines);
238  if (cpl_table_get_nrow(lamplines) < 1) {
239  cpl_table_delete(lamplines);
240  continue;
241  }
242  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_MEAN,
243  aSlice, n);
244  cpl_propertylist_append_float(aHeader, keyword,
245  cpl_table_get_column_mean(lamplines, "peak"));
246  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_STDEV,
247  aSlice, n);
248  cpl_propertylist_append_float(aHeader, keyword,
249  cpl_table_get_column_stdev(lamplines, "peak"));
250  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_MAX,
251  aSlice, n);
252  cpl_propertylist_append_float(aHeader, keyword,
253  cpl_table_get_column_max(lamplines, "peak"));
254  cpl_table_delete(lamplines);
255  } /* for n (all lamps) */
256 } /* muse_wave_calib_qc_peaks() */
257 
258 /*----------------------------------------------------------------------------*/
267 /*----------------------------------------------------------------------------*/
268 static void
269 muse_wave_calib_qc_fwhm_old(cpl_table *aLines, cpl_propertylist *aHeader,
270  cpl_vector *aRefLam, const unsigned short aSlice)
271 {
272  int nlines = cpl_table_get_nrow(aLines);
273 
274  /* track the minimum and maximum resolutions and their wavelengths */
275  double Rmin = DBL_MAX, Rmax = -DBL_MAX, wlmin = DBL_MAX, wlmax = -DBL_MAX;
276 
277  /* convert the properties of the relevant lines from the respective *
278  * table columns into vectors for easy statistics computation */
279  cpl_vector *fwhms = cpl_vector_new(nlines),
280  *resol = cpl_vector_new(cpl_vector_get_size(aRefLam));
281  int i, iresol = 0;
282  for (i = 0; i < nlines; i++) {
283  /* FWHM is directly accessible, but needs to be converted to Angstrom */
284  cpl_errorstate prestate = cpl_errorstate_get();
285  double fwhm = cpl_table_get(aLines, "fwhm", i, NULL), /* in [pix] */
286  sampling = kMuseSpectralSamplingA, /* sensible default in [A/pix] */
287  lambda = cpl_table_get(aLines, "lambda", i, NULL),
288  s1 = (lambda - cpl_table_get(aLines, "lambda", i - 1, NULL))
289  / (cpl_table_get(aLines, "center", i, NULL)
290  - cpl_table_get(aLines, "center", i - 1, NULL)),
291  s2 = (cpl_table_get(aLines, "lambda", i + 1, NULL)
292  - cpl_table_get(aLines, "lambda", i, NULL))
293  / (cpl_table_get(aLines, "center", i + 1, NULL)
294  - cpl_table_get(aLines, "center", i, NULL));
295  if (!cpl_errorstate_is_equal(prestate)) {
296  cpl_errorstate_set(prestate); /* reset "Access beyond boundaries" errors */
297  }
298  if (i == 0) { /* no lower one */
299  sampling = s2;
300  } else if (i == nlines - 1) { /* no higher one */
301  sampling = s1;
302  } else {
303  sampling = (s1 + s2) / 2.;
304  }
305  fwhm *= sampling; /* now in [A] */
306  cpl_vector_set(fwhms, i, fwhm);
307  /* spectral resolution R for this line R = lambda / dlambda */
308  double R = lambda / fwhm;
309  /* compare wavelength to those lines in the FWHM reference list */
310  if (fabs(cpl_vector_get(aRefLam, cpl_vector_find(aRefLam, lambda)) - lambda)
311  < FLT_EPSILON) {
312  if (R < Rmin) {
313  Rmin = R;
314  wlmin = lambda;
315  }
316  if (R > Rmax) {
317  Rmax = R;
318  wlmax = lambda;
319  }
320  cpl_vector_set(resol, iresol++, R);
321  }
322  } /* for i (all arc lines) */
323  cpl_vector_set_size(resol, iresol);
324 
325  char keyword[KEYWORD_LENGTH];
326  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MEAN, aSlice);
327  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_mean(fwhms));
328  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_STDEV, aSlice);
329  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_stdev(fwhms));
330  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MIN, aSlice);
331  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_min(fwhms));
332  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MAX, aSlice);
333  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_max(fwhms));
334  cpl_vector_delete(fwhms);
335 
336  cpl_msg_debug(__func__, "Average spectral resolution is R=%4.0f, ranging from "
337  "%4.0f (at %6.1fA) to %4.0f (at %6.1fA)",
338  cpl_vector_get_mean(resol), Rmin, wlmin, Rmax, wlmax);
339  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL, aSlice);
340  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_mean(resol));
341  cpl_vector_delete(resol);
342 } /* muse_wave_calib_qc_fwhm_old() */
343 
344 /*----------------------------------------------------------------------------*/
352 /*----------------------------------------------------------------------------*/
353 static void
354 muse_wave_calib_qc_fwhm(cpl_table *aFWHM, cpl_propertylist *aHeader,
355  const unsigned short aSlice)
356 {
357  /* compute the spectral resolution at each wavelength, as *
358  * R = lambda / fwhm *
359  * where both lambda and fwhm are [Angstrom] */
360  cpl_table_duplicate_column(aFWHM, "R", aFWHM, "lambda");
361  cpl_table_divide_columns(aFWHM, "R", "fwhm");
362  cpl_table_set_column_unit(aFWHM, "R", "");
363 
364 #if 0 /* AIT Gaussian FWHM */
365  /* range < 600 nm (as in PR5) */
366  cpl_table_unselect_all(aFWHM);
367  cpl_table_or_selected_double(aFWHM, "lambda", CPL_LESS_THAN, 6000.);
368  cpl_table *tblue = cpl_table_extract_selected(aFWHM);
369  /* range > 800 nm (as in PR5) */
370  cpl_table_unselect_all(aFWHM);
371  cpl_table_or_selected_double(aFWHM, "lambda", CPL_GREATER_THAN, 8000.);
372  cpl_table *tred = cpl_table_extract_selected(aFWHM);
373  /* range 600 - 800 nm (as in PR5) */
374  cpl_table_unselect_all(aFWHM);
375  cpl_table_or_selected_double(aFWHM, "lambda", CPL_NOT_LESS_THAN, 6000.);
376  cpl_table_and_selected_double(aFWHM, "lambda", CPL_NOT_GREATER_THAN, 8000.);
377  cpl_table *tgreen = cpl_table_extract_selected(aFWHM);
378 #if 0
379  printf("all:\n");
380  cpl_table_dump(aFWHM, 0, 100000, stdout);
381  fflush(stdout);
382  printf("blue:\n");
383  cpl_table_dump(tblue, 0, 100000, stdout);
384  fflush(stdout);
385  printf("green:\n");
386  cpl_table_dump(tgreen, 0, 100000, stdout);
387  fflush(stdout);
388  printf("red:\n");
389  cpl_table_dump(tred, 0, 100000, stdout);
390  fflush(stdout);
391 #endif
392 #endif /* AIT Gaussian FWHM */
393 
394  /* fill the QC parameters */
395  char keyword[KEYWORD_LENGTH];
396  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL, aSlice);
397  double rmean = cpl_table_get_column_mean(aFWHM, "R");
398  cpl_propertylist_update_float(aHeader, keyword, rmean);
399 
400  double fmean = cpl_table_get_column_mean(aFWHM, "fwhm"),
401  fstdev = cpl_table_get_column_stdev(aFWHM, "fwhm"),
402  flo = cpl_table_get_column_min(aFWHM, "fwhm"),
403  fhi = cpl_table_get_column_max(aFWHM, "fwhm");
404  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MEAN, aSlice);
405  cpl_propertylist_update_float(aHeader, keyword, fmean);
406  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_STDEV, aSlice);
407  cpl_propertylist_update_float(aHeader, keyword, fstdev);
408  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MIN, aSlice);
409  cpl_propertylist_update_float(aHeader, keyword, flo);
410  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MAX, aSlice);
411  cpl_propertylist_update_float(aHeader, keyword, fhi);
412 
413 #if 0 /* AIT Gaussian FWHM */
414  /* output all the stuff as debug messages */
415  cpl_msg_debug(__func__, "Gaussian FWHM [%s]:\n"
416  "\tFWHM all (%d values)\t%.3f +/- %.3f (%.3f) %.3f...%.3f\n"
417  "\tFWHM blue (%d values)\t%.3f +/- %.3f (%.3f)\n"
418  "\tFWHM green (%d values)\t%.3f +/- %.3f (%.3f)\n"
419  "\tFWHM red (%d values)\t%.3f +/- %.3f (%.3f)",
420  cpl_table_get_column_unit(aFWHM, "fwhm"),
421  (int)cpl_table_get_nrow(aFWHM), fmean, fstdev, flo, fhi,
422  cpl_table_get_column_median(aFWHM, "fwhm"),
423  (int)cpl_table_get_nrow(tblue),
424  cpl_table_get_column_mean(tblue, "fwhm"),
425  cpl_table_get_column_stdev(tblue, "fwhm"),
426  cpl_table_get_column_median(tblue, "fwhm"),
427  (int)cpl_table_get_nrow(tgreen),
428  cpl_table_get_column_mean(tgreen, "fwhm"),
429  cpl_table_get_column_stdev(tgreen, "fwhm"),
430  cpl_table_get_column_median(tgreen, "fwhm"),
431  (int)cpl_table_get_nrow(tred),
432  cpl_table_get_column_mean(tred, "fwhm"),
433  cpl_table_get_column_stdev(tred, "fwhm"),
434  cpl_table_get_column_median(tred, "fwhm"));
435  cpl_msg_debug(__func__, "Gaussian spectral resolution R:\n"
436  "\tR all (%d values)\t%.1f +/- %.1f (%.1f) %.1f...%.1f\n"
437  "\tR blue (%d values)\t%.1f +/- %.1f (%.1f)\n"
438  "\tR green (%d values)\t%.1f +/- %.1f (%.1f)\n"
439  "\tR red (%d values)\t%.1f +/- %.1f (%.1f)",
440  (int)cpl_table_get_nrow(aFWHM), rmean,
441  cpl_table_get_column_stdev(aFWHM, "R"),
442  cpl_table_get_column_median(aFWHM, "R"),
443  cpl_table_get_column_min(aFWHM, "R"),
444  cpl_table_get_column_max(aFWHM, "R"),
445  (int)cpl_table_get_nrow(tblue),
446  cpl_table_get_column_mean(tblue, "R"),
447  cpl_table_get_column_stdev(tblue, "R"),
448  cpl_table_get_column_median(tblue, "R"),
449  (int)cpl_table_get_nrow(tgreen),
450  cpl_table_get_column_mean(tgreen, "R"),
451  cpl_table_get_column_stdev(tgreen, "R"),
452  cpl_table_get_column_median(tgreen, "R"),
453  (int)cpl_table_get_nrow(tred),
454  cpl_table_get_column_mean(tred, "R"),
455  cpl_table_get_column_stdev(tred, "R"),
456  cpl_table_get_column_median(tred, "R"));
457  cpl_table_delete(tblue);
458  cpl_table_delete(tgreen);
459  cpl_table_delete(tred);
460 #endif /* AIT Gaussian FWHM */
461  cpl_table_erase_column(aFWHM, "R");
462 } /* muse_wave_calib_qc_fwhm() */
463 
464 /*----------------------------------------------------------------------------*/
476 /*----------------------------------------------------------------------------*/
477 static void
478 muse_wave_calib_qc_lambda(muse_image *aImage, const unsigned short aSlice,
479  cpl_polynomial **aTrace, cpl_polynomial *aFit)
480 {
481  int ny = cpl_image_get_size_y(aImage->data);
482 
483  /* wavelength differences at bottom and top of each slice as QC parameters */
484  double xbotl = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT], 1, NULL),
485  xbotr = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT], 1, NULL),
486  xtopl = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT], ny, NULL),
487  xtopr = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT], ny, NULL);
488  cpl_vector *pos = cpl_vector_new(2);
489  cpl_vector_set(pos, 0, xbotl);
490  cpl_vector_set(pos, 1, 1);
491  double wlbotl = cpl_polynomial_eval(aFit, pos);
492  cpl_vector_set(pos, 0, xbotr);
493  cpl_vector_set(pos, 1, 1);
494  double wlbotr = cpl_polynomial_eval(aFit, pos);
495  cpl_vector_set(pos, 0, xtopl);
496  cpl_vector_set(pos, 1, ny);
497  double wltopl = cpl_polynomial_eval(aFit, pos);
498  cpl_vector_set(pos, 0, xtopr);
499  cpl_vector_set(pos, 1, ny);
500  double wltopr = cpl_polynomial_eval(aFit, pos);
501 #if 0
502  cpl_msg_debug(__func__, "Wavelengths at slice corners: botl(%.2f,1)=%f, "
503  "botr(%.2f,1)=%f, topl(%.2f,%d)=%f, topr(%.2f,%d)=%f",
504  xbotl, wlbotl, xbotr, wlbotr,
505  xtopl, ny, wltopl, xtopr, ny, wltopr);
506 #endif
507  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_DWLEN_BOT, aSlice);
508  cpl_propertylist_append_float(aImage->header, keyword, wlbotl - wlbotr);
509  cpl_free(keyword);
510  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_DWLEN_TOP, aSlice);
511  cpl_propertylist_append_float(aImage->header, keyword, wltopl - wltopr);
512  cpl_free(keyword);
513 
514 #define DWLEN_WARN_LIMIT_N 6.5 /* 5.7 Angstrom differences can happen... */
515 #define DWLEN_WARN_LIMIT_E 5. /* ... at the bottom in nominal mode */
516  double limit = DWLEN_WARN_LIMIT_E;
517  cpl_errorstate prestate = cpl_errorstate_get();
518  if (muse_pfits_get_mode(aImage->header) != MUSE_MODE_WFM_NONAO_X) {
519  limit = DWLEN_WARN_LIMIT_N;
520  }
521  cpl_errorstate_set(prestate); /* swallow possible error about missing INS MODE */
522  if (fabs(wlbotl - wlbotr) > limit) {
523  cpl_msg_warning(__func__, "Wavelength differences at bottom of slice %hu "
524  "are large (%f Angstrom)!", aSlice, wlbotl - wlbotr);
525  }
526  if (fabs(wltopl - wltopr) > DWLEN_WARN_LIMIT_E) { /* strict limit at red end */
527  cpl_msg_warning(__func__, "Wavelength differences at top of slice %hu are "
528  "large (%f Angstrom)!", aSlice, wltopl - wltopr);
529  }
530 
531  /* generate some arbitrary evaluation point, approx. at *
532  * the slice center, and compute the wavelength there. */
533  const double yc = (1. + ny) / 2.,
534  xc = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_CENTER], yc, NULL);
535  cpl_vector_set(pos, 0, xc);
536  cpl_vector_set(pos, 1, yc);
537  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_WLPOS, aSlice);
538  cpl_propertylist_append_float(aImage->header, keyword, yc);
539  cpl_free(keyword);
540  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_WLEN, aSlice);
541  cpl_propertylist_append_float(aImage->header, keyword,
542  cpl_polynomial_eval(aFit, pos));
543  cpl_free(keyword);
544  cpl_vector_delete(pos);
545 } /* muse_wave_calib_qc_lambda() */
546 
547 /*----------------------------------------------------------------------------*/
560 /*----------------------------------------------------------------------------*/
561 static void
562 muse_wave_calib_output_summary(const cpl_propertylist *aHeader,
563  const cpl_table *aWave)
564 {
565  if (!aWave) {
566  cpl_msg_warning(__func__, "Wavelength solution missing, no summary!");
567  return;
568  }
569  cpl_vector *found = cpl_vector_new(kMuseSlicesPerCCD),
570  *ident = cpl_vector_new(kMuseSlicesPerCCD),
571  *used = cpl_vector_new(kMuseSlicesPerCCD),
572  *resolution = cpl_vector_new(kMuseSlicesPerCCD);
573  unsigned short islice;
574  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
575  char keyword[KEYWORD_LENGTH];
576  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_NDET, islice + 1);
577  cpl_vector_set(found, islice, cpl_propertylist_get_int(aHeader, keyword));
578  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_NID, islice + 1);
579  cpl_vector_set(ident, islice, cpl_propertylist_get_int(aHeader, keyword));
580  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_FIT_NLINES, islice + 1);
581  cpl_vector_set(used, islice, cpl_propertylist_get_int(aHeader, keyword));
582  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL, islice + 1);
583  cpl_vector_set(resolution, islice, cpl_propertylist_get_float(aHeader, keyword));
584  } /* for islice (all kMuseSlicesPerCCD) */
585  cpl_msg_info(__func__, "Summary of wavelength solution:\n"
586  "\tDetections per slice: %d ... %d\n"
587  "\tIdentified lines per slice: %d ... %d\n"
588  "\tLines per slice used in fit: %d ... %d\n"
589  "\tRMS of fit [Angstrom]: %5.3f +/- %5.3f (%5.3f ... %5.3f)\n"
590  "\tMean spectral resolution R: %6.1f +/- %5.1f",
591  (int)cpl_vector_get_min(found), (int)cpl_vector_get_max(found),
592  (int)cpl_vector_get_min(ident), (int)cpl_vector_get_max(ident),
593  (int)cpl_vector_get_min(used), (int)cpl_vector_get_max(used),
594  sqrt(cpl_table_get_column_mean(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
595  sqrt(cpl_table_get_column_stdev(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
596  sqrt(cpl_table_get_column_min(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
597  sqrt(cpl_table_get_column_max(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
598  cpl_vector_get_mean(resolution), cpl_vector_get_stdev(resolution));
599  cpl_vector_delete(found);
600  cpl_vector_delete(ident);
601  cpl_vector_delete(used);
602  cpl_vector_delete(resolution);
603 } /* muse_wave_calib_output_summary() */
604 
605 /*----------------------------------------------------------------------------*/
651 /*----------------------------------------------------------------------------*/
652 cpl_table *
653 muse_wave_calib(muse_image *aImage, cpl_table *aTrace, cpl_table *aLinelist,
654  muse_wave_params *aParams)
655 {
656  if (!aImage) {
657  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "arc image missing!");
658  return NULL;
659  }
660  /* variance always has to be larger than zero */
661  double minstat = cpl_image_get_min(aImage->stat);
662  if (minstat <= 0.) {
663  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "arc image %d does"
664  " not have valid STAT extension (minimum is %e)!", 0,
665  minstat);
666  return NULL;
667  }
668 
669  if (!aTrace) {
670  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "trace table missing,"
671  " cannot create wavelength calibration!");
672  return NULL;
673  } else if (cpl_table_get_nrow(aTrace) != kMuseSlicesPerCCD) {
674  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "trace table "
675  "not valid for this dataset!");
676  return NULL;
677  }
678 
679  if (!aLinelist) {
680  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "no arc line list "
681  "supplied!");
682  return NULL;
683  }
684  if (!aParams) {
685  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "wavelength "
686  "calibration parameters missing!");
687  return NULL;
688  }
690  cpl_error_set_message(__func__, CPL_ERROR_UNSUPPORTED_MODE,
691  "unknown weighting scheme");
692  return NULL;
693  }
694 
695  /* do not support smaller mock datasets any more */
696  int nx = cpl_image_get_size_x(aImage->data),
697  ny = cpl_image_get_size_y(aImage->data);
698  if (ny < 4000) {
699  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "this dataset is "
700  "too small (%dx%d pix) to be supported", nx, ny);
701  return NULL;
702  }
703 
704  cpl_msg_info(__func__, "Using polynomial orders %hu (x) and %hu (y), %s "
705  "weighting, assuming initial sampling of %.3f +/- %.3f Angstrom/pix",
706  aParams->xorder, aParams->yorder,
707  muse_wave_weighting_string[aParams->fitweighting],
708  kMuseSpectralSamplingA, aParams->ddisp);
709 
710  cpl_vector *vreflam = muse_wave_lines_get(aLinelist, 5, 0.);
711  if (!vreflam) {
712  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "could not "
713  "create list of FWHM reference arc wavelengths");
714  return NULL;
715  }
716 
717  cpl_propertylist_erase_regexp(aImage->header, "^ESO QC", 0);
718  cpl_table *wavecaltable = NULL;
719 
720  int debug = getenv("MUSE_DEBUG_WAVECAL")
721  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
722  const char *fn_debug = "MUSE_DEBUG_WAVE_LINES.ascii";
723  FILE *fp_debug = NULL;
724  if (debug >= 3) {
725  cpl_msg_info(__func__, "Will write all single line fits to \"%s\"", fn_debug);
726  fp_debug = fopen(fn_debug, "w");
727  if (fp_debug) {
728  fprintf(fp_debug, "#slice x y lambda lambdaerr\n");
729  }
730  }
731 
732  /* loop over all slices */
733  unsigned short islice;
734  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
735  if (debug > 0) {
736  printf("\n\nSlice %hu\n", islice + 1); fflush(stdout);
737  }
738  /* get the tracing polynomials for this slice */
739  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
740  islice + 1);
741  if (!ptrace) {
742  cpl_msg_warning(__func__, "slice %2hu: tracing polynomials missing!",
743  islice + 1);
744  continue;
745  }
746 
747  /* detect all lines in the center of the slice */
748  int imid = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
749  ny/2., NULL));
750  if (imid < 1 || imid > nx) {
751  cpl_msg_warning(__func__, "slice %2hu: faulty polynomial detected",
752  islice + 1);
753  muse_trace_polys_delete(ptrace);
754  continue; /* next slice */
755  }
756  /* create muse_image from the spectrum in which we want to search lines */
757  const int kWidth = 3;
758  int k, kcol1 = imid - kWidth/2, kcol2 = kcol1 + kWidth;
759  muse_imagelist *collist = muse_imagelist_new();
760  for (k = kcol1; k <= kcol2; k++) {
761  muse_image *column = muse_image_new();
762  column->data = cpl_image_extract(aImage->data, k, 1, k, ny);
763  column->dq = cpl_image_extract(aImage->dq, k, 1, k, ny);
764  column->stat = cpl_image_extract(aImage->stat, k, 1, k, ny);
765  column->header = cpl_propertylist_new();
766  cpl_propertylist_append_string(column->header, "BUNIT",
767  cpl_propertylist_get_string(aImage->header,
768  "BUNIT"));
769  muse_imagelist_set(collist, column, k - kcol1);
770  }
771  muse_image *column = muse_combine_median_create(collist);
772  muse_imagelist_delete(collist);
773 
774  /* now search, measure, and store the lines */
775  cpl_table *detlines = muse_wave_lines_search(column, aParams->detsigma,
776  islice + 1);
777  if (!detlines) {
778  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "problem when "
779  "searching for arc lines in slice %hu, columns "
780  "%d..%d", islice + 1, kcol1, kcol2);
781  muse_image_delete(column);
782  cpl_vector_delete(vreflam);
783  muse_trace_polys_delete(ptrace);
784  cpl_table_delete(wavecaltable);
785  cpl_table_delete(aParams->residuals);
786  aParams->residuals = NULL;
787  return NULL;
788  }
789  /* clean up only here, to not disturb error message above */
790  muse_image_delete(column);
791  if (debug >= 2) {
792  printf("Detected arc lines in slice %hu:\n", islice + 1);
793  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
794  fflush(stdout);
795  }
796 
797  /* identify the lines that we detected, and see if there are enough *
798  * for the polynomial order in y-direction to fit a polynomial */
799  int nfound = cpl_table_get_nrow(detlines);
800  /* get all arc lines, taking into account the detection limit */
801  double minflux = cpl_table_get_column_min(detlines, "flux") * 1.15;
802  cpl_vector *vlambda = muse_wave_lines_get(aLinelist, 1, minflux);
803  muse_wave_lines_identify(detlines, vlambda, aParams);
804  cpl_vector_delete(vlambda);
805  int nlines = cpl_table_get_nrow(detlines);
806  cpl_msg_debug(__func__, "Identified %d of %d arc lines in slice %hu (column "
807  "%d, %d..%d)", nlines, nfound, islice + 1, imid, kcol1, kcol2);
808  if (debug >= 2) {
809  printf("Identified arc lines with wavelengths in slice %hu:\n", islice + 1);
810  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
811  fflush(stdout);
812  }
813 
814  /* the first QC parameter does even make sense without lines */
815  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NDET, islice + 1);
816  cpl_propertylist_append_int(aImage->header, keyword, nfound);
817  cpl_free(keyword);
818 
819  if (nlines < aParams->yorder + 1) {
820  /* could apparently not identify enough lines, or an error occured */
821  cpl_msg_error(__func__, "Could not identify enough arc lines in slice %hu"
822  " (%d of %d, required %hu)", islice + 1, nlines, nfound,
823  aParams->yorder + 1);
824  muse_trace_polys_delete(ptrace);
825  cpl_table_delete(detlines);
826  continue; /* work on next slice immediately */
827  }
828 
829  /* the next set of QC parameters */
830  muse_wave_calib_qc_peaks(detlines, aImage->header, islice + 1);
831  muse_wave_calib_qc_fwhm_old(detlines, aImage->header, vreflam, islice + 1);
832  /* above we wrote the number of detected lines, now is the number of *
833  * arc lines really used in the fit, i.e. the identified ones */
834  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NID, islice + 1);
835  cpl_propertylist_append_int(aImage->header, keyword, nlines);
836  cpl_free(keyword);
837 
838  /* positions matrix and wavelengths vector to be used for the 2D polynomial *
839  * fit; just set and initial size of 1, it has to be resized with every line */
840  cpl_matrix *xypos = cpl_matrix_new(2, 1);
841  cpl_vector *lambdas = cpl_vector_new(1),
842  *dlambdas = NULL;
843  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
844  dlambdas = cpl_vector_new(1);
845  }
846  int ientry = 0; /* index for these two structures */
847 
848  /* for all identified arc lines, work on the original input data */
849  int j;
850  for (j = 0; j < nlines; j++) {
851  /* convenient access to some properties of this identified line */
852  double lambda = cpl_table_get(detlines, "lambda", j, NULL);
853  double ypos = cpl_table_get(detlines, "center", j, NULL);
854  /* better fix the Gaussian sigmas, otherwise *
855  * the fit might run astray in low S/N cases */
856  double sigma = cpl_table_get(detlines, "sigma", j, NULL);
857  int n = 0,
858  halfwidth = 2.*cpl_table_get(detlines, "fwhm", j, NULL); /* 2*FWHM */
859  /* get both slice edges and the center */
860  double dleft = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
861  ypos, NULL),
862  dright = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
863  ypos, NULL),
864  dmid = (dleft + dright) / 2.;
865  int ileft = ceil(dleft),
866  iright = floor(dright);
867 #if 0
868  cpl_msg_debug(__func__, "limits at y=%f: %f < %f < %f", ypos, dleft, dmid,
869  dright);
870 #endif
871 
872  /* table to store line fits for this one arc line */
873  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
874  (int)kMuseSliceHiLikelyWidth + 5);
875 
876  /* From the center of the slice move outwards and fit the line *
877  * until we have arrived at the edge of the slice */
878  int i;
879  for (i = dmid; i >= ileft; i--) {
880  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
881  sigma, fittable, ++n);
882  if (rc != CPL_ERROR_NONE) { /* do not count this line */
883  --n;
884  }
885  }
886 #if 0
887  printf("arc line j=%d, columns i=%d...", j + 1, i);
888 #endif
889  for (i = dmid + 1; i <= iright; i++) {
890  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
891  sigma, fittable, ++n);
892  if (rc != CPL_ERROR_NONE) { /* do not count this line */
893  --n;
894  }
895  }
896  /* now remove rows with invalid entries, i.e. those that were not *
897  * filled with the properties of the fit -- cpl_table_erase_invalid() *
898  * does not work, it deletes all columns */
899  cpl_table_select_all(fittable);
900  cpl_table_and_selected_invalid(fittable, "center");
901  cpl_table_erase_selected(fittable);
902 #if 0
903  printf("line %d, %d line fits\n", j + 1, i);
904  cpl_table_dump(fittable, 0, n, stdout);
905  fflush(stdout);
906 #endif
907  cpl_errorstate state = cpl_errorstate_get();
908  muse_wave_line_fit_iterate(fittable, -1, aParams);
909  int npos = cpl_table_get_nrow(fittable);
910  if (npos <= aParams->xorder) {
911  cpl_msg_debug(__func__, "Polynomial fit failed in slice %hu for "
912  "line at %.3fA (y-position near %.2f pix): %s",
913  islice + 1, lambda, ypos, cpl_error_get_message());
914  cpl_errorstate_set(state);
915  }
916 #if 0
917  else {
918  printf("%s: line %2d, %d line fits, %d with low residuals:\n", __func__,
919  j + 1, i, npos);
920  cpl_table_dump(fittable, 0, npos, stdout);
921  fflush(stdout);
922  }
923 #endif
924 
925  /* resize matrix/vector to be able to fit all new entries */
926  cpl_matrix_resize(xypos, 0, 0,
927  0, ientry + npos - cpl_matrix_get_ncol(xypos));
928  cpl_vector_set_size(lambdas, ientry + npos);
929  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
930  cpl_vector_set_size(dlambdas, ientry + npos);
931  }
932 
933  /* now add the final fit positions */
934  int ipos;
935  for (ipos = 0; ipos < npos; ipos++) {
936  /* set x-position (CCD column) in the first matrix row */
937  cpl_matrix_set(xypos, 0, ientry, cpl_table_get(fittable, "x", ipos, NULL));
938  /* y-position on CCD (center of Gauss fit) into second matrix row */
939  cpl_matrix_set(xypos, 1, ientry, cpl_table_get(fittable, "center", ipos, NULL));
940  /* the vector has to contain as many lambda entries */
941  cpl_vector_set(lambdas, ientry, lambda);
942  /* pretend that errors in Gaussian fit are errors in wavelength, *
943  * scale according to the nominal Angstrom/pix sampling of MUSE, *
944  * everything else would be too complicated for little gain */
945  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
946  cpl_vector_set(dlambdas, ientry, cpl_table_get(fittable, "cerr", ipos, NULL)
947  * kMuseSpectralSamplingA);
948  }
949 
950  ientry++; /* next position in this matrix/vector combo */
951  if (fp_debug) {
952  fprintf(fp_debug, " %02hu %04d %9.4f %10.4f %e\n", islice + 1,
953  (int)cpl_table_get(fittable, "x", ipos, NULL),
954  cpl_table_get(fittable, "center", ipos, NULL), lambda,
955  cpl_table_get(fittable, "cerr", ipos, NULL) * kMuseSpectralSamplingA);
956  }
957  }
958  cpl_table_delete(fittable);
959  } /* for j (all identified arc lines) */
960  muse_wave_lines_add_flux_for_lsf(aLinelist, detlines);
961  cpl_table_delete(detlines);
962 
963  /* Compute two-dimensional wavelength solution for each slice. */
964  cpl_polynomial *poly = NULL; /* polynomial for wavelength solution */
965  double mse = 0; /* mean squared error */
966  cpl_error_code rc = muse_wave_poly_fit(xypos, lambdas, dlambdas,
967  &poly, &mse, aParams, islice + 1);
968  cpl_matrix_delete(xypos);
969  int nfinal = muse_cplvector_count_unique(lambdas);
970  cpl_vector_delete(lambdas);
971  cpl_vector_delete(dlambdas);
972  /* above we wrote detected and identified lines, here *
973  * save the ones that actually survived the fit */
974  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_NLINES, islice + 1);
975  cpl_propertylist_append_int(aImage->header, keyword, nfinal);
976  cpl_free(keyword);
977  /* collect QC on wavelengths */
978  muse_wave_calib_qc_lambda(aImage, islice + 1, ptrace, poly);
979  /* trace polynomials are not needed further */
980  muse_trace_polys_delete(ptrace);
981 
982  if (rc != CPL_ERROR_NONE) { /* failure */
983  cpl_msg_warning(__func__, "Something went wrong while fitting: %s",
984  cpl_error_get_message_default(rc));
985  cpl_polynomial_delete(poly);
986  continue; /* try next slice immediately */
987  }
988 
989  if (!wavecaltable) {
990  /* create output table with one row for each slice; no need to *
991  * check return code, it can only fail for negative lengths */
992  wavecaltable = muse_wave_table_create(kMuseSlicesPerCCD,
993  aParams->xorder, aParams->yorder);
994  }
995  rc = muse_wave_table_add_poly(wavecaltable, poly, mse,
996  aParams->xorder, aParams->yorder, islice);
997 
998  /* use the number of wavelengths as indicator of the lines used for the *
999  * fit and record it as QC parameter */
1000  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_RMS, islice + 1);
1001  cpl_propertylist_append_float(aImage->header, keyword, sqrt(mse));
1002  cpl_free(keyword);
1003 
1004  if (rc != CPL_ERROR_NONE) {
1005  cpl_msg_warning(__func__, "Could not write polynomial to wavecal table: %s",
1006  cpl_error_get_message_default(rc));
1007  }
1008  /* clean up, ignore any failure with this */
1009  cpl_polynomial_delete(poly);
1010  } /* for islice (all kMuseSlicesPerCCD) */
1011  cpl_vector_delete(vreflam);
1012  if (fp_debug) {
1013  cpl_msg_info(__func__, "Done writing line fits to \"%s\"", fn_debug);
1014  fclose(fp_debug);
1015  }
1016 
1017  muse_wave_calib_output_summary(aImage->header, wavecaltable);
1018 #if 0
1019  if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
1020  cpl_table_dump(wavecaltable, 0, 3, stdout); fflush(stdout);
1021  }
1022 #endif
1023 
1024  return wavecaltable;
1025 } /* muse_wave_calib() */
1026 
1027 /*----------------------------------------------------------------------------*/
1091 /*----------------------------------------------------------------------------*/
1092 cpl_table *
1093 muse_wave_calib_lampwise(muse_imagelist *aImages, cpl_table *aTrace,
1094  cpl_table *aLinelist, muse_wave_params *aParams)
1095 {
1096  if (!aImages || muse_imagelist_get_size(aImages) < 1) {
1097  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "arc imagelist is "
1098  "missing or empty!");
1099  return NULL;
1100  }
1101  if (!aTrace) {
1102  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "trace table missing,"
1103  " cannot create wavelength calibration!");
1104  return NULL;
1105  } else if (cpl_table_get_nrow(aTrace) != kMuseSlicesPerCCD) {
1106  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "trace table "
1107  "not valid for this dataset!");
1108  return NULL;
1109  }
1110  if (!aLinelist) {
1111  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "no arc line list "
1112  "supplied!");
1113  return NULL;
1114  }
1115  if (!aParams) {
1116  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "wavelength "
1117  "calibration parameters missing!");
1118  return NULL;
1119  }
1121  cpl_error_set_message(__func__, CPL_ERROR_UNSUPPORTED_MODE,
1122  "unknown weighting scheme");
1123  return NULL;
1124  }
1125 
1126  muse_image *firstimage = muse_imagelist_get(aImages, 0);
1127  int nx = cpl_image_get_size_x(firstimage->data),
1128  ny = cpl_image_get_size_y(firstimage->data);
1129  /* do not support binned exposures */
1130  if (ny < 4000) {
1131  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "this dataset is "
1132  "too small (ny = %d pix) to be supported", ny);
1133  return NULL;
1134  }
1135 
1136  cpl_msg_info(__func__, "Using polynomial orders %hu (x) and %hu (y), %s "
1137  "weighting, assuming initial sampling of %.3f +/- %.3f Angstrom/pix",
1138  aParams->xorder, aParams->yorder,
1139  muse_wave_weighting_string[aParams->fitweighting],
1140  kMuseSpectralSamplingA, aParams->ddisp);
1141 
1142  cpl_propertylist_erase_regexp(firstimage->header, "^ESO QC", 0);
1143  char *lamp = muse_utils_header_get_lamp_names(firstimage->header, ',');
1144  cpl_msg_debug(__func__, "Image 1 was taken with lamp %s", lamp);
1145  cpl_free(lamp);
1146  unsigned int k;
1147  for (k = 1; k < muse_imagelist_get_size(aImages); k++) {
1148  int nxk = cpl_image_get_size_x(muse_imagelist_get(aImages, k)->data),
1149  nyk = cpl_image_get_size_y(muse_imagelist_get(aImages, k)->data);
1150  if (nxk != nx || nyk != ny) {
1151  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "arc "
1152  "imagelist is not uniform (image %u is %dx%d, "
1153  "first image is %dx%d)", k + 1, nxk, nyk, nx, ny);
1154  return NULL;
1155  }
1156  cpl_propertylist_erase_regexp(muse_imagelist_get(aImages, k)->header,
1157  "^ESO QC", 0);
1158  lamp = muse_utils_header_get_lamp_names(muse_imagelist_get(aImages, k)->header, ',');
1159  cpl_msg_debug(__func__, "Image %d was taken with lamp %s", k + 1, lamp);
1160  cpl_free(lamp);
1161  } /* for k (all images in list except first) */
1162 
1163  cpl_vector *vreflam = muse_wave_lines_get(aLinelist, 5, 0.);
1164  if (!vreflam) {
1165  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "could not "
1166  "create list of FWHM reference arc wavelengths");
1167  return NULL;
1168  }
1169  cpl_table *wavecal = NULL;
1170  int debug = getenv("MUSE_DEBUG_WAVECAL")
1171  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
1172  const char *fn_debug = "MUSE_DEBUG_WAVE_LINES.ascii";
1173  FILE *fp_debug = NULL;
1174  if (debug >= 3) {
1175  cpl_msg_info(__func__, "Will write all single line fits to \"%s\"", fn_debug);
1176  fp_debug = fopen(fn_debug, "w");
1177  if (fp_debug) {
1178  fprintf(fp_debug, "#slice x y lambda lambdaerr\n");
1179  }
1180  }
1181 
1182  /* loop over all slices */
1183  unsigned short islice;
1184  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
1185  if (debug > 0) {
1186  printf("\n\nSlice %d\n", islice + 1); fflush(stdout);
1187  }
1188  /* get the tracing polynomials for this slice */
1189  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
1190  islice + 1);
1191  if (!ptrace) {
1192  cpl_msg_warning(__func__, "slice %2hu: tracing polynomials missing!",
1193  islice + 1);
1194  continue;
1195  }
1196  /* detect all lines in the center of the slice */
1197  const int imid = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
1198  ny/2., NULL)),
1199  iWidth = 3,
1200  icol1 = imid - iWidth/2,
1201  icol2 = icol1 + iWidth;
1202  if (imid < 1 || imid > nx) {
1203  cpl_msg_warning(__func__, "slice %2hu: faulty polynomial detected",
1204  islice + 1);
1205  muse_trace_polys_delete(ptrace);
1206  continue; /* next slice */
1207  }
1208  cpl_table *detlines = NULL;
1209  int ndet = 0;
1210  for (k = 0; k < muse_imagelist_get_size(aImages); k++) {
1211  muse_image *arc = muse_imagelist_get(aImages, k);
1212  /* create muse_image from the spectrum in which we want to search lines */
1213  muse_imagelist *collist = muse_imagelist_new();
1214  int i;
1215  for (i = icol1; i <= icol2; i++) {
1216  muse_image *column = muse_image_new();
1217  column->data = cpl_image_extract(arc->data, i, 1, i, ny);
1218  column->dq = cpl_image_extract(arc->dq, i, 1, i, ny);
1219  column->stat = cpl_image_extract(arc->stat, i, 1, i, ny);
1220  column->header = cpl_propertylist_new();
1221  cpl_propertylist_append_string(column->header, "BUNIT",
1222  cpl_propertylist_get_string(arc->header,
1223  "BUNIT"));
1224  muse_imagelist_set(collist, column, i - icol1);
1225  } /* for i (neighboring image columns) */
1226  muse_image *column = muse_combine_median_create(collist);
1227  muse_imagelist_delete(collist);
1228 
1229  /* now search, measure, and store the lines */
1230  cpl_table *detections = muse_wave_lines_search(column, aParams->detsigma,
1231  islice + 1);
1232  muse_image_delete(column);
1233  int nlampdet = cpl_table_get_nrow(detections); /* det. with this lamp */
1234  ndet += nlampdet; /* detections of all lamps */
1235  char *lampname = muse_utils_header_get_lamp_names(arc->header, ',');
1236  cpl_table_fill_column_window_string(detections, "lampname", 0, nlampdet,
1237  lampname);
1238  cpl_array *lampnumbers = muse_utils_header_get_lamp_numbers(arc->header);
1239  cpl_table_fill_column_window_int(detections, "lampno", 0, nlampdet,
1240  cpl_array_get_int(lampnumbers, 0, NULL));
1241  cpl_array_delete(lampnumbers);
1242  /* get arc lines for this lamp from original list */
1243  double minflux = cpl_table_get_column_min(detections, "flux") * 1.15;
1244  cpl_vector *ionlambdas = muse_wave_lines_get_for_lamp(aLinelist, lampname,
1245  1, minflux);
1246  cpl_free(lampname);
1247  muse_wave_lines_identify(detections, ionlambdas, aParams);
1248  cpl_vector_delete(ionlambdas);
1249 
1250  if (!detlines) {
1251  detlines = detections;
1252  } else {
1253  cpl_table_insert(detlines, detections, cpl_table_get_nrow(detlines));
1254  cpl_table_delete(detections);
1255  }
1256  } /* for k (all images in list) */
1257  cpl_propertylist *order = cpl_propertylist_new();
1258  cpl_propertylist_append_bool(order, "y", CPL_FALSE);
1259  cpl_table_sort(detlines, order);
1260  cpl_propertylist_delete(order);
1261  int nlines = cpl_table_get_nrow(detlines);
1262  if (debug >= 2) {
1263  printf("Detected and identified %d arc lines in slice %hu:\n", nlines,
1264  islice + 1);
1265  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
1266  fflush(stdout);
1267  }
1268  /* the first QC parameter does even make sense without lines */
1269  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NDET, islice + 1);
1270  cpl_propertylist_append_int(firstimage->header, keyword, ndet);
1271  cpl_free(keyword);
1272 
1273  if (nlines < aParams->yorder + 1) {
1274  /* could apparently not identify enough lines, or an error occured */
1275  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "could not "
1276  "detect and/or identify enough arc lines in slice "
1277  "%hu (%d of %d, required %hu)", islice + 1, nlines,
1278  ndet, aParams->yorder + 1);
1279  muse_trace_polys_delete(ptrace);
1280  cpl_table_delete(detlines);
1281  continue; /* work on next slice immediately */
1282  }
1283  cpl_msg_info(__func__, "Identified %d of %d detected arc lines in slice %hu"
1284  " (column %d, %d..%d)", nlines, ndet, islice + 1, imid, icol1,
1285  icol2);
1286 
1287  /* the next set of QC parameters */
1288  muse_wave_calib_qc_peaks(detlines, firstimage->header, islice + 1);
1289  /* above we wrote the number of detected lines, now is the number of *
1290  * arc lines really used in the fit, i.e. the identified ones */
1291  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NID, islice + 1);
1292  cpl_propertylist_append_int(firstimage->header, keyword, nlines);
1293  cpl_free(keyword);
1294 
1295  /* convert y and lambda into matrix and vector for an initial, *
1296  * vertical, inverse 1D fit */
1297  cpl_vector *cen = cpl_vector_new(nlines);
1298  cpl_matrix *lbda = cpl_matrix_new(1, nlines);
1299  int idx;
1300  for (idx = 0; idx < nlines; idx++) {
1301  cpl_vector_set(cen, idx, cpl_table_get(detlines, "center", idx, NULL));
1302  cpl_matrix_set(lbda, 0, idx, cpl_table_get(detlines, "lambda", idx, NULL));
1303  }
1304 #if 0
1305  char *fn = cpl_sprintf("slice%02hu_lbda1.dat", islice + 1);
1306  FILE *file = fopen(fn, "w");
1307  cpl_matrix_dump(lbda, file);
1308  fclose(file);
1309  cpl_free(fn);
1310 #endif
1311  double mse1d, chisq1d;
1312  cpl_polynomial *fit
1313  = muse_utils_iterate_fit_polynomial(lbda, cen, NULL, detlines,
1314  aParams->yorder, 3.,
1315  &mse1d, &chisq1d);
1316 #if 0
1317  cpl_vector *res = cpl_vector_new(cpl_vector_get_size(cen));
1318  cpl_vector_fill_polynomial_fit_residual(res, cen, NULL, fit, lbda, NULL);
1319  cpl_bivector *biv = cpl_bivector_wrap_vectors(cen, res);
1320  cpl_plot_bivector(NULL, NULL, NULL, biv);
1321  cpl_bivector_unwrap_vectors(biv);
1322  cpl_vector_delete(res);
1323 #endif
1324  if (debug >= 1) {
1325  printf("Initial (inverse) polynomial fit in slice %2hu (RMS = %f, chisq = %e"
1326  ", %d of %d input points left)\n", islice + 1, sqrt(mse1d), chisq1d,
1327  (int)cpl_vector_get_size(cen), nlines);
1328  cpl_polynomial_dump(fit, stdout);
1329  fflush(stdout);
1330  }
1331  nlines = cpl_vector_get_size(cen);
1332 #if 0
1333  fn = cpl_sprintf("slice%02hu_lbda2.dat", islice + 1);
1334  file = fopen(fn, "w");
1335  cpl_matrix_dump(lbda, file);
1336  fclose(file);
1337  cpl_free(fn);
1338 #endif
1339  cpl_vector_delete(cen);
1340  cpl_matrix_delete(lbda);
1341 
1342  /* positions matrix and wavelengths vector to be used for the 2D polynomial *
1343  * fit; just set and initial size of 1, it has to be resized with every line */
1344  cpl_matrix *xypos = cpl_matrix_new(2, 1);
1345  cpl_vector *lambdas = cpl_vector_new(1),
1346  *dlambdas = NULL;
1347  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1348  dlambdas = cpl_vector_new(1);
1349  }
1350  int ientry = 0; /* index for these two structures */
1351 
1352  /* now measure all bright, single lines taken from the input linelist */
1353  cpl_table *fwhmtable = muse_cpltable_new(muse_wavelines_def, 0);
1354  cpl_table_set_column_unit(fwhmtable, "fwhm", "Angstrom");
1355  int j, narclines = cpl_table_get_nrow(aLinelist);
1356  for (j = 0; j < narclines; j++) {
1357  int quality = cpl_table_get_int(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, NULL);
1358  if (quality <= 1) {
1359  continue; /* skip unwanted line */
1360  }
1361 
1362  /* use the lamp name to find the corresponding exposure for arc line */
1363  const char *lampname = muse_wave_lines_get_lampname(aLinelist, j);
1364  muse_image *arc = NULL;
1365  for (k = 0; k < muse_imagelist_get_size(aImages); k++) {
1366  arc = muse_imagelist_get(aImages, k);
1367  char *thislamp = muse_utils_header_get_lamp_names(arc->header, ',');
1368  cpl_boolean match = CPL_FALSE;
1369  if (lampname && thislamp) {
1370  match = strncmp(lampname, thislamp, strlen(lampname)) == 0;
1371  }
1372  cpl_free(thislamp);
1373  if (match) {
1374  break;
1375  }
1376  } /* for k (all images in list) */
1377 
1378  cpl_table *fittable = NULL;
1379  if (quality == 2) { /* multiplet */
1380  fittable = muse_wave_line_handle_multiplet(arc, aLinelist, j, fit, ptrace,
1381  aParams, islice + 1, debug);
1382  } else {
1383  fittable = muse_wave_line_handle_singlet(arc, aLinelist, j, fit, ptrace,
1384  aParams, islice + 1, debug);
1385  }
1386  if (!fittable) {
1387  continue;
1388  }
1389  int nnew = cpl_table_get_nrow(fittable);
1390 
1391  /* resize matrix/vector to be able to fit all new entries */
1392  cpl_matrix_resize(xypos, 0, 0,
1393  0, ientry + nnew - cpl_matrix_get_ncol(xypos));
1394  cpl_vector_set_size(lambdas, ientry + nnew);
1395  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1396  cpl_vector_set_size(dlambdas, ientry + nnew);
1397  }
1398 
1399  /* now add the final fit positions */
1400  cpl_table_set_column_unit(fittable, "fwhm", ""); /* ongoing unit conversion */
1401  int ipos;
1402  for (ipos = 0; ipos < nnew; ipos++) {
1403  /* set x-position (CCD column) in the first matrix row */
1404  cpl_matrix_set(xypos, 0, ientry, cpl_table_get(fittable, "x", ipos, NULL));
1405  /* y-position on CCD (center of Gauss fit) into second matrix row */
1406  cpl_matrix_set(xypos, 1, ientry, cpl_table_get(fittable, "center", ipos, NULL));
1407  /* the vector has to contain as many lambda entries */
1408  double lambda = cpl_table_get(fittable, "lambda", ipos, NULL);
1409  cpl_vector_set(lambdas, ientry, lambda);
1410  /* Pretend that errors in Gaussian fit are errors in wavelength, *
1411  * use the first-guess 1D solution to convert from pix to Angstrom. */
1412  double dlbda;
1413  cpl_polynomial_eval_1d(fit, lambda, &dlbda); /* not interested in result */
1414  double cerr = 0.;
1415  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1416  cerr = cpl_table_get(fittable, "cerr", ipos, NULL) / dlbda;
1417  cpl_vector_set(dlambdas, ientry, cerr);
1418  }
1419  /* now that we have the actual sampling, also convert FWHM to Angstrom */
1420  int err;
1421  double fwhm = cpl_table_get(fittable, "fwhm", ipos, &err) / dlbda;
1422  if (!err) {
1423  cpl_table_set(fittable, "fwhm", ipos, fwhm);
1424  }
1425  ientry++; /* next position in this matrix/vector combo */
1426  if (fp_debug) {
1427  fprintf(fp_debug, " %02hu %04d %9.4f %10.4f %e\n", islice + 1,
1428  (int)cpl_table_get(fittable, "x", ipos, NULL),
1429  cpl_table_get(fittable, "center", ipos, NULL), lambda, cerr);
1430  }
1431  } /* for ipos (all fittable rows) */
1432  cpl_table_set_column_unit(fittable, "fwhm", "Angstrom");
1433  /* append relevant lines to extra table to compute FWHM and R later */
1434  cpl_table_select_all(fittable);
1435  cpl_table_and_selected_invalid(fittable, "fwhm");
1436  cpl_table_erase_selected(fittable);
1437  cpl_table_insert(fwhmtable, fittable, cpl_table_get_nrow(fwhmtable));
1438  cpl_table_delete(fittable);
1439  } /* for j (all arc lines) */
1440  /* reset original line quality info */
1441  cpl_table_abs_column(aLinelist, MUSE_LINE_CATALOG_QUALITY);
1442  muse_wave_lines_add_flux_for_lsf(aLinelist, detlines);
1443  cpl_table_delete(detlines);
1444  cpl_polynomial_delete(fit);
1445  muse_wave_calib_qc_fwhm(fwhmtable, firstimage->header, islice + 1);
1446  cpl_table_delete(fwhmtable);
1447 
1448  /* Compute two-dimensional wavelength solution for each slice. */
1449  cpl_polynomial *poly = NULL; /* polynomial for wavelength solution */
1450  double mse2d;
1451  cpl_error_code rc = muse_wave_poly_fit(xypos, lambdas, dlambdas,
1452  &poly, &mse2d, aParams, islice + 1);
1453  cpl_msg_info(__func__, "Polynomial fit of %"CPL_SIZE_FORMAT" of %d positions"
1454  " in slice %hu gave an RMS of %f Angstrom",
1455  cpl_vector_get_size(lambdas), ientry, islice + 1, sqrt(mse2d));
1456  cpl_matrix_delete(xypos);
1457  int nfinal = muse_cplvector_count_unique(lambdas);
1458  cpl_vector_delete(lambdas);
1459  cpl_vector_delete(dlambdas);
1460  /* above we wrote detected and identified lines, here *
1461  * save the ones that actually survived the fit */
1462  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_NLINES, islice + 1);
1463  cpl_propertylist_append_int(firstimage->header, keyword, nfinal);
1464  cpl_free(keyword);
1465  /* collect QC on wavelengths */
1466  muse_wave_calib_qc_lambda(firstimage, islice + 1, ptrace, poly);
1467  /* trace polynomials are not needed further */
1468  muse_trace_polys_delete(ptrace);
1469 
1470  if (rc != CPL_ERROR_NONE) { /* failure of the polynomial fit */
1471  cpl_error_set_message(__func__, rc, "a failure while computing the two-di"
1472  "mensional polynomial fit in slice %hu", islice + 1);
1473  cpl_polynomial_delete(poly);
1474  continue; /* try next slice immediately */
1475  }
1476 
1477  if (!wavecal) {
1478  /* create output table with one row for each slice; no need to *
1479  * check return code, it can only fail for negative lengths */
1480  wavecal = muse_wave_table_create(kMuseSlicesPerCCD,
1481  aParams->xorder, aParams->yorder);
1482  }
1483  rc = muse_wave_table_add_poly(wavecal, poly, mse2d,
1484  aParams->xorder, aParams->yorder, islice);
1485 
1486  /* use the number of wavelengths as indicator of the lines used for the *
1487  * fit and record it as QC parameter */
1488  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_RMS, islice + 1);
1489  cpl_propertylist_append_float(firstimage->header, keyword, sqrt(mse2d));
1490  cpl_free(keyword);
1491 
1492  if (rc != CPL_ERROR_NONE) {
1493  cpl_msg_warning(__func__, "Could not write polynomial to wavecal table: %s",
1494  cpl_error_get_message_default(rc));
1495  }
1496  /* clean up, ignore any failure with this */
1497  cpl_polynomial_delete(poly);
1498  } /* for islice (all kMuseSlicesPerCCD) */
1499  cpl_vector_delete(vreflam);
1500  if (fp_debug) {
1501  cpl_msg_info(__func__, "Done writing line fits to \"%s\"", fn_debug);
1502  fclose(fp_debug);
1503  }
1504 
1505  muse_wave_calib_output_summary(firstimage->header, wavecal);
1506  /* copy the QC parameters to the other image headers */
1507  for (k = 1; k < muse_imagelist_get_size(aImages); k++) {
1508  cpl_propertylist *header = muse_imagelist_get(aImages, k)->header;
1509  cpl_propertylist_erase_regexp(header, "^ESO QC", 0);
1510  cpl_propertylist_copy_property_regexp(header, firstimage->header, "^ESO QC", 0);
1511  } /* for k (all images in list except first) */
1512 
1513  return wavecal;
1514 } /* muse_wave_calib_lampwise() */
1515 
1516 /*----------------------------------------------------------------------------*/
1534 /*----------------------------------------------------------------------------*/
1535 cpl_boolean
1536 muse_wave_lines_check(cpl_table *aTable, cpl_propertylist *aHeader)
1537 {
1538  cpl_ensure(aTable && aHeader, CPL_ERROR_NULL_INPUT, CPL_FALSE);
1539  /* check the table */
1540  int nrow = cpl_table_get_nrow(aTable);
1541  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, CPL_FALSE);
1542  cpl_ensure(cpl_table_has_column(aTable, MUSE_LINE_CATALOG_LAMBDA) == 1
1543  && cpl_table_has_column(aTable, MUSE_LINE_CATALOG_QUALITY) == 1,
1544  CPL_ERROR_DATA_NOT_FOUND, CPL_FALSE);
1545 
1546  /* check the version */
1547  if (!cpl_propertylist_has(aHeader, MUSE_HDR_LINE_CATALOG_VERSION)) {
1548  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "%s does not "
1549  "contain a VERSION header entry!", MUSE_TAG_LINE_CATALOG);
1550  return CPL_FALSE;
1551  }
1552  int version = cpl_propertylist_get_int(aHeader, MUSE_HDR_LINE_CATALOG_VERSION);
1553 #define LINE_CATALOG_EXPECTED_VERSION 3
1554  if (version != LINE_CATALOG_EXPECTED_VERSION) {
1555  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "VERSION = %d "
1556  "is wrong, we need a %s with VERSION = %d", version,
1557  MUSE_TAG_LINE_CATALOG, LINE_CATALOG_EXPECTED_VERSION);
1558  return CPL_FALSE;
1559  }
1560  return CPL_TRUE;
1561 } /* muse_wave_lines_check() */
1562 
1563 /*----------------------------------------------------------------------------*/
1585 /*----------------------------------------------------------------------------*/
1586 cpl_vector *
1587 muse_wave_lines_get(cpl_table *aTable, int aGoodnessLimit, double aFluxLimit)
1588 {
1589  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, NULL);
1590  int nrow = cpl_table_get_nrow(aTable);
1591  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, NULL);
1592 
1593  cpl_ensure(cpl_table_has_column(aTable, MUSE_LINE_CATALOG_LAMBDA) == 1
1594  && cpl_table_has_column(aTable, MUSE_LINE_CATALOG_QUALITY) == 1,
1595  CPL_ERROR_DATA_NOT_FOUND, NULL);
1596 
1597  /* convert the usable wavelengths into a vector */
1598  cpl_vector *lambdas = cpl_vector_new(nrow);
1599  int i, pos = 0;
1600  for (i = 0; i < nrow; i++) {
1601  double lambda = cpl_table_get(aTable, MUSE_LINE_CATALOG_LAMBDA, i, NULL),
1602  flux = cpl_table_get(aTable, MUSE_LINE_CATALOG_FLUX, i, NULL);
1603 
1604  /* make sure that the arc line table is sorted, to simplify debugging */
1605  if (i > 0 && lambda < cpl_table_get(aTable, MUSE_LINE_CATALOG_LAMBDA, i-1,
1606  NULL)) {
1607  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "%s is not "
1608  "sorted by increasing lambda (at row %d)!",
1609  MUSE_TAG_LINE_CATALOG, i+1);
1610  cpl_vector_delete(lambdas);
1611  return NULL;
1612  }
1613 
1614  if (cpl_table_get(aTable, MUSE_LINE_CATALOG_QUALITY, i, NULL)
1615  < aGoodnessLimit || flux < aFluxLimit) {
1616  /* this is apparently a line that should not be used */
1617  continue;
1618  }
1619  cpl_vector_set(lambdas, pos, lambda);
1620  pos++;
1621  } /* for i (all table rows) */
1622  if (!pos) { /* setting the size to zero does not work! */
1623  cpl_vector_delete(lambdas);
1624  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "No lines with "
1625  "%s >= %d found", MUSE_LINE_CATALOG_QUALITY,
1626  aGoodnessLimit);
1627  return NULL;
1628  }
1629  /* cut vector size to contain the usable wavelengths */
1630  cpl_vector_set_size(lambdas, pos);
1631 
1632  /* remove lines that are too close together, i.e. within 1.5 Angstrom */
1633  for (i = 0; i < cpl_vector_get_size(lambdas) - 1; i++) {
1634  double l1 = cpl_vector_get(lambdas, i),
1635  l2 = cpl_vector_get(lambdas, i + 1);
1636  if ((l2 - l1) < 1.5) {
1637  cpl_msg_debug(__func__, "Excluding lines at %.3f and %.3f (d = %.3f) "
1638  "Angstrom", l1, l2, l2 - l1);
1639  muse_cplvector_erase_element(lambdas, i + 1);
1640  muse_cplvector_erase_element(lambdas, i);
1641  i--; /* stay at this index to see what moved here */
1642  }
1643  } /* for i (all vector entries) */
1644 #if 0
1645  printf("%d arc lines in input list are usable:\n", pos);
1646  cpl_vector_dump(lambdas, stdout);
1647 #endif
1648  cpl_msg_debug(__func__, "Using a list of %d %s arc lines (from %6.1f to %6.1f "
1649  "Angstrom)", pos,
1650  aGoodnessLimit == 1 ? "good"
1651  : (aGoodnessLimit == 5 ? "FWHM reference"
1652  : "all"),
1653  cpl_vector_get(lambdas, 0),
1654  cpl_vector_get(lambdas, cpl_vector_get_size(lambdas) - 1));
1655 
1656  return lambdas;
1657 } /* muse_wave_lines_get() */
1658 
1659 /*----------------------------------------------------------------------------*/
1676 /*----------------------------------------------------------------------------*/
1677 cpl_vector *
1678 muse_wave_lines_get_for_lamp(cpl_table *aTable, const char *aLamp,
1679  int aGoodnessLimit, double aFluxLimit)
1680 {
1681  cpl_ensure(aTable && aLamp, CPL_ERROR_NULL_INPUT, NULL);
1682  int nrow = cpl_table_get_nrow(aTable);
1683  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, NULL);
1684 
1685  cpl_table_unselect_all(aTable);
1686  int i;
1687  for (i = 0; i < nrow; i++) {
1688  const char *lampname = muse_wave_lines_get_lampname(aTable, i);
1689  if (!strcmp(lampname, aLamp)) {
1690  cpl_table_select_row(aTable, i);
1691  }
1692  } /* for i (all table rows) */
1693  cpl_table *lamplines = cpl_table_extract_selected(aTable);
1694 #if 0
1695  printf("lamplines for %s\n", aLamp);
1696  cpl_table_dump(lamplines, 0, 10000, stdout);
1697  fflush(stdout);
1698 #endif
1699  cpl_vector *lamplambdas = muse_wave_lines_get(lamplines, aGoodnessLimit,
1700  aFluxLimit);
1701  cpl_table_delete(lamplines);
1702  return lamplambdas;
1703 } /* muse_wave_lines_get_for_lamp() */
1704 
1705 /*----------------------------------------------------------------------------*/
1719 /*----------------------------------------------------------------------------*/
1720 const char *
1721 muse_wave_lines_get_lampname(cpl_table *aTable, const int aIdx)
1722 {
1723  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, "Unknown_Lamp");
1724  const char *ion = cpl_table_get_string(aTable, "ion", aIdx);
1725  cpl_ensure(ion, CPL_ERROR_ILLEGAL_INPUT, "Unknown_Lamp");
1726 
1727  /* for the HgCd lamp we have "HgI", "CdI", and "Hg_or_Cd" */
1728  if (!strncmp(ion, "Hg", 2) || !strncmp(ion, "Cd", 2)) {
1729  return "HgCd";
1730  }
1731  /* for the Ne lamp we have only "NeI" */
1732  if (!strncmp(ion, "Ne", 2)) {
1733  return "Ne";
1734  }
1735  /* for the Xe lamp we have "XeI" and "XeII" */
1736  if (!strncmp(ion, "Xe", 2)) {
1737  return "Xe";
1738  }
1739  return "Unknown_Lamp";
1740 } /* muse_wave_lines_get_lampname() */
1741 
1742 /*----------------------------------------------------------------------------*/
1754 /*----------------------------------------------------------------------------*/
1755 static void
1756 muse_wave_lines_add_flux_for_lsf(cpl_table *aLineList, cpl_table *aDetlines)
1757 {
1758  if (!cpl_table_has_column(aLineList, "Flux_for_LSF")) {
1759  cpl_table_new_column(aLineList, "Flux_for_LSF",
1760  CPL_TYPE_DOUBLE);
1761  cpl_table_fill_column_window(aLineList, "Flux_for_LSF",
1762  0, cpl_table_get_nrow(aLineList), 0.);
1763  }
1764 
1765  const double sigma = 0.1;
1766  cpl_size i, n = cpl_table_get_nrow(aDetlines);
1767  for (i = 0; i < n; i++) {
1768  int status;
1769  double lbda = cpl_table_get(aDetlines, "lambda", i, &status);
1770  if (status == 1) {
1771  continue;
1772  }
1773  cpl_size i_l = muse_cpltable_find_sorted(aLineList, MUSE_LINE_CATALOG_LAMBDA,
1774  lbda + sigma);
1775  if (cpl_table_get(aLineList, MUSE_LINE_CATALOG_LAMBDA, i_l, &status)
1776  < lbda - sigma) {
1777  continue;
1778  }
1779  double a = cpl_table_get(aLineList, "Flux_for_LSF", i_l, &status);
1780  a += cpl_table_get(aDetlines, "flux", i, &status) / 300.;
1781  cpl_table_set(aLineList, "Flux_for_LSF", i_l, a);
1782  } /* for i (all rows in aDetlines) */
1783 } /* muse_wave_lines_add_flux_for_lsf() */
1784 
1785 /*----------------------------------------------------------------------------*/
1811 /*----------------------------------------------------------------------------*/
1812 cpl_table *
1813 muse_wave_lines_search(muse_image *aColumnImage, double aSigma,
1814  const unsigned short aSlice)
1815 {
1816 #if !SEARCH_DEBUG_FILES
1817  UNUSED_ARGUMENT(aSlice);
1818 #endif
1819  cpl_ensure(aColumnImage, CPL_ERROR_NULL_INPUT, NULL);
1820  cpl_ensure(cpl_image_get_min(aColumnImage->stat) > 0.,
1821  CPL_ERROR_DATA_NOT_FOUND, NULL);
1822  cpl_ensure(aSigma > 0., CPL_ERROR_ILLEGAL_INPUT, NULL);
1823 
1824 #define SEARCH_SUBTRACT_BG 1
1825 #if SEARCH_SUBTRACT_BG
1826  /* Subtract any large scale background using a median-filtered spectrum. *
1827  * In the longer run, maybe a polynomial fit to the median-smoothed data *
1828  * should be used instead, because it does not create extra noise. */
1829  cpl_image *bgmedian = cpl_image_duplicate(aColumnImage->data);
1830  cpl_image_fill_noise_uniform(bgmedian, -FLT_MIN, FLT_MIN);
1831 #define BG_KERNEL_WIDTH 51
1832  cpl_mask *filter = cpl_mask_new(1, BG_KERNEL_WIDTH);
1833  cpl_mask_not(filter);
1834  cpl_image_filter_mask(bgmedian, aColumnImage->data, filter, CPL_FILTER_MEDIAN,
1835  CPL_BORDER_FILTER);
1836  cpl_mask_delete(filter);
1837 #if SEARCH_DEBUG_FILES
1838  char fn[FILENAME_MAX];
1839  sprintf(fn, "column_slice%02hu.fits", aSlice);
1840  muse_image_save(aColumnImage, fn);
1841  sprintf(fn, "column_slice%02hu_bgmedian.fits", aSlice);
1842  cpl_image_save(bgmedian, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
1843 #endif
1844  cpl_image *bgsub = cpl_image_subtract_create(aColumnImage->data, bgmedian);
1845  cpl_image_delete(bgmedian);
1846 #if SEARCH_DEBUG_FILES
1847  sprintf(fn, "column_slice%02hu_bgsub.fits", aSlice);
1848  cpl_image_save(bgsub, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
1849 #endif
1850 #endif
1851 
1852  /* create the S/N image */
1853  cpl_image *N = cpl_image_power_create(aColumnImage->stat, 0.5), /* noise */
1854 #if SEARCH_SUBTRACT_BG
1855  *SN = cpl_image_divide_create(bgsub, N); /* S/N image */
1856  cpl_image_delete(bgsub);
1857 #else
1858  *SN = cpl_image_divide_create(aColumnImage->data, N); /* S/N image */
1859 #endif
1860  cpl_image_delete(N);
1861 #if SEARCH_DEBUG_FILES
1862  sprintf(fn, "column_slice%02hu_SN.fits", aSlice);
1863  cpl_image_save(SN, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
1864 #endif
1865 
1866  /* use median statistics of the S/N image to determine the detection limit */
1867  double mdev, median = cpl_image_get_median_dev(SN, &mdev),
1868  limitSN = fmax(0.1, median + aSigma*mdev);
1869  cpl_mask *mask = cpl_mask_threshold_image_create(SN, limitSN, FLT_MAX);
1870  cpl_size nlines = 0; /* number of detected lines */
1871  cpl_image *label = cpl_image_labelise_mask_create(mask, &nlines);
1872  cpl_mask_delete(mask);
1873 #if SEARCH_DEBUG_FILES
1874  sprintf(fn, "column_slice%02hu_label.fits", aSlice);
1875  cpl_image_save(label, fn, CPL_TYPE_USHORT, NULL, CPL_IO_DEFAULT);
1876 #endif
1877 #if SEARCH_DEBUG || SEARCH_DEBUG_FILES
1878  double mean = cpl_image_get_mean(SN),
1879  stdev = cpl_image_get_stdev(SN),
1880  min = cpl_image_get_min(SN),
1881  max = cpl_image_get_max(SN);
1882  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" lines found; parameters: sigma=%f, "
1883  "med=%f+/-%f " "mean=%f+/-%f min/max=%f/%f limit(S/N)=%f", nlines,
1884  aSigma, median, mdev, mean, stdev, min, max, limitSN);
1885 #endif
1886  cpl_image_delete(SN);
1887 
1888  /* create table to store the fit results of all the lines */
1889  cpl_table *linesresult = muse_cpltable_new(muse_wavelines_def, nlines);
1890 
1891  /* now loop through all the detected lines and measure them */
1892  int i;
1893  for (i = 0; i < nlines; i++) {
1894  /* create mask for this label to isolate line (note that i starts at 0!) */
1895  cpl_mask *linemask = cpl_mask_threshold_image_create(label, i+0.5, i+1.5);
1896  int masksize = cpl_mask_get_size_y(linemask);
1897 
1898  /* determine the edges of the labeled region and the center of the line */
1899  int j = 1;
1900  while (j <= masksize && cpl_mask_get(linemask, 1, j) == CPL_BINARY_0) {
1901  j++;
1902  }
1903  int lopix = j;
1904  while (j <= masksize && cpl_mask_get(linemask, 1, j) == CPL_BINARY_1) {
1905  j++;
1906  }
1907  int hipix = j - 1;
1908  double cenpix = (hipix + lopix) / 2.;
1909 #if SEARCH_DEBUG
1910  int err;
1911  cpl_msg_debug(__func__, "lo/hi/cen=%d,%d,%f: values=%f,%f", lopix, hipix,
1912  cenpix, cpl_image_get(aColumnImage->data, 1, lopix, &err),
1913  cpl_image_get(aColumnImage->data, 1, hipix, &err));
1914 #endif
1915  cpl_mask_delete(linemask);
1916  if (lopix == hipix) {
1917  /* one pixel is not enough for a solid detection: *
1918  * set to some very low area / flux to be deleted below */
1919  cpl_table_set(linesresult, "flux", i, -1.);
1920  continue;
1921  }
1922 
1923  /* Enlarge region to be sure to measure the whole line (correct *
1924  * width!). As long as the pixel values are smaller than on the *
1925  * edges, and we don't go too far this should be OK. */
1926 #define MAXENLARGE 5
1927  /* reference pixel value (lower edge of mask) */
1928  int err0, err1 = 0, err2 = 0;
1929  double valref = cpl_image_get(aColumnImage->data, 1, lopix, &err0);
1930  cpl_errorstate prestate = cpl_errorstate_get();
1931  j = lopix;
1932  double value = 0;
1933  while (err1 == 0 && value < valref && (lopix - j) <= MAXENLARGE) {
1934  j--; /* enlarge downwards */
1935  value = cpl_image_get(aColumnImage->data, 1, j, &err1);
1936  }
1937  lopix = j + 1; /* set new lower edge */
1938 
1939  /* reference pixel value (lower edge of mask) */
1940  valref = cpl_image_get(aColumnImage->data, 1, hipix, &err2);
1941  j = hipix;
1942  value = 0;
1943  while (err2 == 0 && value < valref && (j - hipix) <= MAXENLARGE) {
1944  j++; /* enlarge upwards */
1945  value = cpl_image_get(aColumnImage->data, 1, j, &err2);
1946  }
1947  hipix = j - 1; /* set new upper edge */
1948 #if SEARCH_DEBUG
1949  cpl_msg_debug(__func__, "region=%d...%d, size=%d", lopix, hipix,
1950  hipix - lopix + 1);
1951 #endif
1952  if (err1 < 0 || err2 < 0) {
1953  cpl_errorstate_set(prestate); /* clean possible "Access beyond boundaries" */
1954  }
1955 
1956  /* fill vectors */
1957  cpl_vector *positions = cpl_vector_new(hipix - lopix + 1), /* for positions */
1958  *values = cpl_vector_new(hipix - lopix + 1), /* for pixel values */
1959  *valuessigma = cpl_vector_new(hipix - lopix + 1); /* sigma values */
1960  int k;
1961  for (j = lopix, k = 0; j <= hipix; j++, k++) {
1962  cpl_vector_set(positions, k, j);
1963  cpl_vector_set(values, k, cpl_image_get(aColumnImage->data, 1, j, &err0));
1964  /* when setting pixel value from stat extension, use *
1965  * square root, here we need the sigma not the variance: */
1966  cpl_vector_set(valuessigma, k, sqrt(cpl_image_get(aColumnImage->stat,
1967  1, j, &err0)));
1968  }
1969 
1970 #if SEARCH_DEBUG
1971  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
1972  cpl_bivector_dump(biv, stdout);
1973  cpl_bivector_unwrap_vectors(biv);
1974 #endif
1975 
1976  /* fit Gaussian */
1977  /* compute position, sigma, area, background offset, and mean squared error */
1978  double center, cerr, sigma, area, bglevel, mse;
1979  cpl_matrix *covariance;
1980  prestate = cpl_errorstate_get();
1981  cpl_error_code rc = cpl_vector_fit_gaussian(positions, NULL, values,
1982  valuessigma, CPL_FIT_ALL,
1983  &center, &sigma, &area, &bglevel,
1984  &mse, NULL, &covariance);
1985 
1986  /* try to treat some possible problems */
1987  if (rc == CPL_ERROR_CONTINUE) { /* fit didn't converge */
1988  /* estimate position error as sigma^2/area as CPL docs suggest *
1989  * no warning necessary in this case, print a debug message */
1990  cerr = sqrt(sigma * sigma / area);
1991  cpl_msg_debug(__func__, "Gaussian fit around position %6.1f: %s",
1992  cenpix, cpl_error_get_message());
1993 #if DEBUG_GAUSSFIT
1994  if (cpl_msg_get_log_level() == CPL_MSG_DEBUG) {
1995  /* CPL_ERROR_CONTINUE often occurs if there is another peak on the edge *
1996  * of the input positions/data array, so print it out in debug mode */
1997  cpl_bivector *bv = cpl_bivector_wrap_vectors(positions, values);
1998  printf("Gaussian fit: %f+/-%f, %f, %f, %f %f\nand the input data:\n",
1999  center, cerr, bglevel, area, sigma, mse);
2000  cpl_bivector_dump(bv, stdout);
2001  cpl_bivector_unwrap_vectors(bv);
2002  fflush(stdout);
2003  }
2004 #endif
2005  } else if (rc == CPL_ERROR_SINGULAR_MATRIX || !covariance) {
2006  cerr = sqrt(sigma * sigma / area);
2007  cpl_msg_debug(__func__, "Gaussian fit around position %6.1f: %s",
2008  cenpix, cpl_error_get_message());
2009  } else if (rc != CPL_ERROR_NONE) {
2010  /* although this is unlikely to occur, better print some warning *
2011  * and do something to the data to give this fit less weight in *
2012  * the final computation of the solution */
2013  cpl_msg_debug(__func__, "Gaussian fit around position %6.1f: %s",
2014  cenpix, cpl_error_get_message());
2015  cerr = 100.; /* set a high centering error */
2016  } else {
2017  /* no error returned, everything should have worked nicely *
2018  * take the centering error from the covariance matrix */
2019  cerr = sqrt(cpl_matrix_get(covariance, 0, 0));
2020 #if SEARCH_DEBUG
2021  cpl_matrix_dump(covariance, stdout);
2022 #endif
2023  cpl_matrix_delete(covariance);
2024  }
2025  cpl_errorstate_set(prestate); /* now we can reset the status */
2026  if (fabs(center - cenpix) > MUSE_WAVE_LINES_SEARCH_SHIFT_WARN) {
2027  cpl_msg_debug(__func__, "Large shift in Gaussian centering: initial "
2028  "%7.2f, fit %7.2f", cenpix, center);
2029  }
2030 
2031  /* save all results of this fit into the output table *
2032  * (no need to fill the x-position into the "x" column) */
2033  cpl_table_set(linesresult, "y", i, cenpix);
2034  /* set actual peak value in the table using cpl_vector_get_max() is safe, *
2035  * because the vector is constructed to be small enough to not include *
2036  * neighboring lines and so the likelihood of including cosmic rays is low */
2037  cpl_table_set(linesresult, "peak", i, cpl_vector_get_max(values));
2038  cpl_table_set(linesresult, "center", i, center);
2039  cpl_table_set(linesresult, "cerr", i, cerr);
2040  cpl_table_set(linesresult, "fwhm", i, CPL_MATH_FWHM_SIG * sigma);
2041  cpl_table_set(linesresult, "sigma", i, sigma);
2042  cpl_table_set(linesresult, "flux", i, area);
2043  cpl_table_set(linesresult, "bg", i, bglevel);
2044  cpl_table_set(linesresult, "mse", i, mse);
2045 
2046  cpl_vector_delete(positions);
2047  cpl_vector_delete(values);
2048  cpl_vector_delete(valuessigma);
2049 #if SEARCH_DEBUG
2050  printf("%s: results of fit stored in table row %d:\n", __func__, i + 1);
2051  cpl_table_dump(linesresult, i, 1, stdout);
2052  fflush(stdout);
2053 #endif
2054  } /* for i (all detected lines) */
2055  cpl_image_delete(label);
2056 
2057  /* again loop through all the table lines, and remove unlikely detections */
2058  cpl_table_unselect_all(linesresult);
2059  for (i = 0; i < cpl_table_get_nrow(linesresult); i++) {
2060  if (cpl_table_get(linesresult, "cerr", i, NULL) > kMuseArcMaxCenteringError ||
2061  cpl_table_get(linesresult, "fwhm", i, NULL) < kMuseArcMinFWHM ||
2062  cpl_table_get(linesresult, "fwhm", i, NULL) > kMuseArcMaxFWHM ||
2063  cpl_table_get(linesresult, "flux", i, NULL) < kMuseArcMinFlux) {
2064  cpl_table_select_row(linesresult, i);
2065  }
2066  } /* for i (all detected lines) */
2067  cpl_table_erase_selected(linesresult);
2068 
2069  return linesresult;
2070 } /* muse_wave_lines_search() */
2071 
2072 /*----------------------------------------------------------------------------*/
2092 /*----------------------------------------------------------------------------*/
2093 cpl_error_code
2094 muse_wave_lines_identify(cpl_table *aLines, cpl_vector *aLambdas,
2095  const muse_wave_params *aParams)
2096 {
2097  cpl_ensure_code(aLines && aLambdas, CPL_ERROR_NULL_INPUT);
2098 
2099  /* convert detected lines from the table into a simple vector */
2100  int i, nlines = cpl_table_get_nrow(aLines);
2101  cpl_vector *vcenter = cpl_vector_new(nlines);
2102  for (i = 0; i < nlines; i++) {
2103  cpl_vector_set(vcenter, i, cpl_table_get(aLines, "center", i, NULL));
2104  } /* for i (all lines) */
2105 #if 0
2106  cpl_vector_dump(vcenter, stdout);
2107 #endif
2108  double dmin = kMuseSpectralSamplingA - kMuseSpectralSamplingA * aParams->ddisp,
2109  dmax = kMuseSpectralSamplingA + kMuseSpectralSamplingA * aParams->ddisp;
2110  cpl_bivector *id_lines = cpl_ppm_match_positions(vcenter, aLambdas, dmin, dmax,
2111  aParams->tolerance, NULL, NULL);
2112 #if 0
2113  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" identified lines (of %"
2114  CPL_SIZE_FORMAT" detected lines and %"CPL_SIZE_FORMAT
2115  " from linelist)", cpl_bivector_get_size(id_lines),
2116  cpl_vector_get_size(vcenter), cpl_vector_get_size(aLambdas));
2117  cpl_bivector_dump(id_lines, stdout);
2118  fflush(stdout);
2119 #endif
2120  cpl_vector_delete(vcenter);
2121 
2122 #if 0
2123  printf("detected lines before cleanup:\n");
2124  cpl_table_dump(aLines, 0, nlines, stdout);
2125  fflush(stdout);
2126 #endif
2127  /* now go through the detected lines and remove all unidentified */
2128  const double *id_center = cpl_bivector_get_x_data_const(id_lines),
2129  *id_lambda = cpl_bivector_get_y_data_const(id_lines);
2130  cpl_table_unselect_all(aLines);
2131  int j, nid = cpl_bivector_get_size(id_lines);
2132  for (i = 0, j = 0; i < cpl_table_get_nrow(aLines) && id_center && id_lambda; i++) {
2133 #if 0
2134  cpl_msg_debug(__func__, "c=%f, l=%f, c_det=%f", id_center[j], id_lambda[j],
2135  cpl_table_get(aLines, "center", i, NULL));
2136 #endif
2137  if (j < nid && fabs(id_center[j] - cpl_table_get(aLines, "center", i, NULL))
2138  < DBL_EPSILON) {
2139  /* found the same line, append the wavelength */
2140  cpl_table_set(aLines, "lambda", i, id_lambda[j]);
2141  j++;
2142  continue;
2143  }
2144  cpl_table_select_row(aLines, i); /* not matching, delete this line */
2145  } /* for i (detected lines) and j (identified lines) */
2146  cpl_table_erase_selected(aLines);
2147  cpl_bivector_delete(id_lines);
2148  int debug = getenv("MUSE_DEBUG_WAVECAL")
2149  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
2150  if (debug >= 2) {
2151  printf("identified %d lines, %"CPL_SIZE_FORMAT" after cleanup:\n", nid,
2152  cpl_table_get_nrow(aLines));
2153  cpl_table_dump(aLines, 0, nid, stdout);
2154  fflush(stdout);
2155  }
2156  nid = cpl_table_get_nrow(aLines);
2157  /* check that after identification enough lines are still there */
2158  if (nid <= 0) {
2159  return CPL_ERROR_DATA_NOT_FOUND;
2160  } else if (nid < aParams->yorder + 1) {
2161  return CPL_ERROR_ILLEGAL_OUTPUT;
2162  }
2163  return CPL_ERROR_NONE;
2164 } /* muse_wave_lines_identify() */
2165 
2166 /*----------------------------------------------------------------------------*/
2198 /*----------------------------------------------------------------------------*/
2199 cpl_table *
2200 muse_wave_line_handle_singlet(muse_image *aImage, cpl_table *aLinelist,
2201  unsigned int aIdx, cpl_polynomial *aPoly,
2202  cpl_polynomial **aTrace,
2203  const muse_wave_params *aParams,
2204  const unsigned short aSlice, int aDebug)
2205 {
2206  cpl_ensure(aImage && aLinelist && aPoly && aTrace, CPL_ERROR_NULL_INPUT, NULL);
2207 
2208  /* Fix the Gaussian sigmas for lines that are not FWHM reference lines, *
2209  * otherwise the fit might run astray in low S/N cases. A negative sigma *
2210  * signifies constant value to fitting routine. */
2211  double sigma = kMuseSliceSlitWidthA / kMuseSpectralSamplingA / CPL_MATH_FWHM_SIG;
2212  /* if not reference line, set it negative, so that it is fixed in the fit */
2213  if (cpl_table_get(aLinelist, MUSE_LINE_CATALOG_QUALITY, aIdx, NULL) != 5) {
2214  sigma *= -1;
2215  }
2216  int halfwidth = 3.*kMuseSliceSlitWidthA / kMuseSpectralSamplingA; /* 3*FWHM */
2217 
2218  /* convenient access to some properties of the current line */
2219  double lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, aIdx, NULL),
2220  ypos = cpl_polynomial_eval_1d(aPoly, lambda, NULL);
2221  if ((ypos - halfwidth) < 1 ||
2222  (ypos + halfwidth) > cpl_image_get_size_y(aImage->data)) {
2223  if (aDebug >= 2) {
2224  cpl_msg_debug(__func__, "%f is supposed to lie near %.3f in slice %2hu, "
2225  "i.e. outside!", lambda, ypos, aSlice);
2226  }
2227  return NULL;
2228  }
2229  if (aDebug >= 2) {
2230  cpl_msg_debug(__func__, "%f is supposed to lie near %.3f in slice %2hu",
2231  lambda, ypos, aSlice);
2232  }
2233  /* get both slice edges and the center */
2234  double dleft = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT],
2235  ypos, NULL),
2236  dright = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT],
2237  ypos, NULL),
2238  dmid = (dleft + dright) / 2.;
2239  int ileft = ceil(dleft),
2240  iright = floor(dright);
2241 #if 0
2242  cpl_msg_debug(__func__, "limits at y=%f: %f < %f < %f", ypos, dleft, dmid,
2243  dright);
2244 #endif
2245 
2246  /* table to store line fits for this one arc line */
2247  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
2248  (int)kMuseSliceHiLikelyWidth + 5);
2249 
2250  /* From the center of the slice move outwards and fit the line *
2251  * until we have arrived at the edge of the slice */
2252  double y = ypos;
2253  int i, n = 0;
2254  for (i = dmid; i >= ileft; i--) {
2255  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, y, halfwidth,
2256  sigma, fittable, ++n);
2257  if (rc != CPL_ERROR_NONE) { /* do not count this line */
2258  --n;
2259  } else {
2260  double ynew = cpl_table_get(fittable, "center", n - 1, NULL);
2261  if (fabs(y - ynew) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2262  y = ynew;
2263  }
2264  }
2265  } /* for i (columns to left of slice middle) */
2266 #if 0
2267  printf("arc line aIdx=%u, columns i=%d...", aIdx + 1, i);
2268 #endif
2269  y = ypos; /* start at middle y position again */
2270  for (i = dmid + 1; i <= iright; i++) {
2271  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
2272  sigma, fittable, ++n);
2273  if (rc != CPL_ERROR_NONE) { /* do not count this line */
2274  --n;
2275  } else {
2276  double ynew = cpl_table_get(fittable, "center", n - 1, NULL);
2277  if (fabs(y - ynew) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2278  y = ynew;
2279  }
2280  }
2281  } /* for i (columns to right of slice middle) */
2282  /* now remove rows with invalid entries, i.e. those that were not *
2283  * filled with the properties of the fit -- cpl_table_erase_invalid() *
2284  * does not work, it deletes all columns */
2285  cpl_table_select_all(fittable);
2286  cpl_table_and_selected_invalid(fittable, "center");
2287  cpl_table_erase_selected(fittable);
2288  cpl_table_fill_column_window(fittable, "lambda", 0,
2289  cpl_table_get_nrow(fittable), lambda);
2290 #if 0
2291  printf("line %u, %d line fits\n", aIdx + 1, n);
2292  cpl_table_dump(fittable, 0, n, stdout);
2293  fflush(stdout);
2294 #endif
2295  /* and reject deviant fits */
2296  muse_wave_line_fit_iterate(fittable, lambda, aParams);
2297  int npos = cpl_table_get_nrow(fittable);
2298  if (npos <= 0) {
2299  cpl_msg_warning(__func__, "Polynomial fit failed in slice %hu for "
2300  "line %u (y-position near %.2f pix): %s", aSlice,
2301  aIdx + 1, ypos, cpl_error_get_message());
2302  }
2303 #if 0
2304  else {
2305  printf("%s: line %2u, %d line fits, %d with low residuals:\n", __func__,
2306  aIdx + 1, n, npos);
2307  cpl_table_dump(fittable, 0, npos, stdout);
2308  fflush(stdout);
2309  }
2310 #endif
2311  return fittable;
2312 } /* muse_wave_line_handle_singlet() */
2313 
2314 /*----------------------------------------------------------------------------*/
2350 /*----------------------------------------------------------------------------*/
2351 cpl_table *
2352 muse_wave_line_handle_multiplet(muse_image *aImage, cpl_table *aLinelist,
2353  unsigned int aIdx, cpl_polynomial *aPoly,
2354  cpl_polynomial **aTrace,
2355  const muse_wave_params *aParams,
2356  const unsigned short aSlice, int aDebug)
2357 {
2358  cpl_ensure(aImage && aLinelist && aPoly && aTrace, CPL_ERROR_NULL_INPUT, NULL);
2359 
2360  /* search for the other line(s) of the multiplet */
2361 #define MULTIPLET_SEARCH_RANGE 40. /* 40 Angstrom for NeI 7024...7059 */
2362  unsigned int nlines = 1; /* for the moment we know of one line of the multiplet */
2363  double lbda1 = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, aIdx, NULL);
2364  const char *lamp1 = muse_wave_lines_get_lampname(aLinelist, aIdx);
2365  cpl_vector *lambdas = cpl_vector_new(nlines),
2366  *fluxes = cpl_vector_new(nlines);
2367  cpl_vector_set(lambdas, 0, lbda1);
2368  cpl_vector_set(fluxes, 0, cpl_table_get(aLinelist, MUSE_LINE_CATALOG_FLUX,
2369  aIdx, NULL));
2370  double lambda;
2371  unsigned int j = aIdx;
2372  for (lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, ++j, NULL);
2373  fabs(lambda - lbda1) < MULTIPLET_SEARCH_RANGE;
2374  lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, ++j, NULL)) {
2375  int quality = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, NULL);
2376  const char *lamp = muse_wave_lines_get_lampname(aLinelist, j);
2377  if (quality == 2 && !strcmp(lamp1, lamp)) { /* duplet of the same lamp */
2378  nlines++;
2379  cpl_vector_set_size(lambdas, nlines);
2380  cpl_vector_set_size(fluxes, nlines);
2381  cpl_vector_set(lambdas, nlines - 1, lambda);
2382  double flux = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_FLUX, j, NULL);
2383  cpl_vector_set(fluxes, nlines - 1, flux);
2384  /* invert the sign of the quality, so that this line is not found again */
2385  cpl_table_set(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, -quality);
2386  }
2387  } /* for lambda */
2388 
2389  if (aDebug >= 2) {
2390  printf("found multiplet of lamp %s with %u lines:\n", lamp1, nlines);
2391  cpl_bivector *bv = cpl_bivector_wrap_vectors(lambdas, fluxes);
2392  cpl_bivector_dump(bv, stdout);
2393  cpl_bivector_unwrap_vectors(bv);
2394  fflush(stdout);
2395  }
2396 
2397  /* positive sigma: using negative value (= fixed parameter in fit) actually *
2398  * causes higher deviations, worse RMS in the global fit, so leave it free */
2399  double sigma = kMuseSliceSlitWidthA / kMuseSpectralSamplingA / CPL_MATH_FWHM_SIG;
2400  int halfwidth = 3.*kMuseSliceSlitWidthA / kMuseSpectralSamplingA; /* 3*FWHM */
2401 
2402  cpl_vector *ypos = cpl_vector_new(nlines);
2403  for (j = 0; j < nlines; j++) {
2404  double yp = cpl_polynomial_eval_1d(aPoly, cpl_vector_get(lambdas, j), NULL);
2405  cpl_vector_set(ypos, j, yp);
2406  } /* for j */
2407  double ypos1 = cpl_vector_get(ypos, 0),
2408  ypos2 = cpl_vector_get(ypos, nlines - 1);
2409  cpl_bivector *peaks = cpl_bivector_wrap_vectors(ypos, fluxes);
2410  if ((ypos1 - halfwidth) < 1 ||
2411  (ypos2 + halfwidth) > cpl_image_get_size_y(aImage->data)) {
2412  if (aDebug >= 2) {
2413  cpl_msg_debug(__func__, "%f is supposed to lie at %.3f..%.3f in slice %2hu, "
2414  "i.e. outside!", lambda, ypos1, ypos2, aSlice);
2415  }
2416  cpl_bivector_delete(peaks);
2417  cpl_vector_delete(lambdas);
2418  return NULL;
2419  }
2420  if (aDebug >= 2) {
2421  cpl_msg_debug(__func__, "%f is supposed to lie at %.3f..%.3f in slice %2hu",
2422  lambda, ypos1, ypos2, aSlice);
2423  }
2424  /* get both slice edges and the center */
2425  double dleft = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT],
2426  (ypos1 + ypos2)/2., NULL),
2427  dright = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT],
2428  (ypos1 + ypos2)/2., NULL),
2429  dmid = (dleft + dright) / 2.;
2430  int ileft = ceil(dleft),
2431  iright = floor(dright);
2432 
2433  /* table to store line fits for this one arc line */
2434  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
2435  ((int)kMuseSliceHiLikelyWidth + 5) * nlines);
2436 
2437  /* From the center of the slice move outwards and fit the *
2438  * line until we have arrived at the edge of the slice */
2439  cpl_bivector *bp = cpl_bivector_duplicate(peaks),
2440  *bpgood = cpl_bivector_duplicate(peaks);
2441  int i, n = 0;
2442  for (i = dmid; i >= ileft; i--) {
2443  n += nlines;
2444  cpl_error_code rc = muse_wave_line_fit_multiple(aImage, i, bp, lambdas,
2445  halfwidth, sigma, fittable, n);
2446  if (rc != CPL_ERROR_NONE) { /* do not count this fit */
2447  cpl_bivector_delete(bp);
2448  bp = cpl_bivector_duplicate(bpgood);
2449  n -= nlines;
2450  } else {
2451  cpl_vector *shifts = cpl_vector_duplicate(cpl_bivector_get_x(bp));
2452  cpl_vector_subtract(shifts, cpl_bivector_get_x(bpgood));
2453  double shift = cpl_vector_get_median(shifts); /* may be non-const! */
2454  cpl_vector_delete(shifts);
2455  if (fabs(shift) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2456  cpl_bivector_delete(bpgood);
2457  bpgood = cpl_bivector_duplicate(bp);
2458  } else {
2459  cpl_bivector_delete(bp);
2460  bp = cpl_bivector_duplicate(bpgood);
2461  }
2462  } /* else (good fit) */
2463  } /* for i (columns to left of slice middle) */
2464  cpl_bivector_delete(bp);
2465  cpl_bivector_delete(bpgood);
2466  bp = cpl_bivector_duplicate(peaks);
2467  bpgood = cpl_bivector_duplicate(peaks);
2468  for (i = dmid + 1; i <= iright; i++) {
2469  n += nlines;
2470  cpl_error_code rc = muse_wave_line_fit_multiple(aImage, i, bp, lambdas,
2471  halfwidth, sigma, fittable, n);
2472  if (rc != CPL_ERROR_NONE) { /* do not count this fit */
2473  cpl_bivector_delete(bp);
2474  bp = cpl_bivector_duplicate(bpgood);
2475  n -= nlines;
2476  } else {
2477  cpl_vector *shifts = cpl_vector_duplicate(cpl_bivector_get_x(bp));
2478  cpl_vector_subtract(shifts, cpl_bivector_get_x(bpgood));
2479  double shift = cpl_vector_get_median(shifts); /* may be non-const! */
2480  cpl_vector_delete(shifts);
2481  if (fabs(shift) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2482  cpl_bivector_delete(bpgood);
2483  bpgood = cpl_bivector_duplicate(bp);
2484  } else {
2485  cpl_bivector_delete(bp);
2486  bp = cpl_bivector_duplicate(bpgood);
2487  }
2488  } /* else (good fit) */
2489  } /* for i (columns to right of slice middle) */
2490  cpl_bivector_delete(bp);
2491  cpl_bivector_delete(bpgood);
2492  /* now remove rows with invalid entries, i.e. those that *
2493  * were not filled with the properties of the fit */
2494  cpl_table_select_all(fittable);
2495  cpl_table_and_selected_invalid(fittable, "center");
2496  cpl_table_erase_selected(fittable);
2497  cpl_bivector_delete(peaks);
2498  /* and reject deviant fits */
2499  for (j = 0; j < nlines; j++) {
2500  muse_wave_line_fit_iterate(fittable, cpl_vector_get(lambdas, j), aParams);
2501  } /* for i */
2502  cpl_vector_delete(lambdas);
2503 
2504  return fittable;
2505 } /* muse_wave_line_handle_multiplet() */
2506 
2507 /*----------------------------------------------------------------------------*/
2538 /*----------------------------------------------------------------------------*/
2539 cpl_error_code
2540 muse_wave_line_fit_single(muse_image *aImage, int aX, double aY, int aHalfWidth,
2541  double aSigma, cpl_table *aFitTable, int aRowsNeeded)
2542 {
2543  cpl_ensure_code(aImage && aImage->data && aImage->stat && aFitTable,
2544  CPL_ERROR_NULL_INPUT);
2545 #if 0
2546  cpl_msg_debug(__func__, "%d %d %d -> %d, %d", aX, (int)aY, aHalfWidth,
2547  2*aHalfWidth+1, aRowsNeeded);
2548 #endif
2549  /* determine area on the input image, copy area into the vectors */
2550  cpl_vector *positions = cpl_vector_new(2*aHalfWidth+1),
2551  *values = cpl_vector_new(2*aHalfWidth+1),
2552  *valuessigma = cpl_vector_new(2*aHalfWidth+1);
2553  int i, j, ny = cpl_image_get_size_y(aImage->data);
2554  for (i = (int)aY - aHalfWidth, j = 0; i <= (int)aY + aHalfWidth && i <= ny;
2555  i++, j++) {
2556  int err;
2557  cpl_vector_set(positions, j, i);
2558  cpl_vector_set(values, j, cpl_image_get(aImage->data, aX, i, &err));
2559  cpl_vector_set(valuessigma, j,
2560  sqrt(cpl_image_get(aImage->stat, aX, i, &err)));
2561  }
2562 #if 0
2563  printf("input vectors: positions, values\n");
2564  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
2565  cpl_bivector_dump(biv, stdout);
2566  fflush(stdout);
2567  cpl_bivector_unwrap_vectors(biv);
2568 #if 0
2569  printf("input vectors: values, valuessigma\n");
2570  biv = cpl_bivector_wrap_vectors(values, valuessigma);
2571  cpl_bivector_dump(biv, stdout);
2572  fflush(stdout);
2573  cpl_bivector_unwrap_vectors(biv);
2574 #endif
2575 #endif
2576 
2577  /* do the Gaussfit */
2578  cpl_errorstate prestate = cpl_errorstate_get();
2579  double center, cerror, area, bglevel, mse;
2580  cpl_matrix *covariance = NULL;
2581  cpl_fit_mode mode = CPL_FIT_ALL;
2582  double sigma = aSigma;
2583  if (sigma < 0) {
2584  mode = CPL_FIT_CENTROID | CPL_FIT_AREA | CPL_FIT_OFFSET;
2585  sigma *= -1;
2586  }
2587  cpl_error_code rc
2588  = cpl_vector_fit_gaussian(positions, NULL, values, valuessigma, mode,
2589  &center, &sigma, &area, &bglevel, &mse, NULL,
2590  &covariance);
2591 #if 0
2592  printf("--> rc=%d (%d) %s %#x %#x\n", rc, CPL_ERROR_NONE,
2593  cpl_error_get_message(), covariance, &covariance);
2594  fflush(stdout);
2595 #endif
2596  cpl_vector_delete(positions);
2597  cpl_vector_delete(values);
2598  cpl_vector_delete(valuessigma);
2599  /* Determine goodness of fit and try to treat some possible problems. *
2600  * (This is partly copied from muse_wave_lines_search().) */
2601  /* if no error happened, take the centering error from the covariance matrix */
2602  if (covariance) {
2603  cerror = sqrt(cpl_matrix_get(covariance, 0, 0));
2604  cpl_matrix_delete(covariance);
2605  } else { /* a missing covariance matrix is bad */
2606  cpl_msg_debug(__func__, "Gauss fit produced no covariance matrix (y=%.3f in "
2607  "column=%d): %s", aY, aX, cpl_error_get_message());
2608  cpl_errorstate_set(prestate);
2609  return rc == CPL_ERROR_NONE ? CPL_ERROR_ILLEGAL_OUTPUT : rc;
2610  }
2611  if (rc == CPL_ERROR_CONTINUE) { /* fit didn't converge */
2612  /* estimate position error as sigma^2/area as CPL docs *
2613  * suggest and pretend that everything worked fine */
2614  cerror = sqrt(sigma * sigma / area);
2615  cpl_errorstate_set(prestate);
2616  rc = CPL_ERROR_NONE;
2617  }
2618  if (rc != CPL_ERROR_NONE) {
2619  /* some other, probably more serious error, don't use this fit */
2620  cpl_msg_debug(__func__, "Gauss fit failed with some error (y=%.3f in "
2621  "column=%d): %s", aY, aX, cpl_error_get_message());
2622  cpl_errorstate_set(prestate);
2623  return rc;
2624  }
2625  if (fabs(center - aY) > MUSE_WAVE_LINE_FIT_MAXSHIFT) {
2626  /* too big shift, don't use this fit */
2627  cpl_msg_debug(__func__, "Gauss fit gave unexpectedly large offset "
2628  "(shifted %.3f pix from y=%.3f in column=%d)",
2629  center - aY, aY, aX);
2630  return CPL_ERROR_ACCESS_OUT_OF_RANGE;
2631  }
2632 
2633  /* store the results in the table, after making sure it is large enough */
2634  if (cpl_table_get_nrow(aFitTable) < aRowsNeeded) {
2635  cpl_table_set_size(aFitTable, aRowsNeeded);
2636  }
2637  cpl_table_set(aFitTable, "center", aRowsNeeded - 1, center);
2638  cpl_table_set(aFitTable, "cerr", aRowsNeeded - 1, cerror);
2639  cpl_table_set(aFitTable, "sigma", aRowsNeeded - 1, sigma);
2640  if (mode == CPL_FIT_ALL) { /* is FWHM reference line, store the FWHM */
2641  cpl_table_set(aFitTable, "fwhm", aRowsNeeded - 1, CPL_MATH_FWHM_SIG * sigma);
2642  }
2643  cpl_table_set(aFitTable, "flux", aRowsNeeded - 1, area);
2644  cpl_table_set(aFitTable, "bg", aRowsNeeded - 1, bglevel);
2645  cpl_table_set(aFitTable, "mse", aRowsNeeded - 1, mse);
2646  cpl_table_set(aFitTable, "x", aRowsNeeded - 1, aX);
2647  cpl_table_set(aFitTable, "y", aRowsNeeded - 1, aY);
2648 
2649 #if 0
2650  printf("--> Gaussian fit: %f +/- %f, %f\n", center, cerror, area);
2651  fflush(stdout);
2652 #endif
2653  return rc;
2654 } /* muse_wave_line_fit_single() */
2655 
2656 /*----------------------------------------------------------------------------*/
2694 /*----------------------------------------------------------------------------*/
2695 cpl_error_code
2696 muse_wave_line_fit_multiple(muse_image *aImage, int aX, cpl_bivector *aPeaks,
2697  cpl_vector *aLambdas, int aHalfWidth, double aSigma,
2698  cpl_table *aFitTable, int aRowsNeeded)
2699 {
2700  cpl_ensure_code(aImage && aImage->data && aImage->stat && aFitTable,
2701  CPL_ERROR_NULL_INPUT);
2702  cpl_vector *ypeaks = cpl_bivector_get_x(aPeaks),
2703  *fluxes = cpl_bivector_get_y(aPeaks);
2704  int np = cpl_vector_get_size(ypeaks);
2705  double yp1 = cpl_vector_get(ypeaks, 0),
2706  yp2 = cpl_vector_get(ypeaks, np - 1);
2707  int i1 = floor(yp1),
2708  i2 = ceil(yp2);
2709 #if 0
2710  cpl_msg_debug(__func__, "%d %d..%d %d -> %d", aX, i1, i2, aHalfWidth,
2711  aRowsNeeded);
2712 #endif
2713  /* determine area on the input image, copy area into the vectors */
2714  int npoints = (i2 + aHalfWidth) - (i1 - aHalfWidth) + 1;
2715  cpl_vector *positions = cpl_vector_new(npoints),
2716  *values = cpl_vector_new(npoints),
2717  *valuessigma = cpl_vector_new(npoints);
2718  double minval = DBL_MAX; /* 1st-guess fit value */
2719  int i, j, ny = cpl_image_get_size_y(aImage->data);
2720  for (i = i1 - aHalfWidth, j = 0; i <= i2 + aHalfWidth && i <= ny; i++, j++) {
2721  int err;
2722  cpl_vector_set(positions, j, i);
2723  double value = cpl_image_get(aImage->data, aX, i, &err);
2724  cpl_vector_set(values, j, value);
2725  if (value < minval) {
2726  minval = value;
2727  }
2728  cpl_vector_set(valuessigma, j,
2729  sqrt(cpl_image_get(aImage->stat, aX, i, &err)));
2730  }
2731  minval = minval > 0 ? minval : 0.;
2732 #if 0
2733  printf("input vectors: positions, values\n");
2734  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
2735  cpl_bivector_dump(biv, stdout);
2736  fflush(stdout);
2737  cpl_bivector_unwrap_vectors(biv);
2738 #endif
2739  cpl_bivector *bvalues = cpl_bivector_wrap_vectors(values, valuessigma);
2740 
2741  /* do the multi-Gauss fit */
2742  /* parameters: slope, background, sigma, np * center, np * flux */
2743  cpl_vector *poly = cpl_vector_new(2);
2744  cpl_vector_set(poly, 0, minval);
2745  cpl_vector_set(poly, 1, 0.);
2746  /* record the index of the expected strongest line, and its position */
2747  cpl_array *afluxes = cpl_array_wrap_double(cpl_vector_get_data(fluxes), np);
2748  cpl_size imaxflux;
2749  cpl_array_get_maxpos(afluxes, &imaxflux);
2750  double yposmax = cpl_vector_get(ypeaks, imaxflux);
2751  cpl_array_unwrap(afluxes);
2752  /* do the fit, store the errorstate before */
2753  cpl_errorstate prestate = cpl_errorstate_get();
2754  double sigma = aSigma,
2755  mse, chisq;
2756  cpl_matrix *covariance;
2757  cpl_error_code rc = muse_utils_fit_multigauss_1d(positions, bvalues, ypeaks,
2758  &sigma, fluxes, poly,
2759  &mse, &chisq, &covariance);
2760  cpl_vector_delete(positions);
2761  cpl_bivector_delete(bvalues);
2762 
2763  /* Determine goodness of fit and try to treat some possible problems. *
2764  * (This is partly copied from muse_wave_lines_search().) */
2765  /* if no error happened, take the centering error from the covariance matrix */
2766  if (!covariance) { /* a missing covariance matrix is bad */
2767  cpl_msg_debug(__func__, "Multi-Gauss fit produced no covariance matrix (y=%."
2768  "3f..%.3f in column=%d): %s", yp1, yp2, aX, cpl_error_get_message());
2769  cpl_errorstate_set(prestate);
2770  cpl_vector_delete(poly);
2771  return rc == CPL_ERROR_NONE ? CPL_ERROR_ILLEGAL_OUTPUT : rc;
2772  }
2773  if (rc != CPL_ERROR_NONE) {
2774  /* some other, probably more serious error, don't use this fit */
2775  cpl_msg_debug(__func__, "Multi-Gauss fit failed with some error (y=%.3f..%."
2776  "3f in column=%d): %s", yp1, yp2, aX, cpl_error_get_message());
2777  cpl_errorstate_set(prestate);
2778  cpl_matrix_delete(covariance);
2779  cpl_vector_delete(poly);
2780  return rc;
2781  }
2782  /* did the center of the brightest line shift too much? */
2783  double yfitmax = cpl_vector_get(ypeaks, imaxflux);
2784  if (fabs(yposmax - yfitmax) > MUSE_WAVE_LINE_FIT_MAXSHIFT) {
2785  cpl_msg_debug(__func__, "Multi-Gauss fit gave unexpectedly large offset "
2786  "(shifted %.3f pix from y=%.3f..%.3f in column=%d)",
2787  yposmax - yfitmax, yp1, yp2, aX);
2788  cpl_matrix_delete(covariance);
2789  cpl_vector_delete(poly);
2790  return CPL_ERROR_ACCESS_OUT_OF_RANGE;
2791  }
2792  /* are any of the fluxes negative? */
2793  double yfitmin = cpl_vector_get_min(fluxes);
2794  if (yfitmin < 0) {
2795  cpl_msg_debug(__func__, "Multi-Gauss fit gave negative flux (%e in "
2796  "multiplet from y=%.3f..%.3f in column=%d)", yfitmin,
2797  yp1, yp2, aX);
2798  cpl_matrix_delete(covariance);
2799  cpl_vector_delete(poly);
2800  return CPL_ERROR_ILLEGAL_OUTPUT;
2801  }
2802 
2803  /* store the results in the table, after making sure it is large enough */
2804  if (cpl_table_get_nrow(aFitTable) < aRowsNeeded) {
2805  cpl_table_set_size(aFitTable, aRowsNeeded);
2806  }
2807  /* first record the common values */
2808  cpl_table_fill_column_window(aFitTable, "mse", aRowsNeeded - np, np, mse);
2809  cpl_table_fill_column_window(aFitTable, "x", aRowsNeeded - np, np, aX);
2810  cpl_table_fill_column_window(aFitTable, "sigma", aRowsNeeded - np, np, sigma);
2811  /* multiplets are never FWHM reference lines, do not store FWHM */
2812  for (i = 0, j = aRowsNeeded - np; i < np; i++, j++) {
2813  cpl_table_set(aFitTable, "lambda", j, cpl_vector_get(aLambdas, i));
2814  cpl_table_set(aFitTable, "y", j, cpl_vector_get(ypeaks, i));
2815  double center = cpl_vector_get(ypeaks, i);
2816  cpl_table_set(aFitTable, "center", j, center);
2817  /* centroid errors are starting with row/column index 3 in the matrix, *
2818  * the first ones are the two polynomial coeffs and the sigma */
2819  double cerror = sqrt(cpl_matrix_get(covariance, 3 + i, 3 + i));
2820  cpl_table_set(aFitTable, "cerr", j, cerror);
2821  cpl_table_set(aFitTable, "flux", j, cpl_vector_get(fluxes, i));
2822  /* evaluate the linear background level at the fitted center */
2823  double bg = cpl_vector_get(poly, 0) + cpl_vector_get(poly, 1) * center;
2824  cpl_table_set(aFitTable, "bg", j, bg);
2825  } /* for i, j */
2826  cpl_vector_delete(poly);
2827  cpl_matrix_delete(covariance);
2828 #if 0
2829  printf("stored %d fitted multiplet values:\n", np);
2830  cpl_table_dump(aFitTable, aRowsNeeded - np - 2 < 0 ? 0 : aRowsNeeded - np - 2, np + 5, stdout);
2831  fflush(stdout);
2832 #endif
2833 
2834  return rc;
2835 } /* muse_wave_line_fit_multiple() */
2836 
2837 /*----------------------------------------------------------------------------*/
2876 /*----------------------------------------------------------------------------*/
2877 cpl_error_code
2878 muse_wave_line_fit_iterate(cpl_table *aFitTable, double aLambda,
2879  const muse_wave_params *aParams)
2880 {
2881  cpl_ensure_code(aFitTable, CPL_ERROR_NULL_INPUT);
2882  int npos = cpl_table_get_nrow(aFitTable);
2883  cpl_ensure_code(npos > 0, CPL_ERROR_ILLEGAL_INPUT);
2884  double rsigma = aParams->linesigma < 0 ? 2.5 : aParams->linesigma;
2885 
2886  /* select all entries with the requested wavelength */
2887  cpl_table *fittable = NULL;
2888  if (aLambda > 0) {
2889  cpl_table_unselect_all(aFitTable);
2890  cpl_table_or_selected_double(aFitTable, "lambda", CPL_EQUAL_TO, aLambda);
2891  npos = cpl_table_count_selected(aFitTable);
2892  cpl_ensure_code(npos > 0, CPL_ERROR_ILLEGAL_INPUT);
2893  fittable = cpl_table_extract_selected(aFitTable);
2894  cpl_table_erase_selected(aFitTable);
2895  } else {
2896  fittable = aFitTable;
2897  }
2898 
2899  /* convert "x" and "center" columns into matrix and vector for the fit */
2900  cpl_matrix *pos = cpl_matrix_new(1, npos);
2901  cpl_vector *val = cpl_vector_new(npos);
2902  int i;
2903  for (i = 0; i < npos; i++) {
2904  cpl_matrix_set(pos, 0, i, cpl_table_get(fittable, "x", i, NULL));
2905  cpl_vector_set(val, i, cpl_table_get(fittable, "center", i, NULL));
2906  }
2907 
2908  /* use the iterative polynomial field to throw out bad entries in fittable */
2909  cpl_errorstate state = cpl_errorstate_get();
2910  double mse;
2911  cpl_polynomial *fit = muse_utils_iterate_fit_polynomial(pos, val, NULL, fittable,
2912  aParams->xorder, rsigma,
2913  &mse, NULL);
2914  cpl_matrix_delete(pos);
2915  cpl_vector_delete(val);
2916  cpl_polynomial_delete(fit);
2917  if (!cpl_errorstate_is_equal(state)) {
2918  /* this implies less than xorder entries left, and is really bad; set the *
2919  * error to something high enough to basically exclude these points */
2920  cpl_table_fill_column_window(fittable, "cerr",
2921  0, cpl_table_get_nrow(fittable), 10.);
2922  } else if (aParams->fitweighting == MUSE_WAVE_WEIGHTING_SCATTER) {
2923  /* replace the original centroid error with the fit RMS */
2924  cpl_table_fill_column_window(fittable, "cerr",
2925  0, cpl_table_get_nrow(fittable), sqrt(mse));
2926  } else if (aParams->fitweighting == MUSE_WAVE_WEIGHTING_CERRSCATTER) {
2927  /* add RMS of fit to individual error in quadrature: *
2928  * cerr = sqrt(cerr^2 + mse) */
2929  cpl_table_power_column(fittable, "cerr", 2.); /* cerr^2 */
2930  cpl_table_add_scalar(fittable, "cerr", mse); /* + mse */
2931  cpl_table_power_column(fittable, "cerr", 0.5); /* sqrt() */
2932  }
2933 #if 0
2934  printf("line at %.3f Angstrom: rms = %f (mean error %f)\n", aLambda,
2935  sqrt(mse), cpl_table_get_column_mean(fittable, "cerr"));
2936  cpl_table_dump(fittable, 0, npos, stdout);
2937  fflush(stdout);
2938 #endif
2939  if (aLambda > 0) { /* need to copy results back into input table */
2940  cpl_table_insert(aFitTable, fittable, cpl_table_get_nrow(aFitTable));
2941  cpl_table_delete(fittable);
2942  }
2943  return CPL_ERROR_NONE;
2944 } /* muse_wave_line_fit_iterate() */
2945 
2946 /*----------------------------------------------------------------------------*/
2958 /*----------------------------------------------------------------------------*/
2959 static inline double
2960 muse_wave_ipow(double x, unsigned int y)
2961 {
2962  if (!y) {
2963  return 1;
2964  }
2965  if (y == 1) {
2966  return x;
2967  }
2968  double x2 = x*x;
2969  if (y == 2) {
2970  return x2;
2971  }
2972  if (y == 3) {
2973  return x2*x;
2974  }
2975  if (y == 4) {
2976  return x2*x2;
2977  }
2978  double result;
2979  /* Handle least significant bit in y here in order to avoid an unnecessary *
2980  * multiplication of x - which could cause an over- or underflow */
2981  result = y & 1 ? x : 1.; /* 0^0 is 1, while any other power of 0 is 0 */
2982  while (y >>= 1) {
2983  x *= x;
2984  if (y & 1) { /* Process least significant bit in y */
2985  result *= x;
2986  }
2987  }
2988  return result;
2989 } /* muse_wave_ipow() */
2990 
2991 /*----------------------------------------------------------------------------*/
3006 /*----------------------------------------------------------------------------*/
3007 static int
3008 muse_wave_poly_2d(const double xy[], const double p[], double *f)
3009 {
3010  unsigned short xorder = p[0],
3011  yorder = p[1];
3012  /* add up the polynomial orders */
3013  *f = 0;
3014  double x = xy[0],
3015  y = xy[1];
3016  unsigned int i, idx = 2;
3017  for (i = 0; i <= xorder; i++) {
3018  double xi = muse_wave_ipow(x, i);
3019  unsigned int j; /* y order */
3020  for (j = 0; j <= yorder; j++) {
3021  *f += p[idx++] * xi * muse_wave_ipow(y, j);
3022  } /* for j */
3023  } /* for i */
3024  return 0;
3025 } /* muse_wave_poly_2d() */
3026 
3027 /*----------------------------------------------------------------------------*/
3040 /*----------------------------------------------------------------------------*/
3041 static int
3042 muse_wave_dpoly_2d(const double xy[], const double p[], double f[])
3043 {
3044  unsigned short xorder = p[0],
3045  yorder = p[1];
3046  f[0] = f[1] = 0; /* derivatives regarding the polynomial orders are zero */
3047  /* now the real derivatives, they don't contain the coefficients any more */
3048  double x = xy[0],
3049  y = xy[1];
3050  unsigned int i, idx = 2;
3051  for (i = 0; i <= xorder; i++) {
3052  double xi = muse_wave_ipow(x, i);
3053  unsigned int j; /* y order */
3054  for (j = 0; j <= yorder; j++) {
3055  f[idx++] = xi * muse_wave_ipow(y, j);
3056  } /* for j */
3057  } /* for i */
3058  return 0;
3059 } /* muse_wave_poly_x2y5_deriv() */
3060 
3061 /*----------------------------------------------------------------------------*/
3096 /*----------------------------------------------------------------------------*/
3097 cpl_error_code
3098 muse_wave_poly_fit(cpl_matrix *aXYPos, cpl_vector *aLambdas, cpl_vector *aDLambdas,
3099  cpl_polynomial **aPoly, double *aMSE, muse_wave_params *aParams,
3100  const unsigned short aSlice)
3101 {
3102  cpl_ensure_code(aPoly && aXYPos && aLambdas, CPL_ERROR_NULL_INPUT);
3103 #if 0
3104  printf("aXYPos (%"CPL_SIZE_FORMAT"x%"CPL_SIZE_FORMAT"):\n",
3105  cpl_matrix_get_ncol(aXYPos), cpl_matrix_get_nrow(aXYPos));
3106  cpl_matrix_dump(aXYPos, stdout);
3107  printf("aLambdas (%"CPL_SIZE_FORMAT"):\n", cpl_vector_get_size(aLambdas));
3108  cpl_vector_dump(aLambdas, stdout);
3109  printf("aDLambdas (%"CPL_SIZE_FORMAT"):\n", cpl_vector_get_size(aDLambdas));
3110  cpl_vector_dump(aDLambdas, stdout);
3111  fflush(stdout);
3112 #endif
3113  double rsigma = aParams->fitsigma < 0 ? 3.0 : aParams->fitsigma;
3114  int debug = getenv("MUSE_DEBUG_WAVECAL")
3115  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
3116  /* prepare the polynomial for two dimensions */
3117  *aPoly = cpl_polynomial_new(2);
3118 
3119  double rms = -1; /* start with negative (= invalid) RMS */
3120  int large_residuals = 1, /* init to force a first fit */
3121  niter = 1;
3122  while (large_residuals > 0) {
3123  /* set some typical fit error by default */
3124  cpl_error_code errfit = CPL_ERROR_SINGULAR_MATRIX;
3125 
3126  if (aDLambdas) { /* use our own polynomial fitting with weighting */
3127  /* the lvmq function wants N rows of dimension D=2 *
3128  * instead of the other way around... */
3129  cpl_matrix *xypos = cpl_matrix_transpose_create(aXYPos);
3130 
3131  int npar = 2 + (aParams->xorder + 1) * (aParams->yorder + 1);
3132  cpl_vector *pars = cpl_vector_new(npar);
3133  /* pre-fill all but the zero-order with zero as the first guess value */
3134  cpl_vector_fill(pars, 0.);
3135  /* the polynomial orders go first into the array */
3136  cpl_vector_set(pars, 0, aParams->xorder);
3137  cpl_vector_set(pars, 1, aParams->yorder);
3138  cpl_vector_set(pars, 2, 5000.); /* near lower edge of lambda range */
3139  cpl_array *aflags = cpl_array_new(npar, CPL_TYPE_INT);
3140  /* most are real (free == 1) parameters, except the first two */
3141  cpl_array_set_int(aflags, 0, 0); /* fixed xorder */
3142  cpl_array_set_int(aflags, 1, 0); /* fixed yorder */
3143  cpl_array_fill_window_int(aflags, 2, npar - 2, 1);
3144  int *pflags = cpl_array_get_data_int(aflags);
3145  errfit = cpl_fit_lvmq(xypos, /* sigma_x (unsupported) */NULL,
3146  aLambdas, aDLambdas, pars, pflags,
3147  muse_wave_poly_2d, muse_wave_dpoly_2d,
3148  CPL_FIT_LVMQ_TOLERANCE, CPL_FIT_LVMQ_COUNT,
3149  CPL_FIT_LVMQ_MAXITER, NULL, NULL, NULL);
3150  cpl_array_delete(aflags);
3151  cpl_size k = 2; /* coefficients start at the 3rd vector entry */
3152  unsigned short i;
3153  for (i = 0; i <= aParams->xorder; i++) {
3154  unsigned short j;
3155  for (j = 0; j <= aParams->yorder; j++) {
3156  cpl_size pows[2] = { i, j }; /* trick to access the polynomial */
3157  cpl_polynomial_set_coeff(*aPoly, pows,
3158  cpl_vector_get(pars, k++));
3159  } /* for j */
3160  } /* for i */
3161  cpl_matrix_delete(xypos);
3162  cpl_vector_delete(pars);
3163  } else {
3164  /* use CPL polynomial fitting for everything else */
3165  const cpl_boolean sym = CPL_FALSE;
3166  const cpl_size mindeg2d[] = { 0, 0 },
3167  maxdeg2d[] = { aParams->xorder, aParams->yorder };
3168  errfit = cpl_polynomial_fit(*aPoly, aXYPos, &sym, aLambdas, NULL,
3169  CPL_TRUE, mindeg2d, maxdeg2d);
3170  } /* CPL polynomial fitting */
3171 #if 0
3172  printf("polynomial (orders=%hu/%hu, degree=%"CPL_SIZE_FORMAT"):\n",
3173  aParams->xorder, aParams->yorder, cpl_polynomial_get_degree(*aPoly));
3174  cpl_polynomial_dump(*aPoly, stdout);
3175  fflush(stdout);
3176 #endif
3177 
3178  if (errfit) {
3179  /* the fit failed, complain and abort */
3180  cpl_msg_error(__func__, "The polynomial fit in this slice failed: %s",
3181  cpl_error_get_message());
3182 
3183 #if 0
3184  /* more fancy plotting of input datapoints *
3185  * (color coded lambda over x/y-position) */
3186  FILE *gp = popen("gnuplot -persist", "w");
3187  if (gp) {
3188  /* plot title with fit details */
3189  fprintf(gp, "set title \"2D polynomial fit residuals (failed fit: "
3190  "\'%s\')\n", cpl_error_get_message());
3191  /* set nice palette */
3192  fprintf(gp, "set palette defined ( 0 \"dark-violet\","
3193  "1 \"dark-blue\", 4 \"green\", 6 \"yellow\", 8 \"orange\","
3194  "9 \"red\", 10 \"dark-red\")\n");
3195  fprintf(gp, "unset key\n");
3196  cpl_matrix *xpos = cpl_matrix_extract_row(aXYPos, 0);
3197  fprintf(gp, "set xrange [%f:%f]\n", cpl_matrix_get_min(xpos),
3198  cpl_matrix_get_max(xpos));
3199  cpl_matrix_delete(xpos);
3200  fprintf(gp, "set cbrange [%f:%f]\n", cpl_vector_get_min(aLambdas),
3201  cpl_vector_get_max(aLambdas));
3202  fprintf(gp, "plot \"-\" w p pal\n");
3203 
3204  printf("# X Y lambda\n");
3205  int n;
3206  for (n = 0; n < cpl_vector_get_size(aLambdas); n++) {
3207  printf("%4d %7.2f %8.3f\n", (int)cpl_matrix_get(aXYPos, 0, n),
3208  cpl_matrix_get(aXYPos, 1, n), cpl_vector_get(aLambdas, n));
3209  fprintf(gp, "%f %f %f\n", cpl_matrix_get(aXYPos, 0, n),
3210  cpl_matrix_get(aXYPos, 1, n), cpl_vector_get(aLambdas, n));
3211  }
3212  fflush(stdout);
3213  fprintf(gp, "EOF\n");
3214  pclose(gp);
3215  }
3216 #endif
3217 
3218  /* make sure we delete the polynomial */
3219  cpl_polynomial_delete(*aPoly);
3220  *aPoly = NULL; /* make sure we can detect the failure */
3221  return CPL_ERROR_ILLEGAL_OUTPUT;
3222  } /* if failed fit */
3223 
3224  int npoints = cpl_vector_get_size(aLambdas);
3225  cpl_vector *res = cpl_vector_new(npoints);
3226  /* the aDLambdas parameter (the errors of the datapoints) do not *
3227  * change the filled vector, just the chi^2 which we don't need */
3228  cpl_vector_fill_polynomial_fit_residual(res, aLambdas, aDLambdas, *aPoly,
3229  aXYPos, NULL);
3230  /* compute the mean-squared error, without any weighting, in any case */
3231  double mse = 0;
3232  /* compute a (weighted) mean-squared error, if we know the error bars */
3233  if (aDLambdas) { /* no errors given, equal weights for all points */
3234  double wsum = 0;
3235  int i;
3236  for (i = 0; i < npoints; i++) {
3237  double weight = 1. / cpl_vector_get(aDLambdas, i);
3238  mse += pow(cpl_vector_get(res, i), 2) * weight;
3239  wsum += weight;
3240  } /* for i */
3241  mse /= wsum;
3242  } else {
3243  mse = cpl_vector_product(res, res) / npoints;
3244  }
3245  double rlimit = rsigma * sqrt(mse); /* limit using possibly weighted RMS */
3246  if (debug) {
3247  printf("Resulting 2D polynomial (%d points, RMS = %f dRMS = %f), "
3248  "%.1f-sigma limit now %f (target RMS = %f):\n", npoints, sqrt(mse),
3249  rms < 0 ? 0. : rms - sqrt(mse), rsigma, rlimit, aParams->targetrms);
3250  if (debug > 1) {
3251  cpl_polynomial_dump(*aPoly, stdout);
3252  }
3253  fflush(stdout);
3254  }
3255 #if 0
3256  /* simple plotting of intermediate residuals *
3257  * (value over arbitrary x-position in vector) */
3258  cpl_plot_vector("set title \"res\"\n", "", "", res);
3259 #endif
3260  if (aParams->rflag) { /* dump debug information into table, if requested */
3261  int nnew = cpl_vector_get_size(res),
3262  nrow = aParams->residuals ? cpl_table_get_nrow(aParams->residuals) : 0;
3263  /* create the residuals table if wanted and not yet existing */
3264  if (!aParams->residuals) {
3265  aParams->residuals = muse_cpltable_new(muse_wavedebug_def, nnew);
3266  cpl_table_set_column_savetype(aParams->residuals, "slice", CPL_TYPE_UCHAR);
3267  } else { /* resize table as needed to fill in the new entries */
3268  cpl_table_set_size(aParams->residuals, nrow + nnew);
3269  }
3270  cpl_table_fill_column_window_int(aParams->residuals, "slice",
3271  nrow, nnew, aSlice);
3272  cpl_table_fill_column_window_int(aParams->residuals, "iteration",
3273  nrow, nnew, niter);
3274  cpl_table_fill_column_window_double(aParams->residuals, "rejlimit",
3275  nrow, nnew, rlimit);
3276  int n;
3277  for (n = 0; n < cpl_vector_get_size(res); n++) {
3278  cpl_table_set_int(aParams->residuals, "x", nrow + n,
3279  (int)cpl_matrix_get(aXYPos, 0, n));
3280  cpl_table_set_float(aParams->residuals, "y", nrow + n,
3281  cpl_matrix_get(aXYPos, 1, n));
3282  cpl_table_set_float(aParams->residuals, "lambda", nrow + n,
3283  cpl_vector_get(aLambdas, n));
3284  cpl_table_set_double(aParams->residuals, "residual", nrow + n,
3285  cpl_vector_get(res, n));
3286  } /* for n (all wavelengths) */
3287  if (debug) {
3288  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" entries in residuals table "
3289  "after iteration %d of slice %hu",
3290  cpl_table_get_nrow(aParams->residuals), niter, aSlice);
3291  }
3292  } /* if debug table */
3293  niter++;
3294 
3295  /* if the fit is already good enough, we don't *
3296  * need to go through point rejection any more */
3297  cpl_boolean isgoodenough = (rms < 0 ? CPL_FALSE : rms - sqrt(mse) < 0.001)
3298  || (sqrt(mse) <= aParams->targetrms);
3299  large_residuals = 0;
3300  int i;
3301  for (i = 0; !isgoodenough && i < cpl_vector_get_size(res); i++) {
3302  /* compare this residual value to the RMS value */
3303  if (fabs(cpl_vector_get(res, i)) < rlimit) {
3304  /* good fit at this point */
3305  continue;
3306  }
3307  /* bad residual, remove element, from residuals vector and the data vectors */
3308 #if 0
3309  cpl_msg_debug(__func__, "residual = %f (position %fx%f, lambda=%f+/-%f)",
3310  cpl_vector_get(res, i),
3311  cpl_matrix_get(aXYPos, 0, i), cpl_matrix_get(aXYPos, 1, i),
3312  cpl_vector_get(aLambdas, i),
3313  aDLambdas ? cpl_vector_get(aDLambdas, i) : 1.);
3314 #endif
3315  if (cpl_vector_get_size(res) == 1) {
3316  cpl_msg_debug(__func__, "trying to remove the last vector/matrix "
3317  "element when checking against fit sigma");
3318  break;
3319  }
3320  /* remove bad element from the fit structures... */
3322  cpl_matrix_erase_columns(aXYPos, i, 1);
3323  muse_cplvector_erase_element(aLambdas, i);
3324  if (aDLambdas) {
3325  muse_cplvector_erase_element(aDLambdas, i);
3326  }
3327 
3328  large_residuals++;
3329  i--; /* we stay at this position to see what moved here */
3330  } /* for i */
3331 
3332  rms = sqrt(mse);
3333  if (!large_residuals) {
3334  /* no large residuals (any more), so this is the final solution */
3335 
3336  /* save the mean squared error */
3337  if (aMSE) {
3338  *aMSE = mse;
3339  }
3340  } /* if !large_residuals */
3341 
3342  cpl_vector_delete(res);
3343  } /* while large_residuals */
3344 
3345  return CPL_ERROR_NONE;
3346 } /* muse_wave_poly_fit() */
3347 
3348 /*----------------------------------------------------------------------------*/
3358 /*----------------------------------------------------------------------------*/
3359 cpl_table *
3360 muse_wave_table_create(const unsigned short aNSlices, const unsigned short aXOrder,
3361  const unsigned short aYOrder)
3362 {
3363  cpl_table *table = cpl_table_new(aNSlices);
3364  cpl_ensure(table, CPL_ERROR_UNSPECIFIED, NULL);
3365 
3366  /* create column for slice number*/
3367  cpl_table_new_column(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, CPL_TYPE_INT);
3368  cpl_table_set_column_unit(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, "No");
3369  cpl_table_set_column_format(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, "%2d");
3370 
3371  /* create columns for all coefficients, so loop over fit orders */
3372  unsigned short i;
3373  for (i = 0; i <= aXOrder; i++) {
3374  unsigned short j;
3375  for (j = 0; j <= aYOrder; j++) {
3376  /* create column name, start coefficient names at 0 */
3377  char *colname = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
3378  cpl_table_new_column(table, colname, CPL_TYPE_DOUBLE);
3379  /* fit coeff are in wavelength space */
3380  cpl_table_set_column_unit(table, colname, "Angstrom");
3381  cpl_table_set_column_format(table, colname, "%12.5e");
3382  cpl_free(colname);
3383  }
3384  }
3385  /* create column for mean squared error of fitting parameters */
3386  cpl_table_new_column(table, MUSE_WAVECAL_TABLE_COL_MSE, CPL_TYPE_DOUBLE);
3387  cpl_table_set_column_format(table, MUSE_WAVECAL_TABLE_COL_MSE, "%f");
3388 
3389  return table;
3390 } /* muse_wave_table_create() */
3391 
3392 /*----------------------------------------------------------------------------*/
3406 /*----------------------------------------------------------------------------*/
3407 cpl_error_code
3408 muse_wave_table_add_poly(cpl_table *aTable, cpl_polynomial *aPoly,
3409  double aMSE, unsigned short aXOrder,
3410  unsigned short aYOrder, const unsigned short aRow)
3411 {
3412  cpl_ensure_code(aTable && aPoly, CPL_ERROR_NULL_INPUT);
3413  cpl_ensure_code(cpl_polynomial_get_dimension(aPoly) == 2,
3414  CPL_ERROR_ILLEGAL_INPUT);
3415 
3416  /* row numbers start at 0 but the numbers in the table should start with 1 */
3417  cpl_table_set_int(aTable, MUSE_WAVECAL_TABLE_COL_SLICE_NO, aRow, aRow + 1);
3418  cpl_table_set_double(aTable, MUSE_WAVECAL_TABLE_COL_MSE, aRow, aMSE);
3419 
3420  /* i and j loop over all orders of the polynomial */
3421  unsigned short i;
3422  for (i = 0; i <= aXOrder; i++) {
3423  unsigned short j;
3424  for (j = 0; j <= aYOrder; j++) {
3425  cpl_size pows[2] = { i, j }; /* trick to access the polynomial */
3426 
3427  char *colname = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
3428  cpl_error_code rc = cpl_table_set_double(aTable, colname, aRow,
3429  cpl_polynomial_get_coeff(aPoly,
3430  pows));
3431  if (rc != CPL_ERROR_NONE) {
3432  cpl_msg_warning(__func__, "Problem writing %f to field %s in "
3433  "wavelength table: %s",
3434  cpl_polynomial_get_coeff(aPoly, pows), colname,
3435  cpl_error_get_message());
3436  cpl_polynomial_dump(aPoly, stdout);
3437  cpl_table_dump(aTable, aRow, 1, stdout);
3438  fflush(stdout);
3439  }
3440  cpl_free(colname);
3441  } /* for j (polynomial orders in y) */
3442  } /* for i (polynomial orders in x) */
3443 
3444  return CPL_ERROR_NONE;
3445 } /* muse_wave_table_add_poly() */
3446 
3447 /*----------------------------------------------------------------------------*/
3462 /*----------------------------------------------------------------------------*/
3463 cpl_error_code
3464 muse_wave_table_get_orders(const cpl_table *aWave, unsigned short *aXOrder,
3465  unsigned short *aYOrder)
3466 {
3467  cpl_ensure_code(aWave && aXOrder && aYOrder, CPL_ERROR_NULL_INPUT);
3468  cpl_array *cols = cpl_table_get_column_names(aWave);
3469 
3470  /* second last entry will contain highest order (last is MSE) */
3471  const char *highcol = cpl_array_get_string(cols, cpl_array_get_size(cols) - 2);
3472  char *colname = cpl_strdup(highcol); /* duplicate it so that we can manipulate it */
3473  cpl_array_delete(cols);
3474 
3475  /* we can directly get the value of the last digit */
3476  *aYOrder = atoi(colname+4);
3477  /* now null out the last digit so that we can get the first one only */
3478  colname[4] = '\0';
3479  *aXOrder = atoi(colname+3);
3480  cpl_free(colname);
3481 
3482  return CPL_ERROR_NONE;
3483 } /* muse_wave_table_get_orders() */
3484 
3485 /*----------------------------------------------------------------------------*/
3506 /*----------------------------------------------------------------------------*/
3507 cpl_polynomial *
3508 muse_wave_table_get_poly_for_slice(const cpl_table *aTable,
3509  const unsigned short aSlice)
3510 {
3511  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, NULL);
3512  cpl_ensure(aSlice >= 1 && aSlice <= kMuseSlicesPerCCD,
3513  CPL_ERROR_ILLEGAL_INPUT, NULL);
3514  /* search for row containing the requested slice, first to access it *
3515  * in a possibly incomplete table, and second to check its presence! */
3516  int irow, nrow = cpl_table_get_nrow(aTable);
3517  for (irow = 0; irow < nrow; irow++) {
3518  int err;
3519  unsigned short slice = cpl_table_get_int(aTable,
3520  MUSE_WAVECAL_TABLE_COL_SLICE_NO,
3521  irow, &err);
3522  if (slice == aSlice && !err) {
3523  break;
3524  }
3525  } /* for irow */
3526  cpl_ensure(irow < nrow, CPL_ERROR_DATA_NOT_FOUND, NULL);
3527 
3528  cpl_polynomial *pwave = cpl_polynomial_new(2);
3529  char colname[15]; /* "wlc" plus max 2x5 chars for the numbers */
3530  unsigned short wavexorder, waveyorder;
3531  muse_wave_table_get_orders(aTable, &wavexorder, &waveyorder);
3532  unsigned short l;
3533  for (l = 0; l <= wavexorder; l++) {
3534  unsigned short k;
3535  for (k = 0; k <= waveyorder; k++) {
3536  cpl_size pows[2] = { l, k }; /* trick to access the polynomial */
3537  sprintf(colname, MUSE_WAVECAL_TABLE_COL_COEFF, l, k);
3538  int err;
3539  cpl_polynomial_set_coeff(pwave, pows,
3540  cpl_table_get_double(aTable, colname, irow,
3541  &err));
3542  if (err != 0) { /* broken table entry */
3543  cpl_polynomial_delete(pwave);
3544  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT, "Wavelength "
3545  "calibration table broken in slice %hu (row index"
3546  " %d) column %s", aSlice, irow, colname);
3547  return NULL;
3548  } /* if */
3549  } /* for k */
3550  } /* for l */
3551  return pwave;
3552 } /* muse_wave_table_get_poly_for_slice() */
3553 
3554 /*----------------------------------------------------------------------------*/
3577 /*----------------------------------------------------------------------------*/
3578 cpl_image *
3579 muse_wave_map(muse_image *aImage, const cpl_table *aWave,
3580  const cpl_table *aTrace)
3581 {
3582  cpl_ensure(aImage && aWave && aTrace, CPL_ERROR_NULL_INPUT, NULL);
3583  int nx = cpl_image_get_size_x(aImage->data),
3584  ny = cpl_image_get_size_y(aImage->data);
3585  cpl_image *wavemap = cpl_image_new(nx, ny, CPL_TYPE_FLOAT);
3586  cpl_ensure(wavemap, cpl_error_get_code(), NULL);
3587 
3588  /* get output image buffer as pointer */
3589  float *wdata = cpl_image_get_data_float(wavemap);
3590 
3591  unsigned short wavexorder, waveyorder;
3592  muse_wave_table_get_orders(aWave, &wavexorder, &waveyorder);
3593  cpl_msg_debug(__func__, "order for trace solution is %d, for wavelength "
3594  "solution %hu/%hu", muse_trace_table_get_order(aTrace),
3595  wavexorder, waveyorder);
3596 
3597  /* loop through all slices */
3598  unsigned short islice;
3599  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
3600  cpl_msg_debug(__func__, "Starting to process slice %hu", islice + 1);
3601 
3602  /* fill the wavelength calibration polynomial for this slice */
3603  cpl_polynomial *pwave = muse_wave_table_get_poly_for_slice(aWave, islice + 1);
3604  /* vector for the position within the slice (for evaluation *
3605  * of the wavelength solution in two dimensions */
3606  cpl_vector *pos = cpl_vector_new(2);
3607 
3608  /* get the tracing polynomials for this slice */
3609  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
3610  islice + 1);
3611  if (!ptrace) {
3612  cpl_msg_warning(__func__, "slice %2hu: tracing polynomials missing!",
3613  islice + 1);
3614  continue;
3615  }
3616 #if 0
3617  printf("polynomials for slice %hu:\n", islice + 1);
3618  cpl_polynomial_dump(ptrace[MUSE_TRACE_LEFT], stdout);
3619  cpl_polynomial_dump(ptrace[MUSE_TRACE_RIGHT], stdout);
3620  cpl_polynomial_dump(pwave, stdout);
3621  fflush(stdout);
3622 #endif
3623 
3624  /* within each slice, loop from bottom to top */
3625  int j;
3626  for (j = 1; j <= ny; j++) {
3627  /* determine the slice edges for this vertical *
3628  * position so that we can loop over those pixels */
3629  int ileft = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
3630  j, NULL)),
3631  iright = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
3632  j, NULL));
3633  /* try to detect faulty polynomials */
3634  if (ileft < 1 || iright > nx || ileft > iright) {
3635  cpl_msg_warning(__func__, "slice %2hu: faulty polynomial detected at "
3636  "y=%d", islice + 1, j);
3637  break; /* skip the rest of this slice */
3638  }
3639  cpl_vector_set(pos, 1, j); /* vertical pos. for wavelength evaluation */
3640 
3641  /* now loop over all pixels of this slice horizontally */
3642  int i;
3643  for (i = ileft; i <= iright; i++) {
3644  cpl_vector_set(pos, 0, i); /* horiz. pos. for wavelength evaluation */
3645 
3646  /* compute the wavelength of this pixel */
3647  wdata[(i-1) + (j-1)*nx] = cpl_polynomial_eval(pwave, pos);
3648  } /* for i (horizontal pixels) */
3649  } /* for j (vertical direction) */
3650 
3651  /* we are now done with this slice, clean up */
3652  muse_trace_polys_delete(ptrace);
3653  cpl_polynomial_delete(pwave);
3654  cpl_vector_delete(pos);
3655  } /* for islice */
3656 
3657  return wavemap;
3658 } /* muse_wave_map() */
3659 
3660 /*----------------------------------------------------------------------------*/
3684 /*----------------------------------------------------------------------------*/
3685 cpl_error_code
3686 muse_wave_plot_residuals(cpl_table *aTable, const unsigned short aSlice,
3687  unsigned int aIter, cpl_boolean aPlotLambda,
3688  cpl_vector *aCuts)
3689 {
3690 #if HAVE_POPEN && HAVE_PCLOSE
3691  cpl_ensure_code(aTable, CPL_ERROR_NULL_INPUT);
3692  cpl_error_code rc = muse_cpltable_check(aTable, muse_wavedebug_def);
3693  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
3694 
3695  FILE *gp = popen("gnuplot", "w");
3696  if (!gp) {
3697  return CPL_ERROR_ASSIGNING_STREAM;
3698  }
3699 
3700  /* select relevant rows in the table */
3701  cpl_table_unselect_all(aTable); /* start clean */
3702 
3703  int n, nrow = cpl_table_get_nrow(aTable),
3704  error = 0;
3705  if (aSlice == 0) {
3706  printf("Selecting data of all slices.\n");
3707 
3708  const int *slice = cpl_table_get_data_int_const(aTable, "slice"),
3709  *iter = cpl_table_get_data_int_const(aTable, "iteration");
3710  if (!aIter) { /* last iteration */
3711  fprintf(stderr, "Selecting data of last iteration of all slices\n");
3712  /* the last row of each slice number is the one with the last iteration */
3713  int sliceno = slice[nrow - 1],
3714  iterlast = iter[nrow - 1];
3715  for (n = nrow - 2; n >= 0; n--) {
3716  if (slice[n] == sliceno && iter[n] != iterlast) {
3717  cpl_table_select_row(aTable, n);
3718  }
3719  if (slice[n] != sliceno) { /* previous slice, reset comparison */
3720  sliceno = slice[n];
3721  iterlast = iter[n];
3722  }
3723  } /* for n (table rows) */
3724  cpl_table_erase_selected(aTable);
3725  /* plot title with some details */
3726  fprintf(gp, "set title \"slices %d..%d, iterations %d..%d: 2D polynomial "
3727  "fit residuals (limits %f..%f)\n",
3728  (int)cpl_table_get_column_min(aTable, "slice"),
3729  (int)cpl_table_get_column_max(aTable, "slice"),
3730  (int)cpl_table_get_column_min(aTable, "iteration"),
3731  (int)cpl_table_get_column_max(aTable, "iteration"),
3732  cpl_table_get_column_min(aTable, "rejlimit"),
3733  cpl_table_get_column_max(aTable, "rejlimit"));
3734  } else {
3735  printf("Selecting data of iteration %d.\n", aIter);
3736  for (n = 0; n < nrow; n++) {
3737  if (iter[n] != (int)aIter) {
3738  cpl_table_select_row(aTable, n);
3739  }
3740  } /* for n (table rows) */
3741  cpl_table_erase_selected(aTable);
3742  /* plot title with some details */
3743  fprintf(gp, "set title \"slices %d..%d, iteration %d: 2D polynomial fit "
3744  "residuals (limits %f..%f)\n",
3745  (int)cpl_table_get_column_min(aTable, "slice"),
3746  (int)cpl_table_get_column_max(aTable, "slice"), aIter,
3747  cpl_table_get_column_min(aTable, "rejlimit"),
3748  cpl_table_get_column_max(aTable, "rejlimit"));
3749  }
3750  } else {
3751  printf("Selecting data of slice %hu.\n", aSlice);
3752  const int *slice = cpl_table_get_data_int_const(aTable, "slice");
3753  for (n = 0; n < nrow; n++) {
3754  if (slice[n] != aSlice) {
3755  cpl_table_select_row(aTable, n);
3756  }
3757  } /* for n (table rows) */
3758  cpl_table_erase_selected(aTable);
3759  nrow = cpl_table_get_nrow(aTable);
3760  cpl_table_unselect_all(aTable);
3761 
3762  const int *iter = cpl_table_get_data_int_const(aTable, "iteration");
3763  if (!aIter) { /* last iteration */
3764  /* table is sorted, so last iteration is that of the last row */
3765  aIter = iter[nrow - 1];
3766  }
3767  printf("Selecting data of iteration %d.\n", aIter);
3768  for (n = 0; n < nrow; n++) {
3769  if (iter[n] != (int)aIter) {
3770  cpl_table_select_row(aTable, n);
3771  }
3772  } /* for n (table rows) */
3773  cpl_table_erase_selected(aTable);
3774 
3775  /* plot title with some details */
3776  fprintf(gp, "set title \"slice %hu, iteration %d: 2D polynomial fit residuals"
3777  " (limit=%f)\n", aSlice, aIter,
3778  cpl_table_get_double(aTable, "rejlimit", 0, &error));
3779  } /* single slice */
3780 
3781  /* data access */
3782  nrow = cpl_table_get_nrow(aTable);
3783  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
3784  printf("Plotting %d points.\n", nrow);
3785  const int *x = cpl_table_get_data_int_const(aTable, "x");
3786  const float *y = cpl_table_get_data_float_const(aTable, "y"),
3787  *lambda = cpl_table_get_data_float_const(aTable, "lambda");
3788  const double *r = cpl_table_get_data_double_const(aTable, "residual");
3789  /* *rlimit = cpl_table_get_data_double_const(aTable, "rejlimit"); */
3790 
3791  /* determine plotting limits */
3792  int xmin = cpl_table_get_column_min(aTable, "x") - 2,
3793  xmax = cpl_table_get_column_max(aTable, "x") + 2;
3794  float ymin = cpl_table_get_column_min(aTable, "y") - 2,
3795  ymax = cpl_table_get_column_max(aTable, "y") + 2,
3796  lmin = cpl_table_get_column_min(aTable, "lambda") - 2,
3797  lmax = cpl_table_get_column_max(aTable, "lambda") + 2;
3798  double rmin = cpl_table_get_column_min(aTable, "residual"),
3799  rmax = cpl_table_get_column_max(aTable, "residual");
3800  if (aCuts && cpl_vector_get_size(aCuts) == 2) {
3801  rmin = cpl_vector_get(aCuts, 0);
3802  rmax = cpl_vector_get(aCuts, 1);
3803  }
3804 
3805  /* set nice palette */
3806  fprintf(gp, "set palette defined ( 0 \"dark-violet\","
3807  "1 \"dark-blue\", 4 \"green\", 6 \"yellow\", 8 \"orange\","
3808  "9 \"red\", 10 \"dark-red\")\n");
3809  fprintf(gp, "unset key\n");
3810  printf("Setting plotting limits: [%d:%d][%.2f:%.2f][%.4f:%.4f]\n",
3811  xmin, xmax, aPlotLambda ? lmin : ymin, aPlotLambda ? lmax : ymax,
3812  rmin, rmax);
3813  fprintf(gp, "set xrange [%d:%d]\n", xmin, xmax);
3814  if (aPlotLambda) {
3815  fprintf(gp, "set yrange [%f:%f]\n", lmin, lmax);
3816  } else {
3817  fprintf(gp, "set yrange [%f:%f]\n", ymin, ymax);
3818  }
3819  fprintf(gp, "set cbrange [%f:%f]\n", rmin, rmax);
3820  fprintf(gp, "set view map\n");
3821  fprintf(gp, "splot \"-\" w p pal\n");
3822  for (n = 0; n < nrow; n++) {
3823  if (aPlotLambda) {
3824  fprintf(gp, "%d %f %e\n", x[n], lambda[n], r[n]);
3825  } else {
3826  fprintf(gp, "%d %f %e\n", x[n], y[n], r[n]);
3827  }
3828  }
3829  fprintf(gp, "EOF\n");
3830  fflush(gp);
3831  /* request keypress, so that working with the mouse keeps *
3832  * working and gnuplot has enough time to actually draw all *
3833  * the stuff before the files get removed */
3834  printf("Press ENTER to end program and close plot\n");
3835  getchar();
3836  pclose(gp);
3837  return CPL_ERROR_NONE;
3838 #else /* no HAVE_POPEN && HAVE_PCLOSE */
3839  return CPL_ERROR_UNSUPPORTED_MODE;
3840 #endif /* HAVE_POPEN && HAVE_PCLOSE */
3841 } /* muse_wave_plot_residuals() */
3842 
3843 /*----------------------------------------------------------------------------*/
3869 /*----------------------------------------------------------------------------*/
3870 cpl_error_code
3871 muse_wave_plot_column(cpl_table *aCTable, cpl_table *aRTable,
3872  const unsigned short aSlice, unsigned int aColumn,
3873  unsigned int aIter, cpl_boolean aPlotRes)
3874 {
3875 #if HAVE_POPEN && HAVE_PCLOSE
3876  cpl_ensure_code(aCTable && aRTable, CPL_ERROR_NULL_INPUT);
3877  cpl_error_code rc = muse_cpltable_check(aRTable, muse_wavedebug_def);
3878  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
3879  /* WAVECAL_TABLE doesn't has a _def structure to check, so test orders */
3880  unsigned short xorder, yorder;
3881  muse_wave_table_get_orders(aCTable, &xorder, &yorder);
3882  cpl_ensure_code(xorder > 0 && yorder > 0, CPL_ERROR_ILLEGAL_INPUT);
3883  cpl_ensure_code(aSlice >= 1 && aSlice <= kMuseSlicesPerCCD,
3884  CPL_ERROR_ACCESS_OUT_OF_RANGE);
3885 
3886  FILE *gp = popen("gnuplot", "w");
3887  if (!gp) {
3888  return CPL_ERROR_ASSIGNING_STREAM;
3889  }
3890 
3891  /* select relevant rows in the table */
3892  cpl_table_unselect_all(aRTable); /* start clean */
3893 
3894  printf("Selecting data of slice %hu.\n", aSlice);
3895  const int *slice = cpl_table_get_data_int_const(aRTable, "slice");
3896  int n, nrow = cpl_table_get_nrow(aRTable);
3897  for (n = 0; n < nrow; n++) {
3898  if (slice[n] != aSlice) {
3899  cpl_table_select_row(aRTable, n);
3900  }
3901  } /* for n (table rows) */
3902  cpl_table_erase_selected(aRTable);
3903  nrow = cpl_table_get_nrow(aRTable);
3904  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
3905  cpl_table_unselect_all(aRTable);
3906 
3907  const int *iter = cpl_table_get_data_int_const(aRTable, "iteration");
3908  if (!aIter) { /* last iteration */
3909  /* table is sorted, so last iteration is that of the last row */
3910  aIter = iter[nrow - 1];
3911  }
3912  printf("Selecting data of iteration %d.\n", aIter);
3913  for (n = 0; n < nrow; n++) {
3914  if (iter[n] != (int)aIter) {
3915  cpl_table_select_row(aRTable, n);
3916  }
3917  } /* for n (table rows) */
3918  cpl_table_erase_selected(aRTable);
3919  nrow = cpl_table_get_nrow(aRTable);
3920  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
3921  cpl_table_unselect_all(aRTable);
3922 
3923  unsigned int col1 = cpl_table_get_column_min(aRTable, "x"),
3924  col2 = cpl_table_get_column_max(aRTable, "x");
3925  if (aColumn > 0) {
3926  col1 = col2 = aColumn;
3927  } else if (aColumn > (unsigned)kMuseOutputXRight) {
3928  /* approximate central column of the slice */
3929  col1 = col2 = (col1 + col2) / 2;
3930  }
3931  printf("Plotting data of columns %u..%u.\n", col1, col2);
3932 
3933  /* determine plotting limits */
3934  float ymin = cpl_table_get_column_min(aRTable, "y") - 10,
3935  ymax = cpl_table_get_column_max(aRTable, "y") + 10,
3936  lmin = cpl_table_get_column_min(aRTable, "lambda") - 10,
3937  lmax = cpl_table_get_column_max(aRTable, "lambda") + 10;
3938  double rmin = cpl_table_get_column_min(aRTable, "residual") * 1.03,
3939  rmax = cpl_table_get_column_max(aRTable, "residual") * 1.03;
3940  /* plot title with some details */
3941  fprintf(gp, "set title \"slice %hu, iteration %d, column %u..%u: polynomial "
3942  "and ", aSlice, aIter, col1, col2);
3943  printf("Setting plotting limits: ");
3944  if (aPlotRes) {
3945  fprintf(gp, "residuals (limit=%f)\"\n",
3946  cpl_table_get_double(aRTable, "rejlimit", 0, NULL));
3947  printf("[%.2f:%.2f][%.4f:%.4f]\n", lmin, lmax, rmin, rmax);
3948  fprintf(gp, "set xrange [%f:%f]\n", lmin, lmax);
3949  fprintf(gp, "set yrange [%f:%f]\n", rmin, rmax);
3950  fprintf(gp, "set xlabel \"Wavelength [Angstrom]\"\n");
3951  fprintf(gp, "set ylabel \"Residuals [Angstrom]\"\n");
3952  } else {
3953  fprintf(gp, "arc line positions\"\n");
3954  printf("[%.2f:%.2f][%.2f:%.2f]\n", ymin, ymax, lmin, lmax);
3955  fprintf(gp, "set xrange [%g:%g]\n", ymin, ymax);
3956  fprintf(gp, "set yrange [%f:%f]\n", lmin, lmax);
3957  fprintf(gp, "set xlabel \"y-position [pix]\"\n");
3958  fprintf(gp, "set ylabel \"Wavelength [Angstrom]\"\n");
3959  }
3960  fprintf(gp, "set key outside below\n");
3961  fprintf(gp, "set samples 1000\n"); /* more samples so that one can zoom in */
3962 
3963  /* create gnuplot polynomial from table coefficients */
3964  fprintf(gp, "p(x,y) = 0 ");
3965  if (!aPlotRes) { /* create full polynomial */
3966  unsigned short i;
3967  for (i = 0; i <= xorder; i++) {
3968  unsigned short j;
3969  for (j = 0; j <= yorder; j++) {
3970  char *coeff = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
3971  double cvalue = cpl_table_get(aCTable, coeff, aSlice - 1, NULL);
3972  cpl_free(coeff);
3973  fprintf(gp, " + (%g) * x**(%hu) * y**(%hu)", cvalue, i, j);
3974  } /* for j (y orders) */
3975  } /* for i (x orders) */
3976  }
3977  fprintf(gp, "\n");
3978 
3979  const int *x = cpl_table_get_data_int_const(aRTable, "x");
3980  const float *y = cpl_table_get_data_float_const(aRTable, "y"),
3981  *lambda = cpl_table_get_data_float_const(aRTable, "lambda");
3982  const double *r = cpl_table_get_data_double_const(aRTable, "residual");
3983 
3984  /* distribute columns over 256 color values */
3985  double dcol = (col2 - col1) / 255.;
3986  if (dcol == 0.) { /* take care not to produce NANs below when dividing */
3987  dcol = 1.;
3988  }
3989  /* construct the plot command as a single long line */
3990  fprintf(gp, "plot ");
3991  if (aPlotRes) {
3992  fprintf(gp, "0 t \"\", "); /* plot line to represent solution */
3993  }
3994  unsigned int ncol, npoints = 0;
3995  for (ncol = col1; ncol <= col2; ncol++) {
3996  /* plot polynomial and values for the specified CCD column */
3997  int red = (ncol - col1) / dcol, /* red */
3998  grn = (col2 - ncol) / dcol, /* green */
3999  blu = 0; /* blue */
4000  if (aPlotRes) {
4001  fprintf(gp, "\"-\" u 2:3 t \"col %u\" w p ps 0.8 lt rgb \"#%02x%02x%02x\"",
4002  ncol, red, grn, blu);
4003  } else {
4004  /* create data values from polynomial and residuals for given CCD column */
4005  fprintf(gp, "p(%u, x) t \"\" w l lw 0.7 lt rgb \"#%02x%02x%02x\", "
4006  "\"-\" u 1:(p(%u,$1)+$3) t \"col %u\" w p ps 0.8 lt rgb \"#%02x%02x%02x\"",
4007  ncol, red, grn, blu, ncol, ncol, red, grn, blu);
4008  }
4009  if (ncol == col2) {
4010  fprintf(gp, "\n"); /* end the plot command */
4011  } else {
4012  fprintf(gp, ", "); /* plot command of next column to come */
4013  }
4014  } /* for ncol (relevant columns) */
4015  for (ncol = col1; ncol <= col2; ncol++) {
4016  for (n = 0; n < nrow; n++) {
4017  if (x[n] == (int)ncol) {
4018  fprintf(gp, "%f %f %g\n", y[n], lambda[n], r[n]);
4019  npoints++;
4020  }
4021  }
4022  fprintf(gp, "EOF\n");
4023  } /* for ncol (relevant columns) */
4024  printf("Plotted %u points.\n", npoints);
4025  fflush(gp);
4026  /* request keypress, so that working with the mouse keeps *
4027  * working and gnuplot has enough time to actually draw all *
4028  * the stuff before the files get removed */
4029  printf("Press ENTER to end program and close plot\n");
4030  getchar();
4031  pclose(gp);
4032  return CPL_ERROR_NONE;
4033 #else /* no HAVE_POPEN && HAVE_PCLOSE */
4034  return CPL_ERROR_UNSUPPORTED_MODE;
4035 #endif /* HAVE_POPEN && HAVE_PCLOSE */
4036 } /* muse_wave_plot_column() */
4037 
cpl_table * muse_wave_calib_lampwise(muse_imagelist *aImages, cpl_table *aTrace, cpl_table *aLinelist, muse_wave_params *aParams)
Find wavelength calibration solution using a list of arc images with different lamps.
cpl_table * muse_wave_calib(muse_image *aImage, cpl_table *aTrace, cpl_table *aLinelist, muse_wave_params *aParams)
Find wavelength calibration solution on an arc frame.
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
cpl_error_code muse_wave_table_get_orders(const cpl_table *aWave, unsigned short *aXOrder, unsigned short *aYOrder)
Determine the x- and y-order of the polynomial stored in a wavelength calibration table...
Structure definition for a collection of muse_images.
int muse_trace_table_get_order(const cpl_table *aTable)
determine order of tracing polynomial from table
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:84
muse_wave_weighting_type fitweighting
cpl_polynomial * muse_wave_table_get_poly_for_slice(const cpl_table *aTable, const unsigned short aSlice)
Construct polynomial from the wavelength calibration table entry for the given slice.
cpl_vector * muse_wave_lines_get_for_lamp(cpl_table *aTable, const char *aLamp, int aGoodnessLimit, double aFluxLimit)
Load wavelengths for a given lamp from a linelist table into a vector.
cpl_error_code muse_wave_lines_identify(cpl_table *aLines, cpl_vector *aLambdas, const muse_wave_params *aParams)
Identify the wavelength of arc detected lines using pattern matching.
void muse_wave_params_delete(muse_wave_params *aParams)
Deallocate memory associated to a wavelength parameters structure.
cpl_image * data
the data extension
Definition: muse_image.h:47
cpl_boolean rflag
cpl_error_code muse_wave_poly_fit(cpl_matrix *aXYPos, cpl_vector *aLambdas, cpl_vector *aDLambdas, cpl_polynomial **aPoly, double *aMSE, muse_wave_params *aParams, const unsigned short aSlice)
Compute the wavelength solution from the sample positions and the respective wavelengths.
cpl_table * muse_wave_table_create(const unsigned short aNSlices, const unsigned short aXOrder, const unsigned short aYOrder)
Create the table to save te wave wavelength calibration coefficients.
muse_image * muse_combine_median_create(muse_imagelist *aImages)
Median combine a list of input images.
Definition: muse_combine.c:317
cpl_image * stat
the statistics extension
Definition: muse_image.h:65
void muse_imagelist_delete(muse_imagelist *aList)
Free the memory of the MUSE image list.
cpl_vector * muse_wave_lines_get(cpl_table *aTable, int aGoodnessLimit, double aFluxLimit)
Load usable wavelengths from a linelist table into a vector.
cpl_table * residuals
static void muse_wave_lines_add_flux_for_lsf(cpl_table *, cpl_table *)
Add the measured flux of the detected lines to a linelist.
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:41
int muse_pfits_get_lampnum(const cpl_propertylist *aHeaders)
query the number of lamps installed
Definition: muse_pfits.c:1113
cpl_table * muse_wave_line_handle_multiplet(muse_image *aImage, cpl_table *aLinelist, unsigned int aIdx, cpl_polynomial *aPoly, cpl_polynomial **aTrace, const muse_wave_params *aParams, const unsigned short aSlice, int aDebug)
Handle fitting of all multiplets across the columns a given slice.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:73
cpl_error_code muse_cpltable_check(const cpl_table *aTable, const muse_cpltable_def *aDef)
Check whether the table contains the fields of the definition.
unsigned int muse_imagelist_get_size(muse_imagelist *aList)
Return the number of stored images.
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
cpl_image * dq
the data quality extension
Definition: muse_image.h:57
cpl_table * muse_cpltable_new(const muse_cpltable_def *aDef, cpl_size aLength)
Create an empty table according to the specified definition.
const char * muse_wave_lines_get_lampname(cpl_table *aTable, const int aIdx)
Associate the ion listed in a linelist table row to a lamp name.
muse_image * muse_imagelist_get(muse_imagelist *aList, unsigned int aIdx)
Get the muse_image of given list index.
cpl_error_code muse_cplvector_erase_element(cpl_vector *aVector, int aElement)
delete the given element from the input vector
Structure containing wavelength calibration parameters.
cpl_polynomial * muse_utils_iterate_fit_polynomial(cpl_matrix *aPos, cpl_vector *aVal, cpl_vector *aErr, cpl_table *aExtra, const unsigned int aOrder, const double aRSigma, double *aMSE, double *aChiSq)
Iterate a polynomial fit.
Definition: muse_utils.c:1829
unsigned short xorder
cpl_error_code muse_wave_table_add_poly(cpl_table *aTable, cpl_polynomial *aPoly, double aMSE, unsigned short aXOrder, unsigned short aYOrder, const unsigned short aRow)
Save the given polynomials to the wavelength calibration table.
cpl_image * muse_wave_map(muse_image *aImage, const cpl_table *aWave, const cpl_table *aTrace)
Write out a wavelength map for visual checks.
const muse_cpltable_def muse_wavedebug_def[]
MUSE wavelength calibration residuals table definition.
cpl_error_code muse_utils_fit_multigauss_1d(const cpl_vector *aX, const cpl_bivector *aY, cpl_vector *aCenter, double *aSigma, cpl_vector *aFlux, cpl_vector *aPoly, double *aMSE, double *aRedChisq, cpl_matrix **aCovariance)
Carry out a multi-Gaussian fit of data in a vector.
Definition: muse_utils.c:1153
char * muse_utils_header_get_lamp_names(cpl_propertylist *aHeader, char aSep)
Concatenate names of all active calibration lamps.
Definition: muse_utils.c:665
cpl_size muse_cpltable_find_sorted(const cpl_table *aTable, const char *aColumn, double aValue)
Find a row in a table.
cpl_error_code muse_wave_plot_residuals(cpl_table *aTable, const unsigned short aSlice, unsigned int aIter, cpl_boolean aPlotLambda, cpl_vector *aCuts)
Fancy plotting of wavelength calibration residuals (color coded over x/y-position) using gnuplot...
cpl_array * muse_utils_header_get_lamp_numbers(cpl_propertylist *aHeader)
List numbers of all active calibration lamps.
Definition: muse_utils.c:732
cpl_table * muse_wave_lines_search(muse_image *aColumnImage, double aSigma, const unsigned short aSlice)
Search and store emission lines in a column of an arc frame.
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
Definition: muse_image.c:396
cpl_error_code muse_wave_line_fit_iterate(cpl_table *aFitTable, double aLambda, const muse_wave_params *aParams)
Use a low-order polynomial to find and discard bad values for line centroid fits of single arc line a...
muse_imagelist * muse_imagelist_new(void)
Create a new (empty) MUSE image list.
cpl_error_code muse_wave_line_fit_single(muse_image *aImage, int aX, double aY, int aHalfWidth, double aSigma, cpl_table *aFitTable, int aRowsNeeded)
Fit a Gaussian to a single emission line in an arc frame and do simple error handling.
unsigned short yorder
cpl_size muse_cplvector_count_unique(const cpl_vector *aVector)
Count the number of unique entries in a given vector.
Definition of a cpl table structure.
cpl_boolean muse_wave_lines_check(cpl_table *aTable, cpl_propertylist *aHeader)
Check that a LINE_CATALOG has the expected format.
cpl_error_code muse_wave_plot_column(cpl_table *aCTable, cpl_table *aRTable, const unsigned short aSlice, unsigned int aColumn, unsigned int aIter, cpl_boolean aPlotRes)
Plot wavelength calibration polynomial and data or residuals using gnuplot.
muse_image * muse_image_new(void)
Allocate memory for a new muse_image object.
Definition: muse_image.c:65
cpl_error_code muse_wave_line_fit_multiple(muse_image *aImage, int aX, cpl_bivector *aPeaks, cpl_vector *aLambdas, int aHalfWidth, double aSigma, cpl_table *aFitTable, int aRowsNeeded)
Fit a multi-Gaussian to a multiplet of arc emission lines and do simple error handling.
muse_wave_params * muse_wave_params_new(void)
Allocate a wavelength parameters structure and fill it with defaults.
cpl_error_code muse_imagelist_set(muse_imagelist *aList, muse_image *aImage, unsigned int aIdx)
Set the muse_image of given list index.
const muse_cpltable_def muse_wavelines_def[]
MUSE wavelength calibration arc line fit properties table definition.
cpl_table * muse_wave_line_handle_singlet(muse_image *aImage, cpl_table *aLinelist, unsigned int aIdx, cpl_polynomial *aPoly, cpl_polynomial **aTrace, const muse_wave_params *aParams, const unsigned short aSlice, int aDebug)
Handle fitting of all single lines across the columns a given slice.
muse_ins_mode muse_pfits_get_mode(const cpl_propertylist *aHeaders)
find out the observation mode
Definition: muse_pfits.c:1003