IIINSTRUMENT Pipeline Reference Manual 4.4.3
visir_spc_obs.c
1/* $Id: visir_spc_obs.c,v 1.111 2013-05-13 16:04:49 jtaylor Exp $
2 *
3 * This file is part of the VISIR Pipeline
4 * Copyright (C) 2002,2003,2012 European Southern Observatory
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
19 */
20
21/*
22 * $Author: jtaylor $
23 * $Date: 2013-05-13 16:04:49 $
24 * $Revision: 1.111 $
25 * $Name: not supported by cvs2svn $
26 */
27
28#ifdef HAVE_CONFIG_H
29#include <config.h>
30#endif
31
32/*-----------------------------------------------------------------------------
33 Includes
34 -----------------------------------------------------------------------------*/
35#include "cxlist.h"
36#define SBRM_UNDERSCORE_MACRO // enable the _() macro
37#include "sbrm.h"
38
39#include "visir_inputs.h"
40#include "visir_recipe.h"
41#include "visir_spectro.h"
42#include "visir_spc_distortion.h"
43#include "visir_spc_photom.h"
44
45
46/*-----------------------------------------------------------------------------
47 Defines
48 -----------------------------------------------------------------------------*/
49
50// aliases for SBRM features
51#define INIT SBRM_INIT
52#define SET SBRM_SET
53#define CSET SBRM_CSET
54#define RESET SBRM_RESET
55#define FREE SBRM_FREE
56#define CLEANUP SBRM_CLEANUP
57#define YANK SBRM_YANK
58#define ERR SBRM_ABORT
59#define ERR_TO_WARN SBRM_WARN
60#define ERR_TO_INFO SBRM_INFO
61#define VLIST SBRM_VLIST
62#define VALLOC SBRM_VALLOC
63#define NALLOC SBRM_NALLOC
64
65#define RECIPE_STRING "visir_old_spc_obs"
66/* still used by chain recipe internally, so keep name for output */
67#define RECIPE_SAVE_STRING "visir_spc_obs"
68
69/* FITS keys to be loaded for all raw files */
70#define RECIPE_KEYS_REGEXP_ALL \
71 VISIR_PFITS_REGEXP_IMG_RECOMBINE \
72 "|" VISIR_PFITS_REGEXP_SPC_GET_RES_WL
73
74/* FITS keys to be loaded for first raw file */
75#define RECIPE_KEYS_REGEXP \
76 RECIPE_KEYS_REGEXP_ALL \
77 "|" VISIR_PFITS_REGEXP_CAPA \
78 "|" VISIR_PFITS_REGEXP_SPC_WCAL_PAF
79
80/* FITS keys to be loaded for first raw file, in case WCS is used */
81#define RECIPE_KEYS_REGEXP_WCS \
82 RECIPE_KEYS_REGEXP \
83 "|" IRPLIB_PFITS_WCS_REGEXP
84
85#define IS_PREPROCESSED 1
86#define IS_PHOT 2
87#define IS_ECH 4
88#define RAW_INPUT 0
89#define PREPROCESSED_INPUT IS_PREPROCESSED
90#define RAW_PHOT_INPUT IS_PHOT
91#define PREPROCESSED_PHOT_INPUT (IS_PHOT | IS_PREPROCESSED)
92
93/* the number of orders we try to extract */
94#define MAX_ORDERS 9
95
96/* a flag value used to indicate an unused order in the data structures */
97#define UNUSED_ORDER 99
98
99#define MSG_WARN(...) cpl_msg_warning(cpl_func, __VA_ARGS__)
100#define MSG_INFO(...) cpl_msg_info(cpl_func, __VA_ARGS__)
101#define MSG_ERR(...) cpl_msg_error(cpl_func, __VA_ARGS__)
102#define MSG_DBG(...) cpl_msg_debug(cpl_func, __VA_ARGS__)
103
104/*-----------------------------------------------------------------------------
105 Typedefs
106 -----------------------------------------------------------------------------*/
107/* possibly more than 1 of these per order */
108typedef struct {
109 int extract; /* The extraction status (UNUSED_ORDER =
110 * failed, anything else = succeeded) */
111 cpl_image * weight2d; /* The 2D weight images */
112 cpl_table * spc_tbl; /* The extracted spectrum */
113 cpl_propertylist * qclist; /* The QC properties */
114 visir_apdefs * aps; /* The matching aperture definition */
115} match;
116static void match_delete(match m);
117
118typedef VLIST(match) matchlist;
119#define matchlist_new(size) VALLOC(matchlist, match, size)
120static void matchlist_delete(matchlist * ml);
121
122typedef struct {
123 int cutout; /* The order offset, which is +/-4 from the
124 * central order, doubling as cutout status
125 * (UNUSED_ORDER = failed, anything else =
126 * succeeded) */
127 cpl_image * comnarrow; /* The saved image cutout, per order */
128 matchlist * matches;
129 visir_aplist * aplist;
130} order;
131static void order_delete(order o);
132
133typedef VLIST(order) orderlist;
134#define orderlist_new(size, ...) NALLOC(orderlist, order, size, __VA_ARGS__)
135static void orderlist_delete(orderlist * ol);
136
137/*-----------------------------------------------------------------------------
138 Private Functions prototypes
139 -----------------------------------------------------------------------------*/
140static cpl_error_code visir_spc_obs_save(cpl_frameset *,
141 const cpl_parameterlist *,
142 cpl_propertylist *,
143 cpl_propertylist *,
144 const cpl_propertylist *,
145 const cpl_image *,
146 const cpl_table *,
147 const cpl_image *,
148 const char *,
149 const int, const int, const size_t);
150
151static cpl_error_code visir_spc_obs_extend(cpl_propertylist *,
152 const cpl_image *,
153 const cpl_table *,
154 const cpl_image *,
155 const int, const int, const size_t);
156
157static cpl_error_code visir_spc_obs_extend_combined(const cpl_image *);
158
159static cpl_error_code visir_spc_qc_summarise(cpl_propertylist *, orderlist *,
160 const char *);
161
162#ifdef VISIR_CHAIN
163#define cpl_plugin_get_info visir_old_spc_obs_get_info
164#endif
165VISIR_RECIPE_DEFINE(visir_old_spc_obs,
166 VISIR_PARAM_EMIS_TOL |
167 VISIR_PARAM_REFINE | VISIR_PARAM_XCORR |
168 VISIR_PARAM_OFFSETS | VISIR_PARAM_OBJECTS |
169 VISIR_PARAM_NODPOS | VISIR_PARAM_AUTOBPM |
170 VISIR_PARAM_GLITCH | VISIR_PARAM_PURGE |
171 VISIR_PARAM_UNION | VISIR_PARAM_REJECT |
172 VISIR_PARAM_STRIPITE | VISIR_PARAM_STRIPMOR |
173 VISIR_PARAM_PLOT | VISIR_PARAM_SLITSKEW |
174 VISIR_PARAM_SPECSKEW | VISIR_PARAM_VERTARC |
175 VISIR_PARAM_REJLEFT | VISIR_PARAM_REJRIGHT |
176 VISIR_PARAM_HORIARC | VISIR_PARAM_FIXCOMBI |
177 VISIR_PARAM_BKG_CORRECT | VISIR_PARAM_APERT_FILE |
178 VISIR_PARAM_GAIN | VISIR_PARAM_RONOISE |
179 VISIR_PARAM_OXSIGMA | VISIR_PARAM_OXNITER |
180 VISIR_PARAM_OXSMOOTH | VISIR_PARAM_OXKERNEL |
181 VISIR_PARAM_RESPCAL,
182 "Old DRS detector: Spectroscopic Observation recipe",
183 "This recipe estimates the dispersion relation using the "
184 "atmospheric spectrum\n"
185 "in a long-slit spectroscopy half-cycle frame.\n"
186 "It also extracts the spectrum of an observed object using "
187 "a combined frame.\n"
188 "The files listed in the Set Of Frames (sof-file) "
189 "must be tagged:\n"
190 "VISIR-Long-Slit-Spectroscopy-file.fits "
191 VISIR_SPC_OBS_RAW "\n"
192 "VISIR-Quantum-Efficiency-Calibration-file.fits "
193 VISIR_CALIB_QEFF_SPC "\n"
194 "VISIR-Atmospheric-Emission-Lines-Calibration-file.fits "
195 VISIR_CALIB_LINES_SPC
196 "\n"
197 MAN_VISIR_CALIB_BPM_SPC);
198
199/*-----------------------------------------------------------------------------
200 Static variables
201 -----------------------------------------------------------------------------*/
202
203/*----------------------------------------------------------------------------*/
207/*----------------------------------------------------------------------------*/
208
209/*-----------------------------------------------------------------------------
210 Functions code
211 -----------------------------------------------------------------------------*/
212
213static void biimage_delete(cpl_image ** bi)
214{
215 if (!bi) return;
216 cpl_image_delete(bi[0]);
217 cpl_image_delete(bi[1]);
218 cpl_free(bi);
219}
220
221static void match_delete(match m)
222{
223 cpl_propertylist_delete(m.qclist);
224 // original algorithm apertures (with negative ident) are the only ones
225 // owned, hence are the only ones that need to be freed: the rest are just
226 // references into other owning structs
227 if (m.aps->ident < 0) visir_apdefs_delete(m.aps);
228 cpl_table_delete(m.spc_tbl);
229 cpl_image_delete(m.weight2d);
230}
231
232static void matchlist_delete(matchlist * ml)
233{
234 if (!ml) return;
235 for (size_t i = 0; i < ml->sz; ++i) match_delete(ml->data[i]);
236 cpl_free(ml);
237}
238
239static void order_delete(order o)
240{
241 cpl_image_delete(o.comnarrow);
242 visir_aplist_delete(o.aplist); // free list only (elems are just refs)
243 matchlist_delete(o.matches);
244}
245
246static void orderlist_delete(orderlist * ol)
247{
248 if (!ol) return;
249 for (size_t i = 0; i < ol->sz; ++i) order_delete(ol->data[i]);
250 cpl_free(ol);
251}
252
253/*----------------------------------------------------------------------------*/
260/*----------------------------------------------------------------------------*/
261static const char * pn(const int oo)
262{
263 static char buf[80]; // FIXME: don't use static vars
264 const char * sign = oo ? (oo > 0 ? "+" : "-") : "";
265 snprintf(buf, sizeof(buf), "%s%d", sign, abs(oo));
266 return buf;
267}
268
269/*----------------------------------------------------------------------------*/
276/*----------------------------------------------------------------------------*/
277static void * visir_old_spc_obs_(cpl_frameset * framelist,
278 const cpl_parameterlist * parlist)
279{
280 INIT(20);
281
282 visir_spc_config spc_config;
283 spc_config.recipename = RECIPE_STRING;
284 spc_config.parlist = parlist;
285 spc_config.phot_emis_tol = 1.0;
286
287 /* Initialise the QC property lists */
288 SET(phu, cpl_propertylist) = cpl_propertylist_new();
289 spc_config.phu = phu->o;
290
291 /* Retrieve input parameters */
292 spc_config.plot = visir_parameterlist_get_int _(
293 parlist, RECIPE_STRING, VISIR_PARAM_PLOT);
294 spc_config.phi = visir_parameterlist_get_double _(
295 parlist, RECIPE_STRING, VISIR_PARAM_SLITSKEW);
296 spc_config.ksi = visir_parameterlist_get_double _(
297 parlist, RECIPE_STRING, VISIR_PARAM_SPECSKEW);
298 spc_config.eps = visir_parameterlist_get_double _(
299 parlist, RECIPE_STRING, VISIR_PARAM_VERTARC);
300 spc_config.delta = visir_parameterlist_get_double _(
301 parlist, RECIPE_STRING, VISIR_PARAM_HORIARC);
302 spc_config.gain = visir_parameterlist_get_double _(
303 parlist, RECIPE_STRING, VISIR_PARAM_GAIN);
304 spc_config.ron = visir_parameterlist_get_double _(
305 parlist, RECIPE_STRING, VISIR_PARAM_RONOISE);
306 spc_config.ox_sigma = visir_parameterlist_get_double _(
307 parlist, RECIPE_STRING, VISIR_PARAM_OXSIGMA);
308 spc_config.ox_niters = visir_parameterlist_get_int _(
309 parlist, RECIPE_STRING, VISIR_PARAM_OXNITER);
310 spc_config.ox_smooth = visir_parameterlist_get_int _(
311 parlist, RECIPE_STRING, VISIR_PARAM_OXSMOOTH);
312 spc_config.ox_kernel = visir_parameterlist_get_int _(
313 parlist, RECIPE_STRING, VISIR_PARAM_OXKERNEL);
314 spc_config.do_fixcombi = visir_parameterlist_get_bool _(
315 parlist, RECIPE_STRING, VISIR_PARAM_FIXCOMBI);
316 spc_config.phot_emis_tol = visir_parameterlist_get_double _(
317 parlist, RECIPE_STRING, VISIR_PARAM_EMIS_TOL);
318 spc_config.bkgcorrect = visir_parameterlist_get_bool _(
319 parlist, RECIPE_STRING, VISIR_PARAM_BKG_CORRECT);
320 spc_config.respcal = visir_parameterlist_get_string _(
321 parlist, RECIPE_STRING, VISIR_PARAM_RESPCAL);
322
323 char const * const apfile = visir_parameterlist_get_string _(
324 parlist, RECIPE_STRING, VISIR_PARAM_APERT_FILE);
325 // re-enable this if it is an error to extract without sky-subtraction
326 //if (!visir_str_par_is_empty(apfile) && !spc_config.bkgcorrect)
327 // return ERR(CPL_ERROR_INCOMPATIBLE_INPUT,
328 // "--bkgcorrect must be true when using --apfile=...");
329
330 // parse apfile into a list of apertures
331 SET(aplist, visir_aplist, v, visir_aplist_destroy) = NULL;
332 if (visir_str_par_is_empty(apfile))
333 RESET(aplist) = visir_aplist_new(); // no_
334 else {
335 RESET(aplist) = visir_aplist_new_from_file _(apfile);
336 }
337
342 //FOR_EACH_T(visir_apdefs const * const aps, aplist->o) {
343 // SET(line, char, v, cpl_free) = visir_apdefs_dump _(aps);
344 // SET(key, char, v, cpl_free) = cpl_sprintf _("ESO DRS APDEF%d", aps->ident);
345 // cpl_propertylist_append_string _(phu->o, key->o, line->o);
346 //}
347
348 /* Identify the RAW and CALIB frames in the input frameset */
349 visir_dfs_set_groups _(framelist);
350
351 /* Objects observation */
352 SET(all, irplib_framelist) = irplib_framelist_cast _(framelist);
353 char const * const inraw = "Input: raw %s frame (%s)";
354 char const * const inprep = "Input: preprocessed %s frame (%s)";
355
356 int input_mode;
357 const char * rawtag;
358 SET(raws, irplib_framelist) = NULL;
359 if (irplib_frameset_find_file(framelist, VISIR_SPC_OBS_PP)) {
360 RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_OBS_PP);
361 rawtag = VISIR_SPC_OBS_PP;
362 MSG_INFO(inprep, "observation", VISIR_SPC_OBS_PP);
363 input_mode = PREPROCESSED_INPUT;
364
365 } else if (irplib_frameset_find_file(framelist, VISIR_SPC_PHOT_PP)) {
366 RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_PHOT_PP);
367 rawtag = VISIR_SPC_PHOT_PP;
368 MSG_INFO(inprep, "photometric", VISIR_SPC_PHOT_PP);
369 input_mode = PREPROCESSED_PHOT_INPUT;
370
371 } else if (irplib_frameset_find_file(framelist, VISIR_SPC_OBS_ECH_PP)) {
372 RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_OBS_ECH_PP);
373 rawtag = VISIR_SPC_OBS_ECH_PP;
374 MSG_INFO(inprep, "observation echelle", VISIR_SPC_OBS_ECH_PP);
375 input_mode = PREPROCESSED_INPUT | IS_ECH;
376
377 } else if (irplib_frameset_find_file(framelist, VISIR_SPC_PHOT_ECH_PP)) {
378 RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_PHOT_ECH_PP);
379 rawtag = VISIR_SPC_PHOT_ECH_PP;
380 MSG_INFO(inprep, "photometric echelle", VISIR_SPC_PHOT_ECH_PP);
381 input_mode = PREPROCESSED_PHOT_INPUT | IS_ECH;
382
383 } else if (irplib_frameset_find_file(framelist, VISIR_SPC_OBS_RAW)) {
384 RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_OBS_RAW);
385 rawtag = VISIR_SPC_OBS_RAW;
386 MSG_INFO(inraw, "observation", VISIR_SPC_OBS_RAW);
387 input_mode = RAW_INPUT;
388
389 } else if (irplib_frameset_find_file(framelist, VISIR_SPC_PHOT_RAW)) {
390 RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_PHOT_RAW);
391 rawtag = VISIR_SPC_PHOT_RAW;
392 MSG_INFO(inraw, "photometric", VISIR_SPC_PHOT_RAW);
393 input_mode = RAW_PHOT_INPUT;
394
395 } else if (irplib_frameset_find_file(framelist, VISIR_SPC_OBS_ECH_RAW)) {
396 RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_OBS_ECH_RAW);
397 rawtag = VISIR_SPC_OBS_ECH_RAW;
398 MSG_INFO(inraw, "observation echelle", VISIR_SPC_OBS_ECH_RAW);
399 input_mode = RAW_INPUT | IS_ECH;
400
401 } else if (irplib_frameset_find_file(framelist, VISIR_SPC_PHOT_ECH_RAW)) {
402 RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_PHOT_ECH_RAW);
403 rawtag = VISIR_SPC_PHOT_ECH_RAW;
404 MSG_INFO(inraw, "photometric echelle", VISIR_SPC_PHOT_ECH_RAW);
405 input_mode = RAW_PHOT_INPUT | IS_ECH;
406
407 } else {
408 return ERR(CPL_ERROR_ILLEGAL_INPUT, "Input frame set must contain "
409 "frame(s) tagged as one of: " VISIR_SPC_OBS_RAW ", "
410 VISIR_SPC_PHOT_RAW ", " VISIR_SPC_OBS_PP ","
411 VISIR_SPC_PHOT_PP "," VISIR_SPC_OBS_ECH_RAW ","
412 VISIR_SPC_OBS_ECH_PP "," VISIR_SPC_PHOT_ECH_RAW ","
413 VISIR_SPC_PHOT_ECH_PP);
414 }
415
416 cpl_size const nframes = irplib_framelist_get_size _(raws->o);
417
418 irplib_framelist_load_propertylist_all _(raws->o, 0, "^("
419 VISIR_PFITS_REGEXP_SPC_SENSIT
420 "|" RECIPE_KEYS_REGEXP_ALL
421 "|" RECIPE_KEYS_REGEXP_WCS
422 "|" VISIR_PFITS_REGEXP_DIT
423 "|" VISIR_PFITS_DOUBLE_WLEN
424 "|" VISIR_PFITS_DOUBLE_PWLEN
425 "|" "ESO DRS .*"
426 "|" "ESO PRO DATANCOM"
427 "|" "ESO QC EXPTIME"
428 "|" "ESO QC EXECTIME"
429 "|" "ESO QC BACKGD MEAN"
430 ")$", CPL_FALSE);
431
433
434 cpl_propertylist const * const pl =
435 irplib_framelist_get_propertylist_const _(raws->o, 0);
436 int const has_seq = cpl_propertylist_has _(pl, VISIR_PFITS_DOUBLE_SEQ1_DIT);
437 char const * const dit_key = has_seq ? VISIR_PFITS_DOUBLE_SEQ1_DIT
438 : VISIR_PFITS_DOUBLE_DIT;
439
440 /* Standard star catalog */
441 char const * const star_cat =
442 irplib_frameset_find_file _(framelist, VISIR_CALIB_STDSTAR_SPC);
443 if ((input_mode & IS_PHOT) && star_cat == NULL) return ERR(
444 CPL_ERROR_DATA_NOT_FOUND, "Frame set with a photometric frame (%s) "
445 "must include a standard star catalogue (" VISIR_CALIB_STDSTAR_SPC
446 ")", rawtag);
447
448 visir_data_type data_type;
449 cpl_frame const * const frm = irplib_framelist_get_const _(raws->o, 0);
450 visir_get_data_type _(frm, pl, &data_type, NULL);
451
452 if (visir_data_is_drs(data_type)) {
453 /* if parameters are (aqu) defaults and drs data use old parameters */
454 double tmp;
455 cpl_parameter const * par = cpl_parameterlist_find_const _(
456 parlist, PACKAGE "." RECIPE_STRING ".ksi");
457 tmp = cpl_parameter_get_default_double _(par);
458 if (spc_config.ksi == tmp * CPL_MATH_RAD_DEG)
459 spc_config.ksi = VISIR_DRS_DIST_KSI;
460
461 par = cpl_parameterlist_find_const _(
462 parlist, PACKAGE "." RECIPE_STRING ".eps");
463 tmp = cpl_parameter_get_default_double _(par);
464 if (spc_config.eps == tmp) spc_config.eps = VISIR_DRS_DIST_EPS;
465
466 par = cpl_parameterlist_find_const _(
467 parlist, PACKAGE "." RECIPE_STRING ".delta");
468 tmp = cpl_parameter_get_default_double _(par);
469 if (spc_config.delta == tmp) spc_config.delta = VISIR_DRS_DIST_DELTA;
470
471 par = cpl_parameterlist_find_const _(
472 parlist, PACKAGE "." RECIPE_STRING ".phi");
473 tmp = cpl_parameter_get_default_double _(par);
474 if (spc_config.phi == tmp * CPL_MATH_RAD_DEG)
475 spc_config.phi = VISIR_DRS_DIST_PHI;
476 }
477
478 /* Quantum efficiency file */
479 char const * const spc_cal_qeff =
480 irplib_frameset_find_file _(framelist, VISIR_CALIB_QEFF_SPC);
481
482 /* Spectral lines calibration file */
483 char const * const spc_cal_lines =
484 irplib_frameset_find_file _(framelist, VISIR_CALIB_LINES_SPC);
485
486 /* Bad pixels calibration file */
487 char const * const badpix =
488 irplib_frameset_find_file _(framelist, VISIR_CALIB_BPM);
489
490 /* Flatfield calibration file */
491 char const * const flat =
492 irplib_frameset_find_file _(framelist, VISIR_CALIB_FLAT);
493
494 /* Get Resolution and Central Wavelength */
495 double wlen, slitw, temp, fwhm;
496 visir_spc_resol resol = visir_spc_get_res_wl _(raws->o, &wlen, &slitw,
497 &temp, &fwhm, visir_data_is_aqu(data_type));
498
499 /* The number of orders currently in use */
500 int n_orders = 1;
501 if (resol == VISIR_SPC_R_GHR) {
502 if (!(input_mode & IS_ECH) && !(input_mode & IS_PREPROCESSED))
503 return ERR(CPL_ERROR_TYPE_MISMATCH, "Will not reduce echelle data "
504 "tagged as long slit data");
505 n_orders = MAX_ORDERS; // try to extract all orders
506 } else {
507 MSG_WARN("Reducing non-HR Grism data as main order");
508 }
509
510 cpl_boolean drop_wcs;
511 SET(imhcycle, cpl_image) = NULL;
512 SET(combinedpair, cpl_image*, v, biimage_delete) = NULL;
513 if (input_mode & IS_PREPROCESSED) {
514 CSET(plists, cpl_propertylist*, v, cpl_free) = cpl_malloc(
515 nframes * sizeof(cpl_propertylist *));
516 double bg_sum = 0;
517
518 SET(imagelist, cpl_imagelist) = cpl_imagelist_new();
519 for (cpl_size i = 0; i < nframes; i++) {
520 cpl_frame const * const frame = irplib_framelist_get _(raws->o, i);
521 char const * const fname = cpl_frame_get_filename _(frame);
522 cpl_size const next = cpl_fits_count_extensions _(fname);
523
524 plists->o[i] = irplib_framelist_get_propertylist_const _(raws->o,i);
525 if (cpl_propertylist_has(plists->o[i], "ESO QC BACKGD MEAN")) {
526 bg_sum += cpl_propertylist_get_double _(plists->o[i],
527 "ESO QC BACKGD MEAN");
528 }
529
530 SET(tmplist, cpl_imagelist) = cpl_imagelist_new();
531 for (cpl_size j = 0; j < 1 + next; j++) {
532 SET(img, cpl_image) = cpl_image_load( // no_
533 fname, CPL_TYPE_UNSPECIFIED, 0, j);
534 if (cpl_error_get_code()) {
535 ERR_TO_INFO("No image in extension %d", (int)j);
536 continue;
537 }
538 cpl_image_reject_value _(img->o, CPL_VALUE_NAN);
539 cpl_size const lsz = cpl_imagelist_get_size _(tmplist->o);
540 cpl_imagelist_set _(tmplist->o, img->o, lsz);
541 YANK(img);
542 }
543 SET(tmp, cpl_image) = cpl_imagelist_collapse_create _(tmplist->o);
544 cpl_size const lsz = cpl_imagelist_get_size _(imagelist->o);
545 cpl_imagelist_set _(imagelist->o, tmp->o, lsz);
546 YANK(tmp);
547 }
548
549 RESET(combinedpair) = visir_img_recombine_list _(RECIPE_STRING, parlist,
550 imagelist->o, plists->o, CPL_GEOM_FIRST, &drop_wcs);
551
552 cpl_propertylist_append_double _(phu->o, "ESO QC BACKGD MEAN",
553 bg_sum / nframes);
554 FREE(imagelist);
555 FREE(plists);
556
557 {
558 SET(skys, irplib_framelist) = irplib_framelist_extract( // no_
559 all->o, VISIR_SPC_OBS_SKYFRAME);
560 if (cpl_error_get_code()) return ERR(CPL_ERROR_ILLEGAL_INPUT,
561 "Frame set with a preprocessed frame (%s) must include a "
562 "sky frame (" VISIR_SPC_OBS_SKYFRAME ")", rawtag);
563
564 /* FIXME: make use of multiple skyframes */
565 cpl_frame const * const frame = irplib_framelist_get _(skys->o, 0);
566 char const * const fname = cpl_frame_get_filename _(frame);
567 RESET(imhcycle) = cpl_image_load _(fname, CPL_TYPE_UNSPECIFIED,0,0);
568 cpl_image_reject_value _(imhcycle->o, CPL_VALUE_NAN);
569 }
570
571
572 } else {
573 RESET(combinedpair) = visir_img_recombine _(RECIPE_STRING, parlist,
574 raws->o, badpix, flat, CPL_GEOM_FIRST, &drop_wcs,
575 !spc_config.do_fixcombi, wlen, resol);
576
577 /* Get the hcycle image from the reference file */
578 SET(hcycle, cpl_imagelist) = visir_load_hcycle _(raws->o, 0);
579 RESET(imhcycle) = cpl_imagelist_unset _(hcycle->o, 0);
580
581 visir_spc_det_fix _(&imhcycle->o, 1, CPL_FALSE, wlen, resol,
582 spc_config.phi, spc_config.ksi,
583 spc_config.eps, spc_config.delta,
584 spc_config.plot);
585
586 visir_qc_append_background _(phu->o, raws->o, 0, 0);
587 }
588
589 SET(combined, cpl_image) = cpl_image_cast _(combinedpair->o[0],
590 CPL_TYPE_DOUBLE);
591 FREE(combinedpair);
592
593 bool rev = false;
594 if (!(input_mode & IS_PREPROCESSED)) {
595 /* Convert the combined image
596 * unpreprocessed mode only works with drs data */
597 SET(tmp, cpl_image) = visir_spc_flip _(combined->o, wlen, resol,
598 VISIR_DATA_CUBE2, &rev);
599 RESET(combined) = YANK(tmp);
600
601 /* Convert the half cycle image */
602 RESET(tmp) = visir_spc_flip _(imhcycle->o, wlen, resol,
603 VISIR_DATA_CUBE2, NULL);
604 RESET(imhcycle) = YANK(tmp);
605 }
606
607 int const ncomb = visir_get_ncombine _(raws->o);
608
609 if (spc_config.do_fixcombi && !(input_mode & IS_PREPROCESSED)) {
610 visir_spc_det_fix _(&combined->o, 1, CPL_TRUE, wlen, resol,
611 spc_config.phi, spc_config.ksi,
612 spc_config.eps, spc_config.delta,
613 spc_config.plot);
614 }
615
616 visir_optmod ins_settings;
617 visir_spc_optmod_init _(resol, wlen, &ins_settings,
618 visir_data_is_aqu(data_type));
619
620 MSG_DBG("resol = %d, input_mode = %d", resol, input_mode);
621 SET(paflist, cpl_propertylist) = cpl_propertylist_new();
622
623 // there must be at least n_orders args after n_orders for list init'n
624 SET(orders, orderlist) = orderlist_new(n_orders, {0, 0, 0, 0},
625 {1, 0, 0, 0}, {-1, 0, 0, 0}, {2, 0, 0, 0}, {-2, 0, 0, 0},
626 {3, 0, 0, 0}, {-3, 0, 0, 0}, {4, 0, 0, 0}, {-4, 0, 0, 0});
627
628 // for each order; this loop sets 'cutout' to 99 upon failure, over-writing
629 // the order offset (0, -1, +1, -2, etc...)
630 for (size_t i = 0; i < orders->o->sz; ++i) {
631 order * const oi = &orders->o->data[i];
632 char pfx[80] = "";
633 if (input_mode & IS_ECH) snprintf(pfx, sizeof(pfx), "Echelle order %2d: "
634 "offset %s: ", oi->cutout + visir_spc_optmod_get_echelle_order(
635 &ins_settings), pn(oi->cutout));
636 MSG_INFO("%sattempting extraction", pfx);
637
638 int lcol = -1, rcol = -1;
639 spc_config.orderoffset = oi->cutout;
640 SET(imhcyclenarrow, cpl_image) = NULL;
641 visir_spc_extract_order(&imhcyclenarrow->o, &oi->comnarrow, &lcol, //no_
642 &rcol, combined->o, imhcycle->o, wlen,
643 &spc_config, (input_mode & IS_ECH) == IS_ECH,
644 visir_data_is_aqu(data_type));
645 if (cpl_error_get_code()) {
646 ERR_TO_WARN("%scutout failed", pfx);
647 oi->cutout = UNUSED_ORDER;
648 continue;
649 }
650
651 MSG_INFO("%scutout success", pfx);
652 MSG_DBG("extents mapping:\t[raw] =>\t[relative (to cutout)]");
653 MSG_DBG("=======================================================");
654 MSG_DBG(" cutout:\t\t[%d;%d] =>\t[%d;%d]", lcol, rcol,
655 lcol - lcol + 1, rcol - lcol + 1);
656
657 /* find matching apfile lines for this order */
658 oi->aplist = visir_aplist_new();
659 int const ncol = rcol - lcol + 1;
660 FOR_EACH_T(visir_apdefs const * const aps, aplist->o) {
661 bool matched = false;
662 int const l = visir_norm_coord(rev, -0.0, lcol, rcol, aps);
663 int const r = visir_norm_coord(rev, +0.0, lcol, rcol, aps);
664 if (1 <= l && l <= ncol && 1 <= r && r <= ncol) {
665 matched = true;
666 visir_aplist_push_back(oi->aplist, aps); // shallow copy
667 }
668 MSG_DBG("%capfile entry %d:\t[%d;%d] =>\t[%d;%d]",
669 matched ? '*' : ' ', aps->ident, aps->limits[0].l,
670 aps->limits[0].r, l, r);
671 }
672
673 /* form the list of extraction/match combos to iterate over later */
674 cxsize const nmatches = visir_aplist_size(oi->aplist);
675 if (nmatches > 0) {
676 size_t j = 0;
677 oi->matches = matchlist_new(nmatches);
678 FOR_EACH_T(visir_apdefs * const aps, oi->aplist) {
679 cpl_propertylist * const pl = cpl_propertylist_new();
680 oi->matches->data[j++] = (match){ oi->cutout, 0, 0, pl, aps };
681 }
682 } else {
683 /* extraction proceeds using defaults even if no apfile line
684 * matches, so still need space for at least one result */
685 int const ident = -(i+1); // neg ident: use old extract alg
686 visir_apdefs * const aps = visir_apdefs_new(1, ident, 'O', 0);
687 aps->limits[0] = (visir_aplimits){ lcol, rcol };
688 cpl_propertylist * const pl = cpl_propertylist_new();
689 oi->matches = matchlist_new(1);
690 oi->matches->data[0] = (match){ oi->cutout, 0, 0, pl, aps };
691 }
692
693 // for each extraction requested (potentially >1 if multiple apfile
694 // lines matched); this loop sets 'extract' to 99 upon failure, over-
695 // writing the order offset (0, -1, +1, -2, etc...)
696 for (size_t j = 0; j < oi->matches->sz; ++j) {
697 match * const mj = &oi->matches->data[j]; // alias
698
699 cpl_propertylist_update_int _(mj->qclist, "ESO DRS APDEF",
700 mj->aps->ident);
701
702 if (visir_aplist_size(aplist->o) && nmatches < 1) {
703 MSG_WARN("%sextraction %ld: no apdefs matched", pfx, j);
704 mj->extract = UNUSED_ORDER;
705 /* skip this order if no aperture defintions provided */
706 continue;
707 }
708
709 spc_config.extract = j;
710 visir_spc_extract_wcal(oi->comnarrow, imhcyclenarrow->o, // no_
711 lcol, rcol, wlen, slitw, temp, fwhm, resol, &spc_config,
712 spc_cal_lines, spc_cal_qeff, visir_data_is_aqu(data_type),
713 mj->aps, ncomb, rev, &mj->spc_tbl, &mj->weight2d, mj->qclist);
714 if (cpl_error_get_code()) {
715 ERR_TO_WARN("%sextraction %ld: wlen calib failed", pfx, j);
716 // spot #1 where EMISSIVITY (not part of the product) is removed
717 // use the X() macro in the line below to reset any resulting
718 // CPL error since visir_spc_extract_wcal might've failed before
719 // adding the column
720 cpl_table_erase_column X(mj->spc_tbl, "SPC_EMISSIVITY");
721 mj->extract = UNUSED_ORDER;
722 continue;
723 }
724 MSG_INFO("%sextraction %ld: success", pfx, j);
725
726 visir_spectro_qc _(mj->qclist, paflist->o, drop_wcs, raws->o, NULL,
727 "^(" VISIR_PFITS_REGEXP_SPC_WCAL_PAF ")$");
728
729 if (star_cat) {
730 if (!(input_mode & IS_PHOT))
731 MSG_INFO("The frame set contains observation frame(s) "
732 "(%s) together with a standard star catalogue ("
733 VISIR_CALIB_STDSTAR_SPC "), attempting to "
734 "perform a photometric calibration", rawtag);
735
736 // spot #2 where EMISSIVITY (not part of the product) is removed
737 visir_spc_phot_sensit(raws->o, &spc_config, pl, star_cat, // no_
738 &mj->weight2d, mj->qclist, mj->spc_tbl,
739 resol, dit_key);
740 if (cpl_error_get_code()) {
741 // photometry failed
742 if (input_mode & IS_PHOT) return ERR();
743 else ERR_TO_WARN(
744 "The frame set contains observation frame(s) (%s) "
745 "together with a standard star catalogue ("
746 VISIR_CALIB_STDSTAR_SPC ") but the photometric "
747 "calibration failed", rawtag);
748 } else {
749 input_mode |= IS_PHOT;
750 }
751 } else {
752 // spot #3 where EMISSIVITY (not part of the product) is removed
753 cpl_table_erase_column _(mj->spc_tbl, "SPC_EMISSIVITY");
754 }
755 }
756 }
757
758 /* Summarise the relevant QC headers from multiple orders
759 * into the primary header */
760 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC BACKGD SIGMA");
761 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC GAUSSFIT FWHM");
762 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC GAUSSFIT FWHM_ERR");
763 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC GAUSSFIT PEAK");
764 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC GAUSSFIT PEAK_ERR");
765 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC XFWHM");
766 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC SENS MEDIAN");
767 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC SENS MEAN");
768 visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC SENS STDEV");
769
770 /* PRO.CATG */
771 cpl_propertylist_append_string _(paflist->o, CPL_DFS_PRO_CATG,
772 VISIR_SPC_OBS_COMBINED_PROCATG);
773
774 char const * const procat = input_mode & IS_PHOT ? (input_mode & IS_ECH ?
775 VISIR_SPC_PHOT_ECH_TAB_PROCATG : VISIR_SPC_PHOT_TAB_PROCATG) : (
776 input_mode & IS_ECH ? VISIR_SPC_OBS_ECH_TAB_PROCATG :
777 VISIR_SPC_OBS_TAB_PROCATG);
778
779 bool first = true;
780 // loop only executes once for non-echelle
781 for (size_t i = 0; i < orders->o->sz; ++i) {
782 order const * const oi = &orders->o->data[i]; // alias
783 if (oi->cutout == UNUSED_ORDER) continue; // skip cutout failures
784
785 char pfx[80] = "";
786 if (input_mode & IS_ECH) snprintf(pfx, sizeof(pfx), "Echelle order %2d: "
787 "offset %s: ", oi->cutout + visir_spc_optmod_get_echelle_order(
788 &ins_settings), pn(oi->cutout));
789 for (size_t j = 0; j < oi->matches->sz; ++j) {
790 match const * const mj = &oi->matches->data[j]; // alias
791 if (mj->extract == UNUSED_ORDER) continue; // skip extract failures
792
793 MSG_INFO("%sextraction %ld: saving", pfx, j);
794 if (first) {
795 first = false;
796 visir_spc_obs_save _(framelist, parlist, phu->o, mj->qclist,
797 paflist->o, mj->weight2d, mj->spc_tbl,
798 oi->comnarrow, procat, input_mode,
799 oi->cutout, j);
800 } else {
801 visir_spc_obs_extend _(mj->qclist, mj->weight2d, mj->spc_tbl,
802 oi->comnarrow, input_mode, oi->cutout,
803 j);
804 }
805 }
806 }
807
808 if (!first) {
809 visir_spc_obs_extend_combined _(combined->o);
810 }
811
812 return CLEANUP();
813}
814
815/*----------------------------------------------------------------------------*/
822/*----------------------------------------------------------------------------*/
823static int visir_old_spc_obs(cpl_frameset * framelist,
824 const cpl_parameterlist * parlist)
825{
826 visir_old_spc_obs_(framelist, parlist);
827 return cpl_error_get_code();
828}
829
830
831/*----------------------------------------------------------------------------*/
846/*----------------------------------------------------------------------------*/
847static cpl_error_code visir_spc_obs_save(cpl_frameset * set,
848 const cpl_parameterlist * parlist,
849 cpl_propertylist * phu_qclist,
850 cpl_propertylist * qclist,
851 const cpl_propertylist * paflist,
852 const cpl_image * weight2d,
853 const cpl_table * table,
854 const cpl_image * comnarrow,
855 const char * tab_procatg,
856 const int input_mode,
857 const int ord, const size_t extract)
858{
859 char offset[8] = "";
860 char msgbuf[32];
861 bug_if (0);
862
863 if (input_mode & IS_ECH) snprintf(offset, sizeof(offset), "%s:", pn(ord));
864
865 /* THE TABLE */
866 cpl_propertylist * plist = cpl_propertylist_new();
867 snprintf(msgbuf, sizeof(msgbuf), "TAB_SPECTRUM_%s%ld", offset, extract);
868 cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
869 cpl_propertylist_append(plist, qclist);
870 // Put current qclist into plist instead, and then have the primary qclist
871 // as the new qclist.
872 skip_if (irplib_dfs_save_table(set, parlist, set, table, plist,
873 RECIPE_SAVE_STRING, tab_procatg,
874 phu_qclist, NULL, visir_pipe_id,
875 RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS));
876 cpl_propertylist_delete(plist);
877
878 plist = cpl_propertylist_new();
879 /* save in single file with extensions for consistency with imaging */
880 snprintf(msgbuf, sizeof(msgbuf), "IMG_WEIGHT_%s%ld", offset, extract);
881 cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
882 skip_if(cpl_image_save(weight2d, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
883 CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
884 snprintf(msgbuf, sizeof(msgbuf), "IMG_%s%ld", offset, extract);
885 cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
886 skip_if(cpl_image_save(comnarrow, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
887 CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
888 cpl_propertylist_delete(plist);
889
890#ifdef VISIR_SAVE_PAF
891 /* THE PAF FILE FOR QC PARAMETERS */
892
893 skip_if (cpl_dfs_save_paf("VISIR", RECIPE_SAVE_STRING, paflist,
894 RECIPE_SAVE_STRING CPL_DFS_PAF));
895#else
896 bug_if(paflist == NULL);
897#endif
898
899 end_skip;
900
901 return cpl_error_get_code();
902
903}
904
905/*----------------------------------------------------------------------------*/
914/*----------------------------------------------------------------------------*/
915static cpl_error_code visir_spc_obs_extend(cpl_propertylist * qclist,
916 const cpl_image * weight2d,
917 const cpl_table * table,
918 const cpl_image * comnarrow,
919 const int input_mode,
920 const int ord, const size_t extract)
921{
922 char offset[8] = "";
923 char msgbuf[32];
924 bug_if (0);
925
926 if (input_mode & IS_ECH) snprintf(offset, sizeof(offset), "%s:", pn(ord));
927
928 /* THE TABLE */
929 cpl_propertylist * plist = cpl_propertylist_duplicate(qclist);
930 snprintf(msgbuf, sizeof(msgbuf), "TAB_SPECTRUM_%s%ld", offset, extract);
931 cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
932 skip_if (cpl_table_save(table, NULL, plist, RECIPE_SAVE_STRING "_tab"
933 CPL_DFS_FITS, CPL_IO_EXTEND));
934 cpl_propertylist_delete(plist);
935
936 plist = cpl_propertylist_new();
937 snprintf(msgbuf, sizeof(msgbuf), "IMG_WEIGHT_%s%ld", offset, extract);
938 cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
939 skip_if(cpl_image_save(weight2d, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
940 CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
941 snprintf(msgbuf, sizeof(msgbuf), "IMG_%s%ld", offset, extract);
942 cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
943 skip_if(cpl_image_save(comnarrow, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
944 CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
945 cpl_propertylist_delete(plist);
946
947 end_skip;
948
949 return cpl_error_get_code();
950}
951
952/*----------------------------------------------------------------------------*/
958/*----------------------------------------------------------------------------*/
959static cpl_error_code visir_spc_obs_extend_combined(const cpl_image * combined)
960{
961 bug_if (0);
962
963 cpl_propertylist * plist = cpl_propertylist_new();
964 cpl_propertylist_update_string(plist, "EXTNAME", "IMG_COMBINED");
965 skip_if(cpl_image_save(combined, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
966 CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
967 cpl_propertylist_delete(plist);
968
969 end_skip;
970
971 return cpl_error_get_code();
972}
973
974/*----------------------------------------------------------------------------*/
984/*----------------------------------------------------------------------------*/
985static cpl_error_code visir_spc_qc_summarise(cpl_propertylist * phu_qclist,
986 orderlist * orders,
987 const char * key)
988{
989 double sum = 0.0;
990 int count = 0;
991 for (size_t i = 0; i < orders->sz; ++i) {
992 order const * const oi = &orders->data[i]; // alias
993 if (oi->cutout == UNUSED_ORDER) continue; // skip cutout failures
994
995 for (size_t j = 0; oi->matches && j < oi->matches->sz; ++j) {
996 match const * const mj = &oi->matches->data[j]; // alias
997 if (mj->extract == UNUSED_ORDER) continue; // skip extract failures
998
999 double const val = cpl_propertylist_get_double(mj->qclist, key);
1000 if (cpl_error_get_code()) cpl_error_reset();
1001 else {
1002 count++;
1003 sum += val;
1004 break; // only need one from this order (they're all the same)
1005 }
1006 }
1007 }
1008 if (count > 0) {
1009 cpl_propertylist_append_double(phu_qclist, key, sum / count);
1010 return CPL_ERROR_NONE;
1011 }
1012 return CPL_ERROR_DATA_NOT_FOUND;
1013}
1014
cpl_error_code visir_dfs_check_framelist_tag(const irplib_framelist *self)
Check the tags in a frameset (group raw only)
Definition: visir_dfs.c:234
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.
Definition: visir_dfs.c:72
cpl_image ** visir_img_recombine_list(const char *recipename, const cpl_parameterlist *parlist, cpl_imagelist *nodded, const cpl_propertylist **plists, cpl_geom_combine combine_mode, cpl_boolean *pdid_resize)
The VISIR imaging combination using cross correlation.
cpl_imagelist * visir_load_hcycle(const irplib_framelist *rawframes, int pos)
Load the HCYCLE images from a VISIR file.
cpl_image ** visir_img_recombine(const char *recipename, const cpl_parameterlist *parlist, const irplib_framelist *rawframes, const char *badpix, const char *flat, cpl_geom_combine combine_mode, cpl_boolean *pdid_resize, cpl_boolean do_spc_fix, double wlen, visir_spc_resol resol)
The VISIR imaging combination using cross correlation.
visir_aplist * visir_aplist_new_from_file(const char *apfile)
Load the user-supplied apertures definition file.
int visir_parameterlist_get_int(const cpl_parameterlist *self, const char *recipe, visir_parameter bitmask)
Retrieve the value of a VISIR integer parameter.
const char * visir_parameterlist_get_string(const cpl_parameterlist *self, const char *recipe, visir_parameter bitmask)
Retrieve the value of a VISIR string parameter.
double visir_parameterlist_get_double(const cpl_parameterlist *self, const char *recipe, visir_parameter bitmask)
Retrieve the value of a VISIR parameter of type double.
cpl_boolean visir_parameterlist_get_bool(const cpl_parameterlist *self, const char *recipe, visir_parameter bitmask)
Retrieve the value of a VISIR boolean parameter.
cpl_error_code visir_spc_phot_sensit(const irplib_framelist *rawframes, const visir_spc_config *cfg, const cpl_propertylist *plist, const char *star_cat, cpl_image **pweight2d, cpl_propertylist *qclist, cpl_table *spc_table, const visir_spc_resol resol, const char *dit_key)
Compute the sensitivity from an extracted spectrum.