00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024 #ifdef HAVE_CONFIG_H
00025 #include <config.h>
00026 #endif
00027
00028 #include <math.h>
00029 #include <string.h>
00030
00031 #include "muse_sky.h"
00032 #include "muse_sky_old.h"
00033 #include "muse_instrument.h"
00034 #include "muse_lsf_params.h"
00035 #include "muse_optimize.h"
00036 #include "muse_utils.h"
00037 #include "muse_data_format_z.h"
00038
00043
00050
00051 typedef struct {
00053 cpl_array *line_strength;
00055 cpl_array *continuum;
00057 muse_lsf_params *lsf;
00058 } muse_sky_master_params;
00059
00060
00061
00069
00070 static muse_sky_master_params *
00071 muse_sky_master_params_new(cpl_size n_groups, cpl_size n_continuum) {
00072 muse_sky_master_params *res = cpl_calloc(1, sizeof(muse_sky_master_params));
00073 res->line_strength = cpl_array_new(n_groups, CPL_TYPE_DOUBLE);
00074 cpl_array_fill_window_double(res->line_strength, 0, n_groups, 1.0);
00075 res->continuum = cpl_array_new(n_continuum, CPL_TYPE_DOUBLE);
00076 cpl_array_fill_window_double(res->continuum, 0, n_continuum, 0.0);
00077 return res;
00078 }
00079
00080
00086
00087 static void
00088 muse_sky_master_params_delete(muse_sky_master_params *aParams) {
00089 if (aParams != NULL) {
00090 cpl_array_delete(aParams->line_strength);
00091 cpl_array_delete(aParams->continuum);
00092 muse_lsf_params_delete(aParams->lsf);
00093 cpl_free(aParams);
00094 }
00095 }
00096
00097
00109
00110 static muse_sky_master_params *
00111 muse_sky_master_apply_sky_parametrization(const cpl_array *aPar,
00112 cpl_size *offset,
00113 int ngroups) {
00114 muse_sky_master_params *p = muse_sky_master_params_new(ngroups, 0);
00115
00116 int i;
00117 for (i = 0; i < ngroups; i++) {
00118 double s = (cpl_array_get(aPar, (*offset)++, NULL));
00119 cpl_array_set(p->line_strength, i, s*s);
00120 }
00121 return p;
00122 }
00123
00124
00134
00135 static cpl_array *
00136 muse_sky_master_sky_firstguess(int ngroups) {
00137 cpl_array *pars = cpl_array_new(0 + ngroups, CPL_TYPE_DOUBLE);
00138 cpl_size offset = 0;
00139
00140
00141 int i;
00142 for (i = 0; i < ngroups; i++) {
00143 cpl_array_set(pars, offset++, 1e-1);
00144 }
00145
00146 if (offset != cpl_array_get_size(pars)) {
00147 cpl_msg_error(__func__,
00148 "inconsistent array: size %li; filled with %li values",
00149 (long)cpl_array_get_size(pars), (long)offset);
00150 }
00151 return pars;
00152 }
00153
00154
00165
00166 static muse_lsf_params *
00167 muse_sky_master_apply_lsf_parametrization(const cpl_array *aPar,
00168 cpl_size *offset)
00169 {
00170 muse_lsf_params *lsf = muse_lsf_params_new(1, 3, 1);
00171 cpl_array_set(lsf->sensitivity, 0, 1.0);
00172 lsf->offset = cpl_array_get(aPar, (*offset)++, NULL);
00173 lsf->refraction = 1.0 + cpl_array_get(aPar, (*offset)++, NULL);
00174 cpl_array_set(lsf->lsf_width, 0,
00175 cpl_array_get(aPar, (*offset)++, NULL));
00176 #if 1
00177 cpl_array_set(lsf->lsf_width, 1,
00178 cpl_array_get(aPar, (*offset)++, NULL));
00179 cpl_array_set(lsf->lsf_width, 2,
00180 cpl_array_get(aPar, (*offset)++, NULL));
00181 #endif
00182 cpl_size i;
00183 for (i = 0; i < MAX_HERMIT_ORDER; i++) {
00184 cpl_array_set(lsf->hermit[i], 0,
00185 cpl_array_get(aPar, (*offset)++, NULL));
00186 }
00187 return lsf;
00188 }
00189
00190
00199
00200 static cpl_array *
00201 muse_sky_master_lsf_firstguess(void) {
00202 cpl_array *pars = cpl_array_new(5 + MAX_HERMIT_ORDER, CPL_TYPE_DOUBLE);
00203 cpl_size offset = 0;
00204
00205
00206 cpl_array_set(pars, offset++, 0.0);
00207
00208 cpl_array_set(pars, offset++, 0.0);
00209
00210
00211 cpl_array_set(pars, offset++, 1.0);
00212 #if 1
00213 cpl_array_set(pars, offset++, 0);
00214 cpl_array_set(pars, offset++, 0);
00215 #endif
00216
00217 cpl_size i;
00218 for (i = 0; i < MAX_HERMIT_ORDER; i++) {
00219 cpl_array_set(pars, offset++, 0.0);
00220 }
00221 if (offset != cpl_array_get_size(pars)) {
00222 cpl_msg_error(__func__,
00223 "inconsistent array: size %ld, filled with %ld values",
00224 (long)cpl_array_get_size(pars), (long)offset);
00225 }
00226 return pars;
00227 }
00228
00229
00240
00241 static muse_sky_master_params *
00242 muse_sky_master_apply_parametrization(const cpl_array *aPar, int ngroups) {
00243 cpl_size offset = 0;
00244 muse_sky_master_params *p = muse_sky_master_apply_sky_parametrization(aPar, &offset,
00245 ngroups);
00246 p->lsf = muse_sky_master_apply_lsf_parametrization(aPar, &offset);
00247
00248 if (offset != cpl_array_get_size(aPar)) {
00249 cpl_msg_error(__func__,
00250 "inconsistent array: size %ld, read with %ld values",
00251 (long)cpl_array_get_size(aPar), (long)offset);
00252 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT);
00253 muse_sky_master_params_delete(p);
00254 return NULL;
00255 }
00256 return p;
00257 }
00258
00259
00270
00271
00272 static cpl_array *
00273 simulate_master_sky_parameters(const cpl_table *aLines,
00274 cpl_size aNgroups,
00275 const cpl_array *aLambda,
00276 const cpl_array *aPar) {
00277 muse_sky_master_params *p
00278 = muse_sky_master_apply_parametrization(aPar, aNgroups);
00279
00280 cpl_array *continuum = cpl_array_duplicate(aLambda);
00281 muse_cplarray_poly1d(continuum, p->continuum);
00282
00283 cpl_table *lines = cpl_table_duplicate(aLines);
00284 muse_sky_lines_apply_strength(lines, p->line_strength);
00285 double maxflux = cpl_table_get_column_max(lines, "flux");
00286 muse_sky_lines_cut(lines, 1e-4 * maxflux);
00287
00288 cpl_array *simulated = muse_lsf_params_spectrum(aLambda, lines, p->lsf);
00289 cpl_array_add(simulated, continuum);
00290
00291 cpl_array_delete(continuum);
00292 cpl_table_delete(lines);
00293 muse_sky_master_params_delete(p);
00294 return simulated;
00295
00296 }
00297
00298 typedef struct {
00299 const cpl_array *lambda;
00300 const cpl_array *values;
00301 const cpl_array *stat;
00302 const cpl_table *sky;
00303 const cpl_size ngroups;
00304 } muse_master_fit_struct;
00305
00306
00307
00316
00317 static cpl_error_code
00318 muse_sky_master_eval(void *aData, cpl_array *aPar, cpl_array *aRetval) {
00319 cpl_size size = cpl_array_get_size(aRetval);
00320 muse_master_fit_struct *data = aData;
00321 cpl_array *simulated
00322 = simulate_master_sky_parameters(data->sky, data->ngroups,
00323 data->lambda, aPar);
00324
00325 cpl_array_subtract(simulated, data->values);
00326 cpl_array *dsimulated = muse_cplarray_diff(simulated, 1);
00327 cpl_array_divide(dsimulated, data->stat);
00328
00329 cpl_array_fill_window_double(aRetval, 0, size, 0.0);
00330 memcpy(cpl_array_get_data_double(aRetval),
00331 cpl_array_get_data_double_const(dsimulated),
00332 size * sizeof(double));
00333
00334 cpl_array_delete(simulated);
00335 cpl_array_delete(dsimulated);
00336
00337 return CPL_ERROR_NONE;
00338 }
00339
00340
00357
00358
00359 static cpl_error_code
00360 muse_sky_master_correct_firstguess(const cpl_table *aLines,
00361 cpl_size aNgroups,
00362 const cpl_array *aLambda,
00363 const cpl_array *aData,
00364 cpl_array *aPars) {
00365
00366
00367 cpl_array *simulated
00368 = simulate_master_sky_parameters(aLines, aNgroups, aLambda, aPars);
00369
00370 cpl_size offset = 0;
00371 muse_sky_master_params *msp
00372 = muse_sky_master_apply_sky_parametrization(aPars, &offset, aNgroups);
00373 cpl_table *lines = cpl_table_duplicate(aLines);
00374 muse_sky_lines_apply_strength(lines, msp->line_strength);
00375 muse_sky_master_params_delete(msp);
00376 cpl_size i_group;
00377 double delta = 0.0;
00378 for (i_group = 0; i_group < aNgroups; i_group++) {
00379
00380 cpl_table_unselect_all(lines);
00381 cpl_table_or_selected_int(lines, "group", CPL_EQUAL_TO, i_group);
00382 cpl_table *gtable = cpl_table_extract_selected(lines);
00383 cpl_size row;
00384 cpl_table_get_column_maxpos(gtable, "flux", &row);
00385 double wavelength = cpl_table_get_double(gtable, "lambda", row, NULL);
00386 cpl_table_delete(gtable);
00387
00388
00389 cpl_size i_lbda1 = muse_cplarray_find_sorted(aLambda, wavelength - 2.0);
00390 cpl_size i_lbda2 = muse_cplarray_find_sorted(aLambda, wavelength + 2.0);
00391 double y_data = 0;
00392 double y_sim = 0;
00393 double avg_data = 0;
00394 double avg_sim = 0;
00395 cpl_size i_lbda;
00396 for (i_lbda = i_lbda1; i_lbda <= i_lbda2; i_lbda++) {
00397 double lbda = cpl_array_get(aLambda, i_lbda, NULL);
00398 double wy_data = cpl_array_get(aData, i_lbda, NULL);
00399 double wy_sim = cpl_array_get(simulated, i_lbda, NULL);
00400 y_data += wy_data;
00401 avg_data += wy_data * lbda;
00402 y_sim += wy_sim;
00403 avg_sim += wy_sim * lbda;
00404 }
00405
00406
00407 if (y_sim > 0) {
00408 cpl_array_set(aPars, i_group,
00409 cpl_array_get(aPars, i_group, NULL)*sqrt(y_data/y_sim));
00410 avg_data /= y_data;
00411 avg_sim /= y_sim;
00412 delta += avg_data - avg_sim;
00413 }
00414 }
00415 cpl_array_set(aPars, aNgroups,
00416 cpl_array_get(aPars, aNgroups, NULL) + delta/aNgroups);
00417 cpl_table_delete(lines);
00418 cpl_array_delete(simulated);
00419
00420 return CPL_ERROR_NONE;
00421 }
00422
00423
00434
00435 cpl_error_code
00436 muse_sky_lines_fit_old(cpl_table *aSpectrum, cpl_table *aLines)
00437 {
00438 cpl_ensure_code(aSpectrum, CPL_ERROR_NULL_INPUT);
00439 cpl_ensure_code(aLines, CPL_ERROR_NULL_INPUT);
00440
00441 cpl_array *lambda = muse_cpltable_extract_column(aSpectrum, "lambda");
00442 cpl_array *data = muse_cpltable_extract_column(aSpectrum, "data");
00443 cpl_array *stat2 = muse_cpltable_extract_column(aSpectrum, "stat");
00444
00445 cpl_size nstat = cpl_array_get_size(stat2);
00446 cpl_ensure_code(nstat > 0, CPL_ERROR_DATA_NOT_FOUND);
00447 cpl_array *stat = cpl_array_extract(stat2, 0, nstat - 1);
00448 cpl_array *s2 = cpl_array_extract(stat2, 1, nstat);
00449 cpl_array_add(stat, s2);
00450 cpl_array_delete(s2);
00451 cpl_array_power(stat, 0.5);
00452
00453 muse_master_fit_struct fit_data = {
00454 lambda,
00455 data,
00456 stat,
00457 aLines,
00458 cpl_table_get_column_max(aLines, "group") + 1
00459 };
00460
00461 cpl_array *pars = muse_sky_master_sky_firstguess(fit_data.ngroups);
00462 cpl_array *dpars = muse_sky_master_lsf_firstguess();
00463 cpl_array_insert(pars, dpars, cpl_array_get_size(pars));
00464 cpl_array_delete(dpars);
00465
00466
00467
00468
00469
00470 muse_sky_master_correct_firstguess(aLines, fit_data.ngroups, lambda, data, pars);
00471 muse_sky_master_correct_firstguess(aLines, fit_data.ngroups, lambda, data, pars);
00472 muse_sky_master_correct_firstguess(aLines, fit_data.ngroups, lambda, data, pars);
00473
00474 cpl_size size = cpl_array_get_size(lambda);
00475
00476 int debug = getenv("MUSE_DEBUG_LSF_FIT")
00477 && atoi(getenv("MUSE_DEBUG_LSF_FIT")) > 0;
00478 muse_cpl_optimize_control_t ctrl = {
00479 -1, -1, -1, -1,
00480 debug
00481 };
00482
00483 cpl_msg_info(__func__, "Starting master sky fit");
00484 cpl_error_code r = muse_cpl_optimize_lvmq(&fit_data, pars, size-1,
00485 muse_sky_master_eval, &ctrl);
00486 if (r != CPL_ERROR_NONE) {
00487 cpl_msg_error(__func__, "Master sky fit failed with error code %i: %s",
00488 r, cpl_error_get_message());
00489 } else {
00490 cpl_msg_info(__func__, "Master sky fit finished successfully.");
00491 }
00492
00493 muse_sky_master_params *p = muse_sky_master_apply_parametrization(pars, fit_data.ngroups);
00494
00495 cpl_array_delete(pars);
00496
00497 muse_sky_lines_apply_strength(aLines, p->line_strength);
00498 cpl_propertylist *order = cpl_propertylist_new();
00499
00500 cpl_propertylist_append_bool(order, "flux", TRUE);
00501 cpl_table_sort(aLines, order);
00502 cpl_propertylist_delete(order);
00503
00504 cpl_msg_info(__func__, "refraction index=1%s%g, offset=%f Angstrom",
00505 p->lsf->refraction < 1?"-":"+",
00506 fabs(p->lsf->refraction-1), p->lsf->offset);
00507
00508 cpl_array_delete(stat);
00509 muse_sky_master_params_delete(p);
00510
00511 cpl_array_unwrap(lambda);
00512 cpl_array_unwrap(data);
00513 cpl_array_unwrap(stat2);
00514
00515 return CPL_ERROR_NONE;
00516 }
00531 cpl_error_code
00532 muse_sky_subtract_lines_old(muse_pixtable *aPixtable, cpl_table *aLines,
00533 muse_lsf_params **aLsfParams)
00534 {
00535 cpl_ensure_code(aPixtable != NULL, CPL_ERROR_NULL_INPUT);
00536 cpl_ensure_code(aPixtable->table != NULL, CPL_ERROR_NULL_INPUT);
00537 cpl_ensure_code(aLines != NULL, CPL_ERROR_NULL_INPUT);
00538
00539 muse_pixtable **slice_pixtable = muse_pixtable_extracted_get_slices(aPixtable);
00540 cpl_size n_slices = muse_pixtable_extracted_get_size(slice_pixtable);
00541 cpl_size i_slice;
00542 cpl_msg_info(__func__, "Starting sky subtraction of %"CPL_SIZE_FORMAT" slices",
00543 n_slices);
00544 cpl_boolean debug = getenv("MUSE_DEBUG_SKY")
00545 && atoi(getenv("MUSE_DEBUG_SKY")) > 0;
00546 #pragma omp parallel for default(none) \
00547 shared(aLsfParams, aLines, debug, n_slices, slice_pixtable)
00548 for (i_slice = 0; i_slice < n_slices; i_slice++) {
00549 uint32_t origin
00550 = (uint32_t)cpl_table_get_int(slice_pixtable[i_slice]->table,
00551 MUSE_PIXTABLE_ORIGIN, 0, NULL);
00552 int ifu = muse_pixtable_origin_get_ifu(origin);
00553 int slice = muse_pixtable_origin_get_slice(origin);
00554 muse_lsf_params *slice_params = muse_lsf_params_get(aLsfParams, ifu, slice);
00555 if ((slice_params == NULL) && (aLines != NULL)){
00556 cpl_msg_warning(__func__, "No LSF params for slice #%i.%i."
00557 " Ignoring lines in sky subtraction for this slice.",
00558 ifu, slice);
00559 }
00560
00561 cpl_size nrows = muse_pixtable_get_nrow(slice_pixtable[i_slice]);
00562 if (debug) {
00563 cpl_msg_debug(__func__, "Sky subtraction of %li pixels for slice #%i.%i",
00564 (long)nrows, ifu, slice);
00565 }
00566 cpl_errorstate prestate = cpl_errorstate_get();
00567 muse_pixtable *slice_pt = slice_pixtable[i_slice];
00568 cpl_propertylist *order = cpl_propertylist_new();
00569 cpl_propertylist_append_bool(order, MUSE_PIXTABLE_LAMBDA, CPL_FALSE);
00570 cpl_table_sort(slice_pt->table, order);
00571 cpl_propertylist_delete(order);
00572
00573 cpl_table_cast_column(slice_pt->table, MUSE_PIXTABLE_LAMBDA,
00574 "lambda_double", CPL_TYPE_DOUBLE);
00575 cpl_array *lambda = muse_cpltable_extract_column(slice_pt->table, "lambda_double");
00576 cpl_table_unwrap(slice_pt->table, "lambda_double");
00577 cpl_array *spectrum = muse_lsf_params_spectrum(lambda, aLines, slice_params);
00578
00579 cpl_array *data = muse_cpltable_extract_column(slice_pt->table,
00580 MUSE_PIXTABLE_DATA);
00581 cpl_array_subtract(data, spectrum);
00582
00583 cpl_size ii;
00584 for (ii = 0; ii < cpl_array_get_size(data); ii++) {
00585 if (!cpl_array_is_valid(spectrum, ii)) {
00586 cpl_table_set_invalid(slice_pt->table, MUSE_PIXTABLE_DATA, ii);
00587 }
00588 }
00589 cpl_array_unwrap(data);
00590 cpl_array_delete(spectrum);
00591 cpl_array_delete(lambda);
00592
00593 if (!cpl_errorstate_is_equal(prestate)) {
00594 cpl_errorstate_dump(prestate, CPL_FALSE, NULL);
00595 cpl_errorstate_set(prestate);
00596 }
00597 }
00598 muse_pixtable_extracted_delete(slice_pixtable);
00599
00600 if (aPixtable->header) {
00601
00602 cpl_propertylist_update_bool(aPixtable->header, MUSE_HDR_PT_SKYSUB,
00603 CPL_TRUE);
00604 cpl_propertylist_set_comment(aPixtable->header, MUSE_HDR_PT_SKYSUB,
00605 MUSE_HDR_PT_SKYSUB_COMMENT);
00606 }
00607 return CPL_ERROR_NONE;
00608 }
00609