00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022 #ifdef HAVE_CONFIG_H
00023 #include <config.h>
00024 #endif
00025
00026
00027
00028
00029 #include <string.h>
00030 #include <math.h>
00031 #include <cpl.h>
00032 #include <muse.h>
00033 #include "muse_lingain_z.h"
00034
00035
00036 typedef void (*muse_free_func)(void *);
00037
00038 struct muse_gain_fit_t
00039 {
00040 double gain;
00041 double rms;
00042 };
00043
00044 typedef struct muse_gain_fit_t muse_gain_fit_t;
00045
00046 struct muse_linearity_fit_t
00047 {
00048 cpl_array *coefficients;
00049 double range[2];
00050 double rms;
00051 };
00052
00053 typedef struct muse_linearity_fit_t muse_linearity_fit_t;
00054
00055
00056
00057
00058
00059
00060 static void
00061 muse_vfree(void **array, cpl_size size, muse_free_func deallocator)
00062 {
00063 if (array) {
00064 cpl_size idx;
00065 for (idx = 0; idx < size; ++idx) {
00066 if (deallocator) {
00067 deallocator(array[idx]);
00068 }
00069 }
00070 cpl_free(array);
00071 }
00072 return;
00073 }
00074
00075 static void
00076 muse_linearity_fit_clear(muse_linearity_fit_t *aFit)
00077 {
00078 if (aFit) {
00079 if (aFit->coefficients) {
00080 cpl_array_delete(aFit->coefficients);
00081 }
00082 }
00083 return;
00084 }
00085
00086
00097
00098 static cpl_boolean
00099 muse_lingain_validate_parameters(muse_lingain_params_t *aParams)
00100 {
00101 if (aParams->ybox <= 0) {
00102 cpl_msg_error(__func__, "Invalid measurement window size: window must "
00103 "be larger than 0!");
00104 return CPL_FALSE;
00105 }
00106
00107 if (aParams->xgap < 0) {
00108 cpl_msg_error(__func__, "Invalid tracing edge offset: offset is "
00109 "less than 0!");
00110 return CPL_FALSE;
00111 }
00112
00113 if (aParams->xborder < 0) {
00114 cpl_msg_error(__func__, "Invalid offset from detector edge: offset is "
00115 "less than 0!");
00116 return CPL_FALSE;
00117 }
00118
00119 if (aParams->order < 0) {
00120 cpl_msg_error(__func__, "Invalid polynomial fit order: polynomial "
00121 "degree is less than 0!");
00122 return CPL_FALSE;
00123 }
00124
00125 if (aParams->toffset < 0.) {
00126 cpl_msg_error(__func__, "Invalid exposure time offset: offset is less "
00127 "than 0.!");
00128 return CPL_FALSE;
00129 }
00130
00131 if (aParams->sigma < 0.) {
00132 cpl_msg_error(__func__, "Invalid sigma clipping threshold used for "
00133 "signal value cleaning: threshold is less than 0.!");
00134 return CPL_FALSE;
00135 }
00136
00137 if (aParams->signalmin < 0.) {
00138 cpl_msg_error(__func__, "Invalid signal level grid: minimum signal is "
00139 "less than 0.!");
00140 return CPL_FALSE;
00141 }
00142
00143 if (aParams->signalbin <= 0.) {
00144 cpl_msg_error(__func__, "Invalid signal level grid: signal bin size "
00145 "must larger than 0.!");
00146 return CPL_FALSE;
00147 }
00148
00149 if (aParams->signalmax < aParams->signalmin + aParams->signalbin) {
00150 cpl_msg_error(__func__, "Invalid signal level grid: maximum signal is "
00151 "too small for chosen minimum value and bin size!");
00152 return CPL_FALSE;
00153 }
00154
00155 if (aParams->gainlimit < 0.) {
00156 cpl_msg_error(__func__, "Invalid minimum signal used for gain fit: "
00157 "minimum signal is less than 0.!");
00158 return CPL_FALSE;
00159 }
00160
00161 if (aParams->gainsigma < 0.) {
00162 cpl_msg_error(__func__, "Invalid sigma clipping threshold used for "
00163 "gain value cleaning: threshold is less than 0.!");
00164 return CPL_FALSE;
00165 }
00166
00167 if (aParams->ctsmin < 0.) {
00168 cpl_msg_error(__func__, "Invalid signal level grid used for "
00169 "non-linearity analysis: minimum signal is "
00170 "less than 0.!");
00171 return CPL_FALSE;
00172 }
00173
00174 if (aParams->ctsbin <= 0.) {
00175 cpl_msg_error(__func__, "Invalid signal level grid used for "
00176 "non-linearity analysis: signal bin size must larger "
00177 "than 0.!");
00178 return CPL_FALSE;
00179 }
00180
00181 if (aParams->ctsmax < aParams->ctsmin + aParams->ctsbin) {
00182 cpl_msg_error(__func__, "Invalid signal level grid used for "
00183 "non-linearity analysis: maximum signal is "
00184 "too small for chosen minimum value and bin size!");
00185 return CPL_FALSE;
00186 }
00187
00188 if (aParams->linearmin < 0.) {
00189 cpl_msg_error(__func__, "Invalid signal range used for residual "
00190 "non-linearity fit: minimum signal is less than 0.!");
00191 return CPL_FALSE;
00192 }
00193
00194 if (aParams->linearmax <= aParams->linearmin) {
00195 cpl_msg_error(__func__, "Invalid signal range used for residual "
00196 "non-linearity fit: maximum signal is too small for "
00197 "the cosen minimum value!");
00198 return CPL_FALSE;
00199 }
00200
00201 return CPL_TRUE;
00202 }
00203
00204
00211
00212 static muse_imagelist *
00213 muse_lingain_load_images(muse_processing *aProcessing, unsigned short aIFU)
00214 {
00215 muse_imagelist *images;
00216
00217
00218
00219
00220 cpl_frame *bias = muse_frameset_find_master(aProcessing->inframes,
00221 MUSE_TAG_MASTER_BIAS, aIFU);
00222
00223
00224
00225 cpl_propertylist *properties = cpl_propertylist_load(cpl_frame_get_filename(bias), 0);
00226 cpl_frame_delete(bias);
00227 muse_basicproc_params *bpars = muse_basicproc_params_new_from_propertylist(properties);
00228 cpl_propertylist_delete(properties);
00229
00230 images = muse_basicproc_load(aProcessing, aIFU, bpars);
00231 muse_basicproc_params_delete(bpars);
00232
00233 return images;
00234 }
00235
00236
00246
00247 static cpl_table *
00248 muse_lingain_sort_images(const muse_imagelist *aList)
00249 {
00250 cpl_ensure(aList, CPL_ERROR_NULL_INPUT, NULL);
00251
00252 cpl_errorstate status = cpl_errorstate_get();
00253
00254 cpl_table *exposures = cpl_table_new(aList->size);
00255
00256 cpl_table_new_column(exposures, "INDEX", CPL_TYPE_INT);
00257 cpl_table_new_column(exposures, "TYPE", CPL_TYPE_STRING);
00258 cpl_table_new_column(exposures, "MJDOBS", CPL_TYPE_DOUBLE);
00259 cpl_table_new_column(exposures, "EXPTIME", CPL_TYPE_DOUBLE);
00260
00261 unsigned int iexposure;
00262 for (iexposure = 0; iexposure < aList->size; ++iexposure) {
00263 cpl_propertylist *header = muse_imagelist_get((muse_imagelist *)aList, iexposure)->header;
00264 cpl_table_set_int(exposures, "INDEX", iexposure, iexposure);
00265 if (strncmp(muse_pfits_get_dpr_type(header), "BIAS,DETCHECK", 14) == 0) {
00266 cpl_table_set_string(exposures, "TYPE", iexposure,
00267 MUSE_TAG_LINEARITY_BIAS);
00268 } else {
00269 cpl_table_set_string(exposures, "TYPE", iexposure,
00270 MUSE_TAG_LINEARITY_FLAT);
00271 }
00272 cpl_table_set_double(exposures, "MJDOBS", iexposure,
00273 muse_pfits_get_mjdobs(header));
00274 cpl_table_set_double(exposures, "EXPTIME", iexposure,
00275 muse_pfits_get_exptime(header));
00276 }
00277
00278
00279
00280
00281 cpl_propertylist *order = cpl_propertylist_new();
00282 cpl_propertylist_append_bool(order, "EXPTIME", CPL_FALSE);
00283 cpl_propertylist_append_bool(order, "MJDOBS", CPL_FALSE);
00284
00285 cpl_table_sort(exposures, order);
00286 cpl_propertylist_delete(order);
00287
00288 if (!cpl_errorstate_is_equal(status)) {
00289 cpl_table_delete(exposures);
00290 return NULL;
00291 }
00292
00293
00294
00295 cpl_boolean invalid = CPL_FALSE;
00296 cpl_size npairs = (cpl_size)(aList->size / 2);
00297 cpl_size ipair;
00298 for (ipair = 0; ipair < npairs; ++ipair) {
00299 register cpl_size jpair = 2 * ipair;
00300 register cpl_size kpair = jpair + 1;
00301 if (strcmp(cpl_table_get_string(exposures, "TYPE", jpair),
00302 cpl_table_get_string(exposures, "TYPE", kpair)) != 0) {
00303 cpl_msg_error(__func__, "Invalid pair of exposures: Type of exposure "
00304 "does not match: Got '%s', expected '%s'!",
00305 cpl_table_get_string(exposures, "TYPE", jpair),
00306 cpl_table_get_string(exposures, "TYPE", kpair));
00307 invalid = CPL_TRUE;
00308 break;
00309 }
00310 if ((cpl_table_get_double(exposures, "EXPTIME", jpair, NULL) -
00311 cpl_table_get_double(exposures, "EXPTIME", kpair, NULL)) > 0.1) {
00312 cpl_msg_error(__func__, "Invalid pair of exposures: exposure times "
00313 "do not match: Got '%.4f', expected '%.4f'!",
00314 cpl_table_get_double(exposures, "EXPTIME", jpair, NULL),
00315 cpl_table_get_double(exposures, "EXPTIME", kpair, NULL));
00316 invalid = CPL_TRUE;
00317 break;
00318 }
00319 }
00320
00321 if (invalid) {
00322 cpl_table_delete(exposures);
00323 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_OUTPUT);
00324 return NULL;
00325 }
00326
00327
00328 cpl_size nexposures = 2 * npairs;
00329 if (aList->size > nexposures) {
00330 cpl_table_erase_window(exposures, nexposures, aList->size - nexposures);
00331 }
00332
00333 return exposures;
00334 }
00335
00336
00346
00347 static cpl_table *
00348 muse_lingain_create_windowlist(const cpl_table *aTrace,
00349 const cpl_table *aExposureList,
00350 const muse_imagelist *aList,
00351 muse_lingain_params_t *aParams)
00352 {
00353 cpl_ensure(aTrace && aExposureList && aList && aParams,
00354 CPL_ERROR_NULL_INPUT, NULL);
00355
00356 cpl_table *windowlist = cpl_table_new(0);
00357 cpl_table_new_column(windowlist, "Quadrant", CPL_TYPE_INT);
00358 cpl_table_new_column(windowlist, "SliceNo", CPL_TYPE_INT);
00359 cpl_table_new_column(windowlist, "Xmin", CPL_TYPE_INT);
00360 cpl_table_new_column(windowlist, "Ymin", CPL_TYPE_INT);
00361 cpl_table_new_column(windowlist, "Xmax", CPL_TYPE_INT);
00362 cpl_table_new_column(windowlist, "Ymax", CPL_TYPE_INT);
00363
00364
00365
00366
00367 int idx = cpl_table_get_int(aExposureList, "INDEX", 0, NULL);
00368
00369 cpl_errorstate status = cpl_errorstate_get();
00370
00371 muse_image *image = muse_imagelist_get((muse_imagelist *)aList, idx);
00372 cpl_size firstrow = 0;
00373
00374 unsigned char iquadrant;
00375 for (iquadrant = 1; iquadrant <= 4; ++iquadrant) {
00376 cpl_size *qwindow = muse_quadrants_get_window(image, iquadrant);
00377
00378 unsigned short islice;
00379 for (islice = 1; islice <= kMuseSlicesPerCCD; ++islice) {
00380 cpl_errorstate _status = cpl_errorstate_get();
00381 cpl_polynomial **traces = muse_trace_table_get_polys_for_slice(aTrace,
00382 islice);
00383
00384
00385 if (!traces) {
00386 cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
00387 "missing!", islice, aParams->nifu);
00388 muse_trace_polys_delete(traces);
00389 cpl_errorstate_set(_status);
00390 continue;
00391 }
00392 cpl_size torder = 0;
00393 double ledge = cpl_polynomial_get_coeff(traces[MUSE_TRACE_LEFT], &torder);
00394 double redge = cpl_polynomial_get_coeff(traces[MUSE_TRACE_RIGHT], &torder);
00395 if ((ledge < qwindow[0] + aParams->xborder) ||
00396 (redge > qwindow[1] - aParams->xborder)) {
00397 muse_trace_polys_delete(traces);
00398 continue;
00399 }
00400 else {
00401 unsigned int nwindows = (qwindow[3] - qwindow[2] + 1) / aParams->ybox;
00402
00403
00404 cpl_table_set_size(windowlist, firstrow + nwindows);
00405
00406 unsigned int iwindow;
00407 for (iwindow = 0; iwindow < nwindows; ++iwindow) {
00408 cpl_size irow = firstrow + iwindow;
00409
00410 int ymin = iwindow * aParams->ybox + qwindow[2];
00411 int ymax = ymin + aParams->ybox - 1;
00412
00413 double ymid = ymin + 0.5 * aParams->ybox;
00414 double xleft = cpl_polynomial_eval_1d(traces[MUSE_TRACE_LEFT], ymid, NULL);
00415 double xright = cpl_polynomial_eval_1d(traces[MUSE_TRACE_RIGHT], ymid, NULL);
00416
00417 int xmin = (int)rint(xleft) + aParams->xgap;
00418 int xmax = (int)rint(xright) - aParams->xgap;
00419
00420 cpl_table_set_int(windowlist, "Quadrant", irow, iquadrant);
00421 cpl_table_set_int(windowlist, "SliceNo", irow, islice);
00422 cpl_table_set_int(windowlist, "Xmin", irow, xmin);
00423 cpl_table_set_int(windowlist, "Xmax", irow, xmax);
00424 cpl_table_set_int(windowlist, "Ymin", irow, ymin);
00425 cpl_table_set_int(windowlist, "Ymax", irow, ymax);
00426 }
00427 firstrow += nwindows;
00428 }
00429
00430 muse_trace_polys_delete(traces);
00431 }
00432
00433 cpl_free(qwindow);
00434 }
00435
00436 if (!cpl_errorstate_is_equal(status)) {
00437 cpl_table_delete(windowlist);
00438 return NULL;
00439 }
00440
00441 return windowlist;
00442 }
00443
00444
00455
00456 static cpl_array *
00457 muse_lingain_quadrant_get_windows(cpl_table *aWindowList,
00458 unsigned char aQuadrant)
00459 {
00460 cpl_table_select_all(aWindowList);
00461
00462
00463 cpl_size nwindows = cpl_table_and_selected_int(aWindowList, "Quadrant",
00464 CPL_EQUAL_TO, aQuadrant);
00465 if (nwindows == 0) {
00466 return NULL;
00467 }
00468
00469 cpl_array *windex = cpl_array_new(nwindows, CPL_TYPE_SIZE);
00470
00471 cpl_size iwindow = 0;
00472 cpl_size irow;
00473 for (irow = 0; irow < cpl_table_get_nrow(aWindowList); ++irow) {
00474 if (cpl_table_is_selected(aWindowList, irow)) {
00475 cpl_array_set_cplsize(windex, iwindow, irow);
00476 ++iwindow;
00477 }
00478 }
00479
00480
00481 cpl_table_select_all(aWindowList);
00482
00483 return windex;
00484 }
00485
00486
00498
00499 static cpl_array *
00500 muse_lingain_quadrant_extract_data(const cpl_table *aGainTable,
00501 const cpl_array *aWindows,
00502 const char *aName,
00503 cpl_size aRow)
00504 {
00505
00506 const cpl_array *src = cpl_table_get_array(aGainTable, aName, aRow);
00507
00508 if (!src) {
00509 return NULL;
00510 }
00511
00512 cpl_size nwindows = cpl_array_get_size(aWindows);
00513 cpl_array *data = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
00514
00515 cpl_size iwindow;
00516 for (iwindow = 0; iwindow < nwindows; ++iwindow) {
00517 cpl_size jwindow = cpl_array_get_cplsize(aWindows, iwindow, NULL);
00518 int invalid = 0;
00519 double value = cpl_array_get_double(src, jwindow, &invalid);
00520
00521 if (!invalid) {
00522 cpl_array_set_double(data, iwindow, value);
00523 } else {
00524 cpl_array_set_invalid(data, iwindow);
00525 }
00526 }
00527
00528 return data;
00529
00530 }
00531
00532
00549
00550 static cpl_array *
00551 muse_lingain_compute_ron(const cpl_table *aWindowList,
00552 cpl_table *aExposureList,
00553 muse_imagelist *aImageList,
00554 muse_lingain_params_t *aParams)
00555 {
00556 cpl_table_select_all(aExposureList);
00557 cpl_size nbias = cpl_table_and_selected_string(aExposureList, "TYPE", CPL_EQUAL_TO,
00558 MUSE_TAG_LINEARITY_BIAS);
00559 if (nbias > 2) {
00560 cpl_msg_warning(__func__, "Found more than 2 (%" CPL_SIZE_FORMAT ") "
00561 "input images of type '%s' for IFU %u! Only the first 2 "
00562 "images will be used!", nbias, MUSE_TAG_LINEARITY_BIAS,
00563 aParams->nifu);
00564 }
00565
00566 cpl_size ibias = 0;
00567 cpl_size iexposure;
00568 for (iexposure = 0; iexposure < cpl_table_get_nrow(aExposureList); ++iexposure) {
00569 if (cpl_table_is_selected(aExposureList, iexposure)) {
00570 ibias = cpl_table_get_int(aExposureList, "INDEX", iexposure, NULL);
00571 break;
00572 }
00573 }
00574 cpl_table_select_all(aExposureList);
00575
00576
00577
00578
00579 muse_image *bias1 = muse_imagelist_get(aImageList, ibias);
00580 muse_image *bias2 = muse_imagelist_get(aImageList, ibias + 1);
00581
00582 muse_image_reject_from_dq(bias1);
00583 muse_image_reject_from_dq(bias2);
00584
00585 cpl_image *dbias = cpl_image_subtract_create(bias1->data, bias2->data);
00586
00587 cpl_array *ron = cpl_array_new(cpl_table_get_nrow(aWindowList), CPL_TYPE_DOUBLE);
00588
00589 cpl_size iwindow;
00590 for (iwindow = 0; iwindow < cpl_table_get_nrow(aWindowList); ++ iwindow) {
00591 int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
00592 int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
00593 int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
00594 int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);
00595
00596 double _ron = cpl_image_get_stdev_window(dbias, xmin, ymin, xmax, ymax);
00597
00598 _ron = sqrt(0.5 * _ron * _ron);
00599 cpl_array_set_double(ron, iwindow, _ron);
00600 }
00601 cpl_image_delete(dbias);
00602
00603 return ron;
00604 }
00605
00606
00628
00629 static cpl_table *
00630 muse_lingain_window_get_gain(const cpl_table *aWindowList,
00631 const cpl_array *aRon,
00632 cpl_table *aExposureList,
00633 muse_imagelist *aImageList,
00634 muse_lingain_params_t *aParams)
00635 {
00636
00637
00638
00639 cpl_table_select_all(aExposureList);
00640 cpl_size nflats = cpl_table_and_selected_string(aExposureList, "TYPE", CPL_EQUAL_TO,
00641 MUSE_TAG_LINEARITY_FLAT);
00642 if (nflats % 2) {
00643 cpl_msg_warning(__func__, "Found odd number of input images (%" CPL_SIZE_FORMAT
00644 ") input images of type '%s' for IFU %u! Exposure series "
00645 "may be incorrect!", nflats, MUSE_TAG_LINEARITY_FLAT,
00646 aParams->nifu);
00647 }
00648
00649 cpl_table *flats = cpl_table_extract_selected(aExposureList);
00650 cpl_table_select_all(aExposureList);
00651
00652
00653 cpl_errorstate state = cpl_errorstate_get();
00654
00655 cpl_size npairs = nflats / 2;
00656 cpl_size nwindows = cpl_table_get_nrow(aWindowList);
00657
00658 cpl_table *gain = cpl_table_new(npairs);
00659 cpl_table_new_column(gain, "ExpTime", CPL_TYPE_DOUBLE);
00660 cpl_table_new_column_array(gain, "Signal", CPL_TYPE_DOUBLE, nwindows);
00661 cpl_table_new_column_array(gain, "Variance", CPL_TYPE_DOUBLE, nwindows);
00662 cpl_table_new_column_array(gain, "Gain", CPL_TYPE_DOUBLE, nwindows);
00663
00664 if (!cpl_errorstate_is_equal(state)) {
00665 cpl_table_delete(gain);
00666 cpl_table_delete(flats);
00667 return NULL;
00668 }
00669
00670
00671
00672 double mron_value = cpl_array_get_median(aRon);
00673 double mron_sdev = cpl_array_get_stdev(aRon);
00674
00675
00676
00677 cpl_size kpair = 0;
00678 cpl_size ipair;
00679 for (ipair = 0; ipair < npairs; ++ipair) {
00680 cpl_size iexposure = 2 * ipair;
00681 cpl_size jexposure = iexposure + 1;
00682 int iflat = cpl_table_get_int(flats, "INDEX", iexposure, NULL);
00683 int jflat = cpl_table_get_int(flats, "INDEX", jexposure, NULL);
00684 muse_image *flat1 = muse_imagelist_get(aImageList, iflat);
00685 muse_image *flat2 = muse_imagelist_get(aImageList, jflat);
00686
00687 double exptime1 = cpl_table_get_double(flats, "EXPTIME", iexposure, NULL);
00688 double exptime2 = cpl_table_get_double(flats, "EXPTIME", jexposure, NULL);
00689 double flux1 = cpl_image_get_mean(flat1->data);
00690 double flux2 = cpl_image_get_mean(flat2->data);
00691
00692 if (fabs((flux1 - flux2) / flux1) > aParams->fluxtol) {
00693 cpl_msg_warning(__func__, "Inconsistent overall flux level (%.4f, "
00694 "%.4f) detected for flat field pair (%d, %d) of IFU %u "
00695 "with exposure time %.4f! Skipping this pair!",
00696 flux1, flux2, iflat, jflat, aParams->nifu, exptime1);
00697 continue;
00698 }
00699
00700 cpl_table_set_double(gain, "ExpTime", kpair, 0.5 * (exptime1 + exptime2));
00701
00702
00703 muse_image_reject_from_dq(flat1);
00704 muse_image_reject_from_dq(flat2);
00705
00706
00707
00708
00709
00710 cpl_array *_signal = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
00711 cpl_array *_variance = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
00712 cpl_array *_gain = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
00713
00714 if (aParams->sigma > 0.) {
00715
00716 cpl_size iwindow;
00717 for (iwindow = 0; iwindow < nwindows; ++iwindow) {
00718 int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
00719 int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
00720 int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
00721 int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);
00722
00723
00724
00725
00726 cpl_image *_flat1 = cpl_image_extract(flat1->data,
00727 xmin, ymin, xmax, ymax);
00728 cpl_image *_flat2 = cpl_image_extract(flat2->data,
00729 xmin, ymin, xmax, ymax);
00730 cpl_mask *mask = cpl_image_get_bpm(_flat1);
00731 cpl_mask_or(mask, cpl_image_get_bpm(_flat2));
00732 cpl_image_reject_from_mask(_flat2, mask);
00733
00734 double median1 = cpl_image_get_median(_flat1);
00735 double median2 = cpl_image_get_median(_flat2);
00736 double clip1 = aParams->sigma * cpl_image_get_stdev(_flat1);
00737 double clip2 = aParams->sigma * cpl_image_get_stdev(_flat2);
00738
00739
00740
00741
00742
00743 cpl_mask *mask1 =
00744 cpl_mask_threshold_image_create(_flat1,
00745 median1 - clip1,
00746 median1 + clip1);
00747 cpl_mask *mask2 =
00748 cpl_mask_threshold_image_create(_flat2,
00749 median2 - clip2,
00750 median2 + clip2);
00751
00752
00753
00754
00755 cpl_mask_not(mask1);
00756 cpl_mask_not(mask2);
00757 cpl_mask_or(mask1, mask2);
00758 cpl_mask_delete(mask2);
00759
00760 cpl_mask_or(mask, mask1);
00761 cpl_mask_delete(mask1);
00762
00763
00764
00765
00766 cpl_image_reject_from_mask(_flat2, mask);
00767 cpl_image *dflat = cpl_image_subtract_create(_flat1, _flat2);
00768
00769 if ((median1 < kMuseSaturationLimit) && (median2 < kMuseSaturationLimit)) {
00770 double s1 = cpl_image_get_mean(_flat1);
00771 double s2 = cpl_image_get_mean(_flat2);
00772 double s = 0.5 * (s1 + s2);
00773 double v = 1.;
00774 double g = 0.;
00775
00776 if ((cpl_array_get_double(aRon, iwindow, NULL) - mron_value) <= 5. * mron_sdev) {
00777 double sdev = cpl_image_get_stdev(dflat);
00778 v = 0.5 * sdev * sdev - mron_value * mron_value;
00779 g = s / v;
00780 }
00781 cpl_array_set_double(_signal, iwindow, s);
00782 cpl_array_set_double(_variance, iwindow, v);
00783 cpl_array_set_double(_gain, iwindow, g);
00784 } else {
00785 cpl_array_set_invalid(_signal, iwindow);
00786 cpl_array_set_invalid(_variance, iwindow);
00787 cpl_array_set_invalid(_gain, iwindow);
00788 }
00789
00790 cpl_image_delete(dflat);
00791 cpl_image_delete(_flat2);
00792 cpl_image_delete(_flat1);
00793 }
00794
00795 } else {
00796
00797
00798
00799
00800
00801
00802
00803
00804 cpl_image *dflat = cpl_image_subtract_create(flat1->data, flat2->data);
00805
00806
00807 cpl_image_reject_from_mask(flat1->data, cpl_image_get_bpm(dflat));
00808 cpl_image_reject_from_mask(flat2->data, cpl_image_get_bpm(dflat));
00809
00810 cpl_size iwindow;
00811 for (iwindow = 0; iwindow < nwindows; ++iwindow) {
00812 int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
00813 int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
00814 int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
00815 int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);
00816
00817 double median1 = cpl_image_get_median_window(flat1->data,
00818 xmin, ymin, xmax, ymax);
00819 double median2 = cpl_image_get_median_window(flat2->data,
00820 xmin, ymin, xmax, ymax);
00821
00822 if ((median1 < kMuseSaturationLimit) && (median2 < kMuseSaturationLimit)) {
00823 double s1 = cpl_image_get_mean_window(flat1->data,
00824 xmin, ymin, xmax, ymax);
00825 double s2 = cpl_image_get_mean_window(flat2->data,
00826 xmin, ymin, xmax, ymax);
00827 double s = 0.5 * (s1 + s2);
00828 double v = 1.;
00829 double g = 0.;
00830
00831 if ((cpl_array_get_double(aRon, iwindow, NULL) - mron_value) <= 5. * mron_sdev) {
00832 double sdev = cpl_image_get_stdev_window(dflat,
00833 xmin, ymin, xmax, ymax);
00834 v = 0.5 * sdev * sdev - mron_value * mron_value;
00835 g = s / v;
00836 }
00837 cpl_array_set_double(_signal, iwindow, s);
00838 cpl_array_set_double(_variance, iwindow, v);
00839 cpl_array_set_double(_gain, iwindow, g);
00840 } else {
00841 cpl_array_set_invalid(_signal, iwindow);
00842 cpl_array_set_invalid(_variance, iwindow);
00843 cpl_array_set_invalid(_gain, iwindow);
00844 }
00845 }
00846
00847 cpl_image_delete(dflat);
00848
00849 }
00850
00851 cpl_table_set_array(gain, "Signal", kpair, _signal);
00852 cpl_table_set_array(gain, "Variance", kpair, _variance);
00853 cpl_table_set_array(gain, "Gain", kpair, _gain);
00854 ++kpair;
00855
00856 cpl_array_delete(_signal);
00857 cpl_array_delete(_variance);
00858 cpl_array_delete(_gain);
00859 }
00860
00861 cpl_table_delete(flats);
00862
00863
00864 cpl_table_erase_invalid_rows(gain);
00865 if (cpl_table_get_nrow(gain) == 0) {
00866 cpl_table_delete(gain);
00867 gain = NULL;
00868 }
00869
00870 return gain;
00871 }
00872
00873
00887
00888 static cpl_error_code
00889 muse_lingain_quadrant_get_gain(muse_gain_fit_t *aGain,
00890 const cpl_table *aGainTable,
00891 const cpl_array *aWindows,
00892 muse_lingain_params_t *aParams)
00893 {
00894
00895
00896
00897
00898 cpl_size nrows = cpl_table_get_nrow(aGainTable);
00899 cpl_size nwindows = cpl_array_get_size(aWindows);
00900 cpl_size ndata = nrows * nwindows;
00901
00902 cpl_array *qsignal = cpl_array_new(ndata, CPL_TYPE_DOUBLE);
00903 cpl_array *qgain = cpl_array_new(ndata, CPL_TYPE_DOUBLE);
00904
00905 cpl_size irow;
00906 for (irow = 0; irow < nrows; ++irow) {
00907 cpl_size iwindow;
00908 cpl_size stride = irow * nwindows;
00909
00910 const cpl_array *_qsignal = cpl_table_get_array(aGainTable,
00911 "Signal", irow);
00912 const cpl_array *_qgain = cpl_table_get_array(aGainTable,
00913 "Gain", irow);
00914
00915 for (iwindow = 0; iwindow < nwindows; ++iwindow) {
00916 cpl_size jwindow = cpl_array_get_cplsize(aWindows, iwindow, NULL);
00917
00918 int snull = 0;
00919 int gnull = 0;
00920 double s = cpl_array_get_double(_qsignal, jwindow, &snull);
00921 double g = cpl_array_get_double(_qgain, jwindow, &gnull);
00922
00923 cpl_size idata = stride + iwindow;
00924 if (((snull == 0) && (gnull == 0)) && (s > 0.)) {
00925 cpl_array_set_double(qsignal, idata, log10(s));
00926 cpl_array_set_double(qgain, idata, g);
00927 } else {
00928 cpl_array_set_invalid(qsignal, idata);
00929 cpl_array_set_invalid(qgain, idata);
00930 }
00931 }
00932 }
00933
00934
00935 cpl_size nbins = (cpl_size)((aParams->signalmax - aParams->signalmin) /
00936 aParams->signalbin + 0.5);
00937 cpl_array *gx = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
00938 cpl_array *gy = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
00939
00940 cpl_size ibin;
00941 for (ibin = 0; ibin < nbins; ++ibin) {
00942 double bmin = aParams->signalmin + ibin * aParams->signalbin;
00943 double bmax = bmin + aParams->signalbin;
00944 double bmid = 0.5 * (bmin + bmax);
00945
00946 if (bmid < log10(aParams->gainlimit)) {
00947 cpl_array_set_invalid(gx, ibin);
00948 cpl_array_set_invalid(gy, ibin);
00949 continue;
00950 }
00951
00952
00953 cpl_array_set(gx, ibin, pow(10., bmid));
00954
00955 cpl_array *gdata = cpl_array_duplicate(qgain);
00956
00957 cpl_size idata;
00958 for (idata = 0; idata < ndata; ++idata) {
00959
00960
00961 int invalid = 0;
00962 double s = cpl_array_get_double(qsignal, idata, &invalid);
00963
00964 if (invalid || (s < bmin) || (s >= bmax)) {
00965 cpl_array_set_invalid(gdata, idata);
00966 }
00967 }
00968
00969
00970 if (cpl_array_count_invalid(gdata) == ndata) {
00971 cpl_array_set_invalid(gx, ibin);
00972 cpl_array_set_invalid(gy, ibin);
00973 cpl_array_delete(gdata);
00974 continue;
00975 }
00976
00977
00978
00979
00980 double median = cpl_array_get_median(gdata);
00981 double stdev = cpl_array_get_stdev(gdata);
00982
00983 stdev *= aParams->gainsigma;
00984 for (idata = 0; idata < ndata; ++idata) {
00985
00986
00987 int invalid = 0;
00988 double g = cpl_array_get_double(gdata, idata, &invalid);
00989
00990 if (invalid || (g <= median - stdev) || (g >= median + stdev)) {
00991 cpl_array_set_invalid(gdata, idata);
00992 }
00993 }
00994
00995 if (cpl_array_count_invalid(gdata) == ndata) {
00996 cpl_array_set_invalid(gx, ibin);
00997 cpl_array_set_invalid(gy, ibin);
00998 } else {
00999
01000 double gmean = cpl_array_get_mean(gdata);
01001
01002 cpl_array_set_double(gy, ibin, gmean);
01003 }
01004
01005 cpl_array_delete(gdata);
01006 }
01007
01008 cpl_array_delete(qgain);
01009 cpl_array_delete(qsignal);
01010
01011
01012
01013 muse_cplarray_erase_invalid(gx);
01014 muse_cplarray_erase_invalid(gy);
01015
01016 cpl_size nx = cpl_array_get_size(gx);
01017 cpl_size ny = cpl_array_get_size(gy);
01018
01019 if ((nx == 0) || (ny == 0)) {
01020
01021 cpl_msg_debug(__func__, "Fitting the gain relation failed. No data "
01022 "available!");
01023 cpl_array_delete(gy);
01024 cpl_array_delete(gx);
01025
01026 return CPL_ERROR_DATA_NOT_FOUND;
01027 }
01028
01029 if (nx != ny) {
01030 cpl_msg_debug(__func__, "Fitting the gain relation failed. Gain "
01031 "and signal data sets do not match!");
01032 cpl_array_delete(gy);
01033 cpl_array_delete(gx);
01034
01035 return CPL_ERROR_INCOMPATIBLE_INPUT;
01036 }
01037
01038
01039 if (nx < 3) {
01040 cpl_msg_debug(__func__, "Fitting the gain relation failed. Insufficient "
01041 "data points; cannot fit a first order polynomial!");
01042 cpl_array_delete(gy);
01043 cpl_array_delete(gx);
01044
01045 return CPL_ERROR_DATA_NOT_FOUND;
01046 }
01047
01048
01049 double *_x = cpl_array_get_data_double(gx);
01050 double *_y = cpl_array_get_data_double(gy);
01051
01052 cpl_matrix *x = cpl_matrix_wrap(1, nx, _x);
01053 cpl_vector *y = cpl_vector_wrap(ny, _y);
01054 cpl_boolean symetric = CPL_FALSE;
01055 const cpl_size mindeg = 0;
01056 const cpl_size maxdeg = 1;
01057
01058 cpl_polynomial *gfit = cpl_polynomial_new(1);
01059 cpl_error_code ecode = cpl_polynomial_fit(gfit, x, &symetric, y, NULL,
01060 CPL_FALSE, &mindeg, &maxdeg);
01061
01062 cpl_vector_unwrap(y);
01063 cpl_matrix_unwrap(x);
01064
01065 if (ecode != CPL_ERROR_NONE) {
01066 cpl_polynomial_delete(gfit);
01067 cpl_array_delete(gy);
01068 cpl_array_delete(gx);
01069
01070 cpl_msg_debug(__func__, "Fitting the gain relation failed. Fitting "
01071 "first order polynomial to gain data failed!");
01072
01073 aGain->gain = 0.;
01074 return CPL_ERROR_ILLEGAL_OUTPUT;
01075 }
01076
01077 double rms = 0.;
01078 cpl_size ix;
01079 for (ix = 0; ix < nx; ++ix) {
01080 double rx = cpl_array_get_double(gx, ix, NULL);
01081 double ry = cpl_array_get_double(gy, ix, NULL);
01082 double ryf = cpl_polynomial_eval_1d(gfit, rx, NULL);
01083 rms += (ry - ryf) * (ry - ryf);
01084 }
01085 rms = sqrt(rms);
01086
01087 cpl_array_delete(gy);
01088 cpl_array_delete(gx);
01089
01090 const cpl_size order = 0;
01091 aGain->gain = cpl_polynomial_get_coeff(gfit, &order);
01092 aGain->rms = rms;
01093
01094 cpl_polynomial_delete(gfit);
01095
01096 return CPL_ERROR_NONE;
01097 }
01098
01099
01111
01112 static cpl_error_code
01113 muse_lingain_quadrant_get_nonlinearity(muse_linearity_fit_t *aFit,
01114 const cpl_table *aGainTable,
01115 const cpl_array *aWindows,
01116 double aGain,
01117 muse_lingain_params_t *aParams)
01118 {
01119
01120
01121
01122
01123
01124
01125
01126 cpl_size ntimes = cpl_table_get_nrow(aGainTable);
01127 cpl_size irow;
01128 cpl_error_code ecode = cpl_table_get_column_maxpos(aGainTable, "ExpTime",
01129 &irow);
01130 if (ecode != CPL_ERROR_NONE) {
01131 cpl_msg_debug(__func__, "Maximum exposure time cannot be determined from "
01132 "linearity data set of IFU %u!", aParams->nifu);
01133 return ecode;
01134 }
01135
01136 if (irow != ntimes - 1) {
01137 cpl_msg_debug(__func__, "Linearity data set of IFU %u is not sorted in "
01138 "ascending order of the exposure time!", aParams->nifu);
01139 return CPL_ERROR_ILLEGAL_INPUT;
01140 }
01141
01142
01143
01144
01145
01146 cpl_array *sigref =
01147 muse_lingain_quadrant_extract_data(aGainTable, aWindows, "Signal", irow);
01148 cpl_array_logarithm(sigref, 10.);
01149
01150 cpl_size nsignal = cpl_array_get_size(sigref);
01151 cpl_size nlevels = (cpl_size)((aParams->ctsmax - aParams->ctsmin) /
01152 aParams->ctsbin + 0.5);
01153
01154 cpl_array **counts = cpl_calloc(nlevels, sizeof *counts);
01155 const double qgain = log10(aGain);
01156 cpl_size klevel = 0;
01157 cpl_size ilevel;
01158 for (ilevel = 0; ilevel < nlevels; ++ilevel) {
01159 double shigh = aParams->ctsmax - ilevel * aParams->ctsbin;
01160 double slow = shigh - aParams->ctsbin;
01161
01162 shigh -= qgain;
01163 slow -= qgain;
01164
01165
01166
01167 cpl_size *sindex = cpl_malloc(nsignal * sizeof *sindex);
01168 cpl_size kdata = 0;
01169 cpl_size idata;
01170 for (idata = 0; idata < nsignal; ++idata) {
01171 int invalid = 0;
01172 double s = cpl_array_get_double(sigref, idata, &invalid);
01173
01174 if (!invalid && ((s >= slow) && (s < shigh))) {
01175 sindex[kdata] = cpl_array_get_cplsize(aWindows, idata, NULL);
01176 ++kdata;
01177 }
01178 }
01179
01180 if (kdata == 0) {
01181 cpl_free(sindex);
01182 cpl_msg_debug(__func__, "No valid measurement found within signal "
01183 "range [%.6f, %.6f[ [log10(counts)] of the linearity "
01184 "data set of IFU %u!", slow + qgain, shigh + qgain,
01185 aParams->nifu);
01186 continue;
01187 } else {
01188
01189
01190
01191
01192
01193
01194 cpl_array *smean = cpl_array_new(ntimes, CPL_TYPE_DOUBLE);
01195 cpl_array *windex = cpl_array_wrap_cplsize(sindex, kdata);
01196 cpl_size itime;
01197 for (itime = 0; itime < ntimes; ++itime) {
01198 cpl_array *qsignal =
01199 muse_lingain_quadrant_extract_data(aGainTable, windex,
01200 "Signal", itime);
01201 if (qsignal && (cpl_array_count_invalid(qsignal) < kdata)) {
01202 double mean = cpl_array_get_mean(qsignal);
01203 cpl_array_set_double(smean, itime, mean);
01204 } else {
01205 cpl_array_set_invalid(smean, itime);
01206 }
01207 cpl_array_delete(qsignal);
01208 }
01209 cpl_array_unwrap(windex);
01210
01211
01212
01213 counts[klevel] = smean;
01214 ++klevel;
01215 }
01216 cpl_free(sindex);
01217 }
01218
01219 cpl_array_delete(sigref);
01220
01221
01222 cpl_array *exptime = cpl_array_new(ntimes, CPL_TYPE_DOUBLE);
01223
01224 cpl_size itime;
01225 for (itime = 0; itime < ntimes; ++itime) {
01226 double _exptime = cpl_table_get_double(aGainTable, "ExpTime",
01227 itime, NULL);
01228 cpl_array_set_double(exptime, itime, _exptime);
01229 }
01230 cpl_array_add_scalar(exptime, aParams->toffset);
01231
01232
01233 cpl_array **crate = cpl_calloc(klevel, sizeof *crate);
01234 for (ilevel = 0; ilevel < klevel; ++ilevel) {
01235 crate[ilevel] = cpl_array_duplicate(counts[ilevel]);
01236 cpl_array_divide(crate[ilevel], exptime);
01237 }
01238
01239
01240
01241
01242
01243
01244
01245
01246 double linmin = pow(10., aParams->linearmin - qgain);
01247 double linmax = pow(10., aParams->linearmax - qgain);
01248
01249 cpl_array **cresidual = cpl_malloc(klevel * sizeof *cresidual);
01250
01251 for (ilevel = 0; ilevel < klevel; ++ilevel) {
01252 cpl_array *_exptime = cpl_array_duplicate(exptime);
01253 cpl_array *_crate = cpl_array_duplicate(crate[ilevel]);
01254
01255 for (itime = 0; itime < ntimes; ++itime) {
01256 int invalid = 0;
01257 double cnts = cpl_array_get_double(counts[ilevel], itime, &invalid);
01258
01259 if (invalid || ((cnts < linmin) || (cnts >= linmax))) {
01260 cpl_array_set_invalid(_exptime, itime);
01261 cpl_array_set_invalid(_crate, itime);
01262 }
01263 }
01264
01265 if (cpl_array_get_size(_crate) == cpl_array_count_invalid(_crate)) {
01266
01267
01268
01269 cresidual[ilevel] = NULL;
01270
01271 cpl_array_delete(_crate);
01272 cpl_array_delete(_exptime);
01273
01274 } else {
01275
01276
01277
01278
01279 #ifdef USE_POLYFIT
01280
01281
01282 muse_cplarray_erase_invalid(_exptime);
01283 muse_cplarray_erase_invalid(_crate);
01284
01285 cpl_size nt = cpl_array_get_size(_exptime);
01286 cpl_size nc = cpl_array_get_size(_crate);
01287
01288
01289
01290 if (nt != nc) {
01291 cpl_msg_debug(__func__, "Fitting a constant to the count rate data "
01292 "failed. Exposure time and count rate data sets do not "
01293 "match!");
01294 cpl_array_delete(_crate);
01295 cpl_array_delete(_exptime);
01296
01297 muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
01298 muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
01299 muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);
01300
01301 cpl_array_delete(exptime);
01302
01303 return CPL_ERROR_INCOMPATIBLE_INPUT;
01304 }
01305
01306
01307 double *_et = cpl_array_get_data_double(_exptime);
01308 double *_cr = cpl_array_get_data_double(_crate);
01309
01310 cpl_matrix *et = cpl_matrix_wrap(1, nt, _et);
01311 cpl_vector *cr = cpl_vector_wrap(nc, _cr);
01312 cpl_boolean symetric = CPL_FALSE;
01313 const cpl_size mindeg = 0;
01314 const cpl_size maxdeg = 0;
01315
01316 cpl_polynomial *crfit = cpl_polynomial_new(1);
01317 cpl_error_code ecode = cpl_polynomial_fit(crfit, et, &symetric, cr, NULL,
01318 CPL_FALSE, &mindeg, &maxdeg);
01319
01320 cpl_vector_unwrap(et);
01321 cpl_matrix_unwrap(cr);
01322
01323 cpl_array_delete(_crate);
01324 cpl_array_delete(_exptime);
01325
01326 if (ecode != CPL_ERROR_NONE) {
01327 cpl_polynomial_delete(crfit);
01328 cpl_msg_debug(__func__, "Fitting a constant to the count rate level "
01329 "failed. Fitting a zero order polynomial to count rate "
01330 "data failed!");
01331
01332 muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
01333 muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
01334 muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);
01335
01336 cpl_array_delete(exptime);
01337
01338 return CPL_ERROR_ILLEGAL_OUTPUT;
01339 }
01340
01341 const cpl_size order = 0;
01342 double crlevel = cpl_polynomial_get_coeff(crfit, &order);
01343 cpl_polynomial_delete(crfit);
01344 #else
01345 double crlevel = cpl_array_get_mean(_crate);
01346
01347 cpl_array_delete(_crate);
01348 cpl_array_delete(_exptime);
01349 #endif
01350
01351 cresidual[ilevel] = cpl_array_duplicate(crate[ilevel]);
01352 cpl_array_subtract_scalar(cresidual[ilevel], crlevel);
01353 cpl_array_divide(cresidual[ilevel], crate[ilevel]);
01354 }
01355 }
01356
01357 cpl_array_delete(exptime);
01358
01359
01360
01361
01362
01363
01364
01365 double signalmax = aParams->signalmax;
01366 double signalmin = aParams->signalmin;
01367 double signalbin = aParams->signalbin;
01368
01369 cpl_size nbins = (cpl_size)((signalmax - signalmin) / signalbin + 0.5);
01370 cpl_array *lx = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
01371 cpl_array *ly = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
01372 cpl_array *lyse = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
01373
01374 cpl_size ibin;
01375 for (ibin = 0; ibin < nbins; ++ibin) {
01376 double bmin = signalmin + ibin * signalbin;
01377 double bmax = bmin + signalbin;
01378 double bmid = 0.5 * (bmin + bmax);
01379
01380 cpl_array_set_double(lx, ibin, bmid);
01381
01382 bmin = pow(10., bmin);
01383 bmax = pow(10., bmax);
01384
01385 double *rdata = cpl_malloc((klevel * ntimes) * sizeof *rdata);
01386 cpl_size kresidual = 0;
01387 for (ilevel = 0; ilevel < klevel; ++ilevel) {
01388 cpl_array *residual = cresidual[ilevel];
01389
01390 if (residual != NULL) {
01391 for (itime = 0; itime < ntimes; ++itime) {
01392 int cinvalid = 0;
01393 int rinvalid = 0;
01394 double _counts = cpl_array_get_double(counts[ilevel], itime, &cinvalid);
01395 double _residual = cpl_array_get_double(residual, itime, &rinvalid);
01396
01397 if (!(cinvalid || rinvalid) && ((_counts >= bmin) && (_counts < bmax))) {
01398 rdata[kresidual] = _residual;
01399 ++kresidual;
01400 }
01401 }
01402 }
01403 }
01404 if (kresidual == 0) {
01405 cpl_array_set_invalid(lx, ibin);
01406 cpl_array_set_invalid(ly, ibin);
01407 cpl_array_set_invalid(lyse, ibin);
01408 } else {
01409 cpl_array *r = cpl_array_wrap_double(rdata, kresidual);
01410
01411 double rmean = cpl_array_get_mean(r);
01412 double rsdev = cpl_array_get_stdev(r);
01413 cpl_array_set_double(ly, ibin, rmean);
01414 cpl_array_set_double(lyse, ibin, rsdev);
01415
01416 cpl_array_unwrap(r);
01417 }
01418 cpl_free(rdata);
01419 }
01420
01421 muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
01422 muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
01423 muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);
01424
01425
01426 muse_cplarray_erase_invalid(lx);
01427 muse_cplarray_erase_invalid(ly);
01428 muse_cplarray_erase_invalid(lyse);
01429
01430 cpl_size nx = cpl_array_get_size(lx);
01431 cpl_size ny = cpl_array_get_size(ly);
01432
01433 if ((nx == 0) || (ny == 0)) {
01434
01435 cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
01436 "No data available!");
01437 cpl_array_delete(lyse);
01438 cpl_array_delete(ly);
01439 cpl_array_delete(lx);
01440
01441 return CPL_ERROR_DATA_NOT_FOUND;
01442 }
01443
01444 if (nx != ny) {
01445 cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
01446 "Linearity residual and signal data sets do not match!");
01447 cpl_array_delete(lyse);
01448 cpl_array_delete(ly);
01449 cpl_array_delete(lx);
01450
01451 return CPL_ERROR_INCOMPATIBLE_INPUT;
01452 }
01453
01454
01455 if (nx < aParams->order + 2) {
01456 cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
01457 "Insufficient data points; cannot fit a %d order "
01458 "polynomial!", aParams->order);
01459 cpl_array_delete(lyse);
01460 cpl_array_delete(ly);
01461 cpl_array_delete(lx);
01462
01463 return CPL_ERROR_DATA_NOT_FOUND;
01464 }
01465
01466
01467 double *_x = cpl_array_get_data_double(lx);
01468 double *_y = cpl_array_get_data_double(ly);
01469 double *_yse = cpl_array_get_data_double(lyse);
01470
01471 cpl_matrix *x = cpl_matrix_wrap(1, nx, _x);
01472 cpl_vector *y = cpl_vector_wrap(ny, _y);
01473 cpl_vector *yse = cpl_vector_wrap(ny, _yse);
01474 cpl_boolean symetric = CPL_FALSE;
01475 const cpl_size mindeg = 0;
01476 const cpl_size maxdeg = aParams->order;
01477
01478 cpl_polynomial *lfit = cpl_polynomial_new(1);
01479 ecode = cpl_polynomial_fit(lfit, x, &symetric, y, NULL, CPL_FALSE,
01480 &mindeg, &maxdeg);
01481
01482 cpl_vector_unwrap(yse);
01483 cpl_vector_unwrap(y);
01484 cpl_matrix_unwrap(x);
01485
01486 if (ecode != CPL_ERROR_NONE) {
01487 cpl_polynomial_delete(lfit);
01488 cpl_array_delete(lyse);
01489 cpl_array_delete(ly);
01490 cpl_array_delete(lx);
01491
01492 cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
01493 "Fitting %d order polynomial to residual non-linearity "
01494 "data failed!", aParams->order);
01495 aFit->coefficients = NULL;
01496 aFit->range[0] = 0;
01497 aFit->range[1] = 0;
01498
01499 return CPL_ERROR_ILLEGAL_OUTPUT;
01500 }
01501
01502
01503 cpl_size ncoeff = aParams->order + 1;
01504 cpl_array *coefficients = cpl_array_new(ncoeff, CPL_TYPE_DOUBLE);
01505
01506 cpl_size icoeff;
01507 for (icoeff = 0; icoeff < ncoeff; ++icoeff) {
01508 const cpl_size order = icoeff;
01509 double coeff = cpl_polynomial_get_coeff(lfit, &order);
01510 cpl_array_set_double(coefficients, icoeff, coeff);
01511 }
01512
01513 double rms = 0.;
01514 cpl_size ix;
01515 for (ix = 0; ix < nx; ++ix) {
01516 double rx = cpl_array_get_double(lx, ix, NULL);
01517 double ry = cpl_array_get_double(ly, ix, NULL);
01518 double ryf = cpl_polynomial_eval_1d(lfit, rx, NULL);
01519 rms += (ry - ryf) * (ry - ryf);
01520 }
01521 rms = sqrt(rms);
01522 cpl_polynomial_delete(lfit);
01523
01524 aFit->coefficients = coefficients;
01525 aFit->range[0] = cpl_array_get_min(lx);
01526 aFit->range[1] = cpl_array_get_max(lx);
01527 aFit->rms = rms;
01528
01529 cpl_array_delete(lyse);
01530 cpl_array_delete(ly);
01531 cpl_array_delete(lx);
01532
01533 return CPL_ERROR_NONE;
01534 }
01535
01536
01558
01559 static cpl_table *
01560 muse_lingain_compute_gain(const cpl_table *aGainTable, cpl_table *aWindowList,
01561 muse_lingain_params_t *aParams)
01562 {
01563 const unsigned char nquadrants = 4;
01564 unsigned char iquadrant;
01565
01566 cpl_table *gain = cpl_table_new(nquadrants);
01567
01568 if (!gain) {
01569 return NULL;
01570 }
01571
01572 cpl_table_new_column(gain, "Gain", CPL_TYPE_DOUBLE);
01573 cpl_table_new_column(gain, "FitRms", CPL_TYPE_DOUBLE);
01574
01575 for (iquadrant = 0; iquadrant < nquadrants; ++iquadrant) {
01576 unsigned int quadrant = iquadrant + 1;
01577
01578
01579 cpl_array *windex = muse_lingain_quadrant_get_windows(aWindowList, quadrant);
01580
01581 if (!windex) {
01582 cpl_msg_warning(__func__, "No measurement windows defined for "
01583 "quadrant %hhu of IFU %u!", quadrant, aParams->nifu);
01584 cpl_table_set_invalid(gain, "Gain", iquadrant);
01585 cpl_table_set_invalid(gain, "FitRms", iquadrant);
01586 continue;
01587 }
01588
01589
01590
01591
01592 muse_gain_fit_t qgain = {0., 0.};
01593 cpl_error_code ecode = muse_lingain_quadrant_get_gain(&qgain, aGainTable,
01594 windex, aParams);
01595 if (ecode != CPL_ERROR_NONE) {
01596 cpl_msg_warning(__func__, "Detector gain value for quadrant %hhu of "
01597 "IFU %u is invalid!", quadrant, aParams->nifu);
01598 cpl_table_set_invalid(gain, "Gain", iquadrant);
01599 cpl_table_set_invalid(gain, "FitRms", iquadrant);
01600 } else {
01601 cpl_msg_info(__func__, "Detector gain value for quadrant %hhu of "
01602 "IFU %u: %.6f [counts/ADU]", quadrant, aParams->nifu,
01603 qgain.gain);
01604 }
01605
01606 cpl_table_set_double(gain, "Gain", iquadrant, qgain.gain);
01607 cpl_table_set_double(gain, "FitRms", iquadrant, qgain.rms);
01608 cpl_array_delete(windex);
01609 }
01610
01611 return gain;
01612 }
01613
01614
01629
01630 static cpl_table *
01631 muse_lingain_compute_nonlinearity(const cpl_table *aGainTable,
01632 cpl_table *aWindowList,
01633 const cpl_table *aGain,
01634 muse_lingain_params_t *aParams)
01635 {
01636 const unsigned char nquadrants = 4;
01637 unsigned char iquadrant;
01638
01639 cpl_table *linearity = cpl_table_new(nquadrants);
01640 cpl_table_new_column(linearity, "Gain", CPL_TYPE_DOUBLE);
01641 cpl_table_new_column(linearity, "SignalMin", CPL_TYPE_DOUBLE);
01642 cpl_table_new_column(linearity, "SignalMax", CPL_TYPE_DOUBLE);
01643 cpl_table_new_column_array(linearity, "Coefficients", CPL_TYPE_DOUBLE,
01644 aParams->order + 1);
01645 cpl_table_new_column(linearity, "FitRms", CPL_TYPE_DOUBLE);
01646
01647 cpl_table_set_column_unit(linearity, "Gain", "counts/ADU)");
01648 cpl_table_set_column_unit(linearity, "SignalMin", "log10(ADU)");
01649 cpl_table_set_column_unit(linearity, "SignalMax", "log10(ADU)");
01650
01651 for (iquadrant = 0; iquadrant < nquadrants; ++iquadrant) {
01652 unsigned int quadrant = iquadrant + 1;
01653
01654
01655 cpl_array *windex = muse_lingain_quadrant_get_windows(aWindowList, quadrant);
01656
01657 if (!windex) {
01658 cpl_msg_warning(__func__, "No measurement windows defined for "
01659 "quadrant %hhu of IFU %u!", quadrant, aParams->nifu);
01660 continue;
01661 }
01662
01663
01664
01665
01666 int invalid = 0;
01667 double qgain = cpl_table_get_double(aGain, "Gain", iquadrant, &invalid);
01668
01669 if (invalid) {
01670 cpl_msg_warning(__func__, "Got invalid gain for quadrant %hhu of IFU "
01671 "%u! Skipping non-linearity computation!", quadrant,
01672 aParams->nifu);
01673 cpl_array_delete(windex);
01674 continue;
01675 }
01676
01677 muse_linearity_fit_t qlinearity = {NULL, {0., 0.}, 0.};
01678 cpl_error_code ecode =
01679 muse_lingain_quadrant_get_nonlinearity(&qlinearity, aGainTable,
01680 windex, qgain, aParams);
01681
01682 if (ecode != CPL_ERROR_NONE) {
01683 cpl_msg_warning(__func__, "Computation of the detector non-linearity "
01684 "for quadrant %hhu of IFU %u failed!", quadrant,
01685 aParams->nifu);
01686 cpl_table_set_invalid(linearity, "Gain", iquadrant);
01687 cpl_table_set_invalid(linearity, "SignalMin", iquadrant);
01688 cpl_table_set_invalid(linearity, "SignalMax", iquadrant);
01689 cpl_table_set_invalid(linearity, "Coefficients", iquadrant);
01690 cpl_table_set_invalid(linearity, "FitRms", iquadrant);
01691 } else {
01692 cpl_msg_info(__func__, "RMS of residual non-linearity model over "
01693 "signal range [%.4f, %.4f] [log10(ADU)] for quadrant "
01694 "%hhu of IFU %u: %.6e", qlinearity.range[0],
01695 qlinearity.range[1], quadrant, aParams->nifu,
01696 qlinearity.rms);
01697
01698 cpl_table_set_double(linearity, "Gain", iquadrant, qgain);
01699 cpl_table_set_double(linearity, "SignalMin", iquadrant,
01700 qlinearity.range[0]);
01701 cpl_table_set_double(linearity, "SignalMax", iquadrant,
01702 qlinearity.range[1]);
01703 cpl_table_set_array(linearity, "Coefficients", iquadrant,
01704 qlinearity.coefficients);
01705 cpl_table_set_double(linearity, "FitRms", iquadrant,
01706 qlinearity.rms);
01707 }
01708 muse_linearity_fit_clear(&qlinearity);
01709 cpl_array_delete(windex);
01710 }
01711
01712 return linearity;
01713 }
01714
01715
01723
01724 int
01725 muse_lingain_compute(muse_processing *aProcessing,
01726 muse_lingain_params_t *aParams)
01727 {
01728
01729 if (!muse_lingain_validate_parameters(aParams)) {
01730 return -1;
01731 }
01732
01733
01734 cpl_table *trace = muse_table_load(aProcessing, MUSE_TAG_TRACE_TABLE,
01735 aParams->nifu);
01736
01737 if (!trace) {
01738 cpl_msg_error(__func__, "%s could not be loaded!", MUSE_TAG_TRACE_TABLE);
01739 return -1;
01740 }
01741
01742
01743
01744
01745
01746
01747
01748
01749
01750 muse_imagelist *images = muse_lingain_load_images(aProcessing, aParams->nifu);
01751 if (!images) {
01752 cpl_msg_error(__func__, "Loading and basic processing of the raw input "
01753 "images failed for IFU %u!", aParams->nifu);
01754 cpl_table_delete(trace);
01755 return -1;
01756 }
01757
01758
01759
01760 cpl_table *exposures = muse_lingain_sort_images(images);
01761 if (!exposures) {
01762 cpl_msg_error(__func__, "Creating a sorted list of exposures failed "
01763 "for IFU %u!", aParams->nifu);
01764 muse_imagelist_delete(images);
01765 cpl_table_delete(trace);
01766 return -1;
01767 }
01768
01769
01770
01771 cpl_table *windowlist = muse_lingain_create_windowlist(trace, exposures,
01772 images, aParams);
01773
01774 if (!windowlist) {
01775 cpl_msg_error(__func__, "Creating the list of measurement windows failed "
01776 "for IFU %u!", aParams->nifu);
01777 cpl_table_delete(exposures);
01778 muse_imagelist_delete(images);
01779 cpl_table_delete(trace);
01780 return -1;
01781 }
01782 cpl_table_delete(trace);
01783
01784
01785 cpl_array *ron = muse_lingain_compute_ron(windowlist, exposures, images,
01786 aParams);
01787 if (!ron) {
01788 cpl_msg_error(__func__, "Calculating the RON for individual measurement "
01789 "windows failed for IFU %u!", aParams->nifu);
01790 cpl_table_delete(windowlist);
01791 cpl_table_delete(exposures);
01792 muse_imagelist_delete(images);
01793 return -1;
01794 }
01795
01796
01797
01798 cpl_table *gaindata = muse_lingain_window_get_gain(windowlist, ron,
01799 exposures, images,
01800 aParams);
01801 if (!gaindata) {
01802 cpl_msg_error(__func__, "Calculating the gain for individual measurement "
01803 "windows failed for IFU %u!", aParams->nifu);
01804 cpl_array_delete(ron);
01805 cpl_table_delete(windowlist);
01806 cpl_table_delete(exposures);
01807 muse_imagelist_delete(images);
01808 return -1;
01809 }
01810 cpl_array_delete(ron);
01811 cpl_table_delete(exposures);
01812
01813
01814
01815 cpl_table *gain = muse_lingain_compute_gain(gaindata, windowlist, aParams);
01816
01817
01818
01819
01820
01821 if (cpl_table_count_invalid(gain, "Gain") != 0) {
01822 cpl_msg_error(__func__, "Calculating the detector gain failed for IFU %u!",
01823 aParams->nifu);
01824 cpl_table_delete(gain);
01825 cpl_table_delete(gaindata);
01826 cpl_table_delete(windowlist);
01827 muse_imagelist_delete(images);
01828 return -1;
01829 }
01830
01831
01832 cpl_table *linearity = muse_lingain_compute_nonlinearity(gaindata, windowlist,
01833 gain, aParams);
01834 if (cpl_table_has_invalid(linearity, "Coefficients")) {
01835 cpl_msg_error(__func__, "Computing the detector residual non-linearity "
01836 "failed for IFU %u!", aParams->nifu);
01837 cpl_table_delete(linearity);
01838 cpl_table_delete(gain);
01839 cpl_table_delete(gaindata);
01840 cpl_table_delete(windowlist);
01841 muse_imagelist_delete(images);
01842 return -1;
01843 }
01844
01845 cpl_table_delete(gaindata);
01846 cpl_table_delete(windowlist);
01847
01848
01849 cpl_propertylist *header =
01850 cpl_propertylist_duplicate(muse_imagelist_get(images, 0)->header);
01851
01852 muse_imagelist_delete(images);
01853
01854 cpl_propertylist_erase_regexp(header,
01855 "^SIMPLE$|^BITPIX$|^NAXIS|^EXTEND$|^XTENSION$|"
01856 "^DATASUM$|^DATAMIN$|^DATAMAX$|^DATAMD5$|"
01857 "^PCOUNT$|^GCOUNT$|^HDUVERS$|^BLANK$|"
01858 "^BZERO$|^BSCALE$|^BUNIT$|^CHECKSUM$|^INHERIT$|"
01859 "^PIPEFILE$|^ESO PRO ", 0);
01860
01861 const unsigned char nquadrant = 4;
01862 unsigned char iquadrant;
01863 for (iquadrant = 0; iquadrant < nquadrant; ++iquadrant) {
01864 char keyword[KEYWORD_LENGTH];
01865 char comment[KEYWORD_LENGTH];
01866 unsigned char _iquadrant = iquadrant + 1;
01867 int ncoefficient = (int)cpl_table_get_column_depth(linearity,
01868 "Coefficients");
01869 snprintf(keyword, KEYWORD_LENGTH, "ESO DET OUT%d GAIN", _iquadrant);
01870 cpl_propertylist_update_double(header, keyword,
01871 cpl_table_get_double(gain, "Gain",
01872 iquadrant, NULL));
01873
01874
01875
01876
01877 #if 0
01878 cpl_propertylist_set_comment(header, keyword,
01879 "[e-/ADU] Conversion ADUs to electrons");
01880 snprintf(keyword, KEYWORD_LENGTH, "ESO DET OUT%d CONAD", _iquadrant);
01881 cpl_propertylist_set_comment(header, keyword,
01882 "[ADU/e-] Conversion electrons to ADUs");
01883 #endif
01884
01885 snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_LLO, _iquadrant);
01886 cpl_propertylist_append_double(header, keyword,
01887 cpl_table_get_double(linearity, "SignalMin",
01888 iquadrant, NULL));
01889 cpl_propertylist_set_comment(header, keyword,
01890 "[log10(ADU)] Minimum signal used for fit");
01891
01892 snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_LHI, _iquadrant);
01893 cpl_propertylist_append_double(header, keyword,
01894 cpl_table_get_double(linearity, "SignalMax",
01895 iquadrant, NULL));
01896 cpl_propertylist_set_comment(header, keyword,
01897 "[log10(ADU)] Maximum signal used for fit");
01898
01899 snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_ORDER, _iquadrant);
01900 cpl_propertylist_append_double(header, keyword, ncoefficient - 1);
01901 cpl_propertylist_set_comment(header, keyword,
01902 "Order of the polynomial fit");
01903
01904 const cpl_array *coefficients =
01905 cpl_table_get_array(linearity, "Coefficients", iquadrant);
01906 int icoefficient;
01907 for (icoefficient = 0; icoefficient < ncoefficient; ++icoefficient) {
01908 double c = cpl_array_get_double(coefficients, icoefficient, NULL);
01909 snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_COEFFo,
01910 _iquadrant, (unsigned char)icoefficient);
01911 cpl_propertylist_append_double(header, keyword, c);
01912
01913 snprintf(comment, KEYWORD_LENGTH,
01914 "%d order coefficient of the polynomial fit", icoefficient);
01915 cpl_propertylist_set_comment(header, keyword, comment);
01916 }
01917
01918 snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_GFITi_RMS, _iquadrant);
01919 cpl_propertylist_append_double(header, keyword,
01920 cpl_table_get_double(gain, "FitRms",
01921 iquadrant, NULL));
01922 snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_NLFITi_RMS, _iquadrant);
01923 cpl_propertylist_append_double(header, keyword,
01924 cpl_table_get_double(linearity, "FitRms",
01925 iquadrant, NULL));
01926 }
01927 cpl_table_erase_column(linearity, "FitRms");
01928
01929 muse_processing_save_table(aProcessing, aParams->nifu, linearity, header,
01930 MUSE_TAG_NONLINGAIN, MUSE_TABLE_TYPE_CPL);
01931
01932 cpl_propertylist_delete(header);
01933 cpl_table_delete(linearity);
01934 cpl_table_delete(gain);
01935
01936 return 0;
01937 }