39#include "detmon_pernoise.h"
42#include "irplib_ksigma_clip.h"
43#include "irplib_hist.h"
44#include "irplib_utils.h"
56#define pdist(x1,y1,x2,y2) (((x1-x2)*(x1-x2))+((y1-y2)*(y1-y2)))
58#define cpl_drand() ((double)rand()/(double)RAND_MAX)
98 cpl_boolean direction;
107} detmon_pernoise_config;
120detmon_pernoise_dfs_set_groups(cpl_frameset *,
124detmon_pernoise_retrieve_parlist(
const char *,
126 const cpl_parameterlist *);
129detmon_pernoise_qc(cpl_propertylist *,
134detmon_pernoise_save(
const cpl_parameterlist *,
144 const cpl_frameset *);
147detmon_pernoise_rm_bg(cpl_image *,
163#define HORIZONTAL TRUE
166detmon_pernoise_reduce(cpl_image * image)
174 cpl_image * power_im = 0;
175 cpl_image * output = 0;
176 cpl_image * pos_spec = 0;
177 cpl_table * table = 0;
178 cpl_image* fourier_im = 0;
180 cpl_error_code error = CPL_ERROR_NONE;
181 cpl_image * sub_image = 0;
185 if(detmon_pernoise_config.direction == HORIZONTAL) {
186 error = cpl_image_flip(image, 1);
187 cpl_ensure(!error, error, NULL);
190 nsamples = cpl_image_get_size_x(image);
191 nffts = cpl_image_get_size_y(image);
202 error = detmon_pernoise_rm_bg(image, nsamples, nffts);
203 cpl_ensure(!error, error, NULL);
205 sub_image = cpl_image_extract(image, nsamples/8 + 1, nffts/8+1,
206 nsamples*7/8, nffts*7/8);
208 nsamples = cpl_image_get_size_x(sub_image);
209 nffts = cpl_image_get_size_y(sub_image);
222 hanning = cpl_malloc(
sizeof(
float) * nsamples);
224 for(i = 0; i < nsamples; i++) {
225 *(hanning + i) = 0.5 - 0.5 * cos(2 * CPL_MATH_PI * (
float) i / nsamples);
226 for(j = 0; j < nffts; j++) {
228 cpl_image_get(sub_image, i + 1, j + 1, &status);
229 error = cpl_image_set(sub_image, i + 1, j + 1, (*(hanning + i)) * value);
234 if (error != CPL_ERROR_NONE)
239 power = (
float *) cpl_calloc(
sizeof(
float), nsamples * nffts);
242 fourier_im = cpl_image_new(nsamples,nffts, CPL_TYPE_FLOAT_COMPLEX);
243 error = cpl_fft_image(fourier_im, sub_image, CPL_FFT_FORWARD);
245 for(i = 1; i <= nffts; i++) {
246 for(j = 1; j <= nsamples; j++) {
248 double complex cvalue = cpl_image_get_complex(fourier_im,j, i, &rej );
249 double value = cabs(cvalue);
254 cpl_image_set(power_im, j, i, value);
258 cpl_image_delete(fourier_im);
260 output = cpl_image_collapse_create(power_im, 0);
261 pos_spec = cpl_image_extract(output, 1, 1, nsamples/2, 1);
263 cpl_image_delete(power_im);
266 cpl_image_delete(output);
268 table = cpl_table_new(nsamples/2);
269 cpl_table_new_column(table,
"FREQ", CPL_TYPE_DOUBLE);
270 cpl_table_new_column(table,
"POW", CPL_TYPE_DOUBLE);
272 freq = detmon_pernoise_config.speed*1000/nffts_old;
274 for(i = 0; i < nsamples/2; i++) {
275 error = cpl_table_set(table,
"FREQ", i, freq/(nsamples/2)*i);
276 error = cpl_table_set(table,
"POW", i, cpl_image_get(pos_spec, i+1, 1, &status));
279 for(i= 0; i < 5; i++) {
280 error = cpl_table_set(table,
"POW", i, 0.0);
285 cpl_image_delete(pos_spec);
287 cpl_image_delete(sub_image);
288 if (error != CPL_ERROR_NONE)
290 cpl_table_delete(table);
316detmon_pernoise(cpl_frameset * frameset,
317 const cpl_parameterlist * parlist,
319 const char * recipe_name,
320 const char * pipeline_name,
321 const char * procatg_tbl,
322 const char * package,
323 int (*compare)(
const cpl_frame *,
329 cpl_error_code error;
331 if(detmon_pernoise_dfs_set_groups(frameset, tag)) {
332 cpl_msg_error(cpl_func,
"Cannot identify RAW and CALIB frames");
340 error = detmon_pernoise_retrieve_parlist(pipeline_name,
341 recipe_name, parlist);
342 cpl_ensure_code(!error, error);
348 cpl_msg_info(cpl_func,
"Identify the different settings");
349 cpl_size* selection = cpl_frameset_labelise(frameset, compare, &nsets);
350 if(selection == NULL)
351 cpl_msg_error(cpl_func,
"Cannot labelise input frames");
354 detmon_pernoise_config.nb_extensions = 1;
355 if(detmon_pernoise_config.exts < 0) {
356 const cpl_frame *cur_frame =
357 cpl_frameset_get_position_const(frameset, 0);
359 detmon_pernoise_config.nb_extensions =
360 cpl_frame_get_nextensions(cur_frame);
364 for(i = 0; i < nsets; i++)
367 cpl_table ** freq_table;
368 cpl_propertylist ** qclist =
369 (cpl_propertylist **)
370 cpl_malloc(detmon_pernoise_config.nb_extensions *
371 sizeof(cpl_propertylist *));
373 cpl_imagelist ** raws = (cpl_imagelist **) cpl_malloc(detmon_pernoise_config.nb_extensions *
sizeof(cpl_imagelist *));
374 cpl_image ** input = (cpl_image **) cpl_malloc(detmon_pernoise_config.nb_extensions *
sizeof(cpl_image *));
377 if(detmon_pernoise_config.mode == 1)
380 (cpl_table **) cpl_malloc(detmon_pernoise_config.nb_extensions *
381 4 *
sizeof(cpl_table *));
385 (cpl_table **) cpl_malloc(detmon_pernoise_config.nb_extensions *
386 sizeof(cpl_table *));
389 if(detmon_pernoise_config.exts >= 0)
392 cpl_imagelist_load_frameset(frameset, CPL_TYPE_FLOAT, 1,
393 detmon_pernoise_config.exts);
394 *input = cpl_image_subtract_create(cpl_imagelist_get(*raws,0),
395 cpl_imagelist_get(*raws,1));
398 cpl_imagelist *raws_all_exts =
399 cpl_imagelist_load_frameset(frameset, CPL_TYPE_FLOAT, 1,
401 for(j = 0; j < detmon_pernoise_config.nb_extensions; j++)
403 int nframes = cpl_frameset_get_size(frameset);
405 for(k = 0; k < nframes; k++)
408 cpl_imagelist_unset(raws_all_exts,
409 (detmon_pernoise_config.
410 nb_extensions - 1 - j) * k);
411 cpl_imagelist_set(raws[j], image, k);
414 cpl_image_subtract_create(cpl_imagelist_get(raws[j],0),
415 cpl_imagelist_get(raws[j],1));
419 for(j = 0; j < detmon_pernoise_config.nb_extensions; j++) {
420 cpl_msg_info(cpl_func,
"Starting reduction");
421 qclist[j] = cpl_propertylist_new();
422 if(detmon_pernoise_config.mode == 1)
424 int nx = cpl_image_get_size_x(input[j]);
425 int ny = cpl_image_get_size_y(input[j]);
429 quad[0] = cpl_image_extract(input[j], 1, 1, nx/2, ny/2);
430 quad[1] = cpl_image_extract(input[j], 1, ny/2+1, nx/2, ny);
431 quad[2] = cpl_image_extract(input[j], nx/2+1, 1, nx, ny/2);
432 quad[3] = cpl_image_extract(input[j], nx/2+1, ny/2+1, nx, ny);
434 for (k = 0; k < 4; k++)
436 freq_table[j * 4 + k] = detmon_pernoise_reduce(quad[k]);
438 for(k = 0; k < 4; k++)
440 error = detmon_pernoise_qc(qclist[j], freq_table[j + k], k+1);
441 if (error != CPL_ERROR_NONE)
444 for (k = 0; k < 4; k++)
446 cpl_image_delete(quad[k]);
450 freq_table[j] = detmon_pernoise_reduce(input[j]);
451 if(freq_table[j] != NULL)
453 error = detmon_pernoise_qc(qclist[j], freq_table[j], 0);
456 if (error != CPL_ERROR_NONE)
461 if (error == CPL_ERROR_NONE)
463 error = detmon_pernoise_save(parlist, frameset, recipe_name,
464 pipeline_name, procatg_tbl,
465 package, freq_table, qclist, 0,
469 for(j = 0; j < detmon_pernoise_config.nb_extensions; j++)
471 cpl_propertylist_delete(qclist[j]);
472 cpl_imagelist_delete(raws[j]);
473 cpl_image_delete(input[j]);
478 if(detmon_pernoise_config.mode == 1)
480 for(j= 0; j < detmon_pernoise_config.nb_extensions * 4; j++) {
481 cpl_table_delete(freq_table[j]);
484 for(j= 0; j < detmon_pernoise_config.nb_extensions; j++) {
485 cpl_table_delete(freq_table[j]);
488 cpl_free(freq_table);
489 if (error != CPL_ERROR_NONE)
495 return cpl_error_get_code();
509detmon_pernoise_dfs_set_groups(cpl_frameset * set,
const char *tag)
521 nframes = cpl_frameset_get_size(set);
524 for(i = 0; i < nframes; i++) {
525 cpl_frame* cur_frame = cpl_frameset_get_position(set, i);
526 const char* cur_tag = cpl_frame_get_tag(cur_frame);
529 if(!strcmp(cur_tag, tag))
530 cpl_frame_set_group(cur_frame, CPL_FRAME_GROUP_RAW);
562detmon_fill_pernoise_params(cpl_parameterlist * parlist,
563 const char *recipe_name,
564 const char *pipeline_name,
566 const char * direction,
575 detmon_fill_parlist(parlist, recipe_name, pipeline_name, 9,
579 "CPL_TYPE_INT", mode,
583 "CPL_TYPE_BOOL", direction,
587 "CPL_TYPE_DOUBLE", speed,
590 "(yet unsupported) x coordinate of the lower-left "
591 "point of the region of interest. If not modified, default value will be 1.",
594 "(yet unsupported) y coordinate of the lower-left "
595 "point of the region of interest. If not modified, default value will be 1.",
598 "(yet unsupported) x coordinate of the upper-right "
599 "point of the region of interest. If not modified, default value will be X dimension of the input image.",
602 "(yet unsupported) y coordinate of the upper-right "
603 "point of the region of interest. If not modified, default value will be Y dimension of the input image.",
607 "Kappa used for determining threshold of bad (hot, cold) pixels",
608 "CPL_TYPE_DOUBLE", kappa,
611 "Activate the multi-exts option",
612 "CPL_TYPE_INT", exts);
628detmon_fill_pernoise_params_default(cpl_parameterlist * parlist,
629 const char *recipe_name,
630 const char *pipeline_name)
632 detmon_fill_pernoise_params(parlist, recipe_name, pipeline_name,
658detmon_pernoise_retrieve_parlist(
const char *pipeline_name,
659 const char *recipe_name,
660 const cpl_parameterlist * parlist)
663 const cpl_parameter *par;
666 detmon_pernoise_config.mode =
667 detmon_retrieve_par_int(
"mode", pipeline_name, recipe_name,
671 par_name = cpl_sprintf(
"%s.%s.direction", pipeline_name, recipe_name);
672 assert(par_name != NULL);
673 par = cpl_parameterlist_find_const(parlist, par_name);
674 detmon_pernoise_config.direction = cpl_parameter_get_bool(par);
678 par_name = cpl_sprintf(
"%s.%s.speed", pipeline_name, recipe_name);
679 assert(par_name != NULL);
680 par = cpl_parameterlist_find_const(parlist, par_name);
681 detmon_pernoise_config.speed = cpl_parameter_get_double(par);
685 detmon_pernoise_config.llx =
686 detmon_retrieve_par_int(
"llx", pipeline_name, recipe_name,
690 detmon_pernoise_config.lly =
691 detmon_retrieve_par_int(
"lly", pipeline_name, recipe_name,
694 detmon_pernoise_config.urx =
695 detmon_retrieve_par_int(
"urx", pipeline_name, recipe_name,
698 detmon_pernoise_config.ury =
699 detmon_retrieve_par_int(
"ury", pipeline_name, recipe_name,
702 detmon_pernoise_config.kappa =
703 detmon_retrieve_par_double(
"kappa", pipeline_name, recipe_name,
707 detmon_pernoise_config.exts =
708 detmon_retrieve_par_int(
"exts", pipeline_name, recipe_name,
711 if(cpl_error_get_code()) {
712 cpl_msg_error(cpl_func,
"Failed to retrieve the input parameters");
713 cpl_ensure_code(0, CPL_ERROR_DATA_NOT_FOUND);
716 return cpl_error_get_code();
738detmon_pernoise_save(
const cpl_parameterlist * parlist,
739 cpl_frameset * frameset,
740 const char *recipe_name,
741 const char *pipeline_name,
742 const char *procatg_tbl,
744 cpl_table ** freq_table,
745 cpl_propertylist ** qclist,
748 const cpl_frameset * usedframes)
751 cpl_frame *ref_frame;
752 cpl_propertylist *plist;
755 cpl_propertylist *paflist;
756 cpl_error_code error;
758 cpl_propertylist * pro_tbl = cpl_propertylist_new();
760 cpl_propertylist_append_string(pro_tbl,
761 CPL_DFS_PRO_CATG, procatg_tbl);
763 cpl_propertylist_append(pro_tbl, qclist[0]);
770 if(detmon_pernoise_config.mode != 1) {
773 name_o = cpl_sprintf(
"%s_freq_table.fits", recipe_name);
774 assert(name_o != NULL);
777 cpl_sprintf(
"%s_freq_table_set%02d.fits", recipe_name,
779 assert(name_o != NULL);
783 if(cpl_dfs_save_table(frameset, NULL, parlist, usedframes, NULL, freq_table[0],
784 NULL, recipe_name, pro_tbl, NULL,
786 cpl_msg_error(cpl_func,
"Cannot save the product: %s", name_o);
788 cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
791 if(detmon_pernoise_config.exts < 0) {
793 for(i = 1; i < detmon_pernoise_config.nb_extensions; i++) {
795 cpl_table_save(freq_table[i], NULL, qclist[i], name_o,
797 cpl_ensure_code(!error, error);
805 for (
int j = 1; j <= 4; j++) {
808 name_o = cpl_sprintf(
"%s_freq_table_quad%02d.fits",
810 assert(name_o != NULL);
813 cpl_sprintf(
"%s_freq_table_quad%02d_set%02d.fits",
814 recipe_name, j, which_set);
815 assert(name_o != NULL);
819 if(cpl_dfs_save_table(frameset, NULL, parlist, usedframes, NULL,
821 NULL, recipe_name, pro_tbl, NULL,
823 cpl_msg_error(cpl_func,
"Cannot save the product: %s", name_o);
825 cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
828 if(detmon_pernoise_config.exts < 0) {
829 for(i = 1; i < detmon_pernoise_config.nb_extensions; i++) {
830 error = cpl_table_save(freq_table[(j-1) + 4 * i],
831 NULL, qclist[i], name_o,
833 cpl_ensure_code(!error, error);
847 ref_frame = cpl_frameset_get_position(frameset, 0);
848 if((plist = cpl_propertylist_load(cpl_frame_get_filename(ref_frame),
850 cpl_msg_error(cpl_func,
"getting header from reference frame");
851 cpl_ensure_code(0, cpl_error_get_code());
855 paflist = cpl_propertylist_new();
856 cpl_propertylist_copy_property_regexp(paflist, plist,
857 "^(ARCFILE|MJD-OBS|ESO TPL ID|"
858 "DATE-OBS|ESO DET DIT|ESO DET NDIT|"
860 "ESO DET MODE NAME)$", 0);
862 for(i = 0; i < detmon_pernoise_config.nb_extensions; i++) {
863 cpl_propertylist * c_paflist = cpl_propertylist_duplicate(paflist);
864 error = cpl_propertylist_append(c_paflist, qclist[i]);
865 cpl_ensure_code(!error, error);
868 if(detmon_pernoise_config.exts >= 0) {
870 name_o = cpl_sprintf(
"%s.paf", recipe_name);
871 assert(name_o != NULL);
873 name_o = cpl_sprintf(
"%s_set%02d.paf", recipe_name, which_set);
874 assert(name_o != NULL);
878 name_o = cpl_sprintf(
"%s_ext%02d.paf", recipe_name, i+1);
879 assert(name_o != NULL);
881 name_o = cpl_sprintf(
"%s_set%02d_ext%02d.paf", recipe_name, which_set, i+1);
882 assert(name_o != NULL);
886 if(cpl_dfs_save_paf(pipeline_name, recipe_name, c_paflist, name_o)) {
887 cpl_msg_error(cpl_func,
"Cannot save the product: %s", name_o);
889 cpl_propertylist_delete(paflist);
890 cpl_propertylist_delete(plist);
892 cpl_ensure_code(0, CPL_ERROR_FILE_NOT_CREATED);
894 cpl_propertylist_delete(c_paflist);
898 cpl_propertylist_delete(plist);
899 cpl_propertylist_delete(paflist);
900 cpl_propertylist_delete(pro_tbl);
901 return cpl_error_get_code();
916detmon_pernoise_qc(cpl_propertylist * qclist,
920 cpl_error_code error;
923 double freqs[3] = {0, 0, 0};
924 double pows[3] = {0, 0, 0};
933 int nrows = cpl_table_get_nrow(table);
936 double * all_freqs = cpl_table_get_data_double(table,
"FREQ");
937 double * all_pows = cpl_table_get_data_double(table,
"POW");
939 for ( i= 1; i< nrows-1; i++){
940 if (all_pows[i] > pows[0]) {
941 if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
948 freqs[0]=all_freqs[i];
950 }
else if (all_pows[i] > pows[1]) {
951 if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
956 freqs[1]=all_freqs[i];
959 }
else if(all_pows[i] > pows[2]) {
960 if(all_pows[i-1] < all_pows[i] && all_pows[i] > all_pows[i+1]){
963 freqs[2]=all_freqs[i];
969 if (detmon_pernoise_config.mode == 1) {
970 propname = cpl_sprintf(
"ESO QC FREQ1 %d", iquad);
971 assert(propname != NULL);
973 propname = cpl_sprintf(
"ESO QC FREQ1");
976 error = cpl_propertylist_append_double(qclist, propname, freqs[0]);
977 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
978 cpl_ensure_code(!error, error);
982 if (detmon_pernoise_config.mode == 1) {
983 propname = cpl_sprintf(
"ESO QC FREQ2 %d", iquad);
984 assert(propname != NULL);
986 propname = cpl_sprintf(
"ESO QC FREQ2");
989 error = cpl_propertylist_append_double(qclist, propname, freqs[1]);
990 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
991 cpl_ensure_code(!error, error);
995 if (detmon_pernoise_config.mode == 1) {
996 propname = cpl_sprintf(
"ESO QC FREQ3 %d", iquad);
997 assert(propname != NULL);
999 propname = cpl_sprintf(
"ESO QC FREQ3");
1002 error = cpl_propertylist_append_double(qclist, propname, freqs[2]);
1003 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_FREQ_C);
1004 cpl_ensure_code(!error, error);
1008 if (detmon_pernoise_config.mode == 1) {
1009 propname = cpl_sprintf(
"ESO QC POW1 %d", iquad);
1010 assert(propname != NULL);
1012 propname = cpl_sprintf(
"ESO QC POW1");
1015 error = cpl_propertylist_append_double(qclist, propname, pows[0]);
1016 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1017 cpl_ensure_code(!error, error);
1021 if (detmon_pernoise_config.mode == 1) {
1022 propname = cpl_sprintf(
"ESO QC POW2 %d", iquad);
1023 assert(propname != NULL);
1025 propname = cpl_sprintf(
"ESO QC POW2");
1028 error = cpl_propertylist_append_double(qclist, propname, pows[1]);
1029 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1030 cpl_ensure_code(!error, error);
1034 if (detmon_pernoise_config.mode == 1) {
1035 propname = cpl_sprintf(
"ESO QC POW3 %d", iquad);
1036 assert(propname != NULL);
1038 propname = cpl_sprintf(
"ESO QC POW3");
1041 error = cpl_propertylist_append_double(qclist, propname, pows[2]);
1042 error = cpl_propertylist_set_comment(qclist,propname,DETMON_QC_POW_C);
1043 cpl_ensure_code(!error, error);
1048 return cpl_error_get_code();
1063detmon_pernoise_rm_bg(cpl_image * image,
int nsamples,
int nffts)
1065 cpl_vector *values = cpl_vector_new(nsamples * nffts);
1069 cpl_vector *xy_pos = cpl_vector_new(nsamples * nffts * 2);
1070 cpl_polynomial * poly_2d = 0;
1071 cpl_image * poly_ima = 0;
1072 cpl_size degree = 3;
1073 cpl_error_code error = CPL_ERROR_NONE;
1074 cpl_matrix * samppos = 0;
1076 for(i = 1; i <= nffts; i++) {
1077 for(j = 1; j <= nsamples; j++) {
1078 cpl_vector_set(xy_pos, (i - 1) * nsamples + (j - 1), j);
1079 cpl_vector_set(xy_pos, (i - 1) * nsamples + (j - 1) + nsamples * nffts, i);
1080 cpl_vector_set(values, (i - 1) * nsamples + (j - 1),
1081 cpl_image_get(image, j, i, &rejected));
1082 error = cpl_error_get_code();
1083 if (error != CPL_ERROR_NONE)
1088 if (error != CPL_ERROR_NONE)
1093 if (error != CPL_ERROR_NONE)
1098 poly_2d = cpl_polynomial_new(2);
1100 cpl_matrix_wrap(2, nsamples * nffts, cpl_vector_get_data(xy_pos));
1102 cpl_polynomial_fit(poly_2d, samppos, NULL, values, NULL,
1103 CPL_FALSE, NULL, °ree);
1105 cpl_matrix_unwrap(samppos);
1107 poly_ima = cpl_image_new(nsamples, nffts, CPL_TYPE_FLOAT);
1109 cpl_image_fill_polynomial(poly_ima, poly_2d, 1, 1, 1, 1);
1111 cpl_image_subtract(image, poly_ima);
1114 cpl_polynomial_delete(poly_2d);
1115 cpl_image_delete(poly_ima);
1116 cpl_vector_delete(xy_pos);
1117 cpl_vector_delete(values);
1119 return cpl_error_get_code();