44#include "visir_recipe.h"
50#define RECIPE_STRING "visir_util_img_std_cat"
56typedef int (procfld_func_t)(
const char * src,
const int offset,
void * extra);
57typedef struct { procfld_func_t * func;
void * data; } procfld_ftor_t;
58typedef struct {
int * count;
double * dest; } scan_in_jansky_args_t;
59typedef struct {
char ** dest; } scan_in_filtname_args_t;
67 procfld_ftor_t * ftor;
68} process_fields_args_t;
70typedef struct {
int sz;
double *
const head; } darray_t;
71typedef struct { off_t
const sz;
char const *
const head; } carray_t;
72typedef struct {
int sz;
char ** head; } sarray_t;
79cpl_error_code visir_uisc_save(cpl_frameset *,
80 const cpl_parameterlist * parlist,
const cpl_table * tab);
83cpl_error_code visir_uisc_parse(carray_t src,
char * sname,
double * pra,
84 double * pdec,
char * stype, darray_t dest);
87cpl_error_code visir_uisc_check(cpl_frameset * framelist, sarray_t * filters);
90cpl_error_code visir_uisc_flist_read(sarray_t * filts, carray_t src);
93void visir_uisc_flist_free(sarray_t * flist);
96char * visir_uisc_flist_2_str(sarray_t flist);
99int scan_in_jansky(
const char * src,
const int offset,
void * extra);
102int scan_in_filtname(
const char * src,
const int offset,
void * extra);
105int process_fields(carray_t src,
int skipf,
int maxf, procfld_ftor_t * ftor);
111VISIR_RECIPE_DEFINE(visir_util_img_std_cat, 0,
112 "Convert ASCII-file(s) to a FITS standard star catalog",
113 "This recipe generates a FITS standard star catalog for imaging from one \n"
114 "or more ASCII-files. Each line in the text file must have at least 8 \n"
115 "fields separated by white-space. The first field is the star name, e.g.\n"
116 "'HD108903' which will be stored in a table column labeled 'STARS'. The \n"
117 "next 3 fields are the RA (hh mm ss) which will be stored in degrees in a\n"
118 "table column labeled 'RA' - all three are non-negative and hh and mm are\n"
119 "integer. The 3 next fields are the DEC (dd mm ss) which will be stored \n"
120 "in degrees in a table column labeled 'DEC' - all three are non-negative,\n"
121 "dd and mm are integer, while dd has either a '+' or a '-' prepended (in-\n"
122 "cluding -00). The next field is the spectral type which will be stored \n"
123 "in a table column labeled 'SP_TYPE'. Every field after the first 8 are \n"
124 "the JY values for the supported filters, and may be a mixture of image \n"
125 "or spectral filters. All JY values must be positive. For each filter \n"
126 "the JY value is stored in a table column labeled with the filter name \n"
127 "taken from the first row of column headings or, if there is no 1st row \n"
128 "of column headings, using a list of 44 filter names hard-coded in the \n"
129 "recipe (in which case the number of JY fields must be exactly 42 or else\n"
130 "an error is printed and execution ends). \n"
132 "As mentioned, the 1st row may be a list of column headings. If this row\n"
133 "is supplied, the 1st 4 fields will be ignored (but dummy values must be \n"
134 "supplied anyway), and every field after those will be used as the column\n"
135 "label in the output table for that column's JY values. \n"
137 "Lines beginning with a hash (#) are treated as comments. \n"
139 "The ASCII-input should be tagged "VISIR_IMG_LINES_ASCII
", but all input \n"
140 "files will currently be processed regardless. \n"
143const int sanity_check_failed = 0 + CPL_ERROR_EOL;
156void visir_uisc_flist_free(sarray_t * flist)
158 for (
int j = flist->sz; j-- > 0;) {
159 if (!flist->head[j])
continue;
160 cpl_free(flist->head[j]);
161 flist->head[j] = NULL;
164 cpl_free(flist->head);
175int process_fields(carray_t src,
int skipf,
int maxf, procfld_ftor_t * ftor)
177 if (!src.sz)
return -999;
179 int nfields = -skipf;
186 for (; i < src.sz && src.head[i] !=
'\n' && (!maxf || nfields < maxf); ++i) {
187 if (src.head[i] ==
' ' || src.head[i] ==
'\t') {
193 if (++nfields > 0 && ftor && ftor->func) {
195 (void)ftor->func(src.head + i, nfields - 1, ftor->data);
205int scan_in_filtname(
const char * src,
const int offset,
void * extra)
207 scan_in_filtname_args_t * in = extra;
210 if (EOF == sscanf(src,
"%49s", tmp)) {
211 cpl_msg_error(cpl_func,
"Couldn't read formatted input");
212 return CPL_ERROR_FILE_IO;
216 in->dest[offset] = cpl_sprintf(
"%s", tmp);
217 for (
char * p = in->dest[offset]; *p; ++p) {
218 if (*p ==
'.' || *p ==
'-') *p =
'_';
223 char * here = strstr(in->dest[offset],
"SPEC");
224 if (here)
for (
char * p = here; *p; ++p)
227 return CPL_ERROR_NONE;
232cpl_error_code visir_uisc_flist_read(sarray_t * filts, carray_t src)
237 filts->head = cpl_malloc(filts->sz *
sizeof(
char*));
238 for (
int i = filts->sz; i > 0; filts->head[--i] = NULL);
241 scan_in_filtname_args_t fargs = { .dest = filts->head };
242 procfld_ftor_t ftor = { scan_in_filtname, &fargs };
243 int n_fields_read = process_fields(src, 4, filts->sz, &ftor);
245 if (n_fields_read < filts->sz) {
246 cpl_msg_error(cpl_func,
"buffer had unexpected number of fields");
247 cpl_ensure_code(0, CPL_ERROR_ILLEGAL_INPUT);
248 visir_uisc_flist_free(filts);
251 return cpl_error_get_code();
255char * visir_uisc_flist_2_str(sarray_t flist)
258 for (
int i = 0; i < flist.sz; ++i) {
259 char *
new = cpl_sprintf(
"%s%s, ", slist ? slist :
"", flist.head[i]);
260 if (slist) { cpl_free(slist); slist = NULL; }
275cpl_error_code visir_uisc_check(cpl_frameset * framelist, sarray_t * filters)
278 char * buffer = NULL;
280 int prev_data_cols = -1;
281 for (
int file = 0; file < cpl_frameset_get_size(framelist); ++file) {
282 const cpl_frame * frame = cpl_frameset_get_position_const(framelist,
284 skip_if (cpl_error_get_code() || !frame);
285 const char * fname = cpl_frame_get_filename(frame);
286 skip_if (cpl_error_get_code() || !fname);
288 fd = open(fname, O_RDONLY);
290 cpl_msg_error(cpl_func,
"Could not open file %d: %s", file + 1,
292 cpl_ensure_code(0, CPL_ERROR_ASSIGNING_STREAM);
297 bool fail = (fstat(fd, &stbuf) != 0) || (!S_ISREG(stbuf.st_mode));
299 cpl_msg_error(cpl_func,
"Could not stat file %d, %s, or it is not a"
300 " regular file", file + 1, fname);
301 cpl_ensure_code(0, CPL_ERROR_FILE_IO);
305 off_t fsize = stbuf.st_size;
306 buffer = cpl_malloc(fsize);
308 cpl_msg_error(cpl_func,
"Could not allocate space to hold file "
310 cpl_ensure_code(0, CPL_ERROR_NULL_INPUT);
315 while (count < fsize) {
316 ssize_t batch = read(fd, buffer+count, fsize);
318 cpl_msg_error(cpl_func,
"Error reading from file %d: %s",
320 cpl_ensure_code(0, CPL_ERROR_FILE_IO);
326 if (count != fsize) {
327 cpl_msg_error(cpl_func,
"Mismatch in bytes read vs. file size");
328 cpl_ensure_code(0, sanity_check_failed);
344 for (; head < fsize && buffer[head] ==
'#'; ++head) {
345 while (head < fsize && buffer[head] !=
'\n') ++head;
346 if (head >= fsize)
break;
350 cpl_msg_warning(cpl_func,
"No uncommented lines in file %d: %s",
352 if (buffer) { cpl_free(buffer); buffer = NULL; }
353 if (fd != -1) { close(fd); fd = -1; }
360 while (data < fsize && buffer[data] !=
'\n') ++data;
361 if (data >= fsize)
break;
363 }
while (data < fsize && buffer[data] ==
'#');
366 cpl_msg_warning(cpl_func,
"There must at least 2 uncommented lines "
367 "in file %d: %s", file + 1, fname);
368 if (buffer) { cpl_free(buffer); buffer = NULL; }
369 if (fd != -1) { close(fd); fd = -1; }
373 carray_t head_src = { fsize - head, buffer + head };
374 carray_t data_src = { fsize - data, buffer + data };
375 int nheads = process_fields(head_src, 0, 0, 0);
376 int ndatas = process_fields(data_src, 0, 0, 0);
377 int diff = ndatas - nheads;
381 int nfilts = nheads - 4;
386 if (nfilts != filters->sz) {
387 cpl_msg_error(cpl_func,
"Column headings in file %d, "
388 "%s, don't match a previous file (the "
389 "count differs)", file + 1, fname);
390 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
395 sarray_t filts = { nfilts, NULL };
396 skip_if (visir_uisc_flist_read(&filts, head_src));
398 for (
int i = 0; i < filts.sz; ++i) {
399 if (strcmp(filts.head[i], filters->head[i])) {
400 cpl_msg_error(cpl_func,
"Column headings in file %d, "
401 "%s, don't match a previous file (check "
402 "order and spelling)", file + 1, fname);
403 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
404 visir_uisc_flist_free(&filts);
409 visir_uisc_flist_free(&filts);
413 filters->sz = nfilts;
414 skip_if (visir_uisc_flist_read(filters, head_src));
416 char * flist = visir_uisc_flist_2_str(*filters);
417 cpl_msg_info(cpl_func,
"Found %d filters in file %d, %s, which "
418 "will be used as the column headings in the output "
419 "file. The filters are, in order: %s", nfilts,
420 file + 1, fname, flist);
426 else if (diff == 0) {
427 cpl_msg_info(cpl_func,
"No filter names in file %d: %s", file + 1,
430 if (ndatas-8 != filters->sz) {
431 cpl_msg_error(cpl_func,
"Column structure in file %d, %s, "
432 "doesn't match a previous file", file + 1,
434 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
438 else if (prev_data_cols > -1 && prev_data_cols != ndatas) {
439 cpl_msg_error(cpl_func,
"Column structure in file %d, %s, "
440 "doesn't match a previous file (different number "
441 "data columns)", file + 1, fname);
442 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
449 cpl_msg_error(cpl_func,
"Unexpected column structure in file %d, "
450 "%s (header row doesn't match data rows, or data "
451 "rows don't match each other)", file + 1, fname);
452 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
456 prev_data_cols = ndatas;
458 cpl_free(buffer); buffer = NULL;
463 if (!filters->head) {
464 const char * defaults[] = {
465 "MV",
"N_BAND",
"SIC",
"PAH1_1",
"PAH1",
"ARIII",
"SIV_1",
"SIV",
466 "PAH2_1",
"SIV_2",
"PAH2",
"PAH2_2",
"NEII_1",
"NEII",
"NEII_2",
467 "J7_9",
"J8_9",
"J9_8",
"J12_1",
"J12_2",
"B8_7",
"B9_7",
"B10_7",
468 "B11_7",
"B12_4",
"Q0",
"QH2",
"Q1",
"Q2",
"Q3",
"Q4",
"Q7",
"Q8",
469 "12_4_AGP",
"10_5_4QP",
"11_3_4QP",
"N_SW_spec",
"H2S4_spec",
470 "ARIII_spec",
"NEII_2_spec",
"H2S3_spec",
"H2S1_spec",
473 filters->sz =
sizeof(defaults)/
sizeof(defaults[0]);
474 filters->head = cpl_malloc(filters->sz *
sizeof(defaults[0]));
475 for (
int i = filters->sz; i-- > 0; )
476 filters->head[i] = cpl_strdup(defaults[i]);
479 if (filters->sz > prev_data_cols) {
480 cpl_msg_error(cpl_func,
"Fewer Jansky columns found (%d) than "
481 "there are default filters defined (%d)",
482 prev_data_cols, filters->sz);
483 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
487 char * flist = visir_uisc_flist_2_str(*filters);
488 cpl_msg_warning(cpl_func,
"*** BIG ASSUMPTION ***: No filter names "
489 "were found in the file(s). Defaulting to a pre-defined set "
490 "and hoping they match the column data found! The default "
491 "filters are, in order: %s", flist);
497 if (cpl_error_get_code()) visir_uisc_flist_free(filters);
498 if (buffer) cpl_free(buffer);
499 if (fd != -1) close(fd);
501 return cpl_error_get_code();
514int visir_util_img_std_cat(cpl_frameset * framelist,
515 const cpl_parameterlist * parlist)
519 cpl_table * tab = NULL;
522 const double max_radius = VISIR_STAR_MAX_RADIUS;
530 if (cpl_error_get_code())
return cpl_error_get_code();
536 sarray_t filters = { 0, NULL };
537 skip_if (visir_uisc_check(framelist, &filters));
538 jys = cpl_malloc(filters.sz *
sizeof(
double));
541 tab = cpl_table_new(nrows);
542 skip_if (cpl_table_new_column(tab,
"STARS", CPL_TYPE_STRING));
543 skip_if (cpl_table_new_column(tab,
"SP_TYPE", CPL_TYPE_STRING));
544 skip_if (cpl_table_new_column(tab,
"RA", CPL_TYPE_DOUBLE));
545 skip_if (cpl_table_new_column(tab,
"DEC", CPL_TYPE_DOUBLE));
547 for (
int j = 0; j < filters.sz; j++)
548 skip_if (cpl_table_new_column(tab, filters.head[j], CPL_TYPE_DOUBLE));
551 for (
int i = 0; i < cpl_frameset_get_size(framelist); i++) {
552 const cpl_frame * frame = cpl_frameset_get_position_const(framelist, i);
555 const char * filename = cpl_frame_get_filename(frame);
558 skip_if (filename == NULL);
559 in = fopen(filename,
"r");
561 cpl_msg_error(cpl_func,
"Could not open the %d. file: %s", i+1,
567 while (fgets(line,
sizeof(line), in) != NULL) {
569 if (line[0] !=
'#') {
572 carray_t src = {
sizeof(line), line };
573 darray_t dest = { filters.sz, jys };
574 if (visir_uisc_parse(src, sname, &ra, &dec, stype, dest)) {
575 cpl_msg_warning(cpl_func,
"Unparsable line %d in file %d: "
576 "%s (may be header line, or corrupt)", jrow,
585 skip_if (cpl_table_set_size(tab, nrows));
587 skip_if (cpl_table_set_string(tab,
"STARS", irow, sname));
588 skip_if (cpl_table_set_string(tab,
"SP_TYPE", irow, stype));
589 skip_if (cpl_table_set_double(tab,
"RA", irow, ra));
590 skip_if (cpl_table_set_double(tab,
"DEC", irow, dec));
591 for (
int j = 0; j < filters.sz; j++) skip_if
592 (cpl_table_set_double(tab, filters.head[j], irow, jys[j]));
600 cpl_msg_warning(cpl_func,
"No usable lines in file %s", filename);
608 skip_if (cpl_table_set_size(tab, nrows));
610 mindist = visir_star_dist_min(cpl_table_get_data_double(tab,
"RA"),
611 cpl_table_get_data_double(tab,
"DEC"), nrows,
614 if (mindist < max_radius)
615 cpl_msg_warning(cpl_func,
"The pair of closest stars is %s (%d) and %s "
616 "(%d) with the distance: %g",
617 cpl_table_get_string(tab,
"STARS", iloc1), iloc1,
618 cpl_table_get_string(tab,
"STARS", iloc2), iloc2,
621 cpl_msg_info(cpl_func,
"The pair of closest stars is %s (%d) and %s "
622 "(%d) with the distance: %g",
623 cpl_table_get_string(tab,
"STARS", iloc1), iloc1,
624 cpl_table_get_string(tab,
"STARS", iloc2), iloc2,
628 cpl_msg_info(cpl_func,
"Saving the table with %d rows and %d filters",
631 skip_if (visir_uisc_save(framelist, parlist, tab));
635 visir_uisc_flist_free(&filters);
636 if (in) { fclose(in); in = NULL; }
637 if (jys) { cpl_free(jys); jys = NULL; }
638 cpl_table_delete(tab);
640 return cpl_error_get_code();
644int scan_in_jansky(
const char * src,
const int offset,
void * extra)
646 scan_in_jansky_args_t * in = extra;
647 return *in->count += sscanf(src,
"%lg", in->dest + offset);
665cpl_error_code visir_uisc_parse(carray_t src,
char * sname,
double * pra,
666 double * pdec,
char * stype, darray_t dest)
679 const char * format =
"%s %d %d %lg %c%d %d %lg %s ";
680 int nvals = sscanf(src.head, format, sname, &ra1, &ra2, &ra3, &isign, &dec1,
681 &dec2, &dec3, stype);
684 scan_in_jansky_args_t fargs = { .count = &nvals, .dest = dest.head };
685 procfld_ftor_t ftor = { scan_in_jansky, &fargs };
686 int nfields = process_fields(src, 8, dest.sz, &ftor);
688 if (nvals != dest.sz+9 || nfields < dest.sz) {
689 cpl_msg_warning(cpl_func,
"Read %d items from line (length=%u) instead "
690 "of the expected %d items", nvals,
691 (
unsigned)strlen(src.head), dest.sz+9);
692 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
693 return cpl_error_get_code();
696 return visir_star_convert(src.head, ra1, ra2, ra3, isign, dec1, dec2, dec3,
697 dest.head, dest.sz, pra, pdec);
710cpl_error_code visir_uisc_save(cpl_frameset * set,
711 const cpl_parameterlist * parlist,
const cpl_table * tab)
713 cpl_propertylist * applist = cpl_propertylist_new();
716 bug_if (cpl_propertylist_append_string(applist,
"INSTRUME",
"VISIR"));
718 skip_if (irplib_dfs_save_table(set, parlist, set, tab, NULL, RECIPE_STRING,
719 VISIR_IMA_STD_CAT_PROCATG,
720 applist, NULL, visir_pipe_id,
721 RECIPE_STRING CPL_DFS_FITS));
725 cpl_propertylist_delete(applist);
727 return cpl_error_get_code();
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.