44#include "visir_recipe.h"
50#define RECIPE_STRING "visir_util_img_std_cat"
56typedef int (procfld_func_t)(
const char * src,
const int offset,
void * extra);
57typedef struct { procfld_func_t * func;
void * data; } procfld_ftor_t;
58typedef struct {
int * count;
double * dest; } scan_in_jansky_args_t;
59typedef struct {
char ** dest; } scan_in_filtname_args_t;
67 procfld_ftor_t * ftor;
68} process_fields_args_t;
70typedef struct {
int sz;
double *
const head; } darray_t;
71typedef struct { off_t
const sz;
char const *
const head; } carray_t;
72typedef struct {
int sz;
char ** head; } sarray_t;
79int visir_util_img_std_cat(cpl_frameset * framelist,
80 const cpl_parameterlist * parlist);
83cpl_error_code visir_uisc_save(cpl_frameset *,
84 const cpl_parameterlist * parlist,
const cpl_table * tab);
87cpl_error_code visir_uisc_parse(carray_t src,
char * sname,
double * pra,
88 double * pdec,
char * stype, darray_t dest);
91cpl_error_code visir_uisc_check(cpl_frameset * framelist, sarray_t * filters);
94cpl_error_code visir_uisc_flist_read(sarray_t * filts, carray_t src);
97void visir_uisc_flist_free(sarray_t * flist);
100char * visir_uisc_flist_2_str(sarray_t flist);
103int scan_in_jansky(
const char * src,
const int offset,
void * extra);
106int scan_in_filtname(
const char * src,
const int offset,
void * extra);
109int process_fields(carray_t src,
int skipf,
int maxf, procfld_ftor_t * ftor);
115VISIR_RECIPE_DEFINE(visir_util_img_std_cat, 0,
116 "Convert ASCII-file(s) to a FITS standard star catalog",
117 "This recipe generates a FITS standard star catalog for imaging from one \n"
118 "or more ASCII-files. Each line in the text file must have at least 8 \n"
119 "fields separated by white-space. The first field is the star name, e.g.\n"
120 "'HD108903' which will be stored in a table column labeled 'STARS'. The \n"
121 "next 3 fields are the RA (hh mm ss) which will be stored in degrees in a\n"
122 "table column labeled 'RA' - all three are non-negative and hh and mm are\n"
123 "integer. The 3 next fields are the DEC (dd mm ss) which will be stored \n"
124 "in degrees in a table column labeled 'DEC' - all three are non-negative,\n"
125 "dd and mm are integer, while dd has either a '+' or a '-' prepended (in-\n"
126 "cluding -00). The next field is the spectral type which will be stored \n"
127 "in a table column labeled 'SP_TYPE'. Every field after the first 8 are \n"
128 "the JY values for the supported filters, and may be a mixture of image \n"
129 "or spectral filters. All JY values must be positive. For each filter \n"
130 "the JY value is stored in a table column labeled with the filter name \n"
131 "taken from the first row of column headings or, if there is no 1st row \n"
132 "of column headings, using a list of 44 filter names hard-coded in the \n"
133 "recipe (in which case the number of JY fields must be exactly 42 or else\n"
134 "an error is printed and execution ends). \n"
136 "As mentioned, the 1st row may be a list of column headings. If this row\n"
137 "is supplied, the 1st 4 fields will be ignored (but dummy values must be \n"
138 "supplied anyway), and every field after those will be used as the column\n"
139 "label in the output table for that column's JY values. \n"
141 "Lines beginning with a hash (#) are treated as comments. \n"
143 "The ASCII-input should be tagged "VISIR_IMG_LINES_ASCII
", but all input \n"
144 "files will currently be processed regardless. \n"
147const int sanity_check_failed = 0 + CPL_ERROR_EOL;
160void visir_uisc_flist_free(sarray_t * flist)
162 for (
int j = flist->sz; j-- > 0;) {
163 if (!flist->head[j])
continue;
164 cpl_free(flist->head[j]);
165 flist->head[j] = NULL;
168 cpl_free(flist->head);
179int process_fields(carray_t src,
int skipf,
int maxf, procfld_ftor_t * ftor)
181 if (!src.sz)
return -999;
183 int nfields = -skipf;
190 for (; i < src.sz && src.head[i] !=
'\n' && (!maxf || nfields < maxf); ++i) {
191 if (src.head[i] ==
' ' || src.head[i] ==
'\t') {
197 if (++nfields > 0 && ftor && ftor->func) {
199 (void)ftor->func(src.head + i, nfields - 1, ftor->data);
209int scan_in_filtname(
const char * src,
const int offset,
void * extra)
211 scan_in_filtname_args_t * in = extra;
214 if (EOF == sscanf(src,
"%s", tmp)) {
215 cpl_msg_error(cpl_func,
"Couldn't read formatted input");
216 return CPL_ERROR_FILE_IO;
220 in->dest[offset] = cpl_sprintf(
"%s", tmp);
221 for (
char * p = in->dest[offset]; *p; ++p) {
222 if (*p ==
'.' || *p ==
'-') *p =
'_';
227 char * here = strstr(in->dest[offset],
"SPEC");
228 if (here)
for (
char * p = here; *p; ++p)
231 return CPL_ERROR_NONE;
236cpl_error_code visir_uisc_flist_read(sarray_t * filts, carray_t src)
241 filts->head = cpl_malloc(filts->sz *
sizeof(
char*));
242 for (
int i = filts->sz; i > 0; filts->head[--i] = NULL);
245 scan_in_filtname_args_t fargs = { .dest = filts->head };
246 procfld_ftor_t ftor = { scan_in_filtname, &fargs };
247 int n_fields_read = process_fields(src, 4, filts->sz, &ftor);
249 if (n_fields_read < filts->sz) {
250 cpl_msg_error(cpl_func,
"buffer had unexpected number of fields");
251 cpl_ensure_code(0, CPL_ERROR_ILLEGAL_INPUT);
252 visir_uisc_flist_free(filts);
255 return cpl_error_get_code();
259char * visir_uisc_flist_2_str(sarray_t flist)
262 for (
int i = 0; i < flist.sz; ++i) {
263 char *
new = cpl_sprintf(
"%s%s, ", slist ? slist :
"", flist.head[i]);
264 if (slist) { cpl_free(slist); slist = NULL; }
279cpl_error_code visir_uisc_check(cpl_frameset * framelist, sarray_t * filters)
282 char * buffer = NULL;
284 int prev_data_cols = -1;
285 for (
int file = 0; file < cpl_frameset_get_size(framelist); ++file) {
286 const cpl_frame * frame = cpl_frameset_get_position_const(framelist,
288 skip_if (cpl_error_get_code() || !frame);
289 const char * fname = cpl_frame_get_filename(frame);
290 skip_if (cpl_error_get_code() || !fname);
292 fd = open(fname, O_RDONLY);
294 cpl_msg_error(cpl_func,
"Could not open file %d: %s", file + 1,
296 cpl_ensure_code(0, CPL_ERROR_ASSIGNING_STREAM);
301 bool fail = (fstat(fd, &stbuf) != 0) || (!S_ISREG(stbuf.st_mode));
303 cpl_msg_error(cpl_func,
"Could not stat file %d, %s, or it is not a"
304 " regular file", file + 1, fname);
305 cpl_ensure_code(0, CPL_ERROR_FILE_IO);
309 off_t fsize = stbuf.st_size;
310 buffer = cpl_malloc(fsize);
312 cpl_msg_error(cpl_func,
"Could not allocate space to hold file "
314 cpl_ensure_code(0, CPL_ERROR_NULL_INPUT);
319 while (count < fsize) {
320 ssize_t batch = read(fd, buffer+count, fsize);
322 cpl_msg_error(cpl_func,
"Error reading from file %d: %s",
324 cpl_ensure_code(0, CPL_ERROR_FILE_IO);
330 if (count != fsize) {
331 cpl_msg_error(cpl_func,
"Mismatch in bytes read vs. file size");
332 cpl_ensure_code(0, sanity_check_failed);
348 for (; head < fsize && buffer[head] ==
'#'; ++head) {
349 while (head < fsize && buffer[head] !=
'\n') ++head;
350 if (head >= fsize)
break;
354 cpl_msg_warning(cpl_func,
"No uncommented lines in file %d: %s",
356 if (buffer) { cpl_free(buffer); buffer = NULL; }
357 if (fd != -1) { close(fd); fd = -1; }
364 while (data < fsize && buffer[data] !=
'\n') ++data;
365 if (data >= fsize)
break;
367 }
while (data < fsize && buffer[data] ==
'#');
370 cpl_msg_warning(cpl_func,
"There must at least 2 uncommented lines "
371 "in file %d: %s", file + 1, fname);
372 if (buffer) { cpl_free(buffer); buffer = NULL; }
373 if (fd != -1) { close(fd); fd = -1; }
377 carray_t head_src = { fsize - head, buffer + head };
378 carray_t data_src = { fsize - data, buffer + data };
379 int nheads = process_fields(head_src, 0, 0, 0);
380 int ndatas = process_fields(data_src, 0, 0, 0);
381 int diff = ndatas - nheads;
385 int nfilts = nheads - 4;
390 if (nfilts != filters->sz) {
391 cpl_msg_error(cpl_func,
"Column headings in file %d, "
392 "%s, don't match a previous file (the "
393 "count differs)", file + 1, fname);
394 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
399 sarray_t filts = { nfilts, NULL };
400 skip_if (visir_uisc_flist_read(&filts, head_src));
402 for (
int i = 0; i < filts.sz; ++i) {
403 if (strcmp(filts.head[i], filters->head[i])) {
404 cpl_msg_error(cpl_func,
"Column headings in file %d, "
405 "%s, don't match a previous file (check "
406 "order and spelling)", file + 1, fname);
407 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
408 visir_uisc_flist_free(&filts);
413 visir_uisc_flist_free(&filts);
417 filters->sz = nfilts;
418 skip_if (visir_uisc_flist_read(filters, head_src));
420 char * flist = visir_uisc_flist_2_str(*filters);
421 cpl_msg_info(cpl_func,
"Found %d filters in file %d, %s, which "
422 "will be used as the column headings in the output "
423 "file. The filters are, in order: %s", nfilts,
424 file + 1, fname, flist);
430 else if (diff == 0) {
431 cpl_msg_info(cpl_func,
"No filter names in file %d: %s", file + 1,
434 if (ndatas-8 != filters->sz) {
435 cpl_msg_error(cpl_func,
"Column structure in file %d, %s, "
436 "doesn't match a previous file", file + 1,
438 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
442 else if (prev_data_cols > -1 && prev_data_cols != ndatas) {
443 cpl_msg_error(cpl_func,
"Column structure in file %d, %s, "
444 "doesn't match a previous file (different number "
445 "data columns)", file + 1, fname);
446 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
453 cpl_msg_error(cpl_func,
"Unexpected column structure in file %d, "
454 "%s (header row doesn't match data rows, or data "
455 "rows don't match each other)", file + 1, fname);
456 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
460 prev_data_cols = ndatas;
462 cpl_free(buffer); buffer = NULL;
467 if (!filters->head) {
468 char * defaults[] = {
469 "MV",
"N_BAND",
"SIC",
"PAH1_1",
"PAH1",
"ARIII",
"SIV_1",
"SIV",
470 "PAH2_1",
"SIV_2",
"PAH2",
"PAH2_2",
"NEII_1",
"NEII",
"NEII_2",
471 "J7_9",
"J8_9",
"J9_8",
"J12_1",
"J12_2",
"B8_7",
"B9_7",
"B10_7",
472 "B11_7",
"B12_4",
"Q0",
"QH2",
"Q1",
"Q2",
"Q3",
"Q4",
"Q7",
"Q8",
473 "12_4_AGP",
"10_5_4QP",
"11_3_4QP",
"N_SW_spec",
"H2S4_spec",
474 "ARIII_spec",
"NEII_2_spec",
"H2S3_spec",
"H2S1_spec",
477 filters->sz =
sizeof(defaults)/
sizeof(
char*);
478 filters->head = cpl_malloc(filters->sz *
sizeof(
char*));
479 for (
int i = filters->sz; i-- > 0; )
480 filters->head[i] = cpl_sprintf(
"%s", defaults[i]);
483 if (filters->sz > prev_data_cols) {
484 cpl_msg_error(cpl_func,
"Fewer Jansky columns found (%d) than "
485 "there are default filters defined (%d)",
486 prev_data_cols, filters->sz);
487 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
491 char * flist = visir_uisc_flist_2_str(*filters);
492 cpl_msg_warning(cpl_func,
"*** BIG ASSUMPTION ***: No filter names "
493 "were found in the file(s). Defaulting to a pre-defined set "
494 "and hoping they match the column data found! The default "
495 "filters are, in order: %s", flist);
501 if (cpl_error_get_code()) visir_uisc_flist_free(filters);
502 if (buffer) cpl_free(buffer);
503 if (fd != -1) close(fd);
505 return cpl_error_get_code();
518int visir_util_img_std_cat(cpl_frameset * framelist,
519 const cpl_parameterlist * parlist)
523 cpl_table * tab = NULL;
526 const double max_radius = VISIR_STAR_MAX_RADIUS;
534 if (cpl_error_get_code())
return cpl_error_get_code();
540 sarray_t filters = { 0, NULL };
541 skip_if (visir_uisc_check(framelist, &filters));
542 jys = cpl_malloc(filters.sz *
sizeof(
double));
545 tab = cpl_table_new(nrows);
546 skip_if (cpl_table_new_column(tab,
"STARS", CPL_TYPE_STRING));
547 skip_if (cpl_table_new_column(tab,
"SP_TYPE", CPL_TYPE_STRING));
548 skip_if (cpl_table_new_column(tab,
"RA", CPL_TYPE_DOUBLE));
549 skip_if (cpl_table_new_column(tab,
"DEC", CPL_TYPE_DOUBLE));
551 for (
int j = 0; j < filters.sz; j++)
552 skip_if (cpl_table_new_column(tab, filters.head[j], CPL_TYPE_DOUBLE));
555 for (
int i = 0; i < cpl_frameset_get_size(framelist); i++) {
556 const cpl_frame * frame = cpl_frameset_get_position_const(framelist, i);
559 const char * filename = cpl_frame_get_filename(frame);
562 skip_if (filename == NULL);
563 in = fopen(filename,
"r");
565 cpl_msg_error(cpl_func,
"Could not open the %d. file: %s", i+1,
571 while (fgets(line,
sizeof(line), in) != NULL) {
573 if (line[0] !=
'#') {
576 carray_t src = {
sizeof(line), line };
577 darray_t dest = { filters.sz, jys };
578 if (visir_uisc_parse(src, sname, &ra, &dec, stype, dest)) {
579 cpl_msg_warning(cpl_func,
"Unparsable line %d in file %d: "
580 "%s (may be header line, or corrupt)", jrow,
589 skip_if (cpl_table_set_size(tab, nrows));
591 skip_if (cpl_table_set_string(tab,
"STARS", irow, sname));
592 skip_if (cpl_table_set_string(tab,
"SP_TYPE", irow, stype));
593 skip_if (cpl_table_set_double(tab,
"RA", irow, ra));
594 skip_if (cpl_table_set_double(tab,
"DEC", irow, dec));
595 for (
int j = 0; j < filters.sz; j++) skip_if
596 (cpl_table_set_double(tab, filters.head[j], irow, jys[j]));
604 cpl_msg_warning(cpl_func,
"No usable lines in file %s", filename);
612 skip_if (cpl_table_set_size(tab, nrows));
614 mindist = visir_star_dist_min(cpl_table_get_data_double(tab,
"RA"),
615 cpl_table_get_data_double(tab,
"DEC"), nrows,
618 if (mindist < max_radius)
619 cpl_msg_warning(cpl_func,
"The pair of closest stars is %s (%d) and %s "
620 "(%d) with the distance: %g",
621 cpl_table_get_string(tab,
"STARS", iloc1), iloc1,
622 cpl_table_get_string(tab,
"STARS", iloc2), iloc2,
625 cpl_msg_info(cpl_func,
"The pair of closest stars is %s (%d) and %s "
626 "(%d) with the distance: %g",
627 cpl_table_get_string(tab,
"STARS", iloc1), iloc1,
628 cpl_table_get_string(tab,
"STARS", iloc2), iloc2,
632 cpl_msg_info(cpl_func,
"Saving the table with %d rows and %d filters",
635 skip_if (visir_uisc_save(framelist, parlist, tab));
639 visir_uisc_flist_free(&filters);
640 if (in) { fclose(in); in = NULL; }
641 if (jys) { cpl_free(jys); jys = NULL; }
642 cpl_table_delete(tab);
644 return cpl_error_get_code();
648int scan_in_jansky(
const char * src,
const int offset,
void * extra)
650 scan_in_jansky_args_t * in = extra;
651 return *in->count += sscanf(src,
"%lg", in->dest + offset);
669cpl_error_code visir_uisc_parse(carray_t src,
char * sname,
double * pra,
670 double * pdec,
char * stype, darray_t dest)
683 const char * format =
"%s %d %d %lg %c%d %d %lg %s ";
684 int nvals = sscanf(src.head, format, sname, &ra1, &ra2, &ra3, &isign, &dec1,
685 &dec2, &dec3, stype);
688 scan_in_jansky_args_t fargs = { .count = &nvals, .dest = dest.head };
689 procfld_ftor_t ftor = { scan_in_jansky, &fargs };
690 int nfields = process_fields(src, 8, dest.sz, &ftor);
692 if (nvals != dest.sz+9 || nfields < dest.sz) {
693 cpl_msg_warning(cpl_func,
"Read %d items from line (length=%u) instead "
694 "of the expected %d items", nvals,
695 (
unsigned)strlen(src.head), dest.sz+9);
696 cpl_ensure_code(0, CPL_ERROR_BAD_FILE_FORMAT);
697 return cpl_error_get_code();
700 return visir_star_convert(src.head, ra1, ra2, ra3, isign, dec1, dec2, dec3,
701 dest.head, dest.sz, pra, pdec);
714cpl_error_code visir_uisc_save(cpl_frameset * set,
715 const cpl_parameterlist * parlist,
const cpl_table * tab)
717 cpl_propertylist * applist = cpl_propertylist_new();
720 bug_if (cpl_propertylist_append_string(applist,
"INSTRUME",
"VISIR"));
722 skip_if (irplib_dfs_save_table(set, parlist, set, tab, NULL, RECIPE_STRING,
723 VISIR_IMA_STD_CAT_PROCATG,
724 applist, NULL, visir_pipe_id,
725 RECIPE_STRING CPL_DFS_FITS));
729 cpl_propertylist_delete(applist);
731 return cpl_error_get_code();
int visir_dfs_set_groups(cpl_frameset *set)
Set the group as RAW or CALIB in a frameset.