VIRCAM Pipeline  2.3.10
casu_getstds.c
1 /* $Id: casu_getstds.c,v 1.7 2015/11/27 12:06:41 jim Exp $
2  *
3  * This file is part of the CASU Pipeline utilities
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2015/11/27 12:06:41 $
24  * $Revision: 1.7 $
25  * $Name: $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #ifndef _POSIX_C_SOURCE
35 #define _POSIX_C_SOURCE 200112L
36 #endif
37 
38 #ifndef _XOPEN_SOURCE
39 #define _XOPEN_SOURCE 500 /* posix 2001, mkstemp */
40 #endif
41 
42 #include <stdlib.h>
43 #include <stdio.h>
44 #include <sys/stat.h>
45 #include <sys/types.h>
46 #include <sys/socket.h>
47 #include <netdb.h>
48 #include <netinet/in.h>
49 #include <fcntl.h>
50 #include <ctype.h>
51 #include <assert.h>
52 #include <errno.h>
53 #include <string.h>
54 #include <libgen.h>
55 
56 #include <unistd.h>
57 #include <string.h>
58 #include <cpl.h>
59 
60 #include <math.h>
61 
62 #include "casu_mods.h"
63 #include "catalogue/casu_utils.h"
64 #include "catalogue/casu_wcsutils.h"
65 
66 #define CACHEDIR ".catcache"
67 #define CACHEIND ".catcache/index"
68 #define SZBUF 1024
69 
70 #define CDS "cdsarc.u-strasbg.fr"
71 #define BUFMAX 32768
72 #define PORT 80
73 
74 static cpl_table *casu_2mass_extract(char *path, float ramin, float ramax,
75  float decmin, float decmax);
76 static cpl_table *casu_ppmxl_extract(char *path, float ramin1, float ramax1,
77  float decmin, float decmax);
78 static cpl_table *casu_apass_extract(char *path, float ramin1, float ramax1,
79  float decmin, float decmax);
80 static cpl_table *casu_local_extract(char *path, float ramin1, float ramax1,
81  float decmin, float decmax);
82 static cpl_table *check_cache(char *catname, float ra1_im, float ra2_im,
83  float dec1_im, float dec2_im, char *cacheloc);
84 static void addto_cache(cpl_table *stds, char *catname, float ramin,
85  float ramax, float decmin, float decmax,
86  char *cacheloc);
87 static char *form_request(float ra, float dec, float dra, float ddec,
88  float equinox, const char *catid);
89 static char *url_encode(char *instring);
90 static int connect_site(const char *site, int *sock);
91 static int send_request(int sock, char *req_string, int isbin);
92 static int get_response(int soc, cpl_table **outtab);
93 
96 /*---------------------------------------------------------------------------*/
157 /*---------------------------------------------------------------------------*/
158 
159 extern int casu_getstds(cpl_propertylist *plist, int cache, char *path,
160  char *catname, int cdssearch, char *cacheloc,
161  cpl_table **stds, int *status) {
162  const char *fctid = "casu_getstds";
163  double xx1,xx2,yy1,yy2,r,d,xx,yy,*rad,*decd,dra,ddec;
164  float ramin,ramax,decmin,decmax,*ra,*dec,*x,*y,ramid,decmid;
165  cpl_wcs *wcs;
166  int n,i,sock;
167  cpl_propertylist *p;
168  char *req_string,catname2[64],*cdscatname,*cdscatid,*path2;
169  int isbin = (cdssearch > 6 ? 1 : 0);
170 
171  /* Inherited status */
172 
173  *stds = NULL;
174  if (*status != CASU_OK)
175  return(*status);
176  if (plist == NULL)
177  FATAL_ERROR
178 
179  /* Get the coverage of the input WCS */
180 
181  (void)casu_coverage(plist,0,&xx1,&xx2,&yy1,&yy2,status);
182  ramin = (float)xx1;
183  ramax = (float)xx2;
184  decmin = (float)yy1;
185  decmax = (float)yy2;
186  dra = ramax - ramin;
187  ddec = decmax - decmin;
188  ramid = 0.5*(ramax+ramin);
189  decmid = 0.5*(decmax+decmin);
190 
191  cpl_msg_info(fctid, "Searching catalogue %d, RA/Dec range [%f:%f, %f:%f]", cdssearch, ramin, ramax, decmin, decmax);
192 
193  /* If using the cache, then check if for a previously used table */
194 
195  cdscatid = NULL;
196  if (cdssearch > 0) {
197  if (casu_getstds_cdslist(cdssearch,&cdscatname,&cdscatid,status) !=
198  CASU_OK)
199  return(*status);
200  (void)strcpy(catname2,cdscatname);
201  freespace(cdscatname);
202  } else {
203  (void)strcpy(catname2,catname);
204  }
205  *stds = NULL;
206  if (cache)
207  *stds = check_cache(catname2,ramin,ramax,decmin,decmax,cacheloc);
208 
209  /* If there was nothing in the cache (or you didn't use it) then search
210  the standard catalogues */
211 
212  if (*stds == NULL) {
213 
214  /* Read the standards from local catalogues */
215 
216  if (cdssearch == 0) {
217  cpl_msg_info(fctid, "Reading local catalogue");
218 
219  path2 = cpl_strdup(path);
220  if (! strcmp(catname2,"casu_2mass")) {
221  *stds = casu_2mass_extract(dirname(path2),ramin,ramax,decmin,
222  decmax);
223  } else if (! strcmp(catname2,"casu_ppmxl")) {
224  *stds = casu_ppmxl_extract(dirname(path2),ramin,ramax,decmin,
225  decmax);
226  } else if (! strcmp(catname2,"casu_apass")) {
227  *stds = casu_apass_extract(dirname(path2),ramin,ramax,decmin,
228  decmax);
229  } else {
230  *stds = casu_local_extract(path2,ramin,ramax,decmin,decmax);
231  }
232  cpl_free(path2);
233  if (*stds == NULL) {
234  cpl_msg_error(fctid,"Unable to extract data in %s",path);
235  FATAL_ERROR
236  }
237 
238  /* Read the standards from CDS */
239 
240  } else {
241  cpl_msg_info(fctid, "Requesting catalogue data from CDS");
242 
243  req_string = form_request(ramid,decmid,dra,ddec,2000.0,
244  cdscatid);
245  freespace(cdscatid);
246 
247  /* Now make the connection to the site */
248 
249  if (connect_site(CDS,&sock) != CASU_OK) {
250  cpl_msg_warning(fctid,"Couldn't connect to CDS");
251  FATAL_ERROR
252  }
253 
254  /* Send the request to the site */
255 
256  if (send_request(sock,req_string,isbin) != CASU_OK) {
257  cpl_msg_warning(fctid,"Couldn't send request to site CDS");
258  FATAL_ERROR
259  }
260 
261  /* Get the response from the the server and write it to the
262  output table */
263 
264  if (get_response(sock,stds) != CASU_OK) {
265  cpl_msg_warning(fctid,"Error receiving info from site CDS");
266  FATAL_ERROR
267  }
268 
269  /* Close up the connection */
270 
271  close(sock);
272 
273  /* Do some modifications to the table */
274 
275  cpl_table_name_column(*stds,"RAJ2000","RA");
276  cpl_table_name_column(*stds,"DEJ2000","Dec");
277  }
278 
279  /* Add this table to the cache if you want to */
280 
281  if (cache)
282  addto_cache(*stds,catname2,ramin,ramax,decmin,decmax,cacheloc);
283  } else {
284  freespace(cdscatid);
285  }
286 
287  n = (int)cpl_table_get_nrow(*stds);
288  cpl_msg_info(fctid, "Found %d stars in catalogue", n);
289 
290  /* If there are no rows in the table, this may be a cause for concern.
291  So add the columns into the table and then set a warning return
292  status */
293 
294  if (n == 0) {
295  cpl_table_new_column(*stds,"xpredict",CPL_TYPE_FLOAT);
296  cpl_table_new_column(*stds,"ypredict",CPL_TYPE_FLOAT);
297  WARN_RETURN
298  }
299 
300  /* Now fill the coordinates in */
301 
302  wcs = cpl_wcs_new_from_propertylist((const cpl_propertylist *)plist);
303  if (cpl_table_get_column_type(*stds,"RA") == CPL_TYPE_FLOAT) {
304  ra = cpl_table_get_data_float(*stds,"RA");
305  dec = cpl_table_get_data_float(*stds,"Dec");
306  x = cpl_malloc(n*sizeof(*x));
307  y = cpl_malloc(n*sizeof(*y));
308  for (i = 0; i < n; i++) {
309  r = (double)ra[i];
310  d = (double)dec[i];
311  casu_radectoxy(wcs,r,d,&xx,&yy);
312  x[i] = (float)xx;
313  y[i] = (float)yy;
314  }
315  } else {
316  rad = cpl_table_get_data_double(*stds,"RA");
317  decd = cpl_table_get_data_double(*stds,"Dec");
318  x = cpl_malloc(n*sizeof(*x));
319  y = cpl_malloc(n*sizeof(*y));
320  for (i = 0; i < n; i++) {
321  r = rad[i];
322  d = decd[i];
323  casu_radectoxy(wcs,r,d,&xx,&yy);
324  x[i] = (float)xx;
325  y[i] = (float)yy;
326  }
327  }
328  cpl_wcs_delete(wcs);
329 
330  /* Add the predicted x,y coordinates columns to the table */
331 
332  cpl_table_wrap_float(*stds,x,"xpredict");
333  cpl_table_set_column_unit(*stds,"xpredict","pixels");
334  cpl_table_wrap_float(*stds,y,"ypredict");
335  cpl_table_set_column_unit(*stds,"ypredict","pixels");
336 
337  /* Finally sort this by ypredict */
338 
339  p = cpl_propertylist_new();
340  cpl_propertylist_append_bool(p,"ypredict",0);
341  cpl_table_sort(*stds,p);
342  cpl_propertylist_delete(p);
343 
344  /* Get out of here */
345 
346  GOOD_STATUS
347 }
348 
349 /*---------------------------------------------------------------------------*/
387 /*---------------------------------------------------------------------------*/
388 
389 static const int maxcds = 7;
390 static const char *cdscatnames[8] =
391  {"","2mass","usnob","ppmxl","landolt","wise", "apass", "gaiaedr3"};
392 static const char *cdscats[8] =
393  {"","II/246","I/284","I/317","II/183A","II/311", "II/336", "I/350"};
394 
395 extern int casu_getstds_cdslist(int cdschoice, char **cdscatname,
396  char **cdscatid, int *status) {
397  const char *fctid = "casu_getstds_cdslist";
398 
399  if (*status != CASU_OK)
400  return(*status);
401 
402  /* Check that the choice is within the bounds of possibility */
403 
404  *cdscatname = NULL;
405  *cdscatid = NULL;
406  if (cdschoice < 0 || cdschoice > maxcds) {
407  cpl_msg_error(fctid,"CDS catalogue choice must be >= 0 && <= %d",
408  maxcds);
409  return(CASU_FATAL);
410  }
411  *cdscatname = cpl_strdup(cdscatnames[cdschoice]);
412  *cdscatid = cpl_strdup(cdscats[cdschoice]);
413  GOOD_STATUS
414 }
415 
416 /*---------------------------------------------------------------------------*/
441 /*---------------------------------------------------------------------------*/
442 
443 extern int casu_get_cdschoice(const char* cdscatname)
444 {
445  const char *fctid = "casu_get_cdschoice";
446 
447  for (int i=1; i<=maxcds; i++) {
448  if (!strcmp(cdscatname, cdscatnames[i])) {
449  return i;
450  }
451  }
452  return 0;
453 }
454 
455 
456 /*---------------------------------------------------------------------------*/
484 /*---------------------------------------------------------------------------*/
485 
486 static cpl_table *casu_2mass_extract(char *path, float ramin1, float ramax1,
487  float decmin, float decmax) {
488  cpl_table *t,*s,*o;
489  int i,nrows,start,finish,first_index,last_index,irow,init,j;
490  int first_index_ra,last_index_ra,wrap,iwrap;
491  float dectest,ratest,ramin,ramax;
492  char fullname[SZBUF];
493  cpl_array *a;
494  const char *deccol = "Dec";
495  cpl_propertylist *p;
496 
497  /* Create an output table */
498 
499  o = cpl_table_new(0);
500  init = 1;
501 
502  /* Create a cpl array */
503 
504  a = cpl_array_wrap_string((char **)&deccol,1);
505 
506  /* Is there a wrap around problem? */
507 
508  wrap = (ramin1 < 0.0 && ramax1 > 0.0) ? 2 : 1;
509 
510  /* Loop for each query. If there is a wrap around problem then we need 2
511  queries. If not, then we only need 1 */
512 
513  for (iwrap = 0; iwrap < wrap; iwrap++) {
514  if (wrap == 2) {
515  if (iwrap == 0) {
516  ramin = ramin1 + 360.0;
517  ramax = 360.0;
518  } else {
519  ramin = 0.000001;
520  ramax = ramax1;
521  }
522  } else {
523  ramin = ramin1;
524  ramax = ramax1;
525  }
526 
527  /* Find out where in the index to look */
528 
529  first_index_ra = (int)ramin;
530  last_index_ra = min((int)ramax,359);
531 
532  /* Look at the min and max RA and decide which files need to be
533  opened. */
534 
535  for (i = first_index_ra; i <= last_index_ra; i++) {
536 
537  /* Ok, we've found one that needs opening. Read the file with
538  the relevant CPL call */
539 
540  (void)snprintf(fullname,SZBUF,"%s/npsc%03d.fits",path,i);
541 
542  /* Read the propertylist so that you know how many rows there
543  are in the table */
544 
545  p = cpl_propertylist_load(fullname,1);
546  if (p == NULL) {
547  freetable(o);
548  cpl_array_unwrap(a);
549  return(NULL);
550  }
551  nrows = cpl_propertylist_get_int(p,"NAXIS2");
552  cpl_propertylist_delete(p);
553 
554  /* Load various rows until you find the Dec range that you
555  have specified. First the minimum Dec */
556 
557  start = 0;
558  finish = nrows;
559  first_index = nrows/2;
560  while (finish - start >= 2) {
561  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)first_index,
562  1);
563  dectest = cpl_table_get_double(t,"Dec",0,NULL);
564  cpl_table_delete(t);
565  if (dectest < decmin) {
566  start = first_index;
567  first_index = (first_index + finish)/2;
568  } else {
569  finish = first_index;
570  first_index = (first_index + start)/2;
571  }
572  }
573 
574  /* Load various rows until you find the Dec range that you
575  have specified. Now the maximum Dec */
576 
577  start = first_index;
578  finish = nrows;
579  last_index = start + (finish - start)/2;
580  while (finish - start >= 2) {
581  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)last_index,
582  1);
583  dectest = cpl_table_get_double(t,"Dec",0,NULL);
584  cpl_table_delete(t);
585  if (dectest < decmax) {
586  start = last_index;
587  last_index = (last_index + finish)/2;
588  } else {
589  finish = last_index;
590  last_index = (last_index + start)/2;
591  }
592  }
593  if (last_index < first_index)
594  last_index = first_index;
595 
596  /* Ok now now load all the rows in the relevant dec limits */
597 
598  nrows = last_index - first_index + 1;
599  if ((t = cpl_table_load_window(fullname,1,1,NULL,
600  (cpl_size)first_index,
601  (cpl_size)nrows)) == NULL) {
602  freetable(o);
603  cpl_array_unwrap(a);
604  return(NULL);
605  }
606  cpl_table_unselect_all(t);
607 
608  /* Right, we now know what range of rows to search. Go through
609  these and pick the ones that are in the correct range of RA.
610  If a row qualifies, then 'select' it. */
611 
612  for (j = 0; j < nrows; j++) {
613  ratest = cpl_table_get_double(t,"RA",(cpl_size)j,NULL);
614  if (cpl_error_get_code() != CPL_ERROR_NONE) {
615  cpl_table_delete(t);
616  cpl_array_unwrap(a);
617  freetable(o);
618  return(NULL);
619  }
620  if (ratest >= ramin && ratest <= ramax)
621  cpl_table_select_row(t,(cpl_size)j);
622  }
623 
624  /* Extract the rows that have been selected now and append them
625  onto the output table */
626 
627  s = cpl_table_extract_selected(t);
628  if (init == 1) {
629  cpl_table_copy_structure(o,t);
630  init = 0;
631  }
632  irow = (int)cpl_table_get_nrow(o) + 1;
633  cpl_table_insert(o,s,(cpl_size)irow);
634 
635  /* Tidy up */
636 
637  cpl_table_delete(t);
638  cpl_table_delete(s);
639  }
640  }
641 
642  /* Ok, now just return the table and get out of here */
643 
644  cpl_array_unwrap(a);
645  return(o);
646 }
647 
648 /*---------------------------------------------------------------------------*/
676 /*---------------------------------------------------------------------------*/
677 
678 static cpl_table *casu_ppmxl_extract(char *path, float ramin1, float ramax1,
679  float decmin, float decmax) {
680  cpl_table *t,*s,*o;
681  int i,nrows,start,finish,first_index,last_index,irow,init,j;
682  int first_index_ra,last_index_ra,wrap,iwrap;
683  float dectest,ratest,ramin,ramax;
684  char fullname[SZBUF];
685  cpl_array *a;
686  const char *deccol = "Dec";
687  cpl_propertylist *p;
688 
689  /* Create an output table */
690 
691  o = cpl_table_new(0);
692  init = 1;
693 
694  /* Create a cpl array */
695 
696  a = cpl_array_wrap_string((char **)&deccol,1);
697 
698  /* Is there a wrap around problem? */
699 
700  wrap = (ramin1 < 0.0 && ramax1 > 0.0) ? 2 : 1;
701 
702  /* Loop for each query. If there is a wrap around problem then we need 2
703  queries. If not, then we only need 1 */
704 
705  for (iwrap = 0; iwrap < wrap; iwrap++) {
706  if (wrap == 2) {
707  if (iwrap == 0) {
708  ramin = ramin1 + 360.0;
709  ramax = 360.0;
710  } else {
711  ramin = 0.000001;
712  ramax = ramax1;
713  }
714  } else {
715  ramin = ramin1;
716  ramax = ramax1;
717  }
718 
719  /* Find out where in the index to look */
720 
721  first_index_ra = (int)ramin;
722  last_index_ra = min((int)ramax,359);
723 
724  /* Look at the min and max RA and decide which files need to be
725  opened. */
726 
727  for (i = first_index_ra; i <= last_index_ra; i++) {
728 
729  /* Ok, we've found one that needs opening. Read the file with
730  the relevant CPL call */
731 
732  (void)snprintf(fullname,SZBUF,"%s/nppmxl%03d.fits",path,i);
733 
734  /* Read the propertylist so that you know how many rows there
735  are in the table */
736 
737  p = cpl_propertylist_load(fullname,1);
738  if (p == NULL) {
739  freetable(o);
740  cpl_array_unwrap(a);
741  return(NULL);
742  }
743  nrows = cpl_propertylist_get_int(p,"NAXIS2");
744  cpl_propertylist_delete(p);
745 
746  /* Load various rows until you find the Dec range that you
747  have specified. First the minimum Dec */
748 
749  start = 0;
750  finish = nrows;
751  first_index = nrows/2;
752  while (finish - start >= 2) {
753  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)first_index,
754  1);
755  dectest = (float)cpl_table_get_double(t,"Dec",0,NULL);
756  cpl_table_delete(t);
757  if (dectest < decmin) {
758  start = first_index;
759  first_index = (first_index + finish)/2;
760  } else {
761  finish = first_index;
762  first_index = (first_index + start)/2;
763  }
764  }
765 
766  /* Load various rows until you find the Dec range that you
767  have specified. Now the maximum Dec */
768 
769  start = first_index;
770  finish = nrows;
771  last_index = start + (finish - start)/2;
772  while (finish - start >= 2) {
773  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)last_index,
774  1);
775  dectest = (float)cpl_table_get_double(t,"Dec",0,NULL);
776  cpl_table_delete(t);
777  if (dectest < decmax) {
778  start = last_index;
779  last_index = (last_index + finish)/2;
780  } else {
781  finish = last_index;
782  last_index = (last_index + start)/2;
783  }
784  }
785  if (last_index < first_index)
786  last_index = first_index;
787 
788  /* Ok now now load all the rows in the relevant dec limits */
789 
790  nrows = last_index - first_index + 1;
791  if ((t = cpl_table_load_window(fullname,1,1,NULL,
792  (cpl_size)first_index,
793  (cpl_size)nrows)) == NULL) {
794  freetable(o);
795  cpl_array_unwrap(a);
796  return(NULL);
797  }
798  cpl_table_unselect_all(t);
799 
800  /* Right, we now know what range of rows to search. Go through
801  these and pick the ones that are in the correct range of RA.
802  If a row qualifies, then 'select' it. */
803 
804  for (j = 0; j < nrows; j++) {
805  ratest = (float)cpl_table_get_double(t,"RA",(cpl_size)j,NULL);
806  if (cpl_error_get_code() != CPL_ERROR_NONE) {
807  cpl_table_delete(t);
808  cpl_array_unwrap(a);
809  freetable(o);
810  return(NULL);
811  }
812  if (ratest >= ramin && ratest <= ramax)
813  cpl_table_select_row(t,(cpl_size)j);
814  }
815 
816  /* Extract the rows that have been selected now and append them
817  onto the output table */
818 
819  s = cpl_table_extract_selected(t);
820  if (init == 1) {
821  cpl_table_copy_structure(o,t);
822  init = 0;
823  }
824  irow = (int)cpl_table_get_nrow(o) + 1;
825  cpl_table_insert(o,s,(cpl_size)irow);
826 
827  /* Tidy up */
828 
829  cpl_table_delete(t);
830  cpl_table_delete(s);
831  }
832  }
833 
834  /* Ok, now just return the table and get out of here */
835 
836  cpl_array_unwrap(a);
837  return(o);
838 }
839 
840 /*---------------------------------------------------------------------------*/
868 /*---------------------------------------------------------------------------*/
869 
870 static cpl_table *casu_apass_extract(char *path, float ramin1, float ramax1,
871  float decmin, float decmax) {
872  cpl_table *t,*s,*o;
873  int i,nrows,start,finish,first_index,last_index,irow,init,j;
874  int first_index_ra,last_index_ra,wrap,iwrap;
875  float dectest,ratest,ramin,ramax;
876  char fullname[SZBUF];
877  cpl_array *a;
878  const char *racol = "RA";
879  cpl_propertylist *p;
880 
881  /* Create an output table */
882 
883  o = cpl_table_new(0);
884  init = 1;
885 
886  /* Create a cpl array */
887 
888  a = cpl_array_wrap_string((char **)&racol,1);
889 
890  /* Is there a wrap around problem? */
891 
892  wrap = (ramin1 < 0.0 && ramax1 > 0.0) ? 2 : 1;
893 
894  /* Loop for each query. If there is a wrap around problem then we need 2
895  queries. If not, then we only need 1 */
896 
897  for (iwrap = 0; iwrap < wrap; iwrap++) {
898  if (wrap == 2) {
899  if (iwrap == 0) {
900  ramin = ramin1 + 360.0;
901  ramax = 360.0;
902  } else {
903  ramin = 0.000001;
904  ramax = ramax1;
905  }
906  } else {
907  ramin = ramin1;
908  ramax = ramax1;
909  }
910 
911  /* Find out where in the index to look */
912 
913  first_index_ra = (int)ramin;
914  last_index_ra = min((int)ramax,359);
915 
916  /* Look at the min and max RA and decide which files need to be
917  opened. */
918 
919  for (i = first_index_ra; i <= last_index_ra; i++) {
920 
921  /* Ok, we've found one that needs opening. Read the file with
922  the relevant CPL call */
923 
924  (void)snprintf(fullname,SZBUF,"%s/casuapass_%03d.fits",path,i);
925 
926  /* Read the propertylist so that you know how many rows there
927  are in the table */
928 
929  p = cpl_propertylist_load(fullname,1);
930  if (p == NULL) {
931  freetable(o);
932  cpl_array_unwrap(a);
933  return(NULL);
934  }
935  nrows = cpl_propertylist_get_int(p,"NAXIS2");
936  cpl_propertylist_delete(p);
937 
938  /* Load various rows until you find the RA range that you
939  have specified. First the minimum RA */
940 
941  start = 0;
942  finish = nrows;
943  first_index = nrows/2;
944  while (finish - start >= 2) {
945  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)first_index,
946  1);
947  ratest = (float)cpl_table_get_double(t,"RA",0,NULL);
948  cpl_table_delete(t);
949  if (ratest < ramin) {
950  start = first_index;
951  first_index = (first_index + finish)/2;
952  } else {
953  finish = first_index;
954  first_index = (first_index + start)/2;
955  }
956  }
957 
958  /* Load various rows until you find the RA range that you
959  have specified. Now the maximum RA */
960 
961  start = first_index;
962  finish = nrows;
963  last_index = start + (finish - start)/2;
964  while (finish - start >= 2) {
965  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)last_index,
966  1);
967  ratest = (float)cpl_table_get_double(t,"RA",0,NULL);
968  cpl_table_delete(t);
969  if (ratest < ramax) {
970  start = last_index;
971  last_index = (last_index + finish)/2;
972  } else {
973  finish = last_index;
974  last_index = (last_index + start)/2;
975  }
976  }
977  if (last_index < first_index)
978  last_index = first_index;
979 
980  /* Ok now now load all the rows in the relevant RA limits */
981 
982  nrows = last_index - first_index + 1;
983  if ((t = cpl_table_load_window(fullname,1,1,NULL,
984  (cpl_size)first_index,
985  (cpl_size)nrows)) == NULL) {
986  freetable(o);
987  cpl_array_unwrap(a);
988  return(NULL);
989  }
990  cpl_table_unselect_all(t);
991 
992  /* Right, we now know what range of rows to search. Go through
993  these and pick the ones that are in the correct range of RA.
994  If a row qualifies, then 'select' it. */
995 
996  for (j = 0; j < nrows; j++) {
997  dectest = (float)cpl_table_get_double(t,"Dec",(cpl_size)j,NULL);
998  if (cpl_error_get_code() != CPL_ERROR_NONE) {
999  cpl_table_delete(t);
1000  cpl_array_unwrap(a);
1001  freetable(o);
1002  return(NULL);
1003  }
1004  if (dectest >= decmin && dectest <= decmax)
1005  cpl_table_select_row(t,(cpl_size)j);
1006  }
1007 
1008  /* Extract the rows that have been selected now and append them
1009  onto the output table */
1010 
1011  s = cpl_table_extract_selected(t);
1012  if (init == 1) {
1013  cpl_table_copy_structure(o,t);
1014  init = 0;
1015  }
1016  irow = (int)cpl_table_get_nrow(o) + 1;
1017  cpl_table_insert(o,s,(cpl_size)irow);
1018 
1019  /* Tidy up */
1020 
1021  cpl_table_delete(t);
1022  cpl_table_delete(s);
1023  }
1024  }
1025 
1026  /* Ok, now just return the table and get out of here */
1027 
1028  cpl_array_unwrap(a);
1029  return(o);
1030 }
1031 
1032 /*---------------------------------------------------------------------------*/
1061 /*---------------------------------------------------------------------------*/
1062 
1063 static cpl_table *casu_local_extract(char *path, float ramin1, float ramax1,
1064  float decmin, float decmax) {
1065  cpl_table *whole,*out,*out2;
1066  int wrap,iwrap,nrows;
1067  float ramin,ramax;
1068  cpl_propertylist *p;
1069 
1070  /* First read the input file. It's assumed this isn't so big that it's
1071  going to take very long. */
1072 
1073  whole = cpl_table_load((const char *)path,1,0);
1074 
1075  /* Is there a wrap around problem? */
1076 
1077  wrap = (ramin1 < 0.0 && ramax1 > 0.0) ? 2 : 1;
1078 
1079  /* Loop for each query. If there is a wrap around problem then we need 2
1080  queries. If not, then we only need 1 */
1081 
1082  out = NULL;
1083  for (iwrap = 0; iwrap < wrap; iwrap++) {
1084  if (wrap == 2) {
1085  if (iwrap == 0) {
1086  ramin = ramin1 + 360.0;
1087  ramax = 360.0;
1088  } else {
1089  ramin = 0.000001;
1090  ramax = ramax1;
1091  }
1092  } else {
1093  ramin = ramin1;
1094  ramax = ramax1;
1095  }
1096  cpl_table_unselect_all(whole);
1097 
1098  /* Select on RA and Dec */
1099 
1100  cpl_table_or_selected_double(whole,"RA",CPL_NOT_LESS_THAN,
1101  (double)ramin);
1102  cpl_table_and_selected_double(whole,"RA",CPL_NOT_GREATER_THAN,
1103  (double)ramax);
1104  cpl_table_and_selected_double(whole,"Dec",CPL_NOT_LESS_THAN,
1105  (double)decmin);
1106  cpl_table_and_selected_double(whole,"Dec",CPL_NOT_GREATER_THAN,
1107  (double)decmax);
1108 
1109  /* If this is the first time through, then create the output table */
1110 
1111  if (out == NULL) {
1112  out = cpl_table_extract_selected(whole);
1113  } else {
1114  out2 = cpl_table_extract_selected(whole);
1115  nrows = cpl_table_get_nrow(out);
1116  cpl_table_insert(out,out2,nrows+1);
1117  cpl_table_delete(out2);
1118  }
1119  }
1120 
1121  /* Sort the table by Dec and get out of here*/
1122 
1123  p = cpl_propertylist_new();
1124  cpl_propertylist_append_bool(p,"Dec",0);
1125  cpl_table_sort(out,p);
1126  cpl_propertylist_delete(p);
1127  return(out);
1128 }
1129 
1130 /*---------------------------------------------------------------------------*/
1160 /*---------------------------------------------------------------------------*/
1161 
1162 static cpl_table *check_cache(char *catname, float ra1_im, float ra2_im,
1163  float dec1_im, float dec2_im, char *cacheloc) {
1164  int wrap1,wrap2;
1165  FILE *fd;
1166  char fname[BUFSIZ],catname2[SZBUF];
1167  char * cat_cache = NULL;
1168  float best,ra1_cat,ra2_cat,dec1_cat,dec2_cat,d1,d2,fra,fdec,ftot;
1169  cpl_table *out_cat;
1170 
1171  /* Open the index file. NB the path and file name are hardcoded */
1172 
1173  (void)snprintf(fname,BUFSIZ,"%s/%s",cacheloc,CACHEIND);
1174  fd = fopen(fname,"r");
1175  if (fd == NULL)
1176  return(NULL);
1177 
1178  /* Check to see if there is wrap around in the coordinates */
1179 
1180  wrap1 = (ra1_im < 0.0 ? 1 : 0);
1181 
1182  /* Now see if you have any matching entries */
1183 
1184  best = 0.0;
1185  // Todo: the hard coded numbers should be changed to use BUFSIZ and SZBUF
1186  while (fscanf(fd,"%8191s %1023s %g %g %g %g",
1187  fname,catname2,&ra1_cat,&ra2_cat,
1188  &dec1_cat,&dec2_cat) != EOF) {
1189  wrap2 = (ra1_cat < 0.0 ? 1 : 0);
1190  if (wrap1 != wrap2)
1191  continue;
1192  if (strcmp(catname,catname2))
1193  continue;
1194 
1195  /* Check to see if there is at least some overlap */
1196 
1197  if (!(((ra1_im >= ra1_cat && ra1_im <= ra2_cat) ||
1198  (ra2_im >= ra1_cat && ra2_im <= ra2_cat)) &&
1199  ((dec1_im >= dec1_cat && dec1_im <= dec2_cat) ||
1200  (dec2_im >= dec1_cat && dec2_im <= dec2_cat))))
1201  continue;
1202 
1203  /* Work out exactly how much there is in each coordinate */
1204 
1205  d1 = max(0.0,ra1_cat-ra1_im);
1206  d2 = max(0.0,ra2_im-ra2_cat);
1207  fra = 1.0 - (d1 + d2)/(ra2_im - ra1_im);
1208  d1 = max(0.0,dec1_cat-dec1_im);
1209  d2 = max(0.0,dec2_im-dec2_cat);
1210  fdec = 1.0 - (d1 + d2)/(dec2_im - dec1_im);
1211  ftot = fra*fdec;
1212 
1213  /* Keep track of which is the best one */
1214 
1215  if (ftot > best) {
1216  cpl_free(cat_cache); /* As we are in a loop free old allocation */
1217  cat_cache = cpl_sprintf("%s/%s/%s",cacheloc,CACHEDIR,fname);
1218  best = ftot;
1219  }
1220  }
1221  fclose(fd);
1222 
1223  /* Return a bad status if there isn't sufficient overlap */
1224 
1225  if (best < 0.9)
1226  return(NULL);
1227 
1228  /* If there is enough overlap, then try and read the FITS table. If it
1229  reads successfully, then return the table pointer */
1230 
1231  out_cat = cpl_table_load(cat_cache,1,1);
1232  cpl_free(cat_cache);
1233 
1234  cpl_msg_info(cpl_func, "Using cached catalogue data");
1235 
1236  return(out_cat);
1237 }
1238 
1239 /*---------------------------------------------------------------------------*/
1267 /*---------------------------------------------------------------------------*/
1268 
1269 static void addto_cache(cpl_table *stds, char *catname, float ramin,
1270  float ramax, float decmin, float decmax,
1271  char *cacheloc) {
1272  FILE *fd;
1273  char newname[SZBUF];
1274  int i;
1275 
1276  /* Check to see if the cache directory exists. If it doesn't, then create
1277  it. */
1278 
1279  (void)snprintf(newname,SZBUF,"%s/%s",cacheloc,CACHEDIR);
1280  if (access(newname,0) != 0)
1281  mkdir(newname,0755);
1282 
1283  /* Open the index file with 'append' access */
1284 
1285  (void)snprintf(newname,SZBUF,"%s/%s",cacheloc,CACHEIND);
1286  fd = fopen(newname,"a");
1287 
1288  /* Check the files in the directory to get a number that isn't already
1289  being used */
1290 
1291  i = 0;
1292  while (i >= 0) {
1293  i++;
1294  snprintf(newname,SZBUF,"%s/%s/cch_%08d",cacheloc,CACHEDIR,i);
1295  if (access(newname,F_OK) != 0)
1296  break;
1297  }
1298 
1299  /* Now write the current entry and make a copy of the file into the
1300  directory */
1301 
1302  snprintf(newname,SZBUF,"%s/%s/cch_%08d",cacheloc,CACHEDIR,i);
1303  cpl_table_save(stds,NULL,NULL,newname,CPL_IO_DEFAULT);
1304  snprintf(newname,SZBUF,"cch_%08d",i);
1305  if (cpl_error_get_code() == CPL_ERROR_NONE)
1306  fprintf(fd, "%s %s %g %g %g %g\n",newname,catname,ramin-0.0005,
1307  ramax+0.0005,decmin-0.0005,decmax+0.0005);
1308  fclose(fd);
1309 }
1310 
1311 /*---------------------------------------------------------------------------*/
1334 /*---------------------------------------------------------------------------*/
1335 
1336 static int get_response(int sock, cpl_table **outtab) {
1337  int nnewline,i,nv,irem = 0,fd;
1338  long nrows;
1339  char buf[BUFMAX],outfile[32];
1340  const char *fctid = "get_response";
1341 
1342  /* Read from the socket until we find the double newline */
1343 
1344  nnewline = 0;
1345  while (1) {
1346  nv = recv(sock,buf,sizeof(buf),0);
1347  if (nv <= 0) {
1348  cpl_msg_warning(fctid,"Unable to find double newline");
1349  return(CASU_FATAL);
1350  }
1351  for (i = 0; i < nv; i++) {
1352  if (buf[i] == '\n') {
1353  nnewline++;
1354  if (nnewline == 2)
1355  break;
1356  } else {
1357  if (nnewline && buf[i] != '\r')
1358  nnewline = 0;
1359  }
1360  }
1361  if (nnewline == 2) {
1362  irem = i + 1;
1363  break;
1364  }
1365  }
1366 
1367  /* Shift the buffer */
1368 
1369  for (i = irem; i < nv; i++)
1370  buf[i-irem] = buf[i];
1371  nv -= irem;
1372 
1373  /* Open the output file */
1374 
1375  (void)snprintf(outfile,32,"stdsXXXXXX");
1376  fd = mkstemp(outfile);
1377 
1378  /* Now write the data to the output file */
1379 
1380  while (1) {
1381  ssize_t written = write(fd,buf,nv);
1382  if(written != nv) {
1383  cpl_msg_info(cpl_func, "Not all bytes could be written - "
1384  "check diskspace");
1385  }
1386  nv = recv(sock,buf,sizeof(buf),0);
1387  if (nv == 0)
1388  break;
1389  else if (nv < 0) {
1390  cpl_msg_warning(fctid,"Read from socket failed");
1391  close(fd);
1392  remove(outfile);
1393  return(CASU_FATAL);
1394  }
1395  }
1396 
1397  /* Close the output file. Check and see if a table was created and whether
1398  or not it contains any rows... */
1399 
1400  close(fd);
1401  *outtab = cpl_table_load(outfile,1,0);
1402  if (cpl_error_get_code() != CPL_ERROR_NONE) {
1403  cpl_msg_warning(fctid,"No standards were found");
1404  cpl_error_reset();
1405  remove(outfile);
1406  *outtab = NULL;
1407  return(CASU_FATAL);
1408  } else {
1409  nrows = (int)cpl_table_get_nrow(*outtab);
1410  if (nrows <= 0) {
1411  cpl_msg_warning(fctid,"No standards table had no rows");
1412  remove(outfile);
1413  freetable(*outtab);
1414  return(CASU_FATAL);
1415  }
1416  }
1417 
1418  /* Get out of here */
1419 
1420  remove(outfile);
1421  return(CASU_OK);
1422 }
1423 
1424 /*---------------------------------------------------------------------------*/
1450 /*---------------------------------------------------------------------------*/
1451 
1452 static int send_request(int sock, char *req_string, int isbin) {
1453  char buf[BUFMAX];
1454  const char *fctid = "send_request";
1455 
1456  /* Form the full request contents and send it. */
1457 
1458  if (isbin) {
1459  sprintf(buf,"GET /viz-bin/asu-binfits?%s HTTP/1.0\r\n\r\n",req_string);
1460  } else {
1461  sprintf(buf,"GET /viz-bin/asu-fits?%s HTTP/1.0\r\n\r\n",req_string);
1462  }
1463  if (send(sock,buf,strlen(buf),0) < 0) {
1464  cpl_msg_warning(fctid,"Attempt to send message failed, error: %d\n",
1465  errno);
1466  return(CASU_FATAL);
1467  }
1468  return(CASU_OK);
1469 }
1470 
1471 /*---------------------------------------------------------------------------*/
1494 /*---------------------------------------------------------------------------*/
1495 
1496 static int connect_site(const char *site, int *sock) {
1497  struct hostent *hp;
1498  struct sockaddr_in ssin;
1499  const char *fctid = "connect_site";
1500 
1501  /* Get host pointer */
1502 
1503  hp = gethostbyname(site);
1504  if (hp == NULL) {
1505  cpl_msg_warning(fctid,"Unable to get host information for %s\n",site);
1506  return(CASU_FATAL);
1507  }
1508  ssin.sin_family = hp->h_addrtype;
1509  memcpy(&ssin.sin_addr,hp->h_addr_list[0],hp->h_length);
1510  ssin.sin_port = htons(PORT);
1511 
1512  /* Create an IP-family socket on which to make the connection */
1513 
1514  *sock = socket(hp->h_addrtype,SOCK_STREAM,0);
1515  if (*sock < 0) {
1516  cpl_msg_warning(fctid,"Unable to create socket descriptor for %s\n",
1517  site);
1518  return (CASU_FATAL);
1519  }
1520 
1521  /* Connect to that address...*/
1522 
1523  if (connect(*sock,(struct sockaddr*)&ssin,sizeof(ssin)) < 0) {
1524  cpl_msg_warning(fctid,"Unable to connect to site: %s\n",site);
1525  return(CASU_FATAL);
1526  }
1527 
1528  /* Otherwise send back a good status */
1529 
1530  return(CASU_OK);
1531 }
1532 
1533 /*---------------------------------------------------------------------------*/
1561 /*---------------------------------------------------------------------------*/
1562 
1563 static char *form_request(float ra, float dec, float dra, float ddec,
1564  float equinox, const char *catid) {
1565  static char buf[2*BUFSIZ];
1566  char buf2[BUFSIZ],equi[1];
1567 
1568  /* Format each of the fields and then add them to the buffer that will
1569  become the query string. URLencode them before appending. First
1570  the coordinates */
1571 
1572  (void)snprintf(buf2,BUFSIZ,"-c=%8.3f %8.3f",ra,dec);
1573  (void)strncpy(buf,url_encode(buf2),BUFSIZ);
1574 
1575  /* Now the box size */
1576 
1577  (void)snprintf(buf2,BUFSIZ,"-c.bd=%g/%g",dra,ddec);
1578  (void)strncat(buf,"&",BUFSIZ);
1579  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1580 
1581  /* The catalogue name */
1582 
1583  (void)snprintf(buf2,BUFSIZ,"-source=%s",catid);
1584  (void)strncat(buf,"&",BUFSIZ);
1585  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1586 
1587  /* The equinox */
1588 
1589  equi[0] = (equinox == 1950.0 ? 'B' : 'J');
1590  (void)snprintf(buf2,BUFSIZ,"-c.eq=%c%g",equi[0],equinox);
1591  (void)strncat(buf,"&",BUFSIZ);
1592  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1593 
1594  /* Finally make sure that we don't truncate the result and that we
1595  sort by RA */
1596 
1597  (void)snprintf(buf2,BUFSIZ,"-out.max=unlimited");
1598  (void)strncat(buf,"&",BUFSIZ);
1599  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1600  (void)snprintf(buf2,BUFSIZ,"-sort=_RA*-c.eq");
1601  (void)strncat(buf,"&",BUFSIZ);
1602  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1603  (void)snprintf(buf2,BUFSIZ,"-out.add=_RA*-c.eq");
1604  (void)strncat(buf,"&",BUFSIZ);
1605  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1606  (void)snprintf(buf2,BUFSIZ,"-out.add=_DEC*-c.eq");
1607  (void)strncat(buf,"&",BUFSIZ);
1608  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1609 
1610  /* Right, get out of here */
1611 
1612  return(buf);
1613 }
1614 
1615 /*---------------------------------------------------------------------------*/
1633 /*---------------------------------------------------------------------------*/
1634 
1635 static char *url_encode(char *instring) {
1636  static char buf[BUFSIZ];
1637  int i,j,k,len;
1638 
1639  /* First copy over everything before the equal sign */
1640 
1641  k = 0;
1642  do {
1643  buf[k] = instring[k];
1644  } while (instring[k++] != '=');
1645  len = strlen(instring);
1646 
1647  /* Copy over the string, substituting the things that we don't want in
1648  a URL */
1649 
1650  j = k;
1651  for (i = k; i < len; i++) {
1652  assert(j < (int)sizeof(buf));
1653  if (instring[i] == ' ')
1654  buf[j++] = '+';
1655  else if (isalnum((unsigned char)instring[i]))
1656  buf[j++] = instring[i];
1657  else {
1658  sprintf(buf+j,"%%%2x",(unsigned char)instring[i]);
1659  j += 3;
1660  }
1661  }
1662  buf[j] = '\0';
1663  return(buf);
1664 }
1665 
1669 /*
1670 
1671 $Log: casu_getstds.c,v $
1672 Revision 1.7 2015/11/27 12:06:41 jim
1673 Detabbed a few things
1674 
1675 Revision 1.6 2015/11/25 13:50:03 jim
1676 cacheloc parameter added
1677 
1678 Revision 1.5 2015/11/25 10:24:49 jim
1679 Changed so that catpath always contains the full path of a file and the
1680 file name is stripped off in the event that you're using local
1681 2mass, apass or ppmxl catalogues
1682 
1683 Revision 1.4 2015/10/04 18:43:14 jim
1684 fixed small bug when no standards are returned
1685 
1686 Revision 1.3 2015/08/07 13:06:54 jim
1687 Fixed copyright to ESO
1688 
1689 Revision 1.2 2015/08/06 05:34:02 jim
1690 Fixes to get rid of compiler moans
1691 
1692 Revision 1.1.1.1 2015/06/12 10:44:32 jim
1693 Initial import
1694 
1695 Revision 1.15 2015/05/20 11:58:00 jim
1696 Changed URL for CDS
1697 
1698 Revision 1.14 2015/05/14 11:46:33 jim
1699 All RA and Dec columns are officially double
1700 
1701 Revision 1.13 2015/05/13 11:42:27 jim
1702 superficial changes
1703 
1704 Revision 1.12 2015/03/12 09:16:51 jim
1705 Modified to remove some compiler moans
1706 
1707 Revision 1.11 2015/03/09 19:51:58 jim
1708 Fixed undefined value
1709 
1710 Revision 1.10 2015/03/03 10:48:11 jim
1711 Fixed some memory leaks
1712 
1713 Revision 1.9 2015/01/29 11:48:15 jim
1714 modified comments
1715 
1716 Revision 1.8 2015/01/09 12:13:15 jim
1717 *** empty log message ***
1718 
1719 Revision 1.7 2014/12/11 12:23:33 jim
1720 new version
1721 
1722 Revision 1.6 2014/04/23 05:39:41 jim
1723 Now reads local standard tables
1724 
1725 Revision 1.5 2014/04/09 11:26:36 jim
1726 Order of catalogues for CDS was wrong
1727 
1728 Revision 1.4 2014/04/09 11:08:21 jim
1729 Get rid of a couple of compiler moans
1730 
1731 Revision 1.3 2014/04/09 09:09:51 jim
1732 Detabbed
1733 
1734 Revision 1.2 2013/11/21 09:38:13 jim
1735 detabbed
1736 
1737 Revision 1.1.1.1 2013-08-27 12:07:48 jim
1738 Imported
1739 
1740 
1741 */
int casu_get_cdschoice(const char *cdscatname)
Definition: casu_getstds.c:443
int casu_getstds(cpl_propertylist *plist, int cache, char *path, char *catname, int cdssearch, char *cacheloc, cpl_table **stds, int *status)
Get a table of standard stars that appear on an image from a catalogue.
Definition: casu_getstds.c:159
void casu_radectoxy(cpl_wcs *wcs, double ra, double dec, double *x, double *y)
int casu_coverage(cpl_propertylist *plist, int fudge, double *ra1, double *ra2, double *dec1, double *dec2, int *status)