VIRCAM Pipeline  2.3.12
casu_getstds.c
1 /* $Id: casu_getstds.c,v 1.7 2015/11/27 12:06:41 jim Exp $
2  *
3  * This file is part of the CASU Pipeline utilities
4  * Copyright (C) 2015 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2015/11/27 12:06:41 $
24  * $Revision: 1.7 $
25  * $Name: $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #ifndef _POSIX_C_SOURCE
35 #define _POSIX_C_SOURCE 200112L
36 #endif
37 
38 #ifndef _XOPEN_SOURCE
39 #define _XOPEN_SOURCE 500 /* posix 2001, mkstemp */
40 #endif
41 
42 #include <stdlib.h>
43 #include <stdio.h>
44 #include <sys/stat.h>
45 #include <sys/types.h>
46 #include <sys/socket.h>
47 #include <netdb.h>
48 #include <netinet/in.h>
49 #include <fcntl.h>
50 #include <ctype.h>
51 #include <assert.h>
52 #include <errno.h>
53 #include <string.h>
54 #include <libgen.h>
55 
56 #include <unistd.h>
57 #include <string.h>
58 #include <cpl.h>
59 
60 #include <math.h>
61 
62 #include "casu_mods.h"
63 #include "catalogue/casu_utils.h"
64 #include "catalogue/casu_wcsutils.h"
65 
66 #define CACHEDIR ".catcache"
67 #define CACHEIND ".catcache/index"
68 #define SZBUF 1024
69 
70 #define CDS "cdsarc.u-strasbg.fr"
71 #define BUFMAX 32768
72 #define PORT 80
73 
74 static cpl_table *casu_2mass_extract(char *path, float ramin, float ramax,
75  float decmin, float decmax);
76 static cpl_table *casu_ppmxl_extract(char *path, float ramin1, float ramax1,
77  float decmin, float decmax);
78 static cpl_table *casu_apass_extract(char *path, float ramin1, float ramax1,
79  float decmin, float decmax);
80 static cpl_table *casu_local_extract(char *path, float ramin1, float ramax1,
81  float decmin, float decmax);
82 static cpl_table *check_cache(char *catname, float ra1_im, float ra2_im,
83  float dec1_im, float dec2_im, char *cacheloc);
84 static void addto_cache(cpl_table *stds, char *catname, float ramin,
85  float ramax, float decmin, float decmax,
86  char *cacheloc);
87 static char *form_request(float ra, float dec, float dra, float ddec,
88  float equinox, const char *catid);
89 static char *url_encode(char *instring);
90 static int connect_site(const char *site, int *sock);
91 static int send_request(int sock, char *req_string, int isbin);
92 static int get_response(int soc, cpl_table **outtab);
93 
96 /*---------------------------------------------------------------------------*/
157 /*---------------------------------------------------------------------------*/
158 
159 extern int casu_getstds(cpl_propertylist *plist, int cache, char *path,
160  char *catname, int cdssearch, char *cacheloc,
161  cpl_table **stds, int *status) {
162  const char *fctid = "casu_getstds";
163  double xx1,xx2,yy1,yy2,r,d,xx,yy,*rad,*decd,dra,ddec;
164  float ramin,ramax,decmin,decmax,*ra,*dec,*x,*y,ramid,decmid;
165  cpl_wcs *wcs;
166  int n,i,sock;
167  cpl_propertylist *p;
168  char *req_string,catname2[64],*cdscatname,*cdscatid,*path2;
169  int isbin = (cdssearch > 6 ? 1 : 0);
170 
171  /* Inherited status */
172 
173  *stds = NULL;
174  if (*status != CASU_OK)
175  return(*status);
176  if (plist == NULL)
177  FATAL_ERROR
178 
179  /* Get the coverage of the input WCS */
180 
181  (void)casu_coverage(plist,0,&xx1,&xx2,&yy1,&yy2,status);
182  ramin = (float)xx1;
183  ramax = (float)xx2;
184  decmin = (float)yy1;
185  decmax = (float)yy2;
186  dra = ramax - ramin;
187  ddec = decmax - decmin;
188  ramid = 0.5*(ramax+ramin);
189  decmid = 0.5*(decmax+decmin);
190 
191  cpl_msg_info(fctid, "Searching catalogue %d, RA/Dec range [%f:%f, %f:%f]", cdssearch, ramin, ramax, decmin, decmax);
192 
193  /* If using the cache, then check if for a previously used table */
194 
195  cdscatid = NULL;
196  if (cdssearch > 0) {
197  if (casu_getstds_cdslist(cdssearch,&cdscatname,&cdscatid,status) !=
198  CASU_OK)
199  return(*status);
200  (void)strcpy(catname2,cdscatname);
201  freespace(cdscatname);
202  } else {
203  (void)strcpy(catname2,catname);
204  }
205  *stds = NULL;
206  if (cache)
207  *stds = check_cache(catname2,ramin,ramax,decmin,decmax,cacheloc);
208 
209  /* If there was nothing in the cache (or you didn't use it) then search
210  the standard catalogues */
211 
212  if (*stds == NULL) {
213 
214  /* Read the standards from local catalogues */
215 
216  if (cdssearch == 0) {
217  cpl_msg_info(fctid, "Reading local catalogue");
218 
219  path2 = cpl_strdup(path);
220  if (! strcmp(catname2,"casu_2mass")) {
221  *stds = casu_2mass_extract(dirname(path2),ramin,ramax,decmin,
222  decmax);
223  } else if (! strcmp(catname2,"casu_ppmxl")) {
224  *stds = casu_ppmxl_extract(dirname(path2),ramin,ramax,decmin,
225  decmax);
226  } else if (! strcmp(catname2,"casu_apass")) {
227  *stds = casu_apass_extract(dirname(path2),ramin,ramax,decmin,
228  decmax);
229  } else {
230  *stds = casu_local_extract(path2,ramin,ramax,decmin,decmax);
231  }
232  cpl_free(path2);
233  if (*stds == NULL) {
234  cpl_msg_error(fctid,"Unable to extract data in %s",path);
235  FATAL_ERROR
236  }
237 
238  /* Read the standards from CDS */
239 
240  } else {
241  cpl_msg_info(fctid, "Requesting catalogue data from CDS");
242 
243  req_string = form_request(ramid,decmid,dra,ddec,2000.0,
244  cdscatid);
245  freespace(cdscatid);
246 
247  /* Now make the connection to the site */
248 
249  if (connect_site(CDS,&sock) != CASU_OK) {
250  cpl_msg_warning(fctid,"Couldn't connect to CDS");
251  FATAL_ERROR
252  }
253 
254  /* Send the request to the site */
255 
256  if (send_request(sock,req_string,isbin) != CASU_OK) {
257  cpl_msg_warning(fctid,"Couldn't send request to site CDS");
258  FATAL_ERROR
259  }
260 
261  /* Get the response from the the server and write it to the
262  output table */
263 
264  if (get_response(sock,stds) != CASU_OK) {
265  cpl_msg_warning(fctid,"Error receiving info from site CDS");
266  FATAL_ERROR
267  }
268 
269  /* Close up the connection */
270 
271  close(sock);
272 
273  /* Do some modifications to the table */
274 
275  cpl_table_name_column(*stds,"RAJ2000","RA");
276  cpl_table_name_column(*stds,"DEJ2000","Dec");
277  }
278 
279  /* Add this table to the cache if you want to */
280 
281  if (cache)
282  addto_cache(*stds,catname2,ramin,ramax,decmin,decmax,cacheloc);
283  } else {
284  freespace(cdscatid);
285  }
286 
287  n = (int)cpl_table_get_nrow(*stds);
288  cpl_msg_info(fctid, "Found %d stars in catalogue", n);
289 
290  /* If there are no rows in the table, this may be a cause for concern.
291  So add the columns into the table and then set a warning return
292  status */
293 
294  if (n == 0) {
295  cpl_table_new_column(*stds,"xpredict",CPL_TYPE_FLOAT);
296  cpl_table_new_column(*stds,"ypredict",CPL_TYPE_FLOAT);
297  WARN_RETURN
298  }
299 
300  /* Now fill the coordinates in */
301 
302  wcs = cpl_wcs_new_from_propertylist((const cpl_propertylist *)plist);
303  if (cpl_table_get_column_type(*stds,"RA") == CPL_TYPE_FLOAT) {
304  ra = cpl_table_get_data_float(*stds,"RA");
305  dec = cpl_table_get_data_float(*stds,"Dec");
306  x = cpl_malloc(n*sizeof(*x));
307  y = cpl_malloc(n*sizeof(*y));
308  for (i = 0; i < n; i++) {
309  r = (double)ra[i];
310  d = (double)dec[i];
311  casu_radectoxy(wcs,r,d,&xx,&yy);
312  x[i] = (float)xx;
313  y[i] = (float)yy;
314  }
315  } else {
316  rad = cpl_table_get_data_double(*stds,"RA");
317  decd = cpl_table_get_data_double(*stds,"Dec");
318  x = cpl_malloc(n*sizeof(*x));
319  y = cpl_malloc(n*sizeof(*y));
320  for (i = 0; i < n; i++) {
321  r = rad[i];
322  d = decd[i];
323  casu_radectoxy(wcs,r,d,&xx,&yy);
324  x[i] = (float)xx;
325  y[i] = (float)yy;
326  }
327  }
328  cpl_wcs_delete(wcs);
329 
330  /* Add the predicted x,y coordinates columns to the table */
331 
332  cpl_table_wrap_float(*stds,x,"xpredict");
333  cpl_table_set_column_unit(*stds,"xpredict","pixels");
334  cpl_table_wrap_float(*stds,y,"ypredict");
335  cpl_table_set_column_unit(*stds,"ypredict","pixels");
336 
337  /* Finally sort this by ypredict */
338 
339  p = cpl_propertylist_new();
340  cpl_propertylist_append_bool(p,"ypredict",0);
341  cpl_table_sort(*stds,p);
342  cpl_propertylist_delete(p);
343 
344  /* Get out of here */
345 
346  GOOD_STATUS
347 }
348 
349 /*---------------------------------------------------------------------------*/
387 /*---------------------------------------------------------------------------*/
388 
389 static const int maxcds = 7;
390 static const char *cdscatnames[8] =
391  {"","2mass","usnob","ppmxl","landolt","wise", "apass", "gaiaedr3"};
392 static const char *cdscats[8] =
393  {"","II/246","I/284","I/317","II/183A","II/311", "II/336", "I/350"};
394 
395 extern int casu_getstds_cdslist(int cdschoice, char **cdscatname,
396  char **cdscatid, int *status) {
397  const char *fctid = "casu_getstds_cdslist";
398 
399  if (*status != CASU_OK)
400  return(*status);
401 
402  /* Check that the choice is within the bounds of possibility */
403 
404  *cdscatname = NULL;
405  *cdscatid = NULL;
406  if (cdschoice < 0 || cdschoice > maxcds) {
407  cpl_msg_error(fctid,"CDS catalogue choice must be >= 0 && <= %d",
408  maxcds);
409  return(CASU_FATAL);
410  }
411  *cdscatname = cpl_strdup(cdscatnames[cdschoice]);
412  *cdscatid = cpl_strdup(cdscats[cdschoice]);
413  GOOD_STATUS
414 }
415 
416 /*---------------------------------------------------------------------------*/
441 /*---------------------------------------------------------------------------*/
442 
443 extern int casu_get_cdschoice(const char* cdscatname)
444 {
445 
446  for (int i=1; i<=maxcds; i++) {
447  if (!strcmp(cdscatname, cdscatnames[i])) {
448  return i;
449  }
450  }
451  return 0;
452 }
453 
454 
455 /*---------------------------------------------------------------------------*/
483 /*---------------------------------------------------------------------------*/
484 
485 static cpl_table *casu_2mass_extract(char *path, float ramin1, float ramax1,
486  float decmin, float decmax) {
487  cpl_table *t,*s,*o;
488  int i,nrows,start,finish,first_index,last_index,irow,init,j;
489  int first_index_ra,last_index_ra,wrap,iwrap;
490  float dectest,ratest,ramin,ramax;
491  char fullname[SZBUF];
492  cpl_array *a;
493  const char *deccol = "Dec";
494  cpl_propertylist *p;
495 
496  /* Create an output table */
497 
498  o = cpl_table_new(0);
499  init = 1;
500 
501  /* Create a cpl array */
502 
503  a = cpl_array_wrap_string((char **)&deccol,1);
504 
505  /* Is there a wrap around problem? */
506 
507  wrap = (ramin1 < 0.0 && ramax1 > 0.0) ? 2 : 1;
508 
509  /* Loop for each query. If there is a wrap around problem then we need 2
510  queries. If not, then we only need 1 */
511 
512  for (iwrap = 0; iwrap < wrap; iwrap++) {
513  if (wrap == 2) {
514  if (iwrap == 0) {
515  ramin = ramin1 + 360.0;
516  ramax = 360.0;
517  } else {
518  ramin = 0.000001;
519  ramax = ramax1;
520  }
521  } else {
522  ramin = ramin1;
523  ramax = ramax1;
524  }
525 
526  /* Find out where in the index to look */
527 
528  first_index_ra = (int)ramin;
529  last_index_ra = min((int)ramax,359);
530 
531  /* Look at the min and max RA and decide which files need to be
532  opened. */
533 
534  for (i = first_index_ra; i <= last_index_ra; i++) {
535 
536  /* Ok, we've found one that needs opening. Read the file with
537  the relevant CPL call */
538 
539  (void)snprintf(fullname,SZBUF,"%s/npsc%03d.fits",path,i);
540 
541  /* Read the propertylist so that you know how many rows there
542  are in the table */
543 
544  p = cpl_propertylist_load(fullname,1);
545  if (p == NULL) {
546  freetable(o);
547  cpl_array_unwrap(a);
548  return(NULL);
549  }
550  nrows = cpl_propertylist_get_int(p,"NAXIS2");
551  cpl_propertylist_delete(p);
552 
553  /* Load various rows until you find the Dec range that you
554  have specified. First the minimum Dec */
555 
556  start = 0;
557  finish = nrows;
558  first_index = nrows/2;
559  while (finish - start >= 2) {
560  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)first_index,
561  1);
562  dectest = cpl_table_get_double(t,"Dec",0,NULL);
563  cpl_table_delete(t);
564  if (dectest < decmin) {
565  start = first_index;
566  first_index = (first_index + finish)/2;
567  } else {
568  finish = first_index;
569  first_index = (first_index + start)/2;
570  }
571  }
572 
573  /* Load various rows until you find the Dec range that you
574  have specified. Now the maximum Dec */
575 
576  start = first_index;
577  finish = nrows;
578  last_index = start + (finish - start)/2;
579  while (finish - start >= 2) {
580  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)last_index,
581  1);
582  dectest = cpl_table_get_double(t,"Dec",0,NULL);
583  cpl_table_delete(t);
584  if (dectest < decmax) {
585  start = last_index;
586  last_index = (last_index + finish)/2;
587  } else {
588  finish = last_index;
589  last_index = (last_index + start)/2;
590  }
591  }
592  if (last_index < first_index)
593  last_index = first_index;
594 
595  /* Ok now now load all the rows in the relevant dec limits */
596 
597  nrows = last_index - first_index + 1;
598  if ((t = cpl_table_load_window(fullname,1,1,NULL,
599  (cpl_size)first_index,
600  (cpl_size)nrows)) == NULL) {
601  freetable(o);
602  cpl_array_unwrap(a);
603  return(NULL);
604  }
605  cpl_table_unselect_all(t);
606 
607  /* Right, we now know what range of rows to search. Go through
608  these and pick the ones that are in the correct range of RA.
609  If a row qualifies, then 'select' it. */
610 
611  for (j = 0; j < nrows; j++) {
612  ratest = cpl_table_get_double(t,"RA",(cpl_size)j,NULL);
613  if (cpl_error_get_code() != CPL_ERROR_NONE) {
614  cpl_table_delete(t);
615  cpl_array_unwrap(a);
616  freetable(o);
617  return(NULL);
618  }
619  if (ratest >= ramin && ratest <= ramax)
620  cpl_table_select_row(t,(cpl_size)j);
621  }
622 
623  /* Extract the rows that have been selected now and append them
624  onto the output table */
625 
626  s = cpl_table_extract_selected(t);
627  if (init == 1) {
628  cpl_table_copy_structure(o,t);
629  init = 0;
630  }
631  irow = (int)cpl_table_get_nrow(o) + 1;
632  cpl_table_insert(o,s,(cpl_size)irow);
633 
634  /* Tidy up */
635 
636  cpl_table_delete(t);
637  cpl_table_delete(s);
638  }
639  }
640 
641  /* Ok, now just return the table and get out of here */
642 
643  cpl_array_unwrap(a);
644  return(o);
645 }
646 
647 /*---------------------------------------------------------------------------*/
675 /*---------------------------------------------------------------------------*/
676 
677 static cpl_table *casu_ppmxl_extract(char *path, float ramin1, float ramax1,
678  float decmin, float decmax) {
679  cpl_table *t,*s,*o;
680  int i,nrows,start,finish,first_index,last_index,irow,init,j;
681  int first_index_ra,last_index_ra,wrap,iwrap;
682  float dectest,ratest,ramin,ramax;
683  char fullname[SZBUF];
684  cpl_array *a;
685  const char *deccol = "Dec";
686  cpl_propertylist *p;
687 
688  /* Create an output table */
689 
690  o = cpl_table_new(0);
691  init = 1;
692 
693  /* Create a cpl array */
694 
695  a = cpl_array_wrap_string((char **)&deccol,1);
696 
697  /* Is there a wrap around problem? */
698 
699  wrap = (ramin1 < 0.0 && ramax1 > 0.0) ? 2 : 1;
700 
701  /* Loop for each query. If there is a wrap around problem then we need 2
702  queries. If not, then we only need 1 */
703 
704  for (iwrap = 0; iwrap < wrap; iwrap++) {
705  if (wrap == 2) {
706  if (iwrap == 0) {
707  ramin = ramin1 + 360.0;
708  ramax = 360.0;
709  } else {
710  ramin = 0.000001;
711  ramax = ramax1;
712  }
713  } else {
714  ramin = ramin1;
715  ramax = ramax1;
716  }
717 
718  /* Find out where in the index to look */
719 
720  first_index_ra = (int)ramin;
721  last_index_ra = min((int)ramax,359);
722 
723  /* Look at the min and max RA and decide which files need to be
724  opened. */
725 
726  for (i = first_index_ra; i <= last_index_ra; i++) {
727 
728  /* Ok, we've found one that needs opening. Read the file with
729  the relevant CPL call */
730 
731  (void)snprintf(fullname,SZBUF,"%s/nppmxl%03d.fits",path,i);
732 
733  /* Read the propertylist so that you know how many rows there
734  are in the table */
735 
736  p = cpl_propertylist_load(fullname,1);
737  if (p == NULL) {
738  freetable(o);
739  cpl_array_unwrap(a);
740  return(NULL);
741  }
742  nrows = cpl_propertylist_get_int(p,"NAXIS2");
743  cpl_propertylist_delete(p);
744 
745  /* Load various rows until you find the Dec range that you
746  have specified. First the minimum Dec */
747 
748  start = 0;
749  finish = nrows;
750  first_index = nrows/2;
751  while (finish - start >= 2) {
752  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)first_index,
753  1);
754  dectest = (float)cpl_table_get_double(t,"Dec",0,NULL);
755  cpl_table_delete(t);
756  if (dectest < decmin) {
757  start = first_index;
758  first_index = (first_index + finish)/2;
759  } else {
760  finish = first_index;
761  first_index = (first_index + start)/2;
762  }
763  }
764 
765  /* Load various rows until you find the Dec range that you
766  have specified. Now the maximum Dec */
767 
768  start = first_index;
769  finish = nrows;
770  last_index = start + (finish - start)/2;
771  while (finish - start >= 2) {
772  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)last_index,
773  1);
774  dectest = (float)cpl_table_get_double(t,"Dec",0,NULL);
775  cpl_table_delete(t);
776  if (dectest < decmax) {
777  start = last_index;
778  last_index = (last_index + finish)/2;
779  } else {
780  finish = last_index;
781  last_index = (last_index + start)/2;
782  }
783  }
784  if (last_index < first_index)
785  last_index = first_index;
786 
787  /* Ok now now load all the rows in the relevant dec limits */
788 
789  nrows = last_index - first_index + 1;
790  if ((t = cpl_table_load_window(fullname,1,1,NULL,
791  (cpl_size)first_index,
792  (cpl_size)nrows)) == NULL) {
793  freetable(o);
794  cpl_array_unwrap(a);
795  return(NULL);
796  }
797  cpl_table_unselect_all(t);
798 
799  /* Right, we now know what range of rows to search. Go through
800  these and pick the ones that are in the correct range of RA.
801  If a row qualifies, then 'select' it. */
802 
803  for (j = 0; j < nrows; j++) {
804  ratest = (float)cpl_table_get_double(t,"RA",(cpl_size)j,NULL);
805  if (cpl_error_get_code() != CPL_ERROR_NONE) {
806  cpl_table_delete(t);
807  cpl_array_unwrap(a);
808  freetable(o);
809  return(NULL);
810  }
811  if (ratest >= ramin && ratest <= ramax)
812  cpl_table_select_row(t,(cpl_size)j);
813  }
814 
815  /* Extract the rows that have been selected now and append them
816  onto the output table */
817 
818  s = cpl_table_extract_selected(t);
819  if (init == 1) {
820  cpl_table_copy_structure(o,t);
821  init = 0;
822  }
823  irow = (int)cpl_table_get_nrow(o) + 1;
824  cpl_table_insert(o,s,(cpl_size)irow);
825 
826  /* Tidy up */
827 
828  cpl_table_delete(t);
829  cpl_table_delete(s);
830  }
831  }
832 
833  /* Ok, now just return the table and get out of here */
834 
835  cpl_array_unwrap(a);
836  return(o);
837 }
838 
839 /*---------------------------------------------------------------------------*/
867 /*---------------------------------------------------------------------------*/
868 
869 static cpl_table *casu_apass_extract(char *path, float ramin1, float ramax1,
870  float decmin, float decmax) {
871  cpl_table *t,*s,*o;
872  int i,nrows,start,finish,first_index,last_index,irow,init,j;
873  int first_index_ra,last_index_ra,wrap,iwrap;
874  float dectest,ratest,ramin,ramax;
875  char fullname[SZBUF];
876  cpl_array *a;
877  const char *racol = "RA";
878  cpl_propertylist *p;
879 
880  /* Create an output table */
881 
882  o = cpl_table_new(0);
883  init = 1;
884 
885  /* Create a cpl array */
886 
887  a = cpl_array_wrap_string((char **)&racol,1);
888 
889  /* Is there a wrap around problem? */
890 
891  wrap = (ramin1 < 0.0 && ramax1 > 0.0) ? 2 : 1;
892 
893  /* Loop for each query. If there is a wrap around problem then we need 2
894  queries. If not, then we only need 1 */
895 
896  for (iwrap = 0; iwrap < wrap; iwrap++) {
897  if (wrap == 2) {
898  if (iwrap == 0) {
899  ramin = ramin1 + 360.0;
900  ramax = 360.0;
901  } else {
902  ramin = 0.000001;
903  ramax = ramax1;
904  }
905  } else {
906  ramin = ramin1;
907  ramax = ramax1;
908  }
909 
910  /* Find out where in the index to look */
911 
912  first_index_ra = (int)ramin;
913  last_index_ra = min((int)ramax,359);
914 
915  /* Look at the min and max RA and decide which files need to be
916  opened. */
917 
918  for (i = first_index_ra; i <= last_index_ra; i++) {
919 
920  /* Ok, we've found one that needs opening. Read the file with
921  the relevant CPL call */
922 
923  (void)snprintf(fullname,SZBUF,"%s/casuapass_%03d.fits",path,i);
924 
925  /* Read the propertylist so that you know how many rows there
926  are in the table */
927 
928  p = cpl_propertylist_load(fullname,1);
929  if (p == NULL) {
930  freetable(o);
931  cpl_array_unwrap(a);
932  return(NULL);
933  }
934  nrows = cpl_propertylist_get_int(p,"NAXIS2");
935  cpl_propertylist_delete(p);
936 
937  /* Load various rows until you find the RA range that you
938  have specified. First the minimum RA */
939 
940  start = 0;
941  finish = nrows;
942  first_index = nrows/2;
943  while (finish - start >= 2) {
944  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)first_index,
945  1);
946  ratest = (float)cpl_table_get_double(t,"RA",0,NULL);
947  cpl_table_delete(t);
948  if (ratest < ramin) {
949  start = first_index;
950  first_index = (first_index + finish)/2;
951  } else {
952  finish = first_index;
953  first_index = (first_index + start)/2;
954  }
955  }
956 
957  /* Load various rows until you find the RA range that you
958  have specified. Now the maximum RA */
959 
960  start = first_index;
961  finish = nrows;
962  last_index = start + (finish - start)/2;
963  while (finish - start >= 2) {
964  t = cpl_table_load_window(fullname,1,1,a,(cpl_size)last_index,
965  1);
966  ratest = (float)cpl_table_get_double(t,"RA",0,NULL);
967  cpl_table_delete(t);
968  if (ratest < ramax) {
969  start = last_index;
970  last_index = (last_index + finish)/2;
971  } else {
972  finish = last_index;
973  last_index = (last_index + start)/2;
974  }
975  }
976  if (last_index < first_index)
977  last_index = first_index;
978 
979  /* Ok now now load all the rows in the relevant RA limits */
980 
981  nrows = last_index - first_index + 1;
982  if ((t = cpl_table_load_window(fullname,1,1,NULL,
983  (cpl_size)first_index,
984  (cpl_size)nrows)) == NULL) {
985  freetable(o);
986  cpl_array_unwrap(a);
987  return(NULL);
988  }
989  cpl_table_unselect_all(t);
990 
991  /* Right, we now know what range of rows to search. Go through
992  these and pick the ones that are in the correct range of RA.
993  If a row qualifies, then 'select' it. */
994 
995  for (j = 0; j < nrows; j++) {
996  dectest = (float)cpl_table_get_double(t,"Dec",(cpl_size)j,NULL);
997  if (cpl_error_get_code() != CPL_ERROR_NONE) {
998  cpl_table_delete(t);
999  cpl_array_unwrap(a);
1000  freetable(o);
1001  return(NULL);
1002  }
1003  if (dectest >= decmin && dectest <= decmax)
1004  cpl_table_select_row(t,(cpl_size)j);
1005  }
1006 
1007  /* Extract the rows that have been selected now and append them
1008  onto the output table */
1009 
1010  s = cpl_table_extract_selected(t);
1011  if (init == 1) {
1012  cpl_table_copy_structure(o,t);
1013  init = 0;
1014  }
1015  irow = (int)cpl_table_get_nrow(o) + 1;
1016  cpl_table_insert(o,s,(cpl_size)irow);
1017 
1018  /* Tidy up */
1019 
1020  cpl_table_delete(t);
1021  cpl_table_delete(s);
1022  }
1023  }
1024 
1025  /* Ok, now just return the table and get out of here */
1026 
1027  cpl_array_unwrap(a);
1028  return(o);
1029 }
1030 
1031 /*---------------------------------------------------------------------------*/
1060 /*---------------------------------------------------------------------------*/
1061 
1062 static cpl_table *casu_local_extract(char *path, float ramin1, float ramax1,
1063  float decmin, float decmax) {
1064  cpl_table *whole,*out,*out2;
1065  int wrap,iwrap,nrows;
1066  float ramin,ramax;
1067  cpl_propertylist *p;
1068 
1069  /* First read the input file. It's assumed this isn't so big that it's
1070  going to take very long. */
1071 
1072  whole = cpl_table_load((const char *)path,1,0);
1073 
1074  /* Is there a wrap around problem? */
1075 
1076  wrap = (ramin1 < 0.0 && ramax1 > 0.0) ? 2 : 1;
1077 
1078  /* Loop for each query. If there is a wrap around problem then we need 2
1079  queries. If not, then we only need 1 */
1080 
1081  out = NULL;
1082  for (iwrap = 0; iwrap < wrap; iwrap++) {
1083  if (wrap == 2) {
1084  if (iwrap == 0) {
1085  ramin = ramin1 + 360.0;
1086  ramax = 360.0;
1087  } else {
1088  ramin = 0.000001;
1089  ramax = ramax1;
1090  }
1091  } else {
1092  ramin = ramin1;
1093  ramax = ramax1;
1094  }
1095  cpl_table_unselect_all(whole);
1096 
1097  /* Select on RA and Dec */
1098 
1099  cpl_table_or_selected_double(whole,"RA",CPL_NOT_LESS_THAN,
1100  (double)ramin);
1101  cpl_table_and_selected_double(whole,"RA",CPL_NOT_GREATER_THAN,
1102  (double)ramax);
1103  cpl_table_and_selected_double(whole,"Dec",CPL_NOT_LESS_THAN,
1104  (double)decmin);
1105  cpl_table_and_selected_double(whole,"Dec",CPL_NOT_GREATER_THAN,
1106  (double)decmax);
1107 
1108  /* If this is the first time through, then create the output table */
1109 
1110  if (out == NULL) {
1111  out = cpl_table_extract_selected(whole);
1112  } else {
1113  out2 = cpl_table_extract_selected(whole);
1114  nrows = cpl_table_get_nrow(out);
1115  cpl_table_insert(out,out2,nrows+1);
1116  cpl_table_delete(out2);
1117  }
1118  }
1119 
1120  /* Sort the table by Dec and get out of here*/
1121 
1122  p = cpl_propertylist_new();
1123  cpl_propertylist_append_bool(p,"Dec",0);
1124  cpl_table_sort(out,p);
1125  cpl_propertylist_delete(p);
1126  return(out);
1127 }
1128 
1129 /*---------------------------------------------------------------------------*/
1159 /*---------------------------------------------------------------------------*/
1160 
1161 static cpl_table *check_cache(char *catname, float ra1_im, float ra2_im,
1162  float dec1_im, float dec2_im, char *cacheloc) {
1163  int wrap1,wrap2;
1164  FILE *fd;
1165  char fname[BUFSIZ],catname2[SZBUF];
1166  char * cat_cache = NULL;
1167  float best,ra1_cat,ra2_cat,dec1_cat,dec2_cat,d1,d2,fra,fdec,ftot;
1168  cpl_table *out_cat;
1169 
1170  /* Open the index file. NB the path and file name are hardcoded */
1171 
1172  (void)snprintf(fname,BUFSIZ,"%s/%s",cacheloc,CACHEIND);
1173  fd = fopen(fname,"r");
1174  if (fd == NULL)
1175  return(NULL);
1176 
1177  /* Check to see if there is wrap around in the coordinates */
1178 
1179  wrap1 = (ra1_im < 0.0 ? 1 : 0);
1180 
1181  /* Now see if you have any matching entries */
1182 
1183  best = 0.0;
1184  // Todo: the hard coded numbers should be changed to use BUFSIZ and SZBUF
1185  while (fscanf(fd,"%8191s %1023s %g %g %g %g",
1186  fname,catname2,&ra1_cat,&ra2_cat,
1187  &dec1_cat,&dec2_cat) != EOF) {
1188  wrap2 = (ra1_cat < 0.0 ? 1 : 0);
1189  if (wrap1 != wrap2)
1190  continue;
1191  if (strcmp(catname,catname2))
1192  continue;
1193 
1194  /* Check to see if there is at least some overlap */
1195 
1196  if (!(((ra1_im >= ra1_cat && ra1_im <= ra2_cat) ||
1197  (ra2_im >= ra1_cat && ra2_im <= ra2_cat)) &&
1198  ((dec1_im >= dec1_cat && dec1_im <= dec2_cat) ||
1199  (dec2_im >= dec1_cat && dec2_im <= dec2_cat))))
1200  continue;
1201 
1202  /* Work out exactly how much there is in each coordinate */
1203 
1204  d1 = max(0.0,ra1_cat-ra1_im);
1205  d2 = max(0.0,ra2_im-ra2_cat);
1206  fra = 1.0 - (d1 + d2)/(ra2_im - ra1_im);
1207  d1 = max(0.0,dec1_cat-dec1_im);
1208  d2 = max(0.0,dec2_im-dec2_cat);
1209  fdec = 1.0 - (d1 + d2)/(dec2_im - dec1_im);
1210  ftot = fra*fdec;
1211 
1212  /* Keep track of which is the best one */
1213 
1214  if (ftot > best) {
1215  cpl_free(cat_cache); /* As we are in a loop free old allocation */
1216  cat_cache = cpl_sprintf("%s/%s/%s",cacheloc,CACHEDIR,fname);
1217  best = ftot;
1218  }
1219  }
1220  fclose(fd);
1221 
1222  /* Return a bad status if there isn't sufficient overlap */
1223 
1224  if (best < 0.9)
1225  return(NULL);
1226 
1227  /* If there is enough overlap, then try and read the FITS table. If it
1228  reads successfully, then return the table pointer */
1229 
1230  out_cat = cpl_table_load(cat_cache,1,1);
1231  cpl_free(cat_cache);
1232 
1233  cpl_msg_info(cpl_func, "Using cached catalogue data");
1234 
1235  return(out_cat);
1236 }
1237 
1238 /*---------------------------------------------------------------------------*/
1266 /*---------------------------------------------------------------------------*/
1267 
1268 static void addto_cache(cpl_table *stds, char *catname, float ramin,
1269  float ramax, float decmin, float decmax,
1270  char *cacheloc) {
1271  FILE *fd;
1272  char newname[SZBUF];
1273  int i;
1274 
1275  /* Check to see if the cache directory exists. If it doesn't, then create
1276  it. */
1277 
1278  (void)snprintf(newname,SZBUF,"%s/%s",cacheloc,CACHEDIR);
1279  if (access(newname,0) != 0)
1280  mkdir(newname,0755);
1281 
1282  /* Open the index file with 'append' access */
1283 
1284  (void)snprintf(newname,SZBUF,"%s/%s",cacheloc,CACHEIND);
1285  fd = fopen(newname,"a");
1286 
1287  /* Check the files in the directory to get a number that isn't already
1288  being used */
1289 
1290  i = 0;
1291  while (i >= 0) {
1292  i++;
1293  snprintf(newname,SZBUF,"%s/%s/cch_%08d",cacheloc,CACHEDIR,i);
1294  if (access(newname,F_OK) != 0)
1295  break;
1296  }
1297 
1298  /* Now write the current entry and make a copy of the file into the
1299  directory */
1300 
1301  snprintf(newname,SZBUF,"%s/%s/cch_%08d",cacheloc,CACHEDIR,i);
1302  cpl_table_save(stds,NULL,NULL,newname,CPL_IO_DEFAULT);
1303  snprintf(newname,SZBUF,"cch_%08d",i);
1304  if (cpl_error_get_code() == CPL_ERROR_NONE)
1305  fprintf(fd, "%s %s %g %g %g %g\n",newname,catname,ramin-0.0005,
1306  ramax+0.0005,decmin-0.0005,decmax+0.0005);
1307  fclose(fd);
1308 }
1309 
1310 /*---------------------------------------------------------------------------*/
1333 /*---------------------------------------------------------------------------*/
1334 
1335 static int get_response(int sock, cpl_table **outtab) {
1336  int nnewline,i,nv,irem = 0,fd;
1337  long nrows;
1338  char buf[BUFMAX],outfile[32];
1339  const char *fctid = "get_response";
1340 
1341  /* Read from the socket until we find the double newline */
1342 
1343  nnewline = 0;
1344  while (1) {
1345  nv = recv(sock,buf,sizeof(buf),0);
1346  if (nv <= 0) {
1347  cpl_msg_warning(fctid,"Unable to find double newline");
1348  return(CASU_FATAL);
1349  }
1350  for (i = 0; i < nv; i++) {
1351  if (buf[i] == '\n') {
1352  nnewline++;
1353  if (nnewline == 2)
1354  break;
1355  } else {
1356  if (nnewline && buf[i] != '\r')
1357  nnewline = 0;
1358  }
1359  }
1360  if (nnewline == 2) {
1361  irem = i + 1;
1362  break;
1363  }
1364  }
1365 
1366  /* Shift the buffer */
1367 
1368  for (i = irem; i < nv; i++)
1369  buf[i-irem] = buf[i];
1370  nv -= irem;
1371 
1372  /* Open the output file */
1373 
1374  (void)snprintf(outfile,32,"stdsXXXXXX");
1375  fd = mkstemp(outfile);
1376 
1377  /* Now write the data to the output file */
1378 
1379  while (1) {
1380  ssize_t written = write(fd,buf,nv);
1381  if(written != nv) {
1382  cpl_msg_info(cpl_func, "Not all bytes could be written - "
1383  "check diskspace");
1384  }
1385  nv = recv(sock,buf,sizeof(buf),0);
1386  if (nv == 0)
1387  break;
1388  else if (nv < 0) {
1389  cpl_msg_warning(fctid,"Read from socket failed");
1390  close(fd);
1391  remove(outfile);
1392  return(CASU_FATAL);
1393  }
1394  }
1395 
1396  /* Close the output file. Check and see if a table was created and whether
1397  or not it contains any rows... */
1398 
1399  close(fd);
1400  *outtab = cpl_table_load(outfile,1,0);
1401  if (cpl_error_get_code() != CPL_ERROR_NONE) {
1402  cpl_msg_warning(fctid,"No standards were found");
1403  cpl_error_reset();
1404  remove(outfile);
1405  *outtab = NULL;
1406  return(CASU_FATAL);
1407  } else {
1408  nrows = (int)cpl_table_get_nrow(*outtab);
1409  if (nrows <= 0) {
1410  cpl_msg_warning(fctid,"No standards table had no rows");
1411  remove(outfile);
1412  freetable(*outtab);
1413  return(CASU_FATAL);
1414  }
1415  }
1416 
1417  /* Get out of here */
1418 
1419  remove(outfile);
1420  return(CASU_OK);
1421 }
1422 
1423 /*---------------------------------------------------------------------------*/
1449 /*---------------------------------------------------------------------------*/
1450 
1451 static int send_request(int sock, char *req_string, int isbin) {
1452  char buf[BUFMAX];
1453  const char *fctid = "send_request";
1454 
1455  /* Form the full request contents and send it. */
1456 
1457  if (isbin) {
1458  sprintf(buf,"GET /viz-bin/asu-binfits?%s HTTP/1.0\r\n\r\n",req_string);
1459  } else {
1460  sprintf(buf,"GET /viz-bin/asu-fits?%s HTTP/1.0\r\n\r\n",req_string);
1461  }
1462  if (send(sock,buf,strlen(buf),0) < 0) {
1463  cpl_msg_warning(fctid,"Attempt to send message failed, error: %d\n",
1464  errno);
1465  return(CASU_FATAL);
1466  }
1467  return(CASU_OK);
1468 }
1469 
1470 /*---------------------------------------------------------------------------*/
1493 /*---------------------------------------------------------------------------*/
1494 
1495 static int connect_site(const char *site, int *sock) {
1496  struct hostent *hp;
1497  struct sockaddr_in ssin;
1498  const char *fctid = "connect_site";
1499 
1500  /* Get host pointer */
1501 
1502  hp = gethostbyname(site);
1503  if (hp == NULL) {
1504  cpl_msg_warning(fctid,"Unable to get host information for %s\n",site);
1505  return(CASU_FATAL);
1506  }
1507  ssin.sin_family = hp->h_addrtype;
1508  memcpy(&ssin.sin_addr,hp->h_addr_list[0],hp->h_length);
1509  ssin.sin_port = htons(PORT);
1510 
1511  /* Create an IP-family socket on which to make the connection */
1512 
1513  *sock = socket(hp->h_addrtype,SOCK_STREAM,0);
1514  if (*sock < 0) {
1515  cpl_msg_warning(fctid,"Unable to create socket descriptor for %s\n",
1516  site);
1517  return (CASU_FATAL);
1518  }
1519 
1520  /* Connect to that address...*/
1521 
1522  if (connect(*sock,(struct sockaddr*)&ssin,sizeof(ssin)) < 0) {
1523  cpl_msg_warning(fctid,"Unable to connect to site: %s\n",site);
1524  return(CASU_FATAL);
1525  }
1526 
1527  /* Otherwise send back a good status */
1528 
1529  return(CASU_OK);
1530 }
1531 
1532 /*---------------------------------------------------------------------------*/
1560 /*---------------------------------------------------------------------------*/
1561 
1562 static char *form_request(float ra, float dec, float dra, float ddec,
1563  float equinox, const char *catid) {
1564  static char buf[2*BUFSIZ];
1565  char buf2[BUFSIZ],equi[1];
1566 
1567  /* Format each of the fields and then add them to the buffer that will
1568  become the query string. URLencode them before appending. First
1569  the coordinates */
1570 
1571  (void)snprintf(buf2,BUFSIZ,"-c=%8.3f %8.3f",ra,dec);
1572  (void)strncpy(buf,url_encode(buf2),BUFSIZ);
1573 
1574  /* Now the box size */
1575 
1576  (void)snprintf(buf2,BUFSIZ,"-c.bd=%g/%g",dra,ddec);
1577  (void)strncat(buf,"&",BUFSIZ);
1578  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1579 
1580  /* The catalogue name */
1581 
1582  (void)snprintf(buf2,BUFSIZ,"-source=%s",catid);
1583  (void)strncat(buf,"&",BUFSIZ);
1584  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1585 
1586  /* The equinox */
1587 
1588  equi[0] = (equinox == 1950.0 ? 'B' : 'J');
1589  (void)snprintf(buf2,BUFSIZ,"-c.eq=%c%g",equi[0],equinox);
1590  (void)strncat(buf,"&",BUFSIZ);
1591  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1592 
1593  /* Finally make sure that we don't truncate the result and that we
1594  sort by RA */
1595 
1596  (void)snprintf(buf2,BUFSIZ,"-out.max=unlimited");
1597  (void)strncat(buf,"&",BUFSIZ);
1598  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1599  (void)snprintf(buf2,BUFSIZ,"-sort=_RA*-c.eq");
1600  (void)strncat(buf,"&",BUFSIZ);
1601  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1602  (void)snprintf(buf2,BUFSIZ,"-out.add=_RA*-c.eq");
1603  (void)strncat(buf,"&",BUFSIZ);
1604  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1605  (void)snprintf(buf2,BUFSIZ,"-out.add=_DEC*-c.eq");
1606  (void)strncat(buf,"&",BUFSIZ);
1607  (void)strncat(buf,url_encode(buf2),BUFSIZ);
1608 
1609  /* Right, get out of here */
1610 
1611  return(buf);
1612 }
1613 
1614 /*---------------------------------------------------------------------------*/
1632 /*---------------------------------------------------------------------------*/
1633 
1634 static char *url_encode(char *instring) {
1635  static char buf[BUFSIZ];
1636  int i,j,k,len;
1637 
1638  /* First copy over everything before the equal sign */
1639 
1640  k = 0;
1641  do {
1642  buf[k] = instring[k];
1643  } while (instring[k++] != '=');
1644  len = strlen(instring);
1645 
1646  /* Copy over the string, substituting the things that we don't want in
1647  a URL */
1648 
1649  j = k;
1650  for (i = k; i < len; i++) {
1651  assert(j < (int)sizeof(buf));
1652  if (instring[i] == ' ')
1653  buf[j++] = '+';
1654  else if (isalnum((unsigned char)instring[i]))
1655  buf[j++] = instring[i];
1656  else {
1657  sprintf(buf+j,"%%%2x",(unsigned char)instring[i]);
1658  j += 3;
1659  }
1660  }
1661  buf[j] = '\0';
1662  return(buf);
1663 }
1664 
1668 /*
1669 
1670 $Log: casu_getstds.c,v $
1671 Revision 1.7 2015/11/27 12:06:41 jim
1672 Detabbed a few things
1673 
1674 Revision 1.6 2015/11/25 13:50:03 jim
1675 cacheloc parameter added
1676 
1677 Revision 1.5 2015/11/25 10:24:49 jim
1678 Changed so that catpath always contains the full path of a file and the
1679 file name is stripped off in the event that you're using local
1680 2mass, apass or ppmxl catalogues
1681 
1682 Revision 1.4 2015/10/04 18:43:14 jim
1683 fixed small bug when no standards are returned
1684 
1685 Revision 1.3 2015/08/07 13:06:54 jim
1686 Fixed copyright to ESO
1687 
1688 Revision 1.2 2015/08/06 05:34:02 jim
1689 Fixes to get rid of compiler moans
1690 
1691 Revision 1.1.1.1 2015/06/12 10:44:32 jim
1692 Initial import
1693 
1694 Revision 1.15 2015/05/20 11:58:00 jim
1695 Changed URL for CDS
1696 
1697 Revision 1.14 2015/05/14 11:46:33 jim
1698 All RA and Dec columns are officially double
1699 
1700 Revision 1.13 2015/05/13 11:42:27 jim
1701 superficial changes
1702 
1703 Revision 1.12 2015/03/12 09:16:51 jim
1704 Modified to remove some compiler moans
1705 
1706 Revision 1.11 2015/03/09 19:51:58 jim
1707 Fixed undefined value
1708 
1709 Revision 1.10 2015/03/03 10:48:11 jim
1710 Fixed some memory leaks
1711 
1712 Revision 1.9 2015/01/29 11:48:15 jim
1713 modified comments
1714 
1715 Revision 1.8 2015/01/09 12:13:15 jim
1716 *** empty log message ***
1717 
1718 Revision 1.7 2014/12/11 12:23:33 jim
1719 new version
1720 
1721 Revision 1.6 2014/04/23 05:39:41 jim
1722 Now reads local standard tables
1723 
1724 Revision 1.5 2014/04/09 11:26:36 jim
1725 Order of catalogues for CDS was wrong
1726 
1727 Revision 1.4 2014/04/09 11:08:21 jim
1728 Get rid of a couple of compiler moans
1729 
1730 Revision 1.3 2014/04/09 09:09:51 jim
1731 Detabbed
1732 
1733 Revision 1.2 2013/11/21 09:38:13 jim
1734 detabbed
1735 
1736 Revision 1.1.1.1 2013-08-27 12:07:48 jim
1737 Imported
1738 
1739 
1740 */
int casu_get_cdschoice(const char *cdscatname)
Definition: casu_getstds.c:443
int casu_getstds(cpl_propertylist *plist, int cache, char *path, char *catname, int cdssearch, char *cacheloc, cpl_table **stds, int *status)
Get a table of standard stars that appear on an image from a catalogue.
Definition: casu_getstds.c:159
void casu_radectoxy(cpl_wcs *wcs, double ra, double dec, double *x, double *y)
int casu_coverage(cpl_propertylist *plist, int fudge, double *ra1, double *ra2, double *dec1, double *dec2, int *status)