/*-------------------------------------------------------------------------*/
/**
  @file		cache.c
  @author	N. Devillard
  @date		Mar 2001
  @version	$Revision: 1.2 $
  @brief	FITS caching capabilities

  This modules implements a cache for FITS access routines.
  The first time a FITS file is seen by the library, all corresponding
  pointers are cached here. This speeds up multiple accesses to large
  files by magnitudes.
*/
/*--------------------------------------------------------------------------*/

/*
	$Id: cache.c,v 1.2 2002/01/10 08:53:08 ndevilla Exp $
	$Author: ndevilla $
	$Date: 2002/01/10 08:53:08 $
	$Revision: 1.2 $
*/

/*---------------------------------------------------------------------------
   								Includes
 ---------------------------------------------------------------------------*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>

#include "static_sz.h"
#include "cache.h"
#include "fits_p.h"
#include "fits_std.h"


/*---------------------------------------------------------------------------
   								Defines
 ---------------------------------------------------------------------------*/

/** Define this symbol to get debug symbols -- not recommended! */
#define QFITS_CACHE_DEBUG	0
#if QFITS_CACHE_DEBUG
#define qdebug( code ) { code }
#else
#define qdebug( code )
#endif


/**
 * Minimum cache size: this avoids repeated mallocs to increase
 * the cache size for every new file. The first table already contains
 * space for MINSZ entries, and its size is doubled every time it is
 * overflowed.
 */
#define QFITS_CACHE_MINSZ		128

/**
 * Maximum cache size: the cache will be purged every time the number of
 * input FITS files reaches above this limit. This avoids having the cache
 * grow infinitely.
 */

#define QFITS_CACHE_MAXSZ		4096


/**
 * This static definition declares the maximum possible number of
 * extensions in a FITS file. It only has effects in the qfits_cache_add
 * function where a table is statically allocated for efficiency reasons.
 * If the number of extensions grows over this limit, change the value of
 * this constant. If the number of extensions is a priori unknown but can
 * grow much larger than a predictable value, the best solution is to
 * implement a dynamic memory allocation in qfits_cache_add.
 */

#define QFITS_MAX_EXTS			128


/*---------------------------------------------------------------------------
   								New types
 ---------------------------------------------------------------------------*/


/*-------------------------------------------------------------------------*/
/**
  @brief	Cache cell (private)

  This structure stores all informations about a given FITS file.
  It is strictly internal to this module.
 */
/*-------------------------------------------------------------------------*/

typedef struct _qfits_cache_cell_ {
	char	*	name ;	/* File name 	*/
	unsigned	hash ;
	time_t		mtime;  /* Last modification date */
	time_t		ctime;  /* Last modification date */

	int			exts ;	/* # of extensions in file */

	int		*	ohdr ;	/* Offsets to headers */
	int		*	shdr ;	/* Header sizes */
	int		*	data ;	/* Offsets to data */
	int		*	dsiz ;	/* Data sizes */
} qfits_cache_cell ;


static qfits_cache_cell * qfits_cache = NULL ;
static int qfits_cache_last = -1 ;
static int qfits_cache_size =  0 ;


/*---------------------------------------------------------------------------
					Globals (private to this module)
 ---------------------------------------------------------------------------*/

static void * mem_double(void * ptr, int size);
static void qfits_cache_activate(void);
static unsigned qfits_cache_hash(char * key);
static int qfits_is_cached(char * filename);
static int qfits_cache_add(char * name);


/*---------------------------------------------------------------------------
  							Function codes
 ---------------------------------------------------------------------------*/


/* Doubles the allocated size associated to a pointer */
/* 'size' is the current allocated size. */
static void * mem_double(void * ptr, int size)
{
    void    *   newptr ;

    newptr = calloc(2*size, 1);
    memcpy(newptr, ptr, size);
    free(ptr);
    return newptr ;
}


/*
 * Activate cache: initialize cache buffer with minimum size
 */

static void qfits_cache_activate(void)
{
	qdebug(
		printf("qfits: activating cache...\n");
	);
	/* Allocate new array for cache cells */
	qfits_cache = malloc(QFITS_CACHE_MINSZ * sizeof(qfits_cache_cell));
	qfits_cache_size = QFITS_CACHE_MINSZ ;
	/* Register purge function with atexit */
	atexit(qfits_cache_purge);
	return ;
}


/*-------------------------------------------------------------------------*/
/**
  @brief	Purge the qfits cache.
  @return	void

  This function is useful for programs running for a long period,
  to clean up the cache. Ideally in a daemon, it should be called
  by a timer at regular intervals. Notice that since the cache is
  fairly small, you should not need to care too much about this.
 */
/*--------------------------------------------------------------------------*/

void qfits_cache_purge(void)
{
	int	i ;

	if (qfits_cache==NULL)
		return ;
	qdebug(
		printf("qfits: purging cache...\n");
	);

	for (i=0 ; i<qfits_cache_last ; i++) {
		if (qfits_cache[i].name!=NULL)
			free(qfits_cache[i].name);
		if (qfits_cache[i].ohdr!=NULL)
			free(qfits_cache[i].ohdr);
		if (qfits_cache[i].data!=NULL)
			free(qfits_cache[i].data);
		if (qfits_cache[i].shdr!=NULL)
			free(qfits_cache[i].shdr);
		if (qfits_cache[i].dsiz!=NULL)
			free(qfits_cache[i].dsiz);
	}
	free(qfits_cache);
	qfits_cache = NULL ;
	qfits_cache_size = 0 ;
	qfits_cache_last = -1 ;
	return ;
}

/*
 * Simple hash function to speed up name lookups
 */
static unsigned qfits_cache_hash(char * key)
{
    int         len ;
    unsigned    hash ;
    int         i ;

    len = strlen(key);
    for (hash=0, i=0 ; i<len ; i++) {
        hash += (unsigned)key[i] ;
        hash += (hash<<10);
        hash ^= (hash>>6) ;
    }
    hash += (hash <<3);
    hash ^= (hash >>11);
    hash += (hash <<15);
    return hash ;
}



/*
 * Find out if a file is in the cache already
 */
static int qfits_is_cached(char * filename)
{
	int	i ;
	unsigned h ;
	struct stat sta ;

	if (stat(filename, &sta)!=0) {
		return -1 ;
	}
	h = qfits_cache_hash(filename);
	for (i=0 ; i<=qfits_cache_last ; i++) {
		if (qfits_cache[i].hash == h) {
			if (!strcmp(qfits_cache[i].name, filename)) {
				if ((qfits_cache[i].mtime == sta.st_mtime) &&
					(qfits_cache[i].ctime == sta.st_ctime)) {
					return i ;
				}
			}
		}
	}
	return -1 ;
}


/*-------------------------------------------------------------------------*/
/**
  @brief	Query a FITS file offset from the cache.
  @param	filename	Name of the file to examine.
  @param	what		What should be queried (see below).
  @return	an integer offset, or -1 if an error occurred.

  This function queries the cache for FITS offset information. If the
  requested file name has never been seen before, it is completely parsed
  to extract all offset informations, which are then stored in the cache.
  The next query will get the informations from the cache, avoiding
  a complete re-parsing of the file. This is especially useful for large
  FITS files with lots of extensions, because querying the extensions
  is an expensive operation.

  This operation has side-effects: the cache is an automatically
  allocated structure in memory, that can only grow. Every request
  on a new FITS file will make it grow. The structure is pretty
  light-weight in memory, but nonetheless this is an issue for daemon-type
  programs which must run over long periods. The solution is to clean
  the cache using qfits_cache_purge() at regular intervals. This is left
  to the user of this library.

  To request information about a FITS file, you must pass an integer
  built from the following symbols:

  - @c QFITS_QUERY_N_EXT
  - @c QFITS_QUERY_HDR_START
  - @c QFITS_QUERY_DAT_START
  - @c QFITS_QUERY_HDR_SIZE
  - @c QFITS_QUERY_DAT_SIZE

  Querying the number of extensions present in a file is done
  simply with:

  @code
  next = qfits_query(filename, QFITS_QUERY_N_EXT);
  @endcode

  Querying the offset to the i-th extension header is done with:

  @code
  off = qfits_query(filename, QFITS_QUERY_HDR_START | i);
  @endcode

  i.e. you must OR (|) the extension number with the
  @c QFITS_QUERY_HDR_START symbol. Requesting offsets to extension data is
  done in the same way:

  @code
  off = qfits_query(filename, QFITS_QUERY_DAT_START | i);
  @endcode

  Notice that extension 0 is the main header and main data part
  of the FITS file.
 */
/*--------------------------------------------------------------------------*/
int	qfits_query(char * filename, int what)
{
	int	rank ;
	int	which ;
	int	answer ;

	if (qfits_cache==NULL)
		qfits_cache_activate();

	if (qfits_cache_last > QFITS_CACHE_MAXSZ) {
		/* Call a purge of the cache */
		qfits_cache_purge();
		qfits_cache_activate();
	}

	qdebug(
		printf("qfits: cache req %s\n", filename);
	);
	if ((rank=qfits_is_cached(filename))==-1) {
		rank = qfits_cache_add(filename);
	}
	if (rank==-1) {
		qdebug(
			printf("qfits: error adding %s to cache\n", filename);
		);
		return -1 ;
	}

	/* See what was requested */
	answer=-1 ;
	if (what & QFITS_QUERY_N_EXT) {
		qdebug(
			printf("qfits: query n_exts\n");
		);
		answer = qfits_cache[rank].exts ;
	} else if (what & QFITS_QUERY_HDR_START) {
		which = what & (~QFITS_QUERY_HDR_START);
		if (which>=0 && which<=qfits_cache[rank].exts) {
			answer = qfits_cache[rank].ohdr[which] * FITS_BLOCK_SIZE ;
		}
		qdebug(
			printf("qfits: query offset to header %d\n", which);
		);
	} else if (what & QFITS_QUERY_DAT_START) {
		which = what & (~QFITS_QUERY_DAT_START);
		if (which>=0 && which<=qfits_cache[rank].exts) {
			answer = qfits_cache[rank].data[which] * FITS_BLOCK_SIZE ;
		}
		qdebug(
			printf("qfits: query offset to data %d\n", which);
		);
	} else if (what & QFITS_QUERY_HDR_SIZE) {
		which = what & (~QFITS_QUERY_HDR_SIZE);
		if (which>=0 && which<=qfits_cache[rank].exts) {
			answer = qfits_cache[rank].shdr[which] * FITS_BLOCK_SIZE ;
		}
		qdebug(
			printf("qfits: query sizeof header %d\n", which);
		);
	} else if (what & QFITS_QUERY_DAT_SIZE) {
		which = what & (~QFITS_QUERY_DAT_SIZE);
		if (which>=0 && which<=qfits_cache[rank].exts) {
			answer = qfits_cache[rank].dsiz[which] * FITS_BLOCK_SIZE ;
		}
		qdebug(
			printf("qfits: query sizeof data %d\n", which);
		);
	}
	return answer ;
}


/*-------------------------------------------------------------------------*/
/**
  @brief	Add pointer information about a file into the qfits cache.
  @param	filename	Name of the file to examine.
  @return	index to the file information in the cache, or -1 if failure.

  This is the meat of this whole caching business.

  This function picks a file name, and examines the corresponding FITS file
  to deduce all relevant pointers in the file (byte offsets). These byte
  offsets are later used to speed up header lookups. Example: requesting
  some keyword information in the header of the n-th extension will first
  fseek the file to the header start, then search from this position
  onwards. This means that the input FITS file is only parsed for extension
  positions once.

  What this function does is:

  - Open the file, read the first FITS block (@c FITS_BLOCK_SIZE bytes)
  - Check the file is FITS (must have SIMPLE  = at top)
  - Register start of first header at offset 0.
  - Look for END keyword, register start of first data section
    if NAXIS>0.
  - If the EXTEND=T line was found, continue looking for extensions.
  - For each consecutive extension, register extension header start
    and extension data start.

  The initial implementation of this module made use of an mmap() call
  to make the whole file available for search, but this is highly
  unefficient on HPUX, probably because of a weak mmap() implementation
  combined with very slow filesystems. The current implementation makes
  block reads, which should be much faster.
 */
/*--------------------------------------------------------------------------*/

static int qfits_cache_add(char * filename)
{
	FILE	*	in ;
	int			off_hdr[QFITS_MAX_EXTS];
	int			off_dat[QFITS_MAX_EXTS];
	int			i ;
	char		buf[FITS_BLOCK_SIZE] ;
	char	*	buf_c ;
	int			n_blocks ;
	int			found_it ;
	int			xtend ;
	int			naxis ;
	char	*	read_val ;
	int			last ;
	int			end_of_file ;
	int			npix ;
	int			skip_blocks ;
	struct stat sta ;

	qfits_cache_cell * qc ;

	/* Stat file to get its size */
	if (stat(filename, &sta)!=0) {
		qdebug(
			printf("qfits: cannot stat file %s\n", filename);
		);
		return -1 ;
	}

	/* Open input file */
	if ((in=fopen(filename, "r"))==NULL) {
		qdebug(
			printf("qfits: cannot open file %s\n", filename);
		);
		return -1 ;
	}

	/* Read first block in */
	if (fread(buf, 1, FITS_BLOCK_SIZE, in)!=FITS_BLOCK_SIZE) {
		qdebug(
			printf("qfits: error reading first block from %s\n", filename);
		);
		fclose(in);
		return -1 ;
	}
	/* Identify FITS magic number */
	if (buf[0]!='S' ||
		buf[1]!='I' ||
		buf[2]!='M' ||
		buf[3]!='P' ||
		buf[4]!='L' ||
		buf[5]!='E' ||
		buf[6]!=' ' ||
		buf[7]!=' ' ||
		buf[8]!='=') {
		qdebug(
			printf("qfits: file %s is not FITS\n", filename);
		);
		fclose(in);
		return -1 ;
	}

	/*
	 * Browse through file to identify primary HDU size and see if there
	 * might be some extensions. The size of the primary data zone will
	 * also be estimated from the gathering of the NAXIS?? values and
	 * BITPIX.
	 */

	/* Rewind input file, END card might be in first block */
	rewind(in);

	/* Initialize all counters */
	n_blocks = 0 ;
	found_it = 0 ;
	xtend = 0 ;
	naxis = 0 ;
	npix  = 1 ;

	/* Start looking for END card */
	while (found_it==0) {
		/* Read one FITS block */
		if (fread(buf, 1, FITS_BLOCK_SIZE, in)!=FITS_BLOCK_SIZE) {
			qdebug(
				printf("qfits: error reading file %s\n", filename);
			);
			fclose(in);
			return -1 ;
		}
		/* Browse through current block */
		buf_c = buf ;
		for (i=0 ; i<FITS_NCARDS ; i++) {

			/* Look for BITPIX keyword */
			if (buf_c[0]=='B' &&
				buf_c[1]=='I' &&
				buf_c[2]=='T' &&
				buf_c[3]=='P' &&
				buf_c[4]=='I' &&
				buf_c[5]=='X' &&
				buf_c[6]==' ') {
				read_val = qfits_getvalue(buf_c);
				npix *= (int)atoi(read_val) / 8 ;
				if (npix<0) {
					npix=-npix ;
				}
			} else
			/* Look for NAXIS keyword */
			if (buf_c[0]=='N' &&
				buf_c[1]=='A' &&
				buf_c[2]=='X' &&
				buf_c[3]=='I' &&
				buf_c[4]=='S') {

				if (buf_c[5]==' ') {
					/* NAXIS keyword */
					read_val = qfits_getvalue(buf_c);
					naxis = (int)atoi(read_val);
				} else {
					/* NAXIS?? keyword (axis size) */
					read_val = qfits_getvalue(buf_c);
					npix *= (int)atoi(read_val);
				}
			} else
			/* Look for EXTEND keyword */
			if (buf_c[0]=='E' &&
				buf_c[1]=='X' &&
				buf_c[2]=='T' &&
				buf_c[3]=='E' &&
				buf_c[4]=='N' &&
				buf_c[5]=='D' &&
				buf_c[6]==' ') {
				/* The EXTEND keyword is present: might be some extensions */
				read_val = qfits_getvalue(buf_c);
				if (read_val[0]=='T' || read_val[0]=='1') {
					xtend=1 ;
				}
			} else
			/* Look for END keyword */
			if (buf_c[0] == 'E' &&
				buf_c[1] == 'N' &&
				buf_c[2] == 'D' &&
				buf_c[3] == ' ') {
				found_it = 1 ;
			}
			buf_c += FITS_LINESZ ;
		}
		n_blocks ++ ;
	}

	/*
	 * Prepare qfits cache for addition of a new entry
	 */
	qfits_cache_last++ ;
	if (qfits_cache_last==qfits_cache_size) {
		/* New entry goes over allocated size: double the cache size */
		qdebug(
			printf("qfits: doubling cache size\n");
		);
		qfits_cache =
			mem_double(qfits_cache,
						2 * qfits_cache_size * sizeof(qfits_cache_cell));
		qfits_cache_size *= 2 ;
	}
	/* Alias to current pointer in cache for easier reading */
	qc = &(qfits_cache[qfits_cache_last]);
	
	/* Initialize cache cell */
	qc->exts=0 ;
	qc->name = strdup(filename);
	qc->hash = qfits_cache_hash(filename);

	/* Set first HDU offsets */
	off_hdr[0] = 0 ;
	off_dat[0] = n_blocks ;
	
	/* Last is the pointer to the last added extension, plus one. */
	last = 1 ;

	if (xtend) {
		/* Look for extensions */
		qdebug(
			printf("qfits: searching for extensions in %s\n", filename);
		);

		/*
		 * Skip the first data section if pixels were declared
		 */
		if (naxis>0) {
			/* Skip as many blocks as there are declared pixels */
			skip_blocks = npix/FITS_BLOCK_SIZE ;
			if ((npix % FITS_BLOCK_SIZE)!=0) {
				skip_blocks ++ ;
			}
			fseek(in, skip_blocks*FITS_BLOCK_SIZE, SEEK_CUR);
			/* Increase counter of current seen blocks. */
			n_blocks += skip_blocks ;
		}

		/*
		 * Register all extension offsets
		 */
		end_of_file = 0 ;
		while (end_of_file==0) {
			/* Look for extension start */
			found_it=0 ;
			while ((found_it==0) && (end_of_file==0)) {
				if (fread(buf, 1, FITS_BLOCK_SIZE, in)!=FITS_BLOCK_SIZE) {
					/* Reached end of file */
					end_of_file=1 ;
					break ;
				}
				/* Search for XTENSION at block top */
				if (buf[0]=='X' &&
					buf[1]=='T' &&
					buf[2]=='E' &&
					buf[3]=='N' &&
					buf[4]=='S' &&
					buf[5]=='I' &&
					buf[6]=='O' &&
					buf[7]=='N' &&
					buf[8]=='=') {
					/* Got an extension */
					found_it=1 ;
					off_hdr[last] = n_blocks ;
				}
				n_blocks ++ ;
			}
			if (end_of_file && found_it) {
				/* Reached end of file but was expecting an END */
				qdebug(
					printf("qfits: XTENSION without END in %s\n", filename);
				);
				break ;
			}

			/*
			 * Look for extension END
			 * Rewind one block backwards, END might be in same section as
			 * XTENSION start.
			 */
			if (fseek(in, -FITS_BLOCK_SIZE, SEEK_CUR)!=0) {
				qdebug(
					printf("qfits: error fseeking file backwards\n");
				) ;
				fclose(in);
				return -1 ;
			}
			n_blocks -- ;
			found_it=0 ;
			while ((found_it==0) && (end_of_file==0)) {
				if (fread(buf, 1, FITS_BLOCK_SIZE, in)!=FITS_BLOCK_SIZE) {
					qdebug(
					printf("qfits: XTENSION without END in %s\n", filename);
					);
					end_of_file=1;
					break ;
				}

				/* Browse current block for END */
				buf_c = buf ;
				for (i=0 ; i<FITS_NCARDS ; i++) {
					if (buf_c[0]=='E' &&
						buf_c[1]=='N' &&
						buf_c[2]=='D' &&
						buf_c[3]==' ') {
						/* Got the END card */
						found_it=1 ;
						/* Update registered extension list */
						off_dat[last] = n_blocks+1 ;
						last ++ ;
						qc->exts ++ ;
						break ;
					}
					buf_c+=FITS_LINESZ ;
				}
				n_blocks++ ;
			}
		}
	}

	/* Close file */
	fclose(in);

	/* Allocate buffers in cache */
	qc->ohdr = malloc(last * sizeof(int));
	qc->data = malloc(last * sizeof(int));
	qc->shdr = malloc(last * sizeof(int));
	qc->dsiz = malloc(last * sizeof(int));
	/* Store retrieved pointers in the cache */
	for (i=0 ; i<last ; i++) {
		/* Offsets to start */
		qc->ohdr[i] = off_hdr[i];
		qc->data[i] = off_dat[i];

		/* Sizes */
		qc->shdr[i] = off_dat[i] - off_hdr[i] ;
		if (i==last-1) {	
			qc->dsiz[i] = (sta.st_size/FITS_BLOCK_SIZE) - off_dat[i];
		} else {
			qc->dsiz[i] = off_hdr[i+1] - off_dat[i];
		}
	}
	/* Add last modification date */
	qc->mtime = sta.st_mtime ;
	qc->ctime = sta.st_ctime ;

	/* Return index of the added file in the cache */
	return qfits_cache_last ;
}

/* vim: set ts=4 et sw=4 tw=75 */