#!/diska/home/astro2/SciOpsPy/envs/SciOps/bin/./python # -*- coding: utf-8 -*- ''' --- UT1_QC.py --- Script to extract QC information from pipeline reduced FORS2 data as well as raw frames, which also calculates the S/N for the spectral modes. Execution example: ============================================================== python UT1_QC.py DATE [-p reduced directory] [-p2 raw directory] [-t sleep time] [-v version] ***Please note that the DATE parameter is optional, and if left blank today's date is used. Version Control: ============================================================== 2019-03-02 esedagha Added the option default date as today, so that the code can be run without date given. 2019-03-02 esedagha Corrected the mistake in reading the start and end time of a template instead of the OB, when reading seeing and airmass value from ASM. 2019-03-02 esedagha In IMG and IPOL for moving targets no fitting is done, and seeing is read from the ASM or header. 2019-03-04 esedagha Improved the algorithm calculating seeing by fitting Gaussian profiles to 2D raw LSS and PMOS spectra. 2019-06-16 esedagha Complete revamp of how the code deals with the complicated PMOS frames. Also slight adjustment in calculation of fwhm from the 2D Gaussian fit in the IMG and IPOL files. Threshold of peak fitting in get_snr function changed to 6 since this was causing the code to hang. 2019-06-17 esedagha Added the functionality to calculate ellipticity for spectroscopic modes by analysing the associated acquisition images. Addtionally, the code is also decentralized, by moving various classes and functions, as well as the hardcoded variables, to separate script files. Header & QC parameter definitions: ============================================================== OB name: 'ESO OBS NAME' / OB name Prog ID: 'ESO OBS PROG ID' / ESO program identification OB ID: 'ESO OBS ID' / Observation block ID Date: 'DATE' / file creation date (YYYY-MM-DDThh:mm:ss UT) Requested transparency: 'ESO OBS AMBI TRANS' / Req. sky transparency Airmass max: max('ESO TEL AIRM START','ESO TEL AIRM END') Airmass req: 'ESO OBS AIRM' / Req. max. airmass Airmass per: computed / (airmass_max - airmass_req)/airmass_req Airmass grade: < 0% --> A | <= 10% --> B | > 10% --> C SEEING: if IMG mode: 'ESO QC IMGQU' / Image quality of scientific exposure for IMG mode only (although this seems to be wrong and is to be calculated directly on the reduced images) elif IPOL mode: FWHM * pix_scale * binning / FWHM measured on 40% brightest point sources elif LSS or PMOS modes: FWHM * pix_scale * binning / FWHM measured on the spectral trace elif MOS or MXU mode: DIMM seeing corrected for airmass and adjusted for image analysis of ACQ frames. SEE_REQ: 'ESO OBS AMBI FWHM' / Req. max. seeing SEE_PER: '(seeing - see_requested)/(see_requested)' for IMG mode only. Seeing grade: < 0% --> A | <= 10% --> B | > 10% --> C for IMG mode only SEE_PER: % of DIMM readings during observations out of range. (all other modes) Seeing_grade: <= 10% A | <= 20% --> B | > 20% --> C Ellipticity : if IMG mode: 'ESO QC IMGQUELL' / Mean star ellipticity (again the pipeline value seems wrong and this shall be measured durectly on the images) elif IPOL mode: from 2DGaussian fit to the 40% brightest point sournces else: 0. Ell. grade: <= 0.10 --> A | <= 0.20 --> B | > 0.20 --> C @place: ESO - Paranal Observatory @author: Elyar Sedaghati @year(s): 2018 - 19 @Telescope(s): UT1 @Instrument(s): FORS2 @Valid for SciOpsPy: v0.1-b @Documentation url: ***TBD*** @Last SciOps review: *** @Usage: *** @Licence: ESO-license or GPLv3 (a copy is given in the root directory of this program) @Testable: Yes @Test data place (if any required): any night with reduced science data should work @Version: 19.06-2 ''' from astropy.io import fits import argparse, time, os, sys, glob, datetime import numpy as np import pandas from numpy import array, where, median, abs from raw_IPOL_QC import IPOL_QC from PMOS_QC import PMOS_QC, PMOS2_QC from hardcoded import * from FORS2_QC import FORS2_QC from OrderedSet import OrderedSet import warnings warnings.filterwarnings("ignore") if __name__ == "__main__": UTC_hour = datetime.datetime.utcnow().hour if UTC_hour > 19: today = datetime.datetime.now().strftime("%Y-%m-%d") else: today = (datetime.date.today()-datetime.timedelta(1)).strftime('%Y-%m-%d') ########################################################################### ######### Parser to pass various arguments to the script. ################ ###########################################################################acquisition_old = [] parser = argparse.ArgumentParser() parser.add_argument('obsdate', nargs='?', const=1, help='observation date', default=today) parser.add_argument('-p', '--path', action='store', default='/data-ut1/reduced', help='path to reduced data directory') parser.add_argument('-p2', '--path2', action='store', default='/data-ut1/raw', help='path to raw data directory') parser.add_argument('-t', '--time-sleep', action='store', default=60, dest='time', help='sleep time (sec) for the loop checking for new observations (reduce it if you want more frequent QC updates)') parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.0') args = parser.parse_args() print (break_msg) print(txt_header) reduced_old = []; raw_old = []; acquisition_old = [] pmos_raw = []; pmos_red = []; pmos_red_raw = [] for infile in glob.glob(os.path.join(args.path, args.obsdate, 'r.FORS*')): header = fits.open(infile)[0].header if header['ESO PRO CATG'] in cats and header['ESO DET CHIP1 ID']=='CCID20-14-5-3': reduced_old.append(infile) if header['ESO PRO CATG'] == 'REDUCED_SCI_PMOS' and header['ESO DET CHIP1 ID'] == 'CCID20-14-5-3': pmos_red.append(infile) for i in header: if 'ESO PRO REC1 RAW' in i and 'CATG' not in i: pmos_red_raw.append((infile,header[i])) pmos_red_raw = np.array(pmos_red_raw) reduced_old = sorted(reduced_old) for infile in glob.glob(os.path.join(args.path2, args.obsdate, 'FORS2_*.CHIP1.fits')): header = fits.open(infile)[0].header if header['ESO DPR CATG'] == 'SCIENCE': raw_old.append(infile) elif header ['ESO DPR CATG'] == 'ACQUISITION' and 'SLIT' not in infile: acquisition_old.append(infile) if header['ESO DPR CATG'] == 'SCIENCE' and header['ESO INS MODE'] == 'PMOS': pmos_raw.append((infile,header['DATE-OBS'])) pmos_raw = np.array(pmos_raw) i = 0 for raw_fits in raw_old: header2 = fits.open(raw_fits)[0].header # QC for IPOL. if header2['ESO INS MODE'] == 'IPOL': #print(raw_fits) fors = IPOL_QC(raw_fits) fors.image_grade() fors.qc_onscreen() i+=1 if i==10 : print (break_msg) print (txt_header) i = 0 # QC for PMOS based on whether reduced file exists or not elif header2['ESO INS MODE'] == 'PMOS': if pmos_red_raw.size: pmos_red_raw_datetime = [datetime.datetime.strptime(i[6:-5],"%Y-%m-%dT%H:%M:%S.%f") for i in pmos_red_raw[:,1]] else: pmos_red_raw_datetime = [] pmos_raw_datetime = datetime.datetime.strptime(header2['DATE-OBS'],"%Y-%m-%dT%H:%M:%S.%f") diffs = [abs((pmos_raw_datetime-k).total_seconds()) for k in pmos_red_raw_datetime] acq = [] for infile in acquisition_old: if fits.open(infile)[0].header['ESO OBS ID'] == header2['ESO OBS ID']: acq.append(infile) if any(elem in diffs for elem in [0.0, 0.001]): for row in range(len(pmos_red_raw)): diff = (pmos_raw_datetime - datetime.datetime.strptime(pmos_red_raw[row,1][6:-5],"%Y-%m-%dT%H:%M:%S.%f")) if abs(diff.total_seconds()) in [0.0,0.001]: red_fits = pmos_red_raw[row,0] header_red = fits.open(red_fits)[0].header for h in header_red: if header_red[h] == 'FORS2.'+(pmos_raw_datetime - diff).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]+'.fits' and 'ESO PRO REC1 RAW' in h: extension = int(h[16]) fors = PMOS2_QC(red_fits, extension, acq, raw_fits) fors.image_grade() fors.qc_onscreen() i+=1 if i==10 : print (break_msg) print (txt_header) i = 0 else: pass else: fors = PMOS_QC(acq, raw_fits) fors.image_grade() fors.qc_onscreen() i+=1 if i==10 : print (break_msg) print (txt_header) i = 0 # QC for LSS, MXU, MOS and IMG modes else: acq = [] for infile in acquisition_old: if fits.open(infile)[0].header['ESO OBS ID'] == header2['ESO OBS ID']: acq.append(infile) for red_fits in reduced_old: hdu = fits.open(red_fits) header = fits.open(red_fits)[0].header if header['ESO PRO CATG'] in ['REDUCED_SCI_LSS', 'REDUCED_SCI_MXU', 'REDUCED_SCI_MOS', 'SCIENCE_REDUCED_IMG'] and header2['ARCFILE'] == header['ARCFILE']: fors = FORS2_QC(red_fits, acq, raw_fits) fors.image_grade() fors.qc_onscreen() i+=1 if i==10 : print (break_msg) print (txt_header) i = 0 ################################################################################# ## Looping once every args.time seconds through the files in the ### ## /data-ut1/raw/args.obsdate/ directory to check for new incoming FORS2 ### ## science frames, and calculating QC parameters and grades for the new files ### ################################################################################# while True : time.sleep(float(args.time)) reduced_new = []; raw_new = []; acquisition_new = [] pmos_red_new = []; pmos_red_raw_new = []; pmos_raw_new = [] for infile in glob.glob(os.path.join(args.path, args.obsdate, 'r.FORS*')): header = fits.open(infile)[0].header if header['ESO PRO CATG'] in cats and header['ESO DET CHIP1 ID']=='CCID20-14-5-3': reduced_new.append(infile) if header['ESO PRO CATG'] == 'REDUCED_SCI_PMOS' and header['ESO DET CHIP1 ID'] == 'CCID20-14-5-3': pmos_red_new.append(infile) for k in header: if 'ESO PRO REC1 RAW' in k and 'CATG' not in k: pmos_red_raw_new.append((infile,header[k])) pmos_red_raw_new = np.array(pmos_red_raw_new) reduced_new = sorted(reduced_new) for infile in glob.glob(os.path.join(args.path2, args.obsdate, 'FORS2_*.CHIP1.fits')): header = fits.open(infile)[0].header if header['ESO DPR CATG'] == 'SCIENCE': raw_new.append(infile) elif header ['ESO DPR CATG'] == 'ACQUISITION' and 'SLIT' not in infile: acquisition_new.append(infile) if header['ESO DPR CATG'] == 'SCIENCE' and header['ESO INS MODE'] == 'PMOS': pmos_raw_new.append((infile,header['DATE-OBS'])) pmos_raw_new = np.array(pmos_raw_new) if raw_old != raw_new : difflist = set(reduced_new)-set(reduced_old) difflist_raw = set(raw_new)-set(raw_old) for raw_fits in difflist_raw: header2 = fits.open(raw_fits)[0].header if header2['ESO INS MODE'] == 'IPOL': #print(raw_fits) fors = IPOL_QC(raw_fits) fors.image_grade() fors.qc_onscreen() i+=1 if i==10 : print (break_msg) print (txt_header) i = 0 # QC for PMOS based on whether reduced file exists or not elif header2['ESO INS MODE'] == 'PMOS': difflist_pmos_raw = [list(OrderedSet(pmos_raw_new[:,0])-OrderedSet(pmos_raw[:,0])),list(OrderedSet(pmos_raw_new[:,1])-OrderedSet(pmos_raw[:,1]))] if pmos_red_raw.size: difflist_pmos_red_raw = [list(OrderedSet(pmos_red_raw_new[:,0])-OrderedSet(pmos_red_raw[:,0])),list(OrderedSet(pmos_red_raw_new[:,1])-OrderedSet(pmos_red_raw[:,1]))] pmos_red_raw_new_datetime = [datetime.datetime.strptime(i[6:-5],"%Y-%m-%dT%H:%M:%S.%f") for i in difflist_pmos_red_raw[1]] else: difflist_pmos_red_raw = []; pmos_red_raw_new_datetime = [] pmos_raw_new_datetime = datetime.datetime.strptime(header2['DATE-OBS'],"%Y-%m-%dT%H:%M:%S.%f") diffs = [abs((pmos_raw_new_datetime-k).total_seconds()) for k in pmos_red_raw_new_datetime] acq = [] for infile in acquisition_new: if fits.open(infile)[0].header['ESO OBS ID'] == header2['ESO OBS ID']: acq.append(infile) if any(elem in diffs for elem in [0.0, 0.001]): for row in range(len(pmos_red_raw_new)): diff = (pmos_raw_new_datetime - datetime.datetime.strptime(pmos_red_raw_new[row,1][6:-5],"%Y-%m-%dT%H:%M:%S.%f")) if abs(diff.total_seconds()) in [0.0,0.001]: red_fits = pmos_red_raw_new[row,0] header_red = fits.open(red_fits)[0].header for h in header_red: if header_red[h] == 'FORS2.'+(pmos_raw_new_datetime - diff).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]+'.fits' and 'ESO PRO REC1 RAW' in h: extension = int(h[16]) fors = PMOS2_QC(red_fits, extension, acq, raw_fits) fors.image_grade() fors.qc_onscreen() i+=1 if i==10 : print (break_msg) print (txt_header) i = 0 else: pass else: fors = PMOS_QC(acq, raw_fits) fors.image_grade() fors.qc_onscreen() i+=1 if i==10 : print (break_msg) print (txt_header) i = 0 else: acq = [] for infile in acquisition_new: if fits.open(infile)[0].header['ESO OBS ID'] == header2['ESO OBS ID']: acq.append(infile) for red_fits in difflist: hdu = fits.open(red_fits) header = fits.open(red_fits)[0].header if header['ESO PRO CATG'] in ['REDUCED_SCI_LSS', 'REDUCED_SCI_MXU', 'REDUCED_SCI_MOS', 'SCIENCE_REDUCED_IMG']: if header2['ARCFILE'] == header['ARCFILE']: #print(red_fits, raw_fits,header['ESO PRO CATG']) fors = FORS2_QC(red_fits, acq, raw_fits) fors.image_grade() fors.qc_onscreen() i+=1 if i==10 : print (break_msg) print (txt_header) i = 0 reduced_old = reduced_new raw_old = raw_new pmos_raw = pmos_raw_new pmos_red_raw = pmos_red_raw_new