Source code for src.common.driftScans

from .driftScanAttributes import DriftScanAttributes
from .driftScanData import DriftScanData
from .dataProcessingFlowManager import DataProcessingFlowManager
from .msgConfiguration import msg_wrapper
from .miscellaneousFunctions import set_table_name
from dataclasses import dataclass
import numpy as np
import sys,os 
from .miscellaneousFunctions import create_current_scan_directory
from .sqlite_db import SQLiteDB
from .calibrate import calibrate
import pandas as pd
import matplotlib.pyplot as plt
from config import __DBNAME__
import sqlite3

[docs] @dataclass class DriftScans(DriftScanAttributes): """ Driftscan object Args: data (dict): dictionary of fitsfile data """ __dict__: dict
[docs] def add_missing_values(self,myDict,listOfKeys): """Sometimes the data did not record properly and you get enitre missing scans. If this happens""" dbInfo={} for k,v in myDict: # print(k,v) try: if k in listOfKeys: pass else: if k=='FILEPATH': filename=v['value'].split('/')[-1] msg_wrapper("debug",self.log.debug,f'>>{k}: {v["value"]}') dbInfo['FILENAME']=filename dbInfo[k]=v['value'] msg_wrapper("debug",self.log.debug,f'>> FILENAME: {dbInfo["FILENAME"]}') elif k=='DATE': dbInfo['OBSDATE']=v['value'] msg_wrapper("debug",self.log.debug,f'^^{k}: {v["value"]}') else: dbInfo[k]=v['value'] msg_wrapper("debug",self.log.debug,f'**{k}: {v["value"]}') #self.__dict__)#['FILEPATH']['value'].split('/')[-1]) except: # print('###',k,v) try: msg_wrapper("debug",self.log.debug,f'--{k}: {v["value"]}') dbInfo[k]=v['value'] except: if 'plot' in k: pass else: msg_wrapper("debug",self.log.debug,f'--{k}: {v}') dbInfo[k]=v return dbInfo
[docs] def try_test(self,key,myDict,tag,val=''): try: x=myDict[key] # print(x) myDict[f'{tag}{key}']=x # print('+-+-+',f'{tag}{key}',key, tag, myDict[f'{tag}{key}']) except: if val=='': myDict[f'{tag}{key}']=np.nan # print('+---+',f'{tag}{key}',key, tag, myDict[f'{tag}{key}']) else: myDict[f'{tag}{key}']=val
# print('+-/-+',f'{tag}{key}',key, tag, myDict[f'{tag}{key}'])
[docs] def fill_in_missing_data_sb(self, myDict,tag=''): """ Fill in the missing data from the observing file""" keys=['RMSB','RMSA','TA','TAERR','BRMS','SLOPE', 'MIDOFFSET','FLAG','PEAKLOC','BASELEFT','BASERIGHT', 'S2N'] for key in keys: self.try_test(key,myDict,tag) self.try_test('FLAG',myDict,tag,56)
[docs] def fill_in_missing_data_db_common(self, myDict,tag=''): """ Fill in the missing data from the observing file""" keys=['RMSB','RMSA','BRMS','SLOPE', 'MIDOFFSET','BASELEFT','BASERIGHT'] for key in keys: self.try_test(key,myDict,tag)
# self.try_test('FLAG',myDict,tag,56)
[docs] def fill_in_missing_data_db_beams(self, myDict,tag=''): """ Fill in the missing data from the observing file""" keys=['TA','TAERR','FLAG','PEAKLOC','S2N'] for key in keys: self.try_test(key,myDict,tag)
[docs] def getScanData(self, key): try: return self.__dict__[key]['value'] except: return []
[docs] def get_scans(self, scanDict:dict, scanKey:str): offset=scanDict(f'{scanKey}_OFFSET') lcp=scanDict(f'{scanKey}_TA_LCP') rcp=scanDict(f'{scanKey}_TA_RCP') return offset, lcp, rcp
[docs] def process_data(self): """ Process the drift scan observations. Get observations from the files and prepare it for analysis. """ create_current_scan_directory() frontend=self.__dict__['FRONTEND']['value'] theoFit=self.__dict__['theoFit'] autoFit=self.__dict__['autoFit'] frq=int(self.__dict__['CENTFREQ']['value']) hpbw=self.__dict__['HPBW']['value'] fnbw=self.__dict__['FNBW']['value'] log=self.__dict__['log'] fileName=((self.__dict__['FILEPATH']['value']).split("/")[-1])[:18] self.__dict__['FILENAME']=fileName src=self.__dict__['OBJECT']['value'] src=src.replace(' ','') saveTo=f'plots/{src}/{int(frq)}' # print(saveTo) msg_wrapper("debug",self.log.debug,f"Saving plots to: {saveTo}") msg_wrapper("info",self.log.info,f"Getting drift scans from file") # for k,v in self.__dict__.items(): # print(k,v) # print(frontend) # sys.exit() if 'S' in frontend: #'13.0S' or "18.0S" or '02.5S' or "04.5S" or '01.3S' : # Get driftscan data data=DriftScanData(self.__dict__) # get the driftscan data # print(frontend) # sys.exit() if frontend == '13.0S' or frontend == "18.0S": # onOffset=self.getScanData('ON_OFFSET') #self.__dict__['ON_OFFSET']['value'] # lcpOnScan=self.getScanData('ON_TA_LCP') #self.__dict__['ON_TA_LCP']['value'] # rcpOnScan=self.getScanData('ON_TA_RCP') #self.__dict__['ON_TA_RCP']['value'] onOffset, lcpOnScan, rcpOnScan = self.get_scans(self.getScanData,'ON') dataScans=[onOffset,lcpOnScan,rcpOnScan] else: # hpnOffset=self.getScanData('HPN_OFFSET') #self.__dict__['HPN_OFFSET']['value'] # lcpHpnScan=self.getScanData('HPN_TA_LCP') #self.__dict__['HPN_TA_LCP']['value'] # rcpHpnScan=self.getScanData('HPN_TA_RCP') #self.__dict__['HPN_TA_RCP']['value'] hpnOffset, lcpHpnScan, rcpHpnScan = self.get_scans(self.getScanData,'HPN') # hpsOffset=self.getScanData('HPS_OFFSET') #self.__dict__['HPS_OFFSET']['value'] # lcpHpsScan=self.getScanData('HPS_TA_LCP') #self.__dict__['HPS_TA_LCP']['value'] # rcpHpsScan=self.getScanData('HPS_TA_RCP') #self.__dict__['HPS_TA_RCP']['value'] hpsOffset, lcpHpsScan, rcpHpsScan = self.get_scans(self.getScanData,'HPS') # onOffset=self.getScanData('ON_OFFSET') #self.__dict__['ON_OFFSET']['value'] # lcpOnScan=self.getScanData('ON_TA_LCP') #self.__dict__['ON_TA_LCP']['value'] # rcpOnScan=self.getScanData('ON_TA_RCP') #self.__dict__['ON_TA_RCP']['value'] onOffset, lcpOnScan, rcpOnScan = self.get_scans(self.getScanData,'ON') dataScans=[hpnOffset,lcpHpnScan,rcpHpnScan, hpsOffset,lcpHpsScan,rcpHpsScan, onOffset,lcpOnScan,rcpOnScan] # create tables for scans and database data structure scanData={} tableData={} # print(lcpHpsScan) # print(data) # sys.exit() if len(dataScans)==3: tag="ON" tags=[tag] x=dataScans[0] lcp=dataScans[1] rcp=dataScans[2] # process the data - i.e. run the fitting and plotting algorithms processedLCPData=DataProcessingFlowManager(fileName,frq,src,x,lcp,log,0,'y',saveTo,f'{tag}_LCP','LCP',frontend,hpbw,fnbw,theoFit, autoFit) processedRCPData=DataProcessingFlowManager(fileName,frq,src,x,rcp,log,0,'y',saveTo,f'{tag}_RCP','RCP',frontend,hpbw,fnbw,theoFit, autoFit) # sys.exit() self.fill_in_missing_data_sb(processedLCPData.__dict__,f'{tag[0]}L') self.fill_in_missing_data_sb(processedRCPData.__dict__,f'{tag[0]}R') scanData[tag]={'lcp':processedLCPData, 'rcp':processedRCPData} del processedLCPData del processedRCPData # get missing keys, 'plotDir', listOfKeys=['ON_OFFSET', 'ON_RA_J2000', 'RAW_ON_LCPDATA', 'RAW_ON_RCPDATA', 'ON_TA_LCP', 'ON_TA_RCP', 'FILENAME', 'log', 'HDULIST', 'INFOHEADER', 'theoFit', 'autoFit', 'CARDS'] else: tags=['HPN','HPS','ON'] for i in range(len(dataScans)): if i%3==0: if i == 0: tag=tags[0] elif i==3: tag=tags[1] elif i==6: tag=tags[2] x=dataScans[i] lcp=dataScans[i+1] rcp=dataScans[i+2] # process the data - i.e. run the fitting and plotting algorithms processedLCPData=DataProcessingFlowManager(fileName,frq,src,x,lcp,log,0,'y',saveTo,f'{tag}_LCP','LCP',frontend,hpbw,fnbw,theoFit, autoFit) processedRCPData=DataProcessingFlowManager(fileName,frq,src,x,rcp,log,0,'y',saveTo,f'{tag}_RCP','RCP',frontend,hpbw,fnbw,theoFit, autoFit) # if tag=='ON': # sys.exit() if tag=='ON': tg='O' elif tag=='HPN': tg='N' elif tag=='HPS': tg='S' self.fill_in_missing_data_sb(processedLCPData.__dict__,f'{tg}L') self.fill_in_missing_data_sb(processedRCPData.__dict__,f'{tg}R') scanData[tag]={'lcp':processedLCPData, 'rcp':processedRCPData} del processedLCPData del processedRCPData # get missing keys, 'plotDir' , listOfKeys=['HPN_OFFSET','HPN_RA_J2000','RAW_HPN_LCPDATA','RAW_HPN_RCPDATA','HPN_TA_LCP','HPN_TA_RCP', 'HPS_OFFSET','HPS_RA_J2000','RAW_HPS_LCPDATA','RAW_HPS_RCPDATA','HPS_TA_LCP','HPS_TA_RCP', 'ON_OFFSET' ,'ON_RA_J2000' ,'RAW_ON_LCPDATA','RAW_ON_RCPDATA','ON_TA_LCP' ,'ON_TA_RCP' , 'FILENAME' , 'log' , 'HDULIST', 'INFOHEADER','theoFit' ,'autoFit' , 'CARDS'] myDict=self.__dict__.items() tableData=self.add_missing_values(myDict,listOfKeys) # print(len(scanData)) pols=['lcp','rcp'] # print(tags,pols) # sys.exit() # print(tableData) # sys.exit() for pol in pols: print() for tag in tags: print('\nWorking on:', tag,pol) # sys.exit() scan=scanData[f'{tag}'][pol].__dict__ # print(scan) # sys.exit() for k,v in scan.items(): # print('in - ',k,v) # sys.exit() if 'Cleaned' in k or 'log' == k or 'x' == k or 'y' == k\ or k=='applyRFIremoval' or k=='spl' or k=='pt'\ or k=='srcTag' or k=='flag' or k=='pol': pass elif f'{tag.upper()}_{pol.upper()}' in k: # print(f'{tag.upper()}_{pol.upper()}') # sys.exit() for s,t in v.items(): if 'peakModel' in s or 'correctedData' in s\ or 'peakPts' in s or 'Res' in s or 'baseLocs' in s: # print('+',s) pass else: # print('-<<',s,t) if tag=='ON': tg='O' elif tag=='HPN': tg="N" elif tag=="HPS": tg="S" # tg=tag div='' #divider = '_' or "" # print(tg,pol) if s=='peakFit': # print(f'{tg}{pol[0]}{div}TA'.upper(),t) tableData[f'{tg}{pol[0]}{div}TA'.upper()]=t # dbInfo[f'{c[i]}TA'.upper()]=n elif s=='peakRms': # print(f'{tg}{pol[0]}{div}TAERR'.upper(),t) tableData[f'{tg}{pol[0]}{div}TAERR'.upper()]=t elif s=='s2n': # print(f'${tg}{pol[0]}{div}s2n'.upper(),t) tableData[f'{tg}{pol[0]}{div}s2n'.upper()]=t elif s=='midXValue': # print(f'{tg}{pol[0]}{div}midoffset'.upper(),t) tableData[f'{tg}{pol[0]}{div}midoffset'.upper()]=t elif s=='driftRms': # print(f'{tg}{pol[0]}{div}BRMS'.upper(),t) tableData[f'{tg}{pol[0]}{div}BRMS'.upper()]=t elif 'driftCoeffs' in s: # print(f'{tg}{pol[0]}{div}SLOPE'.upper(),t[0]) coeff=t #.split() try: tableData[f'{tg}{pol[0]}{div}SLOPE'.upper()]=float(coeff[0]) except: tableData[f'{tg}{pol[0]}{div}SLOPE'.upper()]=np.nan # dbInfo[f'{c[i]}intercept']=float(coeff[1]) elif 'base' in s: ch=str(t).replace(',',';').replace('[','').replace(']','') # print(s,ch) tableData[f'{tg}{pol[0]}{div}{s}'.upper()] = ch # print(f'{tg}{pol[0]}{div}{s}'.upper(),ch) else: if 'msg' in s: pass else: tableData[f'{tg}{pol[0]}{div}{s}'.upper()] = t # print(f'==== {tg}{pol[0]}{div}{s}'.upper(),t) else: # if pol==pols[0] and tag==tags[0]: # print(f'{tag[0]}{pol[0]}') # print('===',tag,pol[0]) if tag=='HPN': tg='N' elif tag=='ON': tg='O' elif tag=='HPS': tg='S' key=f'{tg}{pol[0]}'.upper() # print('$$$',key) if k == 'fileName': tableData['OBSNAME']=v # print('<<',k,v) # elif k.startswith(f'{tag[0]}{pol[0]}'.upper()): #'RMS' in k: # tableData[k]=v # print('>>',k,v) elif 'RMSB' in k: if k==f'{key}RMSB' : # print('>>>',k,v) tableData[k.upper()]=v elif 'RMSA' in k: if k==f'{key}RMSA' : # print('<<<',k,v) tableData[k.upper()]=v else: # print('<<-',k,v) if tag=='HPS' or tag=='HPN': # print(f'=*= {k}'.upper(), f'{tag[-1]}{pol[0]}rms'.upper()) if f'{tag[-1]}{pol[0]}rms'.upper() in k: tableData[f'{div}{k}'.upper()]=v # sys.exit() # Calibrate the data for k,v in tableData.items(): if k=='OLTA' and len(tags)>1: pc,ta,taErr=calibrate(tableData['SLTA'], tableData['SLTAERR'], tableData['NLTA'], tableData['NLTAERR'], tableData['OLTA'], tableData['OLTAERR'], tableData, self.log) tableData['OLPC']=pc tableData['COLTA']=ta tableData['COLTAERR']=taErr break for k,v in tableData.items(): if k=='ORTA' and len(tags)>1: pc,ta,taErr=calibrate(tableData['SRTA'], tableData['SRTAERR'], tableData['NRTA'], tableData['NRTAERR'], tableData['ORTA'], tableData['ORTAERR'], tableData, self.log) tableData['ORPC']=pc tableData['CORTA']=ta tableData['CORTAERR']=taErr break # for k,v in tableData.items(): # if k=='keys' or k=='values': # pass # else: # if 'S2N' in k:# or 'RMS' in k: # print('* ',k,': ',v) # sys.exit() tableData['SRC']=tableData['OBJECT'].replace(' ','') freq=int(tableData['CENTFREQ']) dbTable = f"{tableData['SRC']}_{freq}"#.replace('-','m').replace('+','p') dbTable=set_table_name(dbTable,self.log) try: int(dbTable[0]) dbTable=f"_{dbTable}" except: pass print(f'Saving to Table: {dbTable}', freq) # sys.exit() # Get data to save to dictionary # --- Setup database where you will be storing information msg_wrapper("debug",self.log.debug,"Setup database") # df=pd.DataFrame(tableData) # print(df) # if 'S' in tableData['FRONTEND']: #=='13.0S': # tableData.pop('HABMSEP',None) # tableData.pop('SEC_Z',None) # tableData.pop('X_Z',None) # tableData.pop('DRY_ATMOS_TRANSMISSION',None) # tableData.pop('ZENITH_TAU_AT_1400M',None) # tableData.pop('ABSORPTION_AT_ZENITH',None) # print(tableData.keys()) # sys.exit() finalDict={} for m in tableData['keys']: # print('\n',m) for k,v in tableData.items(): if m==k: if m=='keys' or m=='values': pass else: # print(m,k,v) # if tableData['FRONTEND']=='13.0S' and tableData['']: # print(m,k,v) # print(tableData) # sys.exit() try: # print(m,k,v['value']) finalDict[k]=v['value'] except: # print(m,k,v) finalDict[k]=v # print(tableData['keys']) # # print(tableData['values']) # for l,v in finalDict.items(): # print(l,v) # print(finalDict['OBSDATE'].split('T')[0]) # Get date finalDict['OBSDATE']=finalDict['OBSDATE'].split('T')[0] # print(finalDict['OBSDATE']) # print(finalDict) # print(len(tableData['keys']),len(tableData['values'][:-1]),len(finalDict)) # # print('here') # print(finalDict['FILEPATH']) # sys.exit() db= SQLiteDB('HART26DATA.db',self.log) db.create_db() table=db.create_table(finalDict,dbTable) db.populate_table(finalDict, table) db.close_db() # print(finalDict['FILEPATH']) # sys.exit() # sys.exit() elif 'D' in frontend: # '03.5D' or "06.0D" # Get driftscan data data=DriftScanData(self.__dict__) # for k,v in data.__dict__.items(): # print('---',k,v) # sys.exit() hpnOffset=self.getScanData('HPN_OFFSET') #self.__dict__['HPN_OFFSET']['value'] lcpHpnScan=self.getScanData('HPN_TA_LCP') #self.__dict__['HPN_TA_LCP']['value'] rcpHpnScan=self.getScanData('HPN_TA_RCP') #self.__dict__['HPN_TA_RCP']['value'] hpsOffset=self.getScanData('HPS_OFFSET') #self.__dict__['HPS_OFFSET']['value'] lcpHpsScan=self.getScanData('HPS_TA_LCP') #self.__dict__['HPS_TA_LCP']['value'] rcpHpsScan=self.getScanData('HPS_TA_RCP') #self.__dict__['HPS_TA_RCP']['value'] onOffset=self.getScanData('ON_OFFSET') #self.__dict__['ON_OFFSET']['value'] lcpOnScan=self.getScanData('ON_TA_LCP') #self.__dict__['ON_TA_LCP']['value'] rcpOnScan=self.getScanData('ON_TA_RCP') #self.__dict__['ON_TA_RCP']['value'] dataScans=[hpnOffset,lcpHpnScan,rcpHpnScan, hpsOffset,lcpHpsScan,rcpHpsScan, onOffset,lcpOnScan,rcpOnScan] # create tables for scans and database data structure scanData={} tableData={} tags=['HPN','HPS','ON'] for i in range(len(dataScans)): if i%3==0: if i == 0: tag=tags[0] elif i==3: tag=tags[1] elif i==6: tag=tags[2] x=dataScans[i] lcp=dataScans[i+1] rcp=dataScans[i+2] # process the data - i.e. run the fitting and plotting algorithms processedLCPData=DataProcessingFlowManager(fileName,frq,src,x,lcp,log,0,'y',saveTo,f'{tag}_LCP','LCP',frontend,hpbw,fnbw,theoFit, autoFit) processedRCPData=DataProcessingFlowManager(fileName,frq,src,x,rcp,log,0,'y',saveTo,f'{tag}_RCP','RCP',frontend,hpbw,fnbw,theoFit, autoFit) if tag=='ON': tg='O' else: tg=tag[-1] myTag=f'{tg}L' # print(myTag,f'A{myTag}',f'B{myTag}') # print(processedLCPData.__dict__.keys()) # print(processedRCPData.__dict__.keys()) # sys.exit() self.fill_in_missing_data_db_common(processedLCPData.__dict__,myTag) self.fill_in_missing_data_db_beams(processedLCPData.__dict__,f'A{myTag}') self.fill_in_missing_data_db_beams(processedLCPData.__dict__,f'B{myTag}') self.fill_in_missing_data_db_common(processedRCPData.__dict__,myTag) self.fill_in_missing_data_db_beams(processedRCPData.__dict__,f'A{myTag}') self.fill_in_missing_data_db_beams(processedRCPData.__dict__,f'B{myTag}') scanData[tag]={'lcp':processedLCPData, 'rcp':processedRCPData} # print(processedLCPData) # sys.exit() del processedLCPData del processedRCPData # get missing keys, 'plotDir', listOfKeys=['HPN_OFFSET','HPN_RA_J2000','RAW_HPN_LCPDATA','RAW_HPN_RCPDATA','HPN_TA_LCP','HPN_TA_RCP', 'HPS_OFFSET','HPS_RA_J2000','RAW_HPS_LCPDATA','RAW_HPS_RCPDATA','HPS_TA_LCP','HPS_TA_RCP', 'ON_OFFSET' ,'ON_RA_J2000' ,'RAW_ON_LCPDATA','RAW_ON_RCPDATA','ON_TA_LCP' ,'ON_TA_RCP' , 'FILENAME' , 'log' , 'HDULIST', 'INFOHEADER','theoFit' ,'autoFit' , 'CARDS'] myDict=self.__dict__.items() tableData=self.add_missing_values(myDict,listOfKeys) pols=['lcp','rcp'] beams=['A','B'] # print(myDict) # sys.exit() # for beam in beams: # cnt=0 for pol in pols: print() for tag in tags: # print('\nWorking on:', tag,pol) # print(f'{tag.upper()}_{pol.upper()}') # print(scanData.keys()) scan=scanData[f'{tag}'][pol].__dict__ # print(scan.keys()) # sys.exit() for k,v in scan.items(): # print(k) if 'Cleaned' in k or 'log' == k or 'x' == k or 'y' == k\ or k=='applyRFIremoval' or k=='spl' or k=='pt'\ or k=='srcTag' or k=='flag' or k=='pol': pass elif f'{tag.upper()}_{pol.upper()}' in k: # print(f'{tag.upper()}_{pol.upper()}') for s,t in v.items(): # print('-+-',s) if 'PeakModel' in s or 'correctedData' in s\ or 'PeakData' in s or 'Res' in s: # or 'baseLocs' in s: # print('+',s) pass else: # print('-<<',k,s,t) if tag=='ON': tg='O' elif tag=='HPN': tg="N" elif tag=="HPS": tg="S" div='' #divider = '_' or "" if s=='leftPeakFit': # print(f'{beams[0]}{tg}{pol[0]}{div}TA'.upper(),t) tableData[f'{beams[0]}{tg}{pol[0]}{div}TA'.upper()]=t # # dbInfo[f'{c[i]}TA'.upper()]=n elif s=='leftPeakFitErr': # print(f'{beams[0]}{tg}{pol[0]}{div}TAERR'.upper(),t) tableData[f'{beams[0]}{tg}{pol[0]}{div}TAERR'.upper()]=t elif s=='s2na': # print(f'{beams[0]}{tg}{pol[0]}{div}s2n'.upper(),t) tableData[f'{beams[0]}{tg}{pol[0]}{div}s2n'.upper()]=t elif s=='midXValueLeft': # print(f'{beams[0]}{tg}{pol[0]}{div}midoffset'.upper(),t) tableData[f'{beams[0]}{tg}{pol[0]}{div}midoffset'.upper()]=t elif s=='rightPeakFit': # print(f'{beams[1]}{tg}{pol[0]}{div}TA'.upper(),t) tableData[f'{beams[1]}{tg}{pol[0]}{div}TA'.upper()]=t # # dbInfo[f'{c[i]}TA'.upper()]=n elif s=='rightPeakFitErr': # print(f'{beams[1]}{tg}{pol[0]}{div}TAERR'.upper(),t) tableData[f'{beams[1]}{tg}{pol[0]}{div}TAERR'.upper()]=t elif s=='s2nb': # print(f'{beams[1]}{tg}{pol[0]}{div}s2n'.upper(),t) tableData[f'{beams[1]}{tg}{pol[0]}{div}s2n'.upper()]=t elif s=='midXValueRight': # print(f'{beams[1]}{tg}{pol[0]}{div}midoffset'.upper(),t) tableData[f'{beams[1]}{tg}{pol[0]}{div}midoffset'.upper()]=t elif s=='driftRms': # print('drift rms: ', f'{tg}{pol[0]}{div}BRMS'.upper(),t) tableData[f'{tg}{pol[0]}{div}BRMS'.upper()]=t elif 'driftCoeffs' in s: # print('driftCoeffs: ', f'{tg}{pol[0]}{div}BRMS'.upper(),t) # print(f'{tg}{pol[0]}{div}SLOPE'.upper(),t[0]) # coeff=t #.split() # print(t) try: if len(t)==0: # tableData[f'{beams[0]}{tg}{pol[0]}{div}BASELocs'.upper()] = np.nan tableData[f'{tg}{pol[0]}{div}SLOPE'.upper()]=np.nan # print(f'### {tg}{pol[0]}{div}SLOPE'.upper()) else: tableData[f'{tg}{pol[0]}{div}SLOPE'.upper()]=t[0] # print(f'xxx {tg}{pol[0]}{div}SLOPE'.upper()) except: tableData[f'{tg}{pol[0]}{div}SLOPE'.upper()]=np.nan # print(f'--- {tg}{pol[0]}{div}SLOPE'.upper()) # dbInfo[f'{c[i]}intercept']=float(coeff[1]) elif 'baseLocsLeft' in s: # print(f'{beams[0]}{tg}{pol[0]}{div}BASELeft'.upper(), f'{t[0]};{t[-1]}') # ch=str(t).replace(',',';').replace('[','').replace(']','') # # print(s,ch) # print(s) # print(t) # print('out') if len(t)==0: tableData[f'{beams[0]}{tg}{pol[0]}{div}BASELocs'.upper()] = '' else: tableData[f'{beams[0]}{tg}{pol[0]}{div}BASELocs'.upper()] = f'{t[0]};{t[-1]}' # # print(f'{tg}{pol[0]}{div}{s}'.upper(),ch) elif 'baseLocsRight' in s: # print(f'{beams[1]}{tg}{pol[0]}{div}BASERight'.upper(),f'{t[0]};{t[-1]}') # ch=str(t).replace(',',';').replace('[','').replace(']','') # # print(s,ch) if len(t)==0: tableData[f'{beams[1]}{tg}{pol[0]}{div}BASELocs'.upper()] = '' else: tableData[f'{beams[1]}{tg}{pol[0]}{div}BASElocs'.upper()] = f'{t[0]};{t[-1]}' # # print(f'{tg}{pol[0]}{div}{s}'.upper(),ch) elif 'Combined' in s: pass else: if 'msg' in s: pass else: tableData[f'{tg}{pol[0]}{div}{s}'.upper()] = t # print(f'>>>> {tg}{pol[0]}{div}{s}'.upper(),t) else: # pass # print('-----',k,beams[0]) # if k == 'fileName': # tableData['OBSNAME']=v if tag=='ON': tg='O' elif tag=='HPN': tg="N" elif tag=="HPS": tg="S" div='' if 'RMSB' in k: # or 'RMSA' in k: tg=f'{tg}{pol[0]}'.upper() if pol=='rcp' : tg=f'{tg}R'.upper() k=k.replace('L','R') # print(k,v,tg) tableData[f'{k}'.upper()]=v else: if tg in k: # print(pol,tag,k,v, 'L') # print(k,v) tableData[f'{k}'.upper()]=v else: pass # elif 'RMSA' in k: # or 'RMSA' in k: # print(pol,tag,k,v) # # tableData[k]=v elif 'RMSA' in k: # or 'RMSA' in k: # print(tg) if pol=='rcp' : tg=f'{tg}R'.upper() k=k.replace('L','R') # print(k,v,tg) tableData[f'{k}'.upper()]=v # print('>>>',tg,pol) # if tg in k: # print(pol,tag,k,v,'R') # else: # pass else: tg=f'{tg}{pol[0]}'.upper() if tg in k: # print(pol,tag,k,v, 'L') # print(k,v) tableData[f'{k}'.upper()]=v else: pass # elif 'RMSA' in k: # or 'RMSA' in k: # print(pol,tag,k,v) # if pol=='LCP' or pol=='lcp': # p='L' # elif pol=='RCP' or pol=='rcp': # p='R' # else: # print(pol) # print('ERROR: unknown polarization') # sys.exit() # # continue # print('pol: ', pol) # if k=='RMSA' or k=='RMSB': # print('((((()))))',k,f'{tg}{p}{k}'.upper(),v) # else: # print('-->>-',k, tag,f'{k}'.upper(),v) # tableData[f'{k}'.upper()]=v # tableData[f'{beams[0]}{tg}{p}{k}'.upper()]=v # else: # pass # sys.exit() # for k,v in tableData.items(): # print(k,v) for beam in beams: for k,v in tableData.items(): if k==f'{beam}OLTA': pc,ta,taErr=calibrate(tableData[f'{beam}SLTA'], tableData[f'{beam}SLTAERR'], tableData[f'{beam}NLTA'], tableData[f'{beam}NLTAERR'], tableData[f'{beam}OLTA'], tableData[f'{beam}OLTAERR'], tableData, self.log) tableData[f'{beam}OLPC']=pc tableData[f'{beam}COLTA']=ta tableData[f'{beam}COLTAERR']=taErr break for beam in beams: for k,v in tableData.items(): if k==f'{beam}ORTA': # print('----',tableData[f'{beam}SRTAERR']) srta=float(tableData[f'{beam}SRTA']) try: srtaerr= float(tableData[f'{beam}SRTAERR']) except: srtaerr=np.nan tableData[f'{beam}SRTAERR']=np.nan nrta= float(tableData[f'{beam}NRTA']) try: nrtaerr= float(tableData[f'{beam}NRTAERR']) except: nrtaerr= np.nan tableData[f'{beam}NRTAERR']=np.nan orta=float(tableData[f'{beam}ORTA']) try: ortaerr=float( tableData[f'{beam}ORTAERR']) except: ortaerr=np.nan tableData[f'{beam}ORTAERR']=np.nan # print(srta,srtaerr,orta,ortaerr,nrta,nrtaerr) pc,ta,taErr=calibrate(srta, srtaerr, nrta,nrtaerr, orta,ortaerr, tableData, self.log) tableData[f'{beam}ORPC']=pc tableData[f'{beam}CORTA']=ta tableData[f'{beam}CORTAERR']=taErr break # Calibrate the data finalData={} for k,v in tableData.items(): if k=='keys' or k=='values': pass else: if k=='RMSA' or k=='RMSB': pass else: # print('* ',k,': ',v) finalData[k]=v # if 'RMS' in k: # # print(k) # print('* ',k,': ',v) # # sys.exit() # Save to table finalData['SRC']=finalData['OBJECT'].replace(' ','') freq=int(finalData['CENTFREQ']) dbTable = f"{finalData['SRC']}_{freq}" # print(dbTable) # sys.exit() try: int(dbTable[0]) dbTable=f"_{dbTable}" except: pass print(f'Table: {dbTable}', freq, frq) print(self.__dict__['FILEPATH']['value']) # sys.exit() # dbTable=set_table_name(dbTable, self.log) # Get data to save to dictionary # --- Setup database where you will be storing information msg_wrapper("debug",self.log.debug,"Setup database") db= SQLiteDB(__DBNAME__,self.log) db.create_db() table=db.create_table(finalData,dbTable) # print(table) # if str(int(freq)) in self.__dict__['FILEPATH']['value']: # print('same freq') db.populate_table(finalData, table) db.close_db() # else: # print("Frequency of source doesn't match path frequency") # # open database and match # # check if table exists in database # cnx = sqlite3.connect(__DBNAME__) # # dbTables= pd.read_sql_query("SELECT name FROM sqlite_schema WHERE type='table'", cnx) # # tables=list(dbTables['name']) # tableData = pd.read_sql_query(f"SELECT * FROM {table}", cnx) # tableFilenames=sorted(list(tableData['FILENAME'])) # for file in tableFilenames: # if self.__dict__['FILENAME'] in file: # print('Already processed file: ',self.__dict__['FILEPATH']['value'], 'in table: ', table) # else: # print('--') # sys.exit() # print(tableFilenames, self.__dict__['FILENAME']) # if '+' in tables # sys.exit() else: print(f"Unknown source frontend value : {self.__dict__[frontend]['value']}. Contact author to have it included.") sys.exit()
[docs] def process_data_only(self,qv='no'): """ Process the drift scan observations. Get observations from the files and prepare it for analysis. """ create_current_scan_directory() frontend=self.__dict__['FRONTEND']['value'] theoFit=self.__dict__['theoFit'] autoFit=self.__dict__['autoFit'] frq=int(self.__dict__['CENTFREQ']['value']) hpbw=self.__dict__['HPBW']['value'] fnbw=self.__dict__['FNBW']['value'] log=self.__dict__['log'] fileName=((self.__dict__['FILEPATH']['value']).split("/")[-1])[:18] self.__dict__['FILENAME']=fileName src=self.__dict__['OBJECT']['value'] src=src.replace(' ','') saveTo=f'plots/{src}/{int(frq)}' msg_wrapper("debug",self.log.debug,f"Saving plots to: {saveTo}") msg_wrapper("info",self.log.info,f"Getting drift scans from file") if 'S' in frontend: #'13.0S' or "18.0S" or '02.5S' or "04.5S" or '01.3S' : # Get driftscan data data=DriftScanData(self.__dict__) # get the driftscan data if frontend == '13.0S' or frontend == "18.0S": onOffset=self.getScanData('ON_OFFSET') #self.__dict__['ON_OFFSET']['value'] lcpOnScan=self.getScanData('ON_TA_LCP') #self.__dict__['ON_TA_LCP']['value'] rcpOnScan=self.getScanData('ON_TA_RCP') #self.__dict__['ON_TA_RCP']['value'] dataScans=[onOffset,lcpOnScan,rcpOnScan] scans=['ON_OFFSET', 'ON_LCP', 'ON_RCP'] plt.figure(figsize=(10,3)) else: hpnOffset=self.getScanData('HPN_OFFSET') #self.__dict__['HPN_OFFSET']['value'] lcpHpnScan=self.getScanData('HPN_TA_LCP') #self.__dict__['HPN_TA_LCP']['value'] rcpHpnScan=self.getScanData('HPN_TA_RCP') #self.__dict__['HPN_TA_RCP']['value'] hpsOffset=self.getScanData('HPS_OFFSET') #self.__dict__['HPS_OFFSET']['value'] lcpHpsScan=self.getScanData('HPS_TA_LCP') #self.__dict__['HPS_TA_LCP']['value'] rcpHpsScan=self.getScanData('HPS_TA_RCP') #self.__dict__['HPS_TA_RCP']['value'] onOffset=self.getScanData('ON_OFFSET') #self.__dict__['ON_OFFSET']['value'] lcpOnScan=self.getScanData('ON_TA_LCP') #self.__dict__['ON_TA_LCP']['value'] rcpOnScan=self.getScanData('ON_TA_RCP') #self.__dict__['ON_TA_RCP']['value'] dataScans=[hpnOffset,lcpHpnScan,rcpHpnScan, hpsOffset,lcpHpsScan,rcpHpsScan, onOffset,lcpOnScan,rcpOnScan] scans=['HPN_OFFSET', 'HPN_LCP', 'HPN_RCP', 'HPS_OFFSET', 'HPS_LCP', 'HPS_RCP', 'ON_OFFSET', 'ON_LCP', 'ON_RCP'] plt.figure(figsize=(20,10)) cnt=1 # print(cnt) # plt.title(f'Plot of {self.__dict__['FILENAME']}') # print(len(scans)) if qv!='no': if frontend != '13.0S' and frontend != "18.0S": for i in range(len(scans)): # print(i) if i%3==0: # print('--',i,i+1,i+2) plt.subplot(3,2,cnt) plt.ylabel('Ta [K]') plt.xlabel('Offset [deg]') plt.title(f'{scans[i+1]} - scan of {fileName}') # plt.title(f'{scans[i+1]}') plt.plot(dataScans[i],dataScans[i+1]) print(dataScans[i+1]) plt.subplot(3,2,cnt+1) plt.ylabel('Ta [K]') plt.xlabel('Offset [deg]') # plt.title(f'{scans[i+2]}') plt.title(f'{scans[i+2]} - scan of {fileName}') plt.plot(dataScans[i],dataScans[i+2]) cnt=cnt+2 # print('--',cnt) else: cnt=1 for i in range(len(scans)): if cnt<=2: # print(i+1) plt.subplot(1,2,cnt) plt.ylabel('Ta [K]') plt.xlabel('Offset [deg]') plt.title(f'{scans[i+1]} - scan of {fileName}') # plt.title(f'{scans[i+1]}') plt.plot(dataScans[0],dataScans[i+1]) # print(dataScans[i+1]) cnt=cnt+1 plt.tight_layout() if qv=='yes': plt.savefig(f'quickview_{src}_{int(frq)}-{fileName}.png') msg_wrapper("info",self.log.info,f'Quickview file saved to: quickview_{src}_{int(frq)}-{fileName}.png') else: pass # plt.show() plt.close() elif 'D' in frontend: # '03.5D' or "06.0D" # Get driftscan data data=DriftScanData(self.__dict__) # for k,v in data.__dict__.items(): # # print('---',k,v) hpnOffset=self.getScanData('HPN_OFFSET') #self.__dict__['HPN_OFFSET']['value'] lcpHpnScan=self.getScanData('HPN_TA_LCP') #self.__dict__['HPN_TA_LCP']['value'] rcpHpnScan=self.getScanData('HPN_TA_RCP') #self.__dict__['HPN_TA_RCP']['value'] hpsOffset=self.getScanData('HPS_OFFSET') #self.__dict__['HPS_OFFSET']['value'] lcpHpsScan=self.getScanData('HPS_TA_LCP') #self.__dict__['HPS_TA_LCP']['value'] rcpHpsScan=self.getScanData('HPS_TA_RCP') #self.__dict__['HPS_TA_RCP']['value'] onOffset=self.getScanData('ON_OFFSET') #self.__dict__['ON_OFFSET']['value'] lcpOnScan=self.getScanData('ON_TA_LCP') #self.__dict__['ON_TA_LCP']['value'] rcpOnScan=self.getScanData('ON_TA_RCP') #self.__dict__['ON_TA_RCP']['value'] dataScans=[hpnOffset,lcpHpnScan,rcpHpnScan, hpsOffset,lcpHpsScan,rcpHpsScan, onOffset,lcpOnScan,rcpOnScan] scans=['HPN_OFFSET', 'HPN_LCP', 'HPN_RCP', 'HPS_OFFSET', 'HPS_LCP', 'HPS_RCP', 'ON_OFFSET', 'ON_LCP', 'ON_RCP'] cnt=1 # print(cnt) plt.figure(figsize=(20,10)) # plt.title(f'Plot of {self.__dict__['FILENAME']}') for i in range(len(scans)): # print(i) if i%3==0: # print('--',i,i+1,i+2) plt.subplot(3,2,cnt) plt.ylabel('Ta [K]') plt.xlabel('Offset [deg]') plt.title(f'{scans[i+1]} - scan of {fileName}') # plt.title(f'{scans[i+1]}') plt.plot(dataScans[i],dataScans[i+1]) # print(dataScans[i+1]) plt.subplot(3,2,cnt+1) plt.ylabel('Ta [K]') plt.xlabel('Offset [deg]') # plt.title(f'{scans[i+2]}') plt.title(f'{scans[i+2]} - scan of {fileName}') plt.plot(dataScans[i],dataScans[i+2]) cnt=cnt+2 # print('--',cnt) if qv=='no': plt.close() else: plt.tight_layout() plt.savefig(f'quickview_{src}_{int(frq)}-{fileName}.png') # plt.show() plt.close() msg_wrapper("info",self.log.info,f'Quickview file saved to: quickview_{src}_{int(frq)}-{fileName}.png') else: print(f"Unknown source frontend value : {self.__dict__[frontend]['value']}. Contact author to have it included.") sys.exit()