Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2# encoding: utf-8 

3""" 

4*Tools for working with 'set-of-files' (sof) files* 

5 

6:Author: 

7 David Young & Marco Landoni 

8 

9:Date Created: 

10 January 22, 2020 

11""" 

12################# GLOBAL IMPORTS #################### 

13from builtins import object 

14import sys 

15import os 

16os.environ['TERM'] = 'vt100' 

17from fundamentals import tools 

18from astropy.io import fits 

19from ccdproc import ImageFileCollection 

20import codecs 

21from soxspipe.commonutils.keyword_lookup import keyword_lookup 

22 

23 

24class set_of_files(object): 

25 """ 

26 *The worker class for the sof module used to homogenise various frame input formats (sof file, directory of fits fits, list of fits file paths) into a CCDProc ImageFileCollection* 

27 

28 **Key Arguments:** 

29 - ``log`` -- logger 

30 - ``settings`` -- the settings dictionary 

31 - ``inputFrames`` -- can be a directory, a set-of-files (SOF) file or a list of fits frame paths. Default [] 

32 - ``keys`` -- key aliases to report in the ImageFileCollection. Default ['MJDOBS', 'CDELT1', 'CDELT2', 'PSZX', 'DPR_TYPE', 'SEQ_ARM', 'EXPTIME', 'NAXIS1', 'NAXIS2', 'DET_READ_SPEED'] 

33 

34 **Usage** 

35 

36 To initiate a sof object, use the following: 

37 

38 ```python 

39 # inputFrames = "/path/to/a/directory" 

40 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits'] 

41 inputFrames = '/path/to/myfiles.sof' 

42 from soxspipe.commonutils import set_of_files 

43 sof = set_of_files( 

44 log=log, 

45 settings=settings, 

46 inputFrames=inputFrames 

47 ) 

48 ``` 

49 

50 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file 

51 """ 

52 # Initialisation 

53 

54 def __init__( 

55 self, 

56 log, 

57 settings=False, 

58 inputFrames=[], 

59 keys=['MJDOBS', 'CDELT1', 'CDELT2', 'PSZX', 

60 'DPR_TYPE', 'DPR_CATG', 'DPR_TECH', 'SEQ_ARM', 'EXPTIME', 'NAXIS1', 'NAXIS2', 'DET_READ_SPEED', 'CONAD', 'DET_GAIN', 'RON', 'CHIP_RON', 'BUNIT'] 

61 ): 

62 self.log = log 

63 log.debug("instansiating a new 'sof' object") 

64 self.settings = settings 

65 self.inputFrames = inputFrames 

66 

67 # KEYWORD LOOKUP OBJECT - LOOKUP KEYWORD FROM DICTIONARY IN RESOURCES 

68 # FOLDER 

69 kw = keyword_lookup( 

70 log=self.log, 

71 settings=self.settings 

72 ).get 

73 keys = kw(keys) 

74 self.keys = [] 

75 self.keys[:] = [k.lower() for k in keys] 

76 

77 # Initial Actions 

78 # FIX RELATIVE HOME PATHS 

79 from os.path import expanduser 

80 home = expanduser("~") 

81 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~": 

82 self.inputFrames = home + "/" + self.inputFrames[1:] 

83 

84 return None 

85 

86 def _generate_sof_file_from_directory( 

87 self, 

88 directory, 

89 sofPath): 

90 """*generate an sof file from a directory of FITS frames* 

91 

92 **Key Arguments:** 

93 - ``directory`` -- the path to the directory to containing the FITS files. 

94 - ``sofPath`` -- the path to generate the sof file to 

95 

96 **Return:** 

97 - ``sofPath`` -- the path to the sof file 

98 

99 **Usage** 

100 

101 ```python 

102 from soxspipe.commonutils import set_of_files 

103 sof = set_of_files( 

104 log=log, 

105 settings=settings 

106 ) 

107 sofFile = sof._generate_sof_file_from_directory( 

108 directory="path/to/directory", sofPath="/path/to/myFile.sof") 

109 ``` 

110 """ 

111 self.log.debug( 

112 'starting the ``_generate_sof_file_from_directory`` method') 

113 

114 from soxspipe.commonutils import keyword_lookup 

115 kw = keyword_lookup( 

116 log=self.log, 

117 settings=self.settings 

118 ).get 

119 

120 # MAKE RELATIVE HOME PATH ABSOLUTE 

121 from os.path import expanduser 

122 home = expanduser("~") 

123 if directory[0] == "~": 

124 directory = directory.replace("~", home) 

125 if sofPath[0] == "~": 

126 sofPath = sofPath.replace("~", home) 

127 

128 content = "" 

129 for d in sorted(os.listdir(directory)): 

130 if os.path.isfile(os.path.join(directory, d)) and (os.path.splitext(d)[-1].lower() == ".fits"): 

131 fitsPath = os.path.abspath(os.path.join(directory, d)) 

132 # OPEN FITS FILE AT HDULIST - HDU (HEADER DATA UNIT) CONTAINS A HEADER AND A DATA ARRAY (IMAGE) OR 

133 # TABLE. 

134 with fits.open(fitsPath) as hdul: 

135 # READ HEADER INTO MEMORY 

136 hdr = hdul[0].header 

137 # PRINT FULL FITS HEADER TO STDOUT 

138 # print(repr(hdr).strip()) 

139 dpr_type = hdr[kw("DPR_TYPE")].strip() 

140 # CHECK ARM 

141 arm = hdr[kw("SEQ_ARM")] 

142 # CHECK BINNING 

143 if kw('CDELT1') in hdr: 

144 xbin = str(int(hdr[kw('CDELT1')])) 

145 ybin = str(int(hdr[kw('CDELT2')])) 

146 catagory = dpr_type + "_" + arm.strip() 

147 if kw('CDELT1') in hdr: 

148 catagory += "_" + \ 

149 xbin.strip() + "x" + ybin.strip() 

150 

151 content += "%(fitsPath)s %(catagory)s\n" % locals() 

152 

153 # Recursively create missing directories 

154 moduleDirectory = os.path.dirname(sofPath) 

155 if not os.path.exists(moduleDirectory): 

156 os.makedirs(moduleDirectory) 

157 

158 # WRITE TO FILE 

159 with open(sofPath, 'w') as myFile: 

160 myFile.write(content) 

161 

162 self.log.debug( 

163 'completed the ``_generate_sof_file_from_directory`` method') 

164 return sofPath 

165 

166 def get( 

167 self): 

168 """*return the set-of-files as a CCDProc ImageFileCollection* 

169 

170 **Return:** 

171 - ``sof`` -- a ccdproc ImageFileCollection of the frames 

172 

173 **Usage** 

174 

175 To generate a ImageFileCollection from a directory, a list of fits filepaths or a set-of-files (SOF) file try the following: 

176 

177 ```python 

178 # inputFrames = "/path/to/a/directory" 

179 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits'] 

180 inputFrames = '/path/to/myfiles.sof' 

181 from soxspipe.commonutils import set_of_files 

182 sof = set_of_files( 

183 log=log, 

184 settings=settings, 

185 inputFrames=inputFrames 

186 ) 

187 sofFile, supplementarySof = sof.get() 

188 print(sofFile.summary) 

189 ``` 

190 

191 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file. 

192 """ 

193 self.log.debug('starting the ``get`` method') 

194 

195 from os.path import expanduser 

196 home = expanduser("~") 

197 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~": 

198 self.inputFrames = home + "/" + self.inputFrames[1:] 

199 

200 # DIRECTORY OF FRAMES 

201 if isinstance(self.inputFrames, str) and os.path.isdir(self.inputFrames): 

202 sof = ImageFileCollection(self.inputFrames, keywords=self.keys) 

203 supplementaryFilepaths = [] 

204 for d in os.listdir(self.inputFrames): 

205 filepath = os.path.join(self.inputFrames, d) 

206 if os.path.isfile(filepath) and ".fits" not in d.lower() and d[0] != ".": 

207 supplementaryFilepaths.append(filepath) 

208 

209 elif isinstance(self.inputFrames, str) and os.path.isfile(self.inputFrames) and '.sof' in self.inputFrames: 

210 readFile = codecs.open( 

211 self.inputFrames, encoding='utf-8', mode='r') 

212 thisData = readFile.read() 

213 readFile.close() 

214 lines = thisData.split("\n") 

215 

216 # REMOVE COMMENTED LINES 

217 lines = [l for l in lines if len(l) and l[0] != "#"] 

218 

219 fitsFiles = [] 

220 fitsFiles[:] = [l.split(".fits")[0].replace("~/", home + "/") + 

221 ".fits" for l in lines if ".fits" in l] 

222 supplementaryFilepaths = [ 

223 l.replace("~/", home + "/") for l in lines if ".fits" not in l.lower() and len(l) > 3] 

224 # MAKE SURE FILES EXIST 

225 allFiles = fitsFiles.extend(supplementaryFilepaths) 

226 for f in fitsFiles + supplementaryFilepaths: 

227 exists = os.path.exists(f) 

228 if not exists: 

229 raise FileNotFoundError(f"the input file `{f}` does not appear to exist") 

230 

231 locations = [os.path.dirname(f) for f in fitsFiles] 

232 if len(set(locations)) == 1: 

233 location = locations[0] 

234 fitsFiles = [os.path.basename( 

235 f) for f in fitsFiles] 

236 else: 

237 location = None 

238 sof = ImageFileCollection( 

239 filenames=fitsFiles, location=location, keywords=self.keys) 

240 elif isinstance(self.inputFrames, list): 

241 fitsFiles = [f for f in self.inputFrames if ".fits" in f.lower()] 

242 # FIND UNIQUE FILE LOCATIONS 

243 locations = [os.path.dirname(f) for f in fitsFiles] 

244 if len(set(locations)) == 1: 

245 location = locations[0] 

246 fitsFiles = [os.path.basename( 

247 f) for f in fitsFiles] 

248 else: 

249 location = None 

250 sof = ImageFileCollection( 

251 filenames=fitsFiles, location=location, keywords=self.keys) 

252 supplementaryFilepaths = [ 

253 f for f in self.inputFrames if ".fits" not in f.lower() and f[0] != "."] 

254 else: 

255 raise TypeError( 

256 "'inputFrames' should be the path to a directory of files, an SOF file or a list of FITS frame paths") 

257 

258 supplementary_sof = self.create_supplimentary_file_dictionary( 

259 supplementaryFilepaths) 

260 

261 self.log.debug('completed the ``get`` method') 

262 return sof, supplementary_sof 

263 

264 def create_supplimentary_file_dictionary( 

265 self, 

266 supplementaryFilepaths): 

267 """*create supplimentary file dictionary* 

268 

269 **Key Arguments:** 

270 - ``supplementaryFilepaths`` -- the list of filepaths to genereate the dictionary for 

271 

272 **Return:** 

273 - ``supplementary_sof`` -- a dictionary of non-fits files needed for recipe 

274 """ 

275 self.log.debug( 

276 'starting the ``create_supplimentary_file_dictionary`` method') 

277 

278 supplementary_sof = {} 

279 for f in supplementaryFilepaths: 

280 for a in ["NIR", "UVB", "VIS"]: 

281 if a.lower() in f.lower() and a not in supplementary_sof.keys(): 

282 supplementary_sof[a] = {} 

283 

284 for f in supplementaryFilepaths: 

285 if "disp_map" in f.lower(): 

286 for a in ["NIR", "UVB", "VIS"]: 

287 if a.lower() in f.lower(): 

288 supplementary_sof[a]["DISP_MAP"] = f 

289 

290 self.log.debug( 

291 'completed the ``create_supplimentary_file_dictionary`` method') 

292 return supplementary_sof 

293 

294 # use the tab-trigger below for new method 

295 # xt-class-method