Coverage for soxspipe/recipes/_base_recipe_.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2# encoding: utf-8
3"""
4*The base recipe class which all other recipes inherit*
6:Author:
7 David Young & Marco Landoni
9:Date Created:
10 January 22, 2020
11"""
12################# GLOBAL IMPORTS ####################
13from builtins import object
14import sys
15import os
16os.environ['TERM'] = 'vt100'
17from fundamentals import tools
18import numpy as np
19from astropy.nddata import CCDData
20from astropy import units as u
21from astropy.stats import mad_std
22import ccdproc
23from astropy.nddata.nduncertainty import StdDevUncertainty
24from ccdproc import Combiner
25from soxspipe.commonutils import set_of_files
26from soxspipe.commonutils import keyword_lookup
27from soxspipe.commonutils import detector_lookup
28from datetime import datetime
29from soxspipe.commonutils import filenamer
30import shutil
33class _base_recipe_(object):
34 """
35 The base recipe class which all other recipes inherit
37 **Key Arguments:**
38 - ``log`` -- logger
39 - ``settings`` -- the settings dictionary
41 **Usage**
43 To use this base recipe to create a new `soxspipe` recipe, have a look at the code for one of the simpler recipes (e.g. `soxs_mbias`) - copy and modify the code.
44 """
46 def __init__(
47 self,
48 log,
49 settings=False
50 ):
51 self.log = log
52 log.debug("instansiating a new '__init__' object")
53 self.settings = settings
54 self.intermediateRootPath = self._absolute_path(
55 settings["intermediate-data-root"])
56 self.reducedRootPath = self._absolute_path(
57 settings["reduced-data-root"])
58 self.calibrationRootPath = self._absolute_path(
59 settings["calibration-data-root"])
61 # SET LATER WHEN VERIFYING FRAMES
62 self.arm = None
63 self.detectorParams = None
65 # KEYWORD LOOKUP OBJECT - LOOKUP KEYWORD FROM DICTIONARY IN RESOURCES
66 # FOLDER
67 self.kw = keyword_lookup(
68 log=self.log,
69 settings=self.settings
70 ).get
72 return None
74 def _prepare_single_frame(
75 self,
76 frame,
77 save=False):
78 """*prepare a single raw frame by converting pixel data from ADU to electrons and adding mask and uncertainty extensions*
80 **Key Arguments:**
81 - ``frame`` -- the path to the frame to prepare, of a CCDData object
83 **Return:**
84 - ``frame`` -- the prepared frame with mask and uncertainty extensions (CCDData object)
86 ```eval_rst
87 .. todo::
89 - write a command-line tool for this method
90 ```
91 """
92 self.log.debug('starting the ``_prepare_single_frame`` method')
94 kw = self.kw
95 dp = self.detectorParams
97 # STORE FILEPATH FOR LATER USE
98 filepath = frame
100 # CONVERT FILEPATH TO CCDDATA OBJECT
101 if isinstance(frame, str):
102 # CONVERT RELATIVE TO ABSOLUTE PATHS
103 frame = self._absolute_path(frame)
104 # OPEN THE RAW FRAME - MASK AND UNCERT TO BE POPULATED LATER
105 frame = CCDData.read(frame, hdu=0, unit=u.adu, hdu_uncertainty='ERRS',
106 hdu_mask='QUAL', hdu_flags='FLAGS', key_uncertainty_type='UTYPE')
108 # CHECK THE NUMBER OF EXTENSIONS IS ONLY 1 AND "SXSPRE" DOES NOT
109 # EXIST. i.e. THIS IS A RAW UNTOUCHED FRAME
110 if len(frame.to_hdu()) > 1 or "SXSPRE" in frame.header:
111 return filepath
113 # MANIPULATE XSH DATA
114 frame = self.xsh2soxs(frame)
115 frame = self._trim_frame(frame)
117 # CORRECT FOR GAIN - CONVERT DATA FROM ADU TO ELECTRONS
118 frame = ccdproc.gain_correct(frame, dp["gain"])
120 # GENERATE UNCERTAINTY MAP AS EXTENSION
121 if frame.header[kw("DPR_TYPE")] == "BIAS":
122 # ERROR IS ONLY FROM READNOISE FOR BIAS FRAMES
123 errorMap = np.ones_like(frame.data) * dp["ron"]
124 # errorMap = StdDevUncertainty(errorMap)
125 frame.uncertainty = errorMap
126 else:
127 # GENERATE UNCERTAINTY MAP AS EXTENSION
128 frame = ccdproc.create_deviation(
129 frame, readnoise=dp["ron"])
131 # FIND THE APPROPRIATE BAD-PIXEL BITMAP AND APPEND AS 'FLAG' EXTENSION
132 # NOTE FLAGS NOTE YET SUPPORTED BY CCDPROC THIS THIS WON'T GET SAVED OUT
133 # AS AN EXTENSION
134 arm = self.arm
135 if kw('WIN_BINX') in frame.header:
136 binx = int(frame.header[kw('WIN_BINX')])
137 biny = int(frame.header[kw('WIN_BINY')])
138 else:
139 binx = 1
140 biny = 1
142 bitMapPath = self.calibrationRootPath + "/" + dp["bad-pixel map"][f"{binx}x{biny}"]
144 if not os.path.exists(bitMapPath):
145 message = "the path to the bitMapPath %s does not exist on this machine" % (
146 bitMapPath,)
147 self.log.critical(message)
148 raise IOError(message)
149 bitMap = CCDData.read(bitMapPath, hdu=0, unit=u.dimensionless_unscaled)
151 # BIAS FRAMES HAVE NO 'FLUX', JUST READNOISE, SO ADD AN EMPTY BAD-PIXEL
152 # MAP
153 if frame.header[kw("DPR_TYPE")] == "BIAS":
154 bitMap.data = np.zeros_like(bitMap.data)
156 # print(bitMap.data.shape)
157 # print(frame.data.shape)
159 frame.flags = bitMap.data
161 # FLATTEN BAD-PIXEL BITMAP TO BOOLEAN FALSE (GOOD) OR TRUE (BAD) AND
162 # APPEND AS 'UNCERT' EXTENSION
163 boolMask = bitMap.data.astype(bool).data
164 try:
165 # FAILS IN PYTHON 2.7 AS BOOLMASK IS A BUFFER - NEED TO CONVERT TO
166 # 2D ARRAY
167 boolMask.shape
169 except:
170 arr = np.frombuffer(boolMask, dtype=np.uint8)
171 arr.shape = (frame.data.shape)
172 boolMask = arr
174 frame.mask = boolMask
176 if save:
177 outDir = self.intermediateRootPath
178 else:
179 outDir = self.intermediateRootPath + "/tmp"
181 # INJECT THE PRE KEYWORD
182 utcnow = datetime.utcnow()
183 frame.header["SXSPRE"] = (utcnow.strftime(
184 "%Y-%m-%dT%H:%M:%S.%f"), "UTC timestamp")
186 # RECURSIVELY CREATE MISSING DIRECTORIES
187 if not os.path.exists(outDir):
188 os.makedirs(outDir)
189 # CONVERT CCDData TO FITS HDU (INCLUDING HEADER) AND SAVE WITH PRE TAG
190 # PREPENDED TO FILENAME
191 basename = os.path.basename(filepath)
192 filenameNoExtension = os.path.splitext(basename)[0]
193 extension = os.path.splitext(basename)[1]
194 filePath = outDir + "/" + \
195 filenameNoExtension + "_pre" + extension
197 # SAVE TO DISK
198 self._write(
199 frame=frame,
200 filedir=outDir,
201 filename=filenameNoExtension + "_pre" + extension,
202 overwrite=True
203 )
205 self.log.debug('completed the ``_prepare_single_frame`` method')
206 return filePath
208 def _absolute_path(
209 self,
210 path):
211 """*convert paths from home directories to absolute paths*
213 **Key Arguments:**
214 - ``path`` -- path possibly relative to home directory
216 **Return:**
217 - ``absolutePath`` -- absolute path
219 **Usage**
221 ```python
222 myPath = self._absolute_path(myPath)
223 ```
224 """
225 self.log.debug('starting the ``_absolute_path`` method')
227 from os.path import expanduser
228 home = expanduser("~")
229 if path[0] == "~":
230 path = home + "/" + path[1:]
232 self.log.debug('completed the ``_absolute_path`` method')
233 return path.replace("//", "/")
235 def prepare_frames(
236 self,
237 save=False):
238 """*prepare raw frames by converting pixel data from ADU to electrons and adding mask and uncertainty extensions*
240 **Key Arguments:**
241 - ``save`` -- save out the prepared frame to the intermediate products directory. Default False.
243 **Return:**
244 - ``preframes`` -- the new image collection containing the prepared frames
246 **Usage**
248 Usually called within a recipe class once the input frames have been selected and verified (see `soxs_mbias` code for example):
250 ```python
251 self.inputFrames = self.prepare_frames(
252 save=self.settings["save-intermediate-products"])
253 ```
254 """
255 self.log.debug('starting the ``prepare_frames`` method')
257 kw = self.kw
259 filepaths = self.inputFrames.files_filtered(include_path=True)
261 frameCount = len(filepaths)
262 print("# PREPARING %(frameCount)s RAW FRAMES - TRIMMING OVERSCAN, CONVERTING TO ELECTRON COUNTS, GENERATING UNCERTAINTY MAPS AND APPENDING DEFAULT BAD-PIXEL MASK" % locals())
263 preframes = []
264 preframes[:] = [self._prepare_single_frame(
265 frame=frame, save=save) for frame in filepaths]
266 sof = set_of_files(
267 log=self.log,
268 settings=self.settings,
269 inputFrames=preframes
270 )
271 preframes, supplementaryInput = sof.get()
272 preframes.sort([kw('MJDOBS').lower()])
274 print("# PREPARED FRAMES - SUMMARY")
275 print(preframes.summary)
277 self.log.debug('completed the ``prepare_frames`` method')
278 return preframes
280 def _verify_input_frames_basics(
281 self):
282 """*the basic verifications that needs done for all recipes*
284 **Return:**
285 - None
287 If the fits files conform to required input for the recipe everything will pass silently, otherwise an exception shall be raised.
288 """
289 self.log.debug('starting the ``_verify_input_frames_basics`` method')
291 kw = self.kw
293 # CHECK WE ACTUALLY HAVE IMAGES
294 if not len(self.inputFrames.files_filtered(include_path=True)):
295 raise FileNotFoundError(
296 "No image frames where passed to the recipe")
298 arm = self.inputFrames.values(
299 keyword=kw("SEQ_ARM").lower(), unique=True)
300 # MIXED INPUT ARMS ARE BAD
301 if len(arm) > 1:
302 arms = " and ".join(arms)
303 print(self.inputFrames.summary)
304 raise TypeError(
305 "Input frames are a mix of %(imageTypes)s" % locals())
306 else:
307 self.arm = arm[0]
309 # CREATE DETECTOR LOOKUP DICTIONARY - SOME VALUES CAN BE OVERWRITTEN
310 # WITH WHAT IS FOUND HERE IN FITS HEADERS
311 self.detectorParams = detector_lookup(
312 log=self.log,
313 settings=self.settings
314 ).get(self.arm)
316 # MIXED BINNING IS BAD
317 cdelt1 = self.inputFrames.values(
318 keyword=kw("CDELT1").lower(), unique=True)
319 cdelt2 = self.inputFrames.values(
320 keyword=kw("CDELT2").lower(), unique=True)
322 if len(cdelt1) > 1 or len(cdelt2) > 1:
323 print(self.inputFrames.summary)
324 raise TypeError(
325 "Input frames are a mix of binnings" % locals())
327 if cdelt1[0] and cdelt2[0]:
328 self.detectorParams["binning"] = [int(cdelt2[0]), int(cdelt1[0])]
330 # MIXED READOUT SPEEDS IS BAD
331 readSpeed = self.inputFrames.values(
332 keyword=kw("DET_READ_SPEED").lower(), unique=True)
333 if len(readSpeed) > 1:
334 print(self.inputFrames.summary)
335 raise TypeError(
336 "Input frames are a mix of readout speeds" % locals())
338 # MIXED GAIN SPEEDS IS BAD
339 # HIERARCH ESO DET OUT1 CONAD - Electrons/ADU
340 # CONAD IS REALLY GAIN AND HAS UNIT OF Electrons/ADU
341 gain = self.inputFrames.values(
342 keyword=kw("CONAD").lower(), unique=True)
343 if len(gain) > 1:
344 print(self.inputFrames.summary)
345 raise TypeError(
346 "Input frames are a mix of gain" % locals())
347 if gain[0]:
348 # UVB & VIS
349 self.detectorParams["gain"] = gain[0] * u.electron / u.adu
350 else:
351 # NIR
352 self.detectorParams["gain"] = self.detectorParams[
353 "gain"] * u.electron / u.adu
355 # HIERARCH ESO DET OUT1 RON - Readout noise in electrons
356 ron = self.inputFrames.values(
357 keyword=kw("RON").lower(), unique=True)
359 # MIXED NOISE
360 if len(ron) > 1:
361 print(self.inputFrames.summary)
362 raise TypeError("Input frames are a mix of readnoise" % locals())
363 if ron[0]:
364 # UVB & VIS
365 self.detectorParams["ron"] = ron[0] * u.electron
366 else:
367 # NIR
368 self.detectorParams["ron"] = self.detectorParams[
369 "ron"] * u.electron
371 self.log.debug('completed the ``_verify_input_frames_basics`` method')
372 return None
374 def clean_up(
375 self):
376 """*remove intermediate files once recipe is complete*
378 **Usage**
380 ```python
381 recipe.clean_up()
382 ```
383 """
384 self.log.debug('starting the ``clean_up`` method')
386 outDir = self.intermediateRootPath + "/tmp"
388 try:
389 shutil.rmtree(outDir)
390 except:
391 pass
393 self.log.debug('completed the ``clean_up`` method')
394 return None
396 def xsh2soxs(
397 self,
398 frame):
399 """*perform some massaging of the xshooter data so it more closely resembles soxs data - this function can be removed once code is production ready*
401 **Key Arguments:**
402 - ``frame`` -- the CCDDate frame to manipulate
404 **Return:**
405 - ``frame`` -- the manipulated soxspipe-ready frame
407 **Usage:**
409 ```python
410 frame = self.xsh2soxs(frame)
411 ```
412 """
413 self.log.debug('starting the ``xsh2soxs`` method')
415 kw = self.kw
416 dp = self.detectorParams
418 # NP ROTATION OF ARRAYS IS IN COUNTER-CLOCKWISE DIRECTION
419 rotationIndex = int(dp["clockwise-rotation"] / 90.)
421 if self.settings["instrument"] == "xsh" and rotationIndex > 0:
422 frame.data = np.rot90(frame.data, rotationIndex)
424 self.log.debug('completed the ``xsh2soxs`` method')
425 return frame
427 def _trim_frame(
428 self,
429 frame):
430 """*return frame with pre-scan and overscan regions removed*
432 **Key Arguments:**
433 - ``frame`` -- the CCDData frame to be trimmed
434 """
435 self.log.debug('starting the ``_trim_frame`` method')
437 kw = self.kw
438 arm = self.arm
439 dp = self.detectorParams
441 rs, re, cs, ce = dp["science-pixels"]["rows"]["start"], dp["science-pixels"]["rows"][
442 "end"], dp["science-pixels"]["columns"]["start"], dp["science-pixels"]["columns"]["end"]
444 binning = dp["binning"]
445 if binning[0] > 1:
446 rs = int(rs / binning[0])
447 re = int(re / binning[0])
448 if binning[1] > 1:
449 cs = int(cs / binning[0])
450 ce = int(ce / binning[0])
452 trimmed_frame = ccdproc.trim_image(frame[rs: re, cs: ce])
454 self.log.debug('completed the ``_trim_frame`` method')
455 return trimmed_frame
457 def _write(
458 self,
459 frame,
460 filedir,
461 filename=False,
462 overwrite=True):
463 """*write frame to disk at the specified location*
465 **Key Arguments:**
466 - ``frame`` -- the frame to save to disk (CCDData object)
467 - ``filedir`` -- the location to save the frame
468 - ``filename`` -- the filename to save the file as. Default: **False** (standardised filename generated in code)
469 - ``overwrite`` -- if a file exists at the filepath then choose to overwrite the file. Default: True
471 **Usage:**
473 Use within a recipe like so:
475 ```python
476 self._write(frame, filePath)
477 ```
478 """
479 self.log.debug('starting the ``write`` method')
481 if not filename:
483 filename = filenamer(
484 log=self.log,
485 frame=frame,
486 settings=self.settings
487 )
489 filepath = filedir + "/" + filename
491 HDUList = frame.to_hdu(
492 hdu_mask='QUAL', hdu_uncertainty='ERRS', hdu_flags=None)
493 HDUList[0].name = "FLUX"
494 HDUList.writeto(filepath, output_verify='exception',
495 overwrite=overwrite, checksum=True)
497 self.log.debug('completed the ``write`` method')
498 return filepath
500 def clip_and_stack(
501 self,
502 frames,
503 recipe):
504 """*mean combine input frames after sigma-clipping outlying pixels using a median value with median absolute deviation (mad) as the deviation function*
506 **Key Arguments:**
507 - ``frames`` -- an ImageFileCollection of the framers to stack
508 - ``recipe`` -- the name of recipe needed to read the correct settings from the yaml files
510 **Return:**
511 - ``combined_frame`` -- the combined master frame (with updated bad-pixel and uncertainty maps)
513 **Usage:**
515 This snippet can be used within the recipe code to combine individual (using bias frames as an example):
517 ```python
518 combined_bias_mean = self.clip_and_stack(
519 frames=self.inputFrames, recipe="soxs_mbias")
520 ```
522 ---
524 ```eval_rst
525 .. todo::
527 - revisit error propagation when combining frames: https://github.com/thespacedoctor/soxspipe/issues/42
528 ```
529 """
530 self.log.debug('starting the ``clip_and_stack`` method')
532 arm = self.arm
533 kw = self.kw
534 dp = self.detectorParams
535 imageType = self.imageType
537 # ALLOW FOR UNDERSCORE AND HYPHENS
538 recipe = recipe.replace("soxs_", "soxs-")
540 # UNPACK SETTINGS
541 clipping_lower_sigma = self.settings[
542 recipe]["clipping-lower-simga"]
543 clipping_upper_sigma = self.settings[
544 recipe]["clipping-upper-simga"]
545 clipping_iteration_count = self.settings[
546 recipe]["clipping-iteration-count"]
548 # LIST OF CCDDATA OBJECTS NEEDED BY COMBINER OBJECT
549 # ccds = [c for c in self.inputFrames.ccds()]
550 ccds = [c for c in self.inputFrames.ccds(ccd_kwargs={"hdu_uncertainty": 'ERRS',
551 "hdu_mask": 'QUAL', "hdu_flags": 'FLAGS', "key_uncertainty_type": 'UTYPE'})]
553 # COMBINER OBJECT WILL FIRST GENERATE MASKS FOR INDIVIDUAL IMAGES VIA
554 # CLIPPING AND THEN COMBINE THE IMAGES WITH THE METHOD SELECTED. PIXEL
555 # MASKED IN ALL INDIVIDUAL IMAGES ARE MASK IN THE FINAL COMBINED IMAGE
556 combiner = Combiner(ccds)
558 print(f"\n# SIGMA-CLIPPING PIXEL WITH OUTLYING VALUES IN INDIVIDUAL {imageType} FRAMES")
559 # PRINT SOME INFO FOR USER
560 badCount = ccds[0].mask.sum()
561 totalPixels = np.size(ccds[0].mask)
562 percent = (float(badCount) / float(totalPixels)) * 100.
563 print(f"The basic bad-pixel mask for the {arm} detector {imageType} frames contains {badCount} pixels ({percent:0.2}% of all pixels)")
565 # GENERATE A MASK FOR EACH OF THE INDIVIDUAL INOUT FRAMES - USING
566 # MEDIAN WITH MEDIAN ABSOLUTE DEVIATION (MAD) AS THE DEVIATION FUNCTION
567 old_n_masked = -1
568 # THIS IS THE SUM OF BAD-PIXELS IN ALL INDIVIDUAL FRAME MASKS
569 new_n_masked = combiner.data_arr.mask.sum()
570 iteration = 1
571 while (new_n_masked > old_n_masked and iteration <= clipping_iteration_count):
572 combiner.sigma_clipping(
573 low_thresh=clipping_lower_sigma, high_thresh=clipping_upper_sigma, func=np.ma.median, dev_func=mad_std)
574 old_n_masked = new_n_masked
575 # RECOUNT BAD-PIXELS NOW CLIPPING HAS RUN
576 new_n_masked = combiner.data_arr.mask.sum()
577 diff = new_n_masked - old_n_masked
578 extra = ""
579 if diff == 0:
580 extra = " - we're done"
581 print(" Clipping iteration %(iteration)s finds %(diff)s more rogue pixels in the set of input frames%(extra)s" % locals())
582 iteration += 1
584 # GENERATE THE COMBINED MEDIAN
585 print("\n# MEAN COMBINING FRAMES - WITH UPDATED BAD-PIXEL MASKS")
586 combined_frame = combiner.average_combine()
588 # MASSIVE FUDGE - NEED TO CORRECTLY WRITE THE HEADER FOR COMBINED
589 # IMAGES
591 combined_frame.header = ccds[0].header
592 try:
593 combined_frame.wcs = ccds[0].wcs
594 except:
595 pass
596 combined_frame.header[
597 kw("DPR_CATG")] = "MASTER_%(imageType)s_%(arm)s" % locals()
599 # CALCULATE NEW PIXELS ADDED TO MASK
600 newBadCount = combined_frame.mask.sum()
601 diff = newBadCount - badCount
602 print("%(diff)s new pixels made it into the combined bad-pixel map" % locals())
604 self.log.debug('completed the ``clip_and_stack`` method')
605 return combined_frame
607 def subtract_calibrations(
608 self,
609 inputFrame,
610 master_bias=False,
611 dark=False):
612 """*subtract calibration frames from an input frame*
614 **Key Arguments:**
615 - ``inputFrame`` -- the input frame to have calibrations subtracted. CCDData object.
616 - ``master_bias`` -- the master bias frame to be subtracted. CCDData object. Default *False*.
617 - ``dark`` -- a dark frame to be subtracted. CCDData object. Default *False*.
619 **Return:**
620 - ``calibration_subtracted_frame`` -- the input frame with the calibration frame(s) subtracted. CCDData object.
622 **Usage:**
624 Within a soxspipe recipe use `subtract_calibrations` like so:
626 ```python
627 myCalibratedFrame = self.subtract_calibrations(
628 inputFrame=inputFrameCCDObject, master_bias=masterBiasCCDObject, dark=darkCCDObject)
629 ```
631 ---
633 ```eval_rst
634 .. todo::
636 - code needs written to scale dark frame to exposure time of science/calibration frame
637 ```
638 """
639 self.log.debug('starting the ``subtract_calibrations`` method')
641 arm = self.arm
642 kw = self.kw
643 dp = self.detectorParams
645 # VERIFY DATA IS IN ORDER
646 if master_bias == False and dark == False:
647 raise TypeError(
648 "subtract_calibrations method needs a master-bias frame and/or a dark frame to subtract")
649 if master_bias == False and dark.header[kw("EXPTIME")] != inputFrame.header[kw("EXPTIME")]:
650 raise AttributeError(
651 "Dark and science/calibration frame have differing exposure-times. A master-bias frame needs to be supplied to scale the dark frame to same exposure time as input science/calibration frame")
652 if master_bias != False and dark != False and dark.header[kw("EXPTIME")] != inputFrame.header[kw("EXPTIME")]:
653 raise AttributeError(
654 "CODE NEEDS WRITTEN HERE TO SCALE DARK FRAME TO EXPOSURE TIME OF SCIENCE/CALIBRATION FRAME")
656 # DARK WITH MATCHING EXPOSURE TIME
657 if dark != False and dark.header[kw("EXPTIME")] == inputFrame.header[kw("EXPTIME")]:
658 calibration_subtracted_frame = inputFrame.subtract(dark)
659 calibration_subtracted_frame.header = inputFrame.header
660 try:
661 calibration_subtracted_frame.wcs = inputFrame.wcs
662 except:
663 pass
665 # ONLY A MASTER BIAS FRAME, NO DARK
666 if dark == False and master_bias != False:
667 calibration_subtracted_frame = inputFrame.subtract(master_bias)
668 calibration_subtracted_frame.header = inputFrame.header
669 try:
670 calibration_subtracted_frame.wcs = inputFrame.wcs
671 except:
672 pass
674 self.log.debug('completed the ``subtract_calibrations`` method')
675 return calibration_subtracted_frame
677 # use the tab-trigger below for new method
678 # xt-class-method