"""
class Reduce provides one (1) public method:
runr()
which calls on the mapper classes and passes the received data to them.
"""
# ------------------------------------------------------------------------------
# DRAGONS
#
# coreReduce.py
# ------------------------------------------------------------------------------
import os
import sys
import inspect
#import traceback
from importlib import import_module
import astrodata
import gemini_instruments
from gempy.utils import logutils
from gempy.library import config
from astrodata import AstroDataError
from recipe_system import __version__
from recipe_system.utils.errors import ModeError
from recipe_system.utils.errors import RecipeNotFound
from recipe_system.utils.errors import PrimitivesNotFound
from recipe_system.utils.reduce_utils import buildParser
from recipe_system.utils.reduce_utils import normalize_ucals
from recipe_system.utils.reduce_utils import set_btypes
from recipe_system.utils.rs_utilities import log_traceback
from recipe_system.mappers.recipeMapper import RecipeMapper
from recipe_system.mappers.primitiveMapper import PrimitiveMapper
[docs]
class UnrecognizedParameterException(Exception):
""" Exception for unrecognized user parameters. """
pass
# ------------------------------------------------------------------------------
log = logutils.get_logger(__name__)
# ------------------------------------------------------------------------------
[docs]
class Reduce:
"""
The Reduce class encapsulates the core processing to be done by reduce.
__init__ may receive one (1) parameter, nominally, an argparse Namespace
instance. However, this object type is not required, but only that any
passed object *must* present an equivalent interface to that of an
<argparse.Namespace> instance, i.e. a duck type.
The class provides one (1) public method, runr(), the only call needed to
run reduce on the supplied argument set.
Parameters
----------
sys_args : :class:`argparse.Namespace` (optional) or <duck-type object>
This object type is not required, per se, but only that any passed
object *must* present an equivalent interface to that of an
:class:`argparse.Namespace` instance.
Attributes
----------
adinputs: <list>
attribute is a list of the input astrodata objects as made from
the 'files' list (see 'files' below).
output_filenames: <list>
read-only property is a list of final output filenames.
mode: <str>
operational mode. Currently, only 'qa', 'sq' modes are supported.
drpkg: <str>
Data reduction package name. Default is 'geminidr', the Gemini
Observatory data reduction package.
files: <list>
List of input filenames. Passed to Reduce.__init__(), these are
converted to astrodata objects.
suffix: <str>
User supplied suffix to be applied as a final suffix to output
filenames.
ucals: <dict>
Dictionary of calibration files passed by --user_cals flag.
uparms: <dict>
Dictionary of user parameters as passed by -p, --param flag.
upload : <list>
List of products to upload to fitsstore as passed by --upload.
E.g.,
--upload metrics calibs
(==> upload == ['metrics', 'calibs'])
will upload QA metrics to fitsstore and processing calibration
files.
recipename: <str> or callable
The name of the recipe that will be run. If None, the 'default'
recipe is used, as specified in the appropriate recipe library.
"""
def __init__(self, sys_args=None):
if sys_args:
args = sys_args
elif self._confirm_args():
args = buildParser(__version__).parse_args()
else:
args = buildParser(__version__).parse_args([])
# acquire any new astrodata classes.
if args.adpkg:
import_module(args.adpkg)
self.mode = args.mode
self.drpkg = args.drpkg
self.files = args.files
self.suffix = args.suffix
self.ucals = normalize_ucals(args.user_cal)
self.uparms = set_btypes(args.userparam)
self.config_file = args.config
self._upload = args.upload
self._output_filenames = None
self.recipename = args.recipename if args.recipename else '_default'
@property
def upload(self):
return self._upload
@upload.setter
def upload(self, upl):
if upl is None:
self._upload = None
elif isinstance(upl, str):
self._upload = [seg.lower().strip() for seg in upl.split(',')]
elif isinstance(upl, list):
self._upload = upl
return
@property
def output_filenames(self):
return self._output_filenames
[docs]
def runr(self):
"""
Map and run the requested or defaulted recipe.
Parameters
----------
<void>
Returns
-------
<void>
"""
recipe = None
try:
ffiles = self._check_files(self.files)
except OSError as err:
log.error(str(err))
raise
try:
adinputs = self._convert_inputs(ffiles)
except OSError as err:
log.error(str(err))
raise
# build mapper inputs, pass no 'ad' objects.
# mappers now receive tags and instr pkg name, e.g., 'gmos'
datatags = set(list(adinputs[0].tags)[:])
instpkg = adinputs[0].instrument(generic=True).lower()
rm = RecipeMapper(datatags, instpkg, mode=self.mode, drpkg=self.drpkg,
recipename=self.recipename)
try:
recipe = rm.get_applicable_recipe()
except ModeError as err:
log.warning("WARNING: {}".format(err))
pass
except RecipeNotFound:
log.warning("No recipe can be found in {} recipe libs.".format(instpkg))
log.warning("Searching primitives ...")
# If it's not a primitive, we'll crash later on, so assume it's a
# primitive and update uparms to prepend the name to any parameters
# without the primitive named explicitly
self.uparms = [((k if ':' in k else f"{self.recipename}:{k}"), v)
for k, v in self.uparms]
rm = None
# PrimitiveMapper now returns the primitive class, not an instance.
pm = PrimitiveMapper(datatags, instpkg, mode=self.mode, drpkg=self.drpkg)
try:
pclass = pm.get_applicable_primitives()
except PrimitivesNotFound as err:
log.error(str(err))
raise
p = pclass(adinputs, mode=self.mode, ucals=self.ucals, uparms=self.uparms,
upload=self.upload, config_file=self.config_file)
# Clean references to avoid keeping adinputs objects in memory one
# there are no more needed.
adinputs = None
# If the RecipeMapper was unable to find a specified user recipe,
# it is possible that the recipe passed was a primitive name.
# Here we examine the primitive set to see if this recipe is actually
# a primitive name.
norec_msg = "{} recipes do not define a '{}' recipe for these data."
if recipe is None and self.recipename == '_default':
raise RecipeNotFound(norec_msg.format(instpkg.upper(), self.mode))
if recipe is None:
try:
primitive_as_recipe = getattr(p, self.recipename)
except AttributeError as err:
err = "Recipe {} Not Found".format(self.recipename)
log.error(str(err))
raise RecipeNotFound("No primitive named {}".format(self.recipename))
pname = primitive_as_recipe.__name__
log.stdinfo("Found '{}' as a primitive.".format(pname))
self._logheader(pname)
try:
primitive_as_recipe()
except Exception as err:
log_traceback(log)
log.error(str(err))
raise
else:
self._logheader(recipe)
try:
recipe(p)
except Exception:
log.error("Reduce received an unhandled exception. Aborting ...")
log_traceback(log)
log.stdinfo("Writing final outputs ...")
try:
for ad in p.streams['main']:
ad.update_filename(suffix="_crash")
except: # in case something has gone really wrong!
log.stdinfo("Problem updating filenames")
self._write_final(p.streams['main'])
self._output_filenames = [ad.filename for ad in p.streams['main']]
raise
self._write_final(p.streams['main'])
self._output_filenames = [ad.filename for ad in p.streams['main']]
log.stdinfo("\nreduce completed successfully.")
# -------------------------------- prive -----------------------------------
def _check_files(self, ffiles):
"""
Sanity check on submitted files.
Parameters
--------
ffiles: <list>
list of passed FITS files.
Return
------
input_files: <list>
list of 'good' input fits datasets.
"""
try:
assert ffiles
except AssertionError:
log.error("NO INPUT FILE(s) specified")
log.stdinfo("type 'reduce -h' for usage information")
raise OSError("NO INPUT FILE(s) specified")
input_files = []
bad_files = []
for image in ffiles:
if not os.access(image, os.R_OK):
log.error('Cannot read file: '+str(image))
bad_files.append(image)
else:
input_files.append(image)
try:
assert bad_files
err = "\n\t".join(bad_files)
log.warning("Files not found or cannot be loaded:\n\t%s" % err)
if input_files:
found = "\n\t".join(input_files)
log.stdinfo("These datasets were loaded:\n\t%s" % found)
else:
log.error("Caller passed no valid input files")
sys.exit(1)
except AssertionError:
log.stdinfo("All submitted files appear valid:")
if len(input_files) > 1:
filestr = input_files[0]
filestr += " ... " + input_files[-1]
filestr += ", {} files submitted.".format(len(input_files))
else:
filestr = input_files[0]
log.stdinfo(filestr)
return input_files
def _convert_inputs(self, inputs):
"""
Convert files into AstroData objects.
Parameters
----------
inputs: <list>, list of FITS file names
Return
------
allinputs: <list>, list of AstroData objects
"""
allinputs = []
for inp in inputs:
try:
ad = astrodata.open(inp)
except AstroDataError as err:
log.warning("Can't Load Dataset: %s" % inp)
log.warning(err)
continue
except OSError as err:
log.warning("Can't Load Dataset: %s" % inp)
log.warning(err)
continue
if not len(ad):
log.warning("%s contains no extensions." % ad.filename)
continue
allinputs.append(ad)
return allinputs
def _confirm_args(self):
"""
Confirm that the first executable frame in the call stack is a reduce
command line. This asserts that a nominal reduce parser, as returned by
buildParser() function, is an equivalent Namespace object to that
of an 'args' key in the stack's 'f_locals' namespace. If the Namespace
objects are not equal, reduce is not calling this class.
Parameters
----------
<void>
Returns
-------
is_reduce: <bool>,
Did 'reduce' call this?
"""
is_reduce = False
exe_path = sys.argv[0]
red_namespace = buildParser(__version__).parse_args([])
if exe_path:
cstack = inspect.stack()
for local, value in list(cstack[-1][0].f_locals.items()):
if local == 'args':
try:
assert(
list(value.__dict__.keys()) ==
list(red_namespace.__dict__.keys())
)
is_reduce = True
except AttributeError:
pass
except AssertionError:
log.stdinfo("A non-reduce command line was detected.")
pass
return is_reduce
def _logheader(self, recipe):
if self.recipename == '_default':
r_actual = recipe.__name__
else:
r_actual = self.recipename
logstring = "RECIPE: {}".format(r_actual)
log.status("="*80)
log.status(logstring)
log.status("="*80)
return
def _write_final(self, outputs):
"""
Write final outputs. Write only if filename is not == orig_filename, or
if there is a user suffix (self.suffix)
Parameters
----------
outputs: <list>, List of AstroData objects
Return
------
<void>
"""
outstr = "\tWrote {} in output directory"
def _sname(name):
head, tail = os.path.splitext(name)
ohead = head.split("_")[0]
newname = ohead + self.suffix + tail
return newname
for ad in outputs:
if self.suffix:
username = _sname(ad.filename)
ad.write(username, overwrite=True)
log.stdinfo(outstr.format(username))
elif ad.filename != ad.orig_filename:
ad.write(ad.filename, overwrite=True)
log.stdinfo(outstr.format(ad.filename))
return