import math
import json
import os
from collections.abc import Mapping
from datetime import datetime, timedelta
from getpass import getuser
from io import StringIO
from urllib.parse import urlparse
import yaml
from . import proto as _proto
from .objects import Enum, ProtobufMessage, Specification, required
from .exceptions import context, FileNotFoundError, FileExistsError
from .utils import (implements, format_list, datetime_from_millis, runtime,
xor, lock_file)
__all__ = ('ApplicationSpec', 'Service', 'Resources', 'File', 'FileType',
'FileVisibility', 'ACLs', 'Master', 'Security', 'ApplicationState',
'FinalStatus', 'ResourceUsageReport', 'ApplicationReport',
'ContainerState', 'Container', 'LogLevel', 'NodeState', 'NodeReport',
'QueueState', 'Queue', 'ApplicationLogs')
def _check_is_filename(target):
orig = target
# Allow local relative paths
if target.startswith("./"):
target = target[2:]
if "/" in target:
raise context.ValueError(
"Keys in `files` must be filenames only (no directories allowed), "
"found %r" % orig
)
def _pop_origin(kwargs):
_origin = kwargs.pop('_origin', None)
if kwargs:
raise TypeError("from_dict() got an unexpected keyword argument "
"%s" % next(iter(kwargs)))
return _origin
def container_instance_from_string(id):
"""Create a ContainerInstance from an id string"""
try:
service, instance = id.rsplit('_', 1)
instance = int(instance)
except (TypeError, ValueError):
raise context.ValueError("Invalid container id %r" % id)
return _proto.ContainerInstance(service_name=service, instance=instance)
def container_instance_to_string(id):
"""Create an id string from a ContainerInstance"""
return '%s_%d' % (id.service_name, id.instance)
def parse_memory(s):
"""Converts bytes expression to number of mebibytes.
If no unit is specified, ``MiB`` is used."""
if isinstance(s, int):
out = s
elif isinstance(s, float):
out = math.ceil(s)
elif isinstance(s, str):
s = s.replace(' ', '')
if not s:
raise context.ValueError("Could not interpret %r as a byte unit" % s)
if s[0].isdigit():
for i, c in enumerate(reversed(s)):
if not c.isalpha():
break
index = len(s) - i
prefix = s[:index]
suffix = s[index:]
try:
n = float(prefix)
except ValueError:
raise context.ValueError("Could not interpret %r as a number" % prefix)
else:
n = 1
suffix = s
try:
multiplier = _byte_sizes[suffix.lower()]
except KeyError:
raise context.ValueError("Could not interpret %r as a byte unit" % suffix)
out = math.ceil(n * multiplier / (2 ** 20))
else:
raise context.TypeError("memory must be an integer, got %r"
% type(s).__name__)
if out < 0:
raise context.ValueError("memory must be positive")
return out
_byte_sizes = {
'kb': 10**3,
'mb': 10**6,
'gb': 10**9,
'tb': 10**12,
'pb': 10**15,
'kib': 2**10,
'mib': 2**20,
'gib': 2**30,
'tib': 2**40,
'pib': 2**50,
'b': 1,
'': 2 ** 20
}
_byte_sizes.update({k[:-1]: v for k, v in _byte_sizes.items() if len(k) >= 2})
def check_no_cycles(dependencies):
completed = set()
seen = set()
for key in dependencies:
if key in completed:
continue
nodes = [key]
while nodes:
# Keep current node on the stack until all descendants are visited
cur = nodes[-1]
if cur in completed:
# Already fully traversed descendants of cur
nodes.pop()
continue
seen.add(cur)
# Add direct descendants of cur to nodes stack
next_nodes = []
for nxt in dependencies[cur]:
if nxt not in completed:
if nxt in seen:
cycle = [nxt]
while nodes[-1] != nxt:
cycle.append(nodes.pop())
cycle.append(nodes.pop())
raise context.ValueError(
'Dependency cycle detected between services: %s' %
'->'.join(str(x) for x in reversed(cycle)))
next_nodes.append(nxt)
if next_nodes:
nodes.extend(next_nodes)
else:
completed.add(cur)
seen.remove(cur)
nodes.pop()
def _infer_format(path, format='infer'):
if format == 'infer':
_, ext = os.path.splitext(path)
if ext == '.json':
format = 'json'
elif ext in {'.yaml', '.yml'}:
format = 'yaml'
else:
if context.is_cli:
msg = "Unsupported file type %r" % ext
else:
msg = ("Can't infer format from filepath %r, please "
"specify manually" % path)
raise context.ValueError(msg)
elif format not in {'json', 'yaml'}:
raise ValueError("Unknown file format: %r" % format)
return format
class Security(Specification):
"""Security configuration.
Secrets may be specified either as file paths or raw bytes, but not both.
Parameters
----------
cert_file : str, File, optional
The TLS certificate file, in pem format. Either a path or a fully
specified File object.
key_file : str, File, optional
The TLS private key file, in pem format. Either a path or a fully
specified File object.
cert_bytes : bytes, optional
The contents of the TLS certificate file, in pem format.
key_bytes : bytes, optional
The contents of the TLS private key file, in pem format.
"""
__slots__ = ('_cert_file', '_key_file', '_cert_bytes', '_key_bytes')
_params = ('cert_file', 'key_file', 'cert_bytes', 'key_bytes')
_protobuf_cls = _proto.Security
def __init__(self, cert_file=None, key_file=None, cert_bytes=None,
key_bytes=None):
self.cert_file = cert_file
self.key_file = key_file
self.cert_bytes = cert_bytes
self.key_bytes = key_bytes
self._validate()
def _get_bytes(self, kind):
self._validate()
out = getattr(self, '%s_bytes' % kind)
if out is not None:
return out
file = getattr(self, '%s_file' % kind)
if not file.source.startswith('file://'):
raise context.ValueError(
"Cannot read security file %r locally" % file.source
)
path = file.source[7:]
if not os.path.exists(path):
raise context.FileNotFoundError(
"Security %s file not found at %r" % (kind, path)
)
with open(path, 'rb') as fil:
return fil.read()
def _validate(self):
if not xor(self.cert_file is None, self.cert_bytes is None):
raise context.ValueError("Must specify exactly one of "
"cert_file or cert_bytes")
if not xor(self.key_file is None, self.key_bytes is None):
raise context.ValueError("Must specify exactly one of "
"key_file or key_bytes")
def _assign_file(self, field, value):
if isinstance(value, str):
value = File(value)
elif not (value is None or isinstance(value, File)):
raise context.TypeError("%s must be a File or None" % field)
setattr(self, '_%s' % field, value)
def _assign_bytes(self, field, value):
if value is not None:
if isinstance(value, str):
value = value.encode()
elif not isinstance(value, bytes):
raise context.TypeError("%s must be a bytes or None" % field)
setattr(self, '_%s' % field, value)
@property
def cert_file(self):
return self._cert_file
@cert_file.setter
def cert_file(self, value):
self._assign_file('cert_file', value)
@property
def key_file(self):
return self._key_file
@key_file.setter
def key_file(self, value):
self._assign_file('key_file', value)
@property
def cert_bytes(self):
return self._cert_bytes
@cert_bytes.setter
def cert_bytes(self, value):
self._assign_bytes('cert_bytes', value)
@property
def key_bytes(self):
return self._key_bytes
@key_bytes.setter
def key_bytes(self, value):
self._assign_bytes('key_bytes', value)
def __repr__(self):
return 'Security<...>'
@classmethod
@implements(Specification.from_dict)
def from_dict(cls, obj, **kwargs):
_origin = _pop_origin(kwargs)
cls._check_keys(obj)
cert_file = obj.pop('cert_file', None)
if cert_file is not None:
cert_file = File.from_dict(cert_file, _origin=_origin)
key_file = obj.pop('key_file', None)
if key_file is not None:
key_file = File.from_dict(key_file, _origin=_origin)
return cls(cert_file=cert_file, key_file=key_file, **obj)
@classmethod
@implements(Specification.from_protobuf)
def from_protobuf(cls, obj):
kwargs = {}
for name in ['cert', 'key']:
which = obj.WhichOneof(name)
if which == '%s_bytes' % name:
kwargs[which] = getattr(obj, '%s_bytes' % name)
elif which == '%s_file' % name:
kwargs[which] = File.from_protobuf(getattr(obj, '%s_file' % name))
return cls(**kwargs)
@implements(Specification.to_dict)
def to_dict(self, skip_nulls=True):
obj = super(Security, self).to_dict(skip_nulls=skip_nulls)
return {k: v.decode() if isinstance(v, bytes) else v
for k, v in obj.items()}
@classmethod
def from_default(cls):
"""The default security configuration.
Usually this loads the credentials stored in the configuration
directory (``~/.skein`` by default). If these credentials don't already
exist, new ones will be created.
When run in a YARN container started by Skein, this loads the same
security credentials as used for the current application.
"""
from .core import properties
# Are we in a container started by skein?
if properties.application_id is not None:
if properties.container_dir is not None:
cert_path = os.path.join(properties.container_dir, '.skein.crt')
key_path = os.path.join(properties.container_dir, '.skein.pem')
if os.path.exists(cert_path) and os.path.exists(key_path):
return Security(cert_file=cert_path, key_file=key_path)
raise context.FileNotFoundError(
"Failed to resolve .skein.{crt,pem} in 'LOCAL_DIRS'")
# Try to load from config_dir, and fallback to minting new credentials
try:
return cls.from_directory(properties.config_dir)
except FileNotFoundError:
pass
new = cls.new_credentials()
try:
out = new.to_directory(properties.config_dir)
context.warn("Skein global security credentials not found, "
"writing now to %r." % properties.config_dir)
except FileExistsError:
# Race condition between competing processes, use the credentials
# written by the other process.
out = cls.from_directory(properties.config_dir)
return out
@classmethod
def from_directory(cls, directory):
"""Create a security object from a directory.
Relies on standard names for each file (``skein.crt`` and
``skein.pem``)."""
cert_path = os.path.join(directory, 'skein.crt')
key_path = os.path.join(directory, 'skein.pem')
for path, name in [(cert_path, 'cert'), (key_path, 'key')]:
if not os.path.exists(path):
raise context.FileNotFoundError(
"Security %s file not found at %r" % (name, path)
)
return Security(cert_file=cert_path, key_file=key_path)
def to_directory(self, directory, force=False):
"""Write this security object to a directory.
Parameters
----------
directory : str
The directory to write the configuration to.
force : bool, optional
If security credentials already exist at this location, an error
will be raised by default. Set to True to overwrite existing files.
Returns
-------
security : Security
A new security object backed by the written files.
"""
self._validate()
# Create directory if it doesn't exist
os.makedirs(directory, exist_ok=True)
cert_path = os.path.join(directory, 'skein.crt')
key_path = os.path.join(directory, 'skein.pem')
cert_bytes = self._get_bytes('cert')
key_bytes = self._get_bytes('key')
lock_path = os.path.join(directory, 'skein.lock')
with lock_file(lock_path):
for path, name in [(cert_path, 'skein.crt'), (key_path, 'skein.pem')]:
if os.path.exists(path):
if force:
os.unlink(path)
else:
msg = ("%r file already exists, use `%s` to overwrite" %
(name, '--force' if context.is_cli else 'force'))
raise context.FileExistsError(msg)
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
for path, data in [(cert_path, cert_bytes), (key_path, key_bytes)]:
with os.fdopen(os.open(path, flags, 0o600), 'wb') as fil:
fil.write(data)
return Security(cert_file=cert_path, key_file=key_path)
@classmethod
def new_credentials(cls):
"""Create a new Security object with a new certificate/key pair."""
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
key = rsa.generate_private_key(public_exponent=65537,
key_size=2048,
backend=default_backend())
key_bytes = key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption())
subject = issuer = x509.Name(
[x509.NameAttribute(NameOID.COMMON_NAME, u'skein-internal')])
now = datetime.utcnow()
cert = (x509.CertificateBuilder()
.subject_name(subject)
.issuer_name(issuer)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now)
.not_valid_after(now + timedelta(days=365))
.sign(key, hashes.SHA256(), default_backend()))
cert_bytes = cert.public_bytes(serialization.Encoding.PEM)
return cls(cert_bytes=cert_bytes, key_bytes=key_bytes)
class ApplicationState(Enum):
"""Enum of application states.
Attributes
----------
NEW : ApplicationState
Application was just created.
NEW_SAVING : ApplicationState
Application is being saved.
SUBMITTED : ApplicationState
Application has been submitted.
ACCEPTED : ApplicationState
Application has been accepted by the scheduler.
RUNNING : ApplicationState
Application is currently running.
FINISHED : ApplicationState
Application finished successfully.
FAILED : ApplicationState
Application failed.
KILLED : ApplicationState
Application was terminated by a user or admin.
"""
_values = ('NEW',
'NEW_SAVING',
'SUBMITTED',
'ACCEPTED',
'RUNNING',
'FINISHED',
'FAILED',
'KILLED')
class FinalStatus(Enum):
"""Enum of application final statuses.
Attributes
----------
SUCCEEDED : FinalStatus
Application finished successfully.
KILLED : FinalStatus
Application was terminated by a user or admin.
FAILED : FinalStatus
Application failed.
UNDEFINED : FinalStatus
Application has not yet finished.
"""
_values = ('SUCCEEDED',
'KILLED',
'FAILED',
'UNDEFINED')
class Resources(Specification):
"""Resource requests per container.
Parameters
----------
memory : str or int
The amount of memory to request. Can be either a string with units
(e.g. ``"5 GiB"``), or numeric. If numeric, specifies the amount of
memory in *MiB*. Note that the units are in mebibytes (MiB) NOT
megabytes (MB) - the former being binary based (1024 MiB in a GiB), the
latter being decimal based (1000 MB in a GB).
Requests smaller than the minimum allocation will receive the minimum
allocation (1024 MiB by default). Requests larger than the maximum
allocation will error on application submission.
vcores : int
The number of virtual cores to request. Depending on your system
configuration one virtual core may map to a single actual core, or a
fraction of a core. Requests larger than the maximum allocation will
error on application submission.
gpus : int, optional
The number of gpus to request. Requires Hadoop >= 3.1, sets
resource requirements for ``yarn.io/gpu``. Default is 0.
fpgas : int, optional
The number of fpgas to request. Requires Hadoop >= 3.1, sets
resource requirements for ``yarn.io/fpga``. Default is 0.
"""
__slots__ = ('_memory', 'vcores', 'gpus', 'fpgas')
_params = ('memory', 'vcores', 'gpus', 'fpgas')
_protobuf_cls = _proto.Resources
def __init__(self, memory=required, vcores=required, gpus=0, fpgas=0):
self._assign_required('memory', memory)
self._assign_required('vcores', vcores)
self.gpus = gpus
self.fpgas = fpgas
self._validate()
@property
def memory(self):
return self._memory
@memory.setter
def memory(self, value):
self._memory = parse_memory(value)
def __repr__(self):
return 'Resources<memory=%d, vcores=%d>' % (self.memory, self.vcores)
def _validate(self, is_request=False):
min = 1 if is_request else 0
self._check_is_bounded_int('vcores', min=min)
self._check_is_bounded_int('memory', min=min)
self._check_is_bounded_int('gpus', min=0)
self._check_is_bounded_int('fpgas', min=0)
class FileVisibility(Enum):
"""Enum of possible file visibilities.
Determines how the file can be shared between containers.
Attributes
----------
APPLICATION : FileVisibility
Shared only among containers of the same application on the node.
PUBLIC : FileVisibility
Shared by all users on the node.
PRIVATE : FileVisibility
Shared among all applications of the same user on the node.
"""
_values = ('APPLICATION', 'PUBLIC', 'PRIVATE')
class FileType(Enum):
"""Enum of possible file types to distribute with the application.
Attributes
----------
FILE : FileType
Regular file
ARCHIVE : FileType
A ``.zip``, ``.tar.gz``, or ``.tgz`` file to be automatically
unarchived in the containers.
"""
_values = ('FILE', 'ARCHIVE')
class File(Specification):
"""A file/archive to distribute with the service.
Parameters
----------
source : str
The path to the file/archive. If no scheme is specified, path is
assumed to be on the local filesystem (``file://`` scheme).
type : FileType or str, optional
The type of file to distribute. Archive's are automatically extracted
by yarn into a directory with the same name as their destination.
By default the type is inferred from the file extension.
visibility : FileVisibility or str, optional
The resource visibility, default is ``FileVisibility.APPLICATION``
size : int, optional
The resource size in bytes. If not provided will be determined by the
file system.
timestamp : int, optional
The time the resource was last modified. If not provided will be
determined by the file system.
"""
__slots__ = ('_source', '_type', '_visibility', 'size', 'timestamp')
_params = ('source', 'type', 'visibility', 'size', 'timestamp')
_protobuf_cls = _proto.File
def __init__(self, source=required, type='infer',
visibility=FileVisibility.APPLICATION, size=0, timestamp=0):
self._assign_required('source', source)
self.type = type
self.visibility = visibility
self.size = size
self.timestamp = timestamp
self._validate()
def __repr__(self):
return 'File<source=%r, type=%r>' % (self.source, self.type)
def _validate(self):
self._check_is_type('source', str)
self._check_is_type('type', FileType)
self._check_is_type('visibility', FileVisibility)
self._check_is_bounded_int('size')
self._check_is_bounded_int('timestamp')
@property
def source(self):
return self._source
@source.setter
def source(self, val):
if not isinstance(val, str):
raise context.TypeError("'source' must be a string")
self._source = self._normpath(val)
@property
def type(self):
if self._type == 'infer':
return (FileType.ARCHIVE
if any(map(self._source.endswith, ('.zip', '.tar.gz', '.tgz')))
else FileType.FILE)
return self._type
@type.setter
def type(self, val):
self._type = val if val == 'infer' else FileType(val)
@property
def visibility(self):
return self._visibility
@visibility.setter
def visibility(self, val):
self._visibility = FileVisibility(val)
@staticmethod
def _normpath(path, origin=None):
url = urlparse(path)
if not url.scheme:
if not os.path.isabs(url.path):
if origin is not None:
path = os.path.normpath(os.path.join(origin, url.path))
else:
path = os.path.abspath(url.path)
else:
path = url.path
return 'file://%s%s' % (url.netloc, path)
return path
@implements(ProtobufMessage.to_protobuf)
def to_protobuf(self):
self._validate()
url = urlparse(self.source)
urlmsg = _proto.Url(scheme=url.scheme,
host=url.hostname,
port=url.port,
file=url.path)
return _proto.File(source=urlmsg,
type=str(self.type),
visibility=str(self.visibility),
size=self.size,
timestamp=self.timestamp)
@classmethod
def from_dict(cls, obj, **kwargs):
"""Create an instance from a dict.
Keys in the dict should match parameter names"""
_origin = _pop_origin(kwargs)
if isinstance(obj, str):
obj = {'source': obj}
cls._check_keys(obj)
if _origin:
if 'source' not in obj:
raise context.TypeError("parameter 'source' is required but "
"wasn't provided")
obj = dict(obj)
obj['source'] = cls._normpath(obj['source'], _origin)
return cls(**obj)
@classmethod
@implements(Specification.from_protobuf)
def from_protobuf(cls, obj):
if not isinstance(obj, cls._protobuf_cls):
raise TypeError("Expected message of type "
"%r" % cls._protobuf_cls.__name__)
url = obj.source
netloc = '%s:%d' % (url.host, url.port) if url.host else ''
source = '%s://%s%s' % (url.scheme, netloc, url.file)
return cls(source=source,
type=_proto.File.Type.Name(obj.type),
visibility=_proto.File.Visibility.Name(obj.visibility),
size=obj.size,
timestamp=obj.timestamp)
class Service(Specification):
"""Description of a Skein service.
Parameters
----------
resources : Resources
Describes the resources needed to run the service.
script : str
A bash script to run the service.
instances : int, optional
The number of instances to create on startup. Default is 1.
files : dict, optional
Describes any files needed to run the service. A mapping of destination
relative paths to ``File`` or ``str`` objects describing the sources
for these paths. If a ``str``, the file type is inferred from the
extension.
env : dict, optional
A mapping of environment variables needed to run the service.
depends : set, optional
A set of service names that this service depends on. The service will
only be started after all its dependencies have been started.
max_restarts : int, optional
The maximum number of restarts to allow for this service. Containers
are only restarted on failure, and the cap is set for all containers in
the service, not per container. Set to -1 to allow infinite restarts.
Default is 0.
allow_failures : bool, optional
If False (default), the whole application will shutdown if the number
of failures for this service exceeds ``max_restarts``. Set to True to
keep the application running even if this service exceeds its failure
limit.
node_label : str, optional
The node label expression to use when requesting containers for this
service. If not set, defaults to the application-level ``node_label``
(if set).
nodes : list, optional
A list of node host names to restrict containers for this service to.
If not set, defaults to no node restrictions.
racks : list, optional
A list of rack names to restrict containers for this service to. The
racks corresponding to any nodes requested will be automatically added
to this list. If not set, defaults to no rack restrictions.
relax_locality : bool, optional
If true, containers for this request may be assigned on hosts and racks
other than the ones explicitly requested. If False, those restrictions
are strictly enforced. Default is False.
"""
__slots__ = ('resources', 'script', 'instances', 'files', 'env',
'depends', 'max_restarts', 'allow_failures', 'node_label',
'nodes', 'racks', 'relax_locality')
_protobuf_cls = _proto.Service
def __init__(self, resources=required, script=required, instances=1,
files=None, env=None, depends=None, max_restarts=0,
allow_failures=False, node_label='', nodes=None, racks=None,
relax_locality=False):
self._assign_required('resources', resources)
self._assign_required('script', script)
self.instances = instances
self.files = ({k: v if isinstance(v, File) else File(v)
for (k, v) in files.items()}
if files is not None else {})
self.env = {} if env is None else env
self.depends = set() if depends is None else set(depends)
self.max_restarts = max_restarts
self.allow_failures = allow_failures
self.node_label = node_label
self.nodes = [] if nodes is None else nodes
self.racks = [] if racks is None else racks
self.relax_locality = relax_locality
self._validate()
def __repr__(self):
return 'Service<instances=%d, ...>' % self.instances
def _validate(self):
self._check_is_bounded_int('instances', min=0)
self._check_is_type('node_label', str)
self._check_is_list_of('nodes', str)
self._check_is_list_of('racks', str)
self._check_is_type('relax_locality', bool)
self._check_is_bounded_int('max_restarts', min=-1)
self._check_is_type('allow_failures', bool)
self._check_is_type('resources', Resources)
self.resources._validate(is_request=True)
self._check_is_dict_of('files', str, File)
for target, f in self.files.items():
_check_is_filename(target)
f._validate()
self._check_is_dict_of('env', str, str)
self._check_is_type('script', str)
if not self.script:
raise context.ValueError("A script must be provided")
self._check_is_set_of('depends', str)
@classmethod
@implements(Specification.from_dict)
def from_dict(cls, obj, **kwargs):
_origin = _pop_origin(kwargs)
obj = obj.copy()
cls._check_keys(obj, cls.__slots__)
resources = obj.pop('resources', None)
if resources is not None:
resources = Resources.from_dict(resources)
files = obj.pop('files', None)
if files is not None:
files = {k: File.from_dict(v, _origin=_origin)
for k, v in files.items()}
return cls(resources=resources,
files=files,
**obj)
@classmethod
@implements(Specification.from_protobuf)
def from_protobuf(cls, obj):
resources = Resources.from_protobuf(obj.resources)
files = {k: File.from_protobuf(v) for k, v in obj.files.items()}
kwargs = {'instances': obj.instances,
'node_label': obj.node_label,
'nodes': list(obj.nodes),
'racks': list(obj.racks),
'relax_locality': obj.relax_locality,
'max_restarts': obj.max_restarts,
'allow_failures': obj.allow_failures,
'resources': resources,
'files': files,
'env': dict(obj.env),
'script': obj.script,
'depends': set(obj.depends)}
return cls(**kwargs)
class ACLs(Specification):
"""Skein Access Control Lists.
Maps access types to users/groups to provide that access.
The following access types are supported:
- VIEW : view application details
- MODIFY : modify the application via YARN (e.g. killing the application)
- UI : access the application Web UI
The VIEW and MODIFY access types are handled by YARN directly; permissions
for these can be set by users and/or groups. Authorizing UI access is
handled by Skein internally, and only user-level access control is
supported.
The application owner (the user who submitted the application) will always
have permission for all access types.
By default, ACLs are disabled - to enable, set ``enable=True``. If enabled,
access is restricted only to the application owner by default - add
users/groups to the access types you wish to expand to other users.
Parameters
----------
enable : bool, optional
If True, the ACLs will be enforced. Default is False.
view_users, view_groups : list, optional
Lists of users/groups to give VIEW access to this application. If both
are empty, only the application owner has access (default). If either
contains ``"*"``, all users are given access (default). See the YARN
documentation for more information on what VIEW access entails.
modify_users, modify_groups : list, optional
Lists of users/groups to give MODIFY access to this application. If
both are empty, only the application owner has access (default). If
either contains ``"*"``, all users are given access (default). See the
YARN documentation for more information on what MODIFY access entails.
ui_users : list, optional
A list of users to give access to the application Web UI. If empty,
only the application owner has access (default). If it contains
``"*"``, all users are given access.
Examples
--------
By default ACLs are disabled, and all users have access.
>>> import skein
>>> acls = skein.ACLs()
Enabling ACLs results in only the application owner having access (provided
YARN is also configured with ACLs enabled).
>>> acls = skein.ACLs(enable=True)
To give access to other users, add users/groups to the desired access
types. Here we enable view access for all users in group ``engineering``,
and modify access for user ``nancy``.
>>> acls = skein.ACLs(enable=True,
... view_groups=['engineering'],
... modify_users=['nancy'])
You can use the wildcard character ``"*"`` to enable access for all users.
Here we give view access to all users:
>>> acls = skein.ACLs(enable=True,
... view_users=['*'])
"""
__slots__ = ('enable', 'view_users', 'view_groups', 'modify_users',
'modify_groups', 'ui_users')
_protobuf_cls = _proto.Acls
def __init__(self, enable=False, view_users=None, view_groups=None,
modify_users=None, modify_groups=None, ui_users=None):
self.enable = enable
self.view_users = [] if view_users is None else view_users
self.view_groups = [] if view_groups is None else view_groups
self.modify_users = [] if modify_users is None else modify_users
self.modify_groups = [] if modify_groups is None else modify_groups
self.ui_users = [] if ui_users is None else ui_users
self._validate()
def __repr__(self):
return 'ACLs<enable=%r, ...>' % self.enable
def _validate(self):
self._check_is_type('enable', bool)
self._check_is_list_of('view_users', str)
self._check_is_list_of('view_groups', str)
self._check_is_list_of('modify_users', str)
self._check_is_list_of('modify_groups', str)
self._check_is_list_of('ui_users', str)
@classmethod
@implements(Specification.from_protobuf)
def from_protobuf(cls, obj):
if not isinstance(obj, cls._protobuf_cls):
raise TypeError("Expected message of type "
"%r" % cls._protobuf_cls.__name__)
return cls(enable=obj.enable,
view_users=list(obj.view_users),
view_groups=list(obj.view_groups),
modify_users=list(obj.modify_users),
modify_groups=list(obj.modify_groups),
ui_users=list(obj.ui_users))
class LogLevel(Enum):
"""Enum of log levels.
Corresponds with ``log4j`` logging levels.
Attributes
----------
OFF : LogLevel
Turns on all logging.
TRACE : LogLevel
The finest level of events.
DEBUG : LogLevel
Fine-grained informational events that are most useful to debug an
application.
INFO : LogLevel
Informational messages that highlight the progress of the application
at a coarse-grained level. The default LogLevel.
WARN : LogLevel
Potentially harmful situations that still allow the application to
continue running.
ERROR : LogLevel
Error events that might still allow the application to continue
running.
FATAL : LogLevel
Severe error events that will lead the application to abort.
OFF : LogLevel
Turns off all logging.
"""
_values = ('ALL',
'TRACE',
'DEBUG',
'INFO',
'WARN',
'ERROR',
'FATAL',
'OFF')
class Master(Specification):
"""Configuration for the Application Master.
Parameters
----------
resources : Resources, optional
Describes the resources needed to run the application master. Default
is 512 MiB, 1 virtual core.
script : str, optional
An optional bash script to run after starting the application master.
If provided, the application will terminate once the script has
completed.
files : dict, optional
Describes any additional files needed to run the application master. A
mapping of destination relative paths to ``File`` or ``str`` objects
describing the sources for these paths. If a ``str``, the file type is
inferred from the extension.
env : dict, optional
A mapping of environment variables to set on the application master.
log_level : str or LogLevel, optional
The application master log level. Sets the ``skein.log.level`` system
property. One of {'ALL', 'TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR',
'FATAL', 'OFF'} (from most to least verbose). Default is 'INFO'.
log_config : str or File, optional
A custom ``log4j.properties`` file to use for the application master.
If not provided, the default logging configuration will be used.
security : Security, optional
The security credentials to use for the application master. If not
provided, these will be the same as those used by the submitting
client.
"""
__slots__ = ('resources', 'script', 'files', 'env',
'_log_level', 'log_config', 'security')
_params = ('resources', 'script', 'files', 'env',
'log_level', 'log_config', 'security')
_protobuf_cls = _proto.Master
def __init__(self, resources=None, script="", files=None, env=None,
log_level=LogLevel.INFO, log_config=None, security=None):
self.resources = (Resources(memory='512 MiB', vcores=1)
if resources is None else resources)
self.script = script
self.files = ({k: v if isinstance(v, File) else File(v)
for (k, v) in files.items()}
if files is not None else {})
self.env = {} if env is None else env
self.log_level = log_level
self.log_config = (File(log_config) if isinstance(log_config, str)
else log_config)
self.security = security
self._validate()
def _validate(self):
self._check_is_type('resources', Resources)
self.resources._validate(is_request=True)
self._check_is_dict_of('files', str, File)
for target, f in self.files.items():
_check_is_filename(target)
f._validate()
self._check_is_dict_of('env', str, str)
self._check_is_type('script', str)
if self.log_config is not None:
self._check_is_type('log_config', File)
self.log_config._validate()
if self.security is not None:
self._check_is_type('security', Security)
self.security._validate()
@property
def log_level(self):
return self._log_level
@log_level.setter
def log_level(self, log_level):
self._log_level = LogLevel(log_level)
def __repr__(self):
return 'Master<...>'
@classmethod
@implements(Specification.from_dict)
def from_dict(cls, obj, **kwargs):
cls._check_keys(obj)
obj = obj.copy()
log_config = obj.pop('log_config', None)
if log_config is not None:
log_config = File.from_dict(log_config, **kwargs)
security = obj.pop('security', None)
if security is not None:
security = Security.from_dict(security, **kwargs)
resources = obj.pop('resources', None)
if resources is not None:
resources = Resources.from_dict(resources)
files = obj.pop('files', None)
if files is not None:
files = {k: File.from_dict(v, **kwargs) for k, v in files.items()}
return cls(log_config=log_config,
security=security,
resources=resources,
files=files,
**obj)
@classmethod
@implements(Specification.from_protobuf)
def from_protobuf(cls, obj):
resources = Resources.from_protobuf(obj.resources)
files = {k: File.from_protobuf(v) for k, v in obj.files.items()}
log_level = _proto.Log.Level.Name(obj.log_level)
log_config = (File.from_protobuf(obj.log_config)
if obj.HasField('log_config')
else None)
security = (Security.from_protobuf(obj.security)
if obj.HasField('security')
else None)
return cls(resources=resources,
files=files,
script=obj.script,
env=dict(obj.env),
log_level=log_level,
log_config=log_config,
security=security)
class ApplicationSpec(Specification):
"""A complete description of an application.
Parameters
----------
services : dict, optional
A mapping of service-name to services. Applications must either specify
at least one service, or a script for the application master to run
(see ``skein.Master`` for more information).
master : Master, optional
Additional configuration for the application master service. See
``skein.Master`` for more information.
name : str, optional
The name of the application, defaults to 'skein'.
queue : str, optional
The queue to submit to. Defaults to the default queue.
user : str, optional
The user name to submit the application as. Requires that the
submitting user have permission to proxy as this user name. Default is
the submitter's user name.
node_label : str, optional
The node label expression to use when requesting containers for this
application. Services can override this setting by specifying
``node_label`` on the service directly. Default is no label.
tags : set, optional
A set of strings to use as tags for this application.
file_systems : list, optional
A list of Hadoop file systems to acquire delegation tokens for.
A token is always acquired for the ``defaultFS``.
acls : ACLs, optional
Allows restricting users/groups to subsets of application access. See
``skein.ACLs`` for more information.
max_attempts : int, optional
The maximum number of submission attempts before marking the
application as failed. Note that this only considers failures of the
application master during startup. Default is 1.
"""
__slots__ = ('services', 'master', 'name', 'queue', 'user', 'node_label',
'tags', 'file_systems', 'acls', 'max_attempts')
_protobuf_cls = _proto.ApplicationSpec
def __init__(self, services=None, master=None, name='skein',
queue='default', user='', node_label='', tags=None,
file_systems=None, acls=None, max_attempts=1):
self.services = {} if services is None else services
self.master = Master() if master is None else master
self.name = name
self.queue = queue
self.user = user
self.node_label = node_label
self.tags = set() if tags is None else set(tags)
self.file_systems = [] if file_systems is None else file_systems
self.acls = ACLs() if acls is None else acls
self.max_attempts = max_attempts
self._validate()
def __repr__(self):
return ('ApplicationSpec<name=%r, queue=%r, services=...>' %
(self.name, self.queue))
def _validate(self):
self._check_is_type('name', str)
self._check_is_type('queue', str)
self._check_is_type('user', str)
self._check_is_type('node_label', str)
self._check_is_set_of('tags', str)
self._check_is_list_of('file_systems', str)
self._check_is_bounded_int('max_attempts', min=1)
self._check_is_type('acls', ACLs)
self.acls._validate()
self._check_is_type('master', Master)
self.master._validate()
self._check_is_dict_of('services', str, Service)
if not self.services and not self.master.script:
raise context.ValueError("There must be at least one service")
for name, service in self.services.items():
service._validate()
missing = set(service.depends).difference(self.services)
if missing:
raise context.ValueError(
"Unknown service dependencies for service %r:\n"
"%s" % (name, format_list(missing)))
dependencies = {name: service.depends
for name, service in self.services.items()}
check_no_cycles(dependencies)
@classmethod
def _from_any(cls, spec):
"""Generic creation method for all types accepted as ``spec``"""
if isinstance(spec, str):
spec = cls.from_file(spec)
elif isinstance(spec, dict):
spec = cls.from_dict(spec)
elif not isinstance(spec, cls):
raise context.TypeError("spec must be either an ApplicationSpec, "
"path, or dict, got "
"%s" % type(spec).__name__)
return spec
@classmethod
@implements(Specification.from_dict)
def from_dict(cls, obj, **kwargs):
_origin = _pop_origin(kwargs)
cls._check_keys(obj)
obj = obj.copy()
services = obj.pop('services', None)
if services is not None and isinstance(services, dict):
services = {k: Service.from_dict(v, _origin=_origin)
for k, v in services.items()}
acls = obj.pop('acls', None)
if acls is not None and isinstance(acls, dict):
acls = ACLs.from_dict(acls)
master = obj.pop('master', None)
if master is not None and isinstance(master, dict):
master = Master.from_dict(master, _origin=_origin)
return cls(services=services, acls=acls, master=master, **obj)
@classmethod
@implements(Specification.from_protobuf)
def from_protobuf(cls, obj):
services = {k: Service.from_protobuf(v)
for k, v in obj.services.items()}
return cls(name=obj.name,
queue=obj.queue,
user=obj.user,
node_label=obj.node_label,
tags=set(obj.tags),
file_systems=list(obj.file_systems),
max_attempts=min(1, obj.max_attempts),
acls=ACLs.from_protobuf(obj.acls),
master=Master.from_protobuf(obj.master),
services=services)
@classmethod
def from_file(cls, path, format='infer'):
"""Create an instance from a json or yaml file.
Parameters
----------
path : str
The path to the file to load.
format : {'infer', 'json', 'yaml'}, optional
The file format. By default the format is inferred from the file
extension.
"""
format = _infer_format(path, format=format)
origin = os.path.abspath(os.path.dirname(path))
with open(path) as f:
data = f.read()
if format == 'json':
obj = json.loads(data)
else:
obj = yaml.safe_load(data)
return cls.from_dict(obj, _origin=origin)
def to_file(self, path, format='infer', skip_nulls=True):
"""Write object to a file.
Parameters
----------
path : str
The path to the file to load.
format : {'infer', 'json', 'yaml'}, optional
The file format. By default the format is inferred from the file
extension.
skip_nulls : bool, optional
By default null values are skipped in the output. Set to True to
output all fields.
"""
format = _infer_format(path, format=format)
data = getattr(self, 'to_' + format)(skip_nulls=skip_nulls)
with open(path, mode='w') as f:
f.write(data)
class ResourceUsageReport(ProtobufMessage):
"""Resource usage report.
Parameters
----------
memory_seconds : int
The total amount of memory (in MBs) the application has allocated times
the number of seconds the application has been running.
vcore_seconds : int
The total number of vcores that the application has allocated times the
number of seconds the application has been running.
num_used_containers : int
Current number of containers in use.
needed_resources : Resources
The needed resources.
reserved_resources : Resources
The reserved resources.
used_resources : Resources
The used resources.
"""
__slots__ = ('memory_seconds', 'vcore_seconds', 'num_used_containers',
'needed_resources', 'reserved_resources', 'used_resources')
_protobuf_cls = _proto.ResourceUsageReport
def __init__(self, memory_seconds, vcore_seconds, num_used_containers,
needed_resources, reserved_resources, used_resources):
self.memory_seconds = memory_seconds
self.vcore_seconds = vcore_seconds
self.num_used_containers = num_used_containers
self.needed_resources = needed_resources
self.reserved_resources = reserved_resources
self.used_resources = used_resources
self._validate()
def __repr__(self):
return 'ResourceUsageReport<...>'
def _validate(self):
for k in ['memory_seconds', 'vcore_seconds', 'num_used_containers']:
self._check_is_bounded_int(k)
for k in ['needed_resources', 'reserved_resources', 'used_resources']:
self._check_is_type(k, Resources)
getattr(self, k)._validate()
@classmethod
@implements(Specification.from_protobuf)
def from_protobuf(cls, obj):
kwargs = dict(memory_seconds=obj.memory_seconds,
vcore_seconds=obj.vcore_seconds,
num_used_containers=obj.num_used_containers)
for k in ['needed_resources', 'reserved_resources', 'used_resources']:
kwargs[k] = Resources.from_protobuf(getattr(obj, k))
return cls(**kwargs)
class ApplicationReport(ProtobufMessage):
"""Report of application status.
Parameters
----------
id : str
The application ID.
name : str
The application name.
user : str
The user that started the application.
queue : str
The application queue.
tags : set of strings
The application tags.
host : str
The host the application master is running on.
port : int
The rpc port for the application master
tracking_url : str
The application tracking url.
state : ApplicationState
The application state.
final_status : FinalStatus
The application final status.
progress : float
The progress of the application, from 0.0 to 1.0.
usage : ResourceUsageReport
Report on application resource usage.
diagnostics : str
The diagnostic message in the case of failures.
start_time : datetime
The application start time.
finish_time : datetime
The application finish time.
"""
__slots__ = ('id', 'name', 'user', 'queue', 'tags', 'host', 'port',
'tracking_url', '_state', '_final_status', 'progress', 'usage',
'diagnostics', 'start_time', 'finish_time')
_params = ('id', 'name', 'user', 'queue', 'tags', 'host', 'port',
'tracking_url', 'state', 'final_status', 'progress', 'usage',
'diagnostics', 'start_time', 'finish_time')
_protobuf_cls = _proto.ApplicationReport
def __init__(self, id, name, user, queue, tags, host, port,
tracking_url, state, final_status, progress, usage,
diagnostics, start_time, finish_time):
self.id = id
self.name = name
self.user = user
self.queue = queue
self.tags = set() if tags is None else set(tags)
self.host = host
self.port = port
self.tracking_url = tracking_url
self.state = state
self.final_status = final_status
self.progress = progress
self.usage = usage
self.diagnostics = diagnostics
self.start_time = start_time
self.finish_time = finish_time
self._validate()
def __repr__(self):
return 'ApplicationReport<id=%r, name=%r>' % (self.id, self.name)
@property
def state(self):
return self._state
@state.setter
def state(self, state):
self._state = ApplicationState(state)
@property
def final_status(self):
return self._final_status
@final_status.setter
def final_status(self, state):
self._final_status = FinalStatus(state)
@property
def runtime(self):
"""The total runtime of the container."""
return runtime(self.start_time, self.finish_time)
def _validate(self):
self._check_is_type('id', str)
self._check_is_type('name', str)
self._check_is_type('user', str)
self._check_is_type('queue', str)
self._check_is_set_of('tags', str)
self._check_is_type('host', str)
self._check_is_type('port', int)
self._check_is_type('tracking_url', str)
self._check_is_type('state', ApplicationState)
self._check_is_type('final_status', FinalStatus)
self._check_is_type('progress', float)
self._check_is_type('usage', ResourceUsageReport)
self.usage._validate()
self._check_is_type('diagnostics', str)
self._check_is_type('start_time', datetime, nullable=True)
self._check_is_type('finish_time', datetime, nullable=True)
@classmethod
@implements(ProtobufMessage.from_protobuf)
def from_protobuf(cls, obj):
state = ApplicationState(_proto.ApplicationState.Type.Name(obj.state))
final_status = FinalStatus(_proto.FinalStatus.Type.Name(obj.final_status))
return cls(id=obj.id,
name=obj.name,
user=obj.user,
queue=obj.queue,
tags=obj.tags,
host=obj.host,
port=obj.port,
tracking_url=obj.tracking_url,
state=state,
final_status=final_status,
progress=obj.progress,
usage=ResourceUsageReport.from_protobuf(obj.usage),
diagnostics=obj.diagnostics,
start_time=datetime_from_millis(obj.start_time),
finish_time=datetime_from_millis(obj.finish_time))
class ContainerState(Enum):
"""Enum of container states.
Attributes
----------
WAITING : ContainerState
Container is waiting on another service to startup before being
requested.
REQUESTED : ContainerState
Container has been requested but is not currently running.
RUNNING : ContainerState
Container is currently running.
SUCCEEDED : ContainerState
Container finished successfully.
FAILED : ContainerState
Container failed.
KILLED : ContainerState
Container was terminated by a user or admin.
"""
_values = ('WAITING',
'REQUESTED',
'RUNNING',
'SUCCEEDED',
'FAILED',
'KILLED')
class Container(ProtobufMessage):
"""Current container state.
Parameters
----------
service_name : str
The name of the service this container is running.
instance : int
The container instance number.
state : ContainerState
The current container state.
yarn_container_id : str
The YARN container id.
yarn_node_http_address : str
The YARN node HTTP address given as ``host:port``.
start_time : datetime
The start time, None if container has not started.
finish_time : datetime
The finish time, None if container has not finished.
exit_message : str
The diagnostic exit message for completed containers.
"""
__slots__ = ('service_name', 'instance', '_state', 'yarn_container_id',
'yarn_node_http_address', 'start_time', 'finish_time',
'exit_message')
_params = ('service_name', 'instance', 'state', 'yarn_container_id',
'yarn_node_http_address', 'start_time', 'finish_time',
'exit_message')
_protobuf_cls = _proto.Container
def __init__(self, service_name, instance, state, yarn_container_id,
yarn_node_http_address, start_time, finish_time, exit_message):
self.service_name = service_name
self.instance = instance
self.state = state
self.yarn_container_id = yarn_container_id
self.yarn_node_http_address = yarn_node_http_address
self.start_time = start_time
self.finish_time = finish_time
self.exit_message = exit_message
self._validate()
def __repr__(self):
return ('Container<service_name=%r, instance=%d, state=%s>'
% (self.service_name, self.instance, self.state))
@property
def state(self):
return self._state
@state.setter
def state(self, state):
self._state = ContainerState(state)
def _validate(self):
self._check_is_type('service_name', str)
self._check_is_type('instance', int)
self._check_is_type('state', ContainerState)
self._check_is_type('yarn_container_id', str)
self._check_is_type('yarn_node_http_address', str)
self._check_is_type('start_time', datetime, nullable=True)
self._check_is_type('finish_time', datetime, nullable=True)
self._check_is_type('exit_message', str)
@property
def id(self):
"""The complete service_name & instance identity of this container."""
return '%s_%d' % (self.service_name, self.instance)
@property
def runtime(self):
"""The total runtime of the application."""
return runtime(self.start_time, self.finish_time)
@property
def yarn_container_logs(self):
if not self.yarn_node_http_address or not self.yarn_container_id:
return ""
return "/".join([
self.yarn_node_http_address,
"node",
"containerlogs",
self.yarn_container_id,
getuser()
])
@classmethod
@implements(ProtobufMessage.from_protobuf)
def from_protobuf(cls, obj):
return cls(service_name=obj.service_name,
instance=obj.instance,
state=ContainerState(_proto.Container.State.Name(obj.state)),
yarn_container_id=obj.yarn_container_id,
yarn_node_http_address=obj.yarn_node_http_address,
start_time=datetime_from_millis(obj.start_time),
finish_time=datetime_from_millis(obj.finish_time),
exit_message=obj.exit_message)
class NodeState(Enum):
"""Enum of node states.
Attributes
----------
DECOMMISSIONED : NodeState
Node is out of service.
DECOMMISSIONING : NodeState
Node is currently decommissioning.
LOST : NodeState
Node has not sent responded for some time.
NEW : NodeState
New has just started.
REBOOTED : NodeState
Node is just rebooted.
RUNNING : NodeState
Node is currently running.
SHUTDOWN : NodeState
Node has been shutdown gracefully.
UNHEALTHY : NodeState
Node is unhealthy.
"""
_values = ('DECOMMISSIONED',
'DECOMMISSIONING',
'LOST',
'NEW',
'REBOOTED',
'RUNNING',
'SHUTDOWN',
'UNHEALTHY')
class NodeReport(ProtobufMessage):
"""Report of node status.
Attributes
----------
id : str
The node id.
http_address : str
The http address to the node manager.
rack_name : str
The rack name for this node.
labels : set
Node labels for this node.
state : NodeState
The node's current state.
health_report : str
The diagnostic health report for this node.
total_resources : Resources
Total resources available on this node.
used_resources : Resources
Used resources available on this node.
"""
__slots__ = ('id', 'http_address', 'rack_name', 'labels', '_state',
'health_report', 'total_resources', 'used_resources')
_params = ('id', 'http_address', 'rack_name', 'labels', 'state',
'health_report', 'total_resources', 'used_resources')
_protobuf_cls = _proto.NodeReport
def __init__(self, id, http_address, rack_name, labels, state,
health_report, total_resources, used_resources):
self.id = id
self.http_address = http_address
self.rack_name = rack_name
self.labels = labels
self.state = state
self.health_report = health_report
self.total_resources = total_resources
self.used_resources = used_resources
self._validate()
def __repr__(self):
return 'NodeReport<id=%r>' % self.id
@property
def host(self):
"""The node manager host for this node."""
return self.id.split(':')[0]
@property
def port(self):
"""The node manager port for this node."""
return int(self.id.split(':')[1])
@property
def state(self):
return self._state
@state.setter
def state(self, state):
self._state = NodeState(state)
def _validate(self):
self._check_is_type('id', str)
self._check_is_type('http_address', str)
self._check_is_type('rack_name', str)
self._check_is_set_of('labels', str)
self._check_is_type('health_report', str)
self._check_is_type('total_resources', Resources)
self.total_resources._validate()
self._check_is_type('used_resources', Resources)
self.used_resources._validate()
@classmethod
@implements(ProtobufMessage.from_protobuf)
def from_protobuf(cls, obj):
return cls(id=obj.id,
http_address=obj.http_address,
rack_name=obj.rack_name,
labels=set(obj.labels),
state=NodeState(_proto.NodeState.Type.Name(obj.state)),
health_report=obj.health_report,
total_resources=Resources.from_protobuf(obj.total_resources),
used_resources=Resources.from_protobuf(obj.used_resources))
class QueueState(Enum):
"""Enum of queue states.
Attributes
----------
RUNNING : QueueState
Queue is running, normal operation.
STOPPED : QueueState
Queue is stopped, no longer taking new requests.
"""
_values = ('RUNNING',
'STOPPED')
class Queue(ProtobufMessage):
"""Information about a specific YARN queue.
Attributes
----------
name : str
The queue's name.
state : QueueState
The queue's state.
capacity : float
The queue's capacity as a percentage. For the capacity scheduler, the
queue is guaranteed access to this percentage of the parent queue's
resources (if sibling queues are running over their limit, there may be
a lag accessing resources as those applications scale down). For the
fair scheduler, this number is the percentage of the total cluster this
queue currently has in its fair share (this will shift dynamically
during cluster use).
max_capacity : float
The queue's max capacity as a percentage. For the capacity scheduler,
this queue may elastically expand to use up to this percentage of its
parent's resources if its siblings aren't running at their capacity.
For the fair scheduler this is always 100%.
percent_used : float
The percent of this queue's capacity that's currently in use. This may
be over 100% if elasticity is in effect.
node_labels : set
A set of all accessible node labels for this queue. If all node labels
are accessible this is the set ``{"*"}``.
default_node_label : str
The default node label for this queue. This will be used if the
application doesn't specify a node label itself.
"""
__slots__ = ('name', '_state', 'capacity', 'max_capacity', 'percent_used',
'node_labels', 'default_node_label')
_params = ('name', 'state', 'capacity', 'max_capacity', 'percent_used',
'node_labels', 'default_node_label')
_protobuf_cls = _proto.Queue
def __init__(self, name, state, capacity, max_capacity, percent_used,
node_labels, default_node_label):
self.name = name
self.state = state
self.capacity = capacity
self.max_capacity = max_capacity
self.percent_used = percent_used
self.node_labels = node_labels
self.default_node_label = default_node_label
self._validate()
def __repr__(self):
return ('Queue<name=%r, percent_used=%.2f>'
% (self.name, self.percent_used))
@property
def state(self):
return self._state
@state.setter
def state(self, state):
self._state = QueueState(state)
def _validate(self):
self._check_is_type('name', str)
self._check_is_type('capacity', float)
self._check_is_type('max_capacity', float)
self._check_is_type('percent_used', float)
self._check_is_set_of('node_labels', str)
self._check_is_type('default_node_label', str)
@classmethod
@implements(ProtobufMessage.from_protobuf)
def from_protobuf(cls, obj):
return cls(name=obj.name,
state=QueueState(_proto.Queue.State.Name(obj.state)),
capacity=obj.capacity,
max_capacity=obj.max_capacity,
percent_used=obj.percent_used,
node_labels=set(obj.node_labels),
default_node_label=obj.default_node_label)
class ApplicationLogs(Mapping):
"""A mapping of ``yarn_container_id`` to their logs for an application"""
def __init__(self, app_id, logs):
self.app_id = app_id
self.logs = logs
def __getitem__(self, k):
return self.logs[k]
def __iter__(self):
return iter(self.logs)
def __len__(self):
return len(self.logs)
def __repr__(self):
return "ApplicationLogs<%s>" % self.app_id
def _ipython_key_completions_(self):
return list(self.logs)
def _repr_html_(self):
elements = ["<h3 style='margin-bottom: 10px'>%s</h3>\n" % self.app_id]
elements.extend(
"<details>\n<summary style='display:list-item'>{title}</summary>\n"
"<pre><code>\n{log}\n\n</code></pre>\n"
"</details>".format(title=title, log=log)
for title, log in sorted(self.logs.items())
)
return "\n".join(elements)
def dump(self, file=None):
"""Write the logs to a file or stdout.
Parameters
----------
file : file-like, optional
A file-like object to write the logs to. Defaults to ``sys.stdout``.
"""
if file is not None:
write = lambda s: print(s, file=file)
else:
write = print
N = len(self.logs) - 1
write("** Logs for %s **" % self.app_id)
write("")
for n, (k, v) in enumerate(sorted(self.logs.items())):
write(k)
write("=" * len(k))
write(v)
if n < N:
write("")
def dumps(self):
"""Write the logs to a string."""
s = StringIO()
self.dump(s)
s.seek(0)
return s.read()
