# -*- coding: utf-8 -*- """ This provides a "toolchain" for the calmjs framework. The toolchain module provides two classes, a ``Spec`` which instances of act as the orchestration object for the ``Toolchain`` instances, where the states of a single workflow through it is tracked. The other being the ``Toolchain`` class, which `compiles`, `assembles` and `links` the stuff as specified in the ``Spec`` into a standalone `module`, which can be treated as an artifact file. This whole thing jumpstarts that process. This whole thing started simply because the author thought it is a lot easier to deal with JavaScript when one treats that as a compilation target. However given that the generic method works better, this module encapsulates the generalized implementation of the previous version. To better convey the understanding of how this came to place, the following terms and variable name, prefix and suffixes have been defined to carry the following meanings that is applicable throughout the entire calmjs system: modname A JavaScript path for a given module import system that it might have. The Python analogue is the name of a given import module. Using the default mapper, one might map a Python module with the name ``calmjs.toolchain`` to ``calmjs/toolchain``. While relative modpaths (i.e. identifiers beginning with './') are supported by most JavaScript/Node.js based import systems, its usage from within calmjs framework is discouraged. sourcepath An absolute path on the local filesystem to the source file for a given modpath. These two values (modpath and sourcepath) serves as the foundational mapping from a JavaScript module name to its corresponding source file. Previously, this was simply named 'source', so certain arguments remain named like so. targetpath A relative path to a build directory (build_dir) serving as the write target for whatever proccessing done by the toolchain implementation to the file provided at the associated sourcepath. Note that by default this is NOT normalized to the currently running OS platform at generation time, due to varying and potentially complex internal/external integration usages. The relative path version (again, from build_dir) is typically recorded by instances of ``Spec`` objects that have undergone a ``Toolchain`` run. Previously, this was simply named 'target'. modpath Typically identical to modname, however the slight difference is that this is the transformed value that is to be better understood by the underlying tools. Think of this as the post compiled value, or an alternative import location that is only applicable in the post-compiled context, specific to the toolchain class that it intends to encapsulate. """ from __future__ import absolute_import from __future__ import unicode_literals import codecs import errno import logging import re import shutil import sys import warnings from collections import namedtuple from functools import partial from inspect import currentframe from traceback import format_stack from os import mkdir from os import makedirs from os.path import basename from os.path import join from os.path import dirname from os.path import exists from os.path import isfile from os.path import isdir from os.path import normpath from os.path import realpath from tempfile import mkdtemp from pkg_resources import Requirement from pkg_resources import working_set as default_working_set from calmjs.parse.io import read from calmjs.parse.io import write from calmjs.parse.parsers.es5 import parse from calmjs.parse.unparsers.base import BaseUnparser from calmjs.parse.unparsers.es5 import pretty_printer from calmjs.parse.sourcemap import encode_sourcemap from calmjs.base import BaseDriver from calmjs.base import BaseRegistry from calmjs.base import BaseLoaderPluginRegistry from calmjs.base import PackageKeyMapping from calmjs.registry import get as get_registry from calmjs.exc import AdviceAbort from calmjs.exc import AdviceCancel from calmjs.exc import ValueSkip from calmjs.exc import ToolchainAbort from calmjs.exc import ToolchainCancel from calmjs.utils import raise_os_error from calmjs.utils import pdb_set_trace from calmjs.vlqsm import SourceWriter logger = logging.getLogger(__name__) __all__ = [ 'AdviceRegistry', 'Spec', 'Toolchain', 'null_transpiler', 'dict_setget', 'dict_setget_dict', 'dict_update_overwrite_check', 'spec_update_loaderplugin_registry', 'spec_update_sourcepath_filter_loaderplugins', 'toolchain_spec_prepare_loaderplugins', 'toolchain_spec_compile_entries', 'ToolchainSpecCompileEntry', 'CALMJS_TOOLCHAIN_ADVICE', 'SETUP', 'CLEANUP', 'SUCCESS', 'AFTER_FINALIZE', 'BEFORE_FINALIZE', 'AFTER_LINK', 'BEFORE_LINK', 'AFTER_ASSEMBLE', 'BEFORE_ASSEMBLE', 'AFTER_COMPILE', 'BEFORE_COMPILE', 'AFTER_PREPARE', 'BEFORE_PREPARE', 'AFTER_TEST', 'BEFORE_TEST', 'ADVICE_PACKAGES', 'ARTIFACT_PATHS', 'BUILD_DIR', 'CALMJS_MODULE_REGISTRY_NAMES', 'CALMJS_LOADERPLUGIN_REGISTRY_NAME', 'CALMJS_LOADERPLUGIN_REGISTRY', 'CALMJS_TEST_REGISTRY_NAMES', 'CONFIG_JS_FILES', 'DEBUG', 'EXPORT_MODULE_NAMES', 'EXPORT_PACKAGE_NAMES', 'EXPORT_TARGET', 'EXPORT_TARGET_OVERWRITE', 'SOURCE_MODULE_NAMES', 'SOURCE_PACKAGE_NAMES', 'TEST_MODULE_NAMES', 'TEST_MODULE_PATHS_MAP', 'TEST_PACKAGE_NAMES', 'TOOLCHAIN_BIN_PATH', 'WORKING_DIR', ] # these are the only non-key entities that should be in this module, as # they currently reference auxilary registry classes that are currently # residing in this module. CALMJS_TOOLCHAIN_ADVICE = 'calmjs.toolchain.advice' CALMJS_TOOLCHAIN_ADVICE_APPLY_SUFFIX = '.apply' # define these as reserved advice names SETUP = 'setup' CLEANUP = 'cleanup' SUCCESS = 'success' AFTER_TEST = 'after_test' # reserved however unused in this module BEFORE_TEST = 'before_test' # reserved however unused in this module AFTER_FINALIZE = 'after_finalize' BEFORE_FINALIZE = 'before_finalize' AFTER_LINK = 'after_link' BEFORE_LINK = 'before_link' AFTER_ASSEMBLE = 'after_assemble' BEFORE_ASSEMBLE = 'before_assemble' AFTER_COMPILE = 'after_compile' BEFORE_COMPILE = 'before_compile' AFTER_PREPARE = 'after_prepare' BEFORE_PREPARE = 'before_prepare' # define these as reserved spec keys # packages that have extra _optional_ advices supplied that have to be # manually included. ADVICE_PACKAGES = 'advice_packages' # advice packages that have been applied to the spec via advice registry # apply_toolchain_spec method. ADVICE_PACKAGES_APPLIED_REQUIREMENTS = 'advice_packages_applied_requirements' # listing of absolute locations on the file system where these bundled # artifact files are. ARTIFACT_PATHS = 'artifact_paths' # build directory BUILD_DIR = 'build_dir' # the key for overriding the advice registry to be use CALMJS_TOOLCHAIN_ADVICE_REGISTRY = 'calmjs_toolchain_advice_registry' # source registries that have been used CALMJS_MODULE_REGISTRY_NAMES = 'calmjs_module_registry_names' CALMJS_TEST_REGISTRY_NAMES = 'calmjs_test_registry_names' # loaderplugin registry related. CALMJS_LOADERPLUGIN_REGISTRY_NAME = 'calmjs_loaderplugin_registry_name' CALMJS_LOADERPLUGIN_REGISTRY = 'calmjs_loaderplugin_registry' # configuration file for enabling execution of code in build directory CONFIG_JS_FILES = 'config_js_files' # for debug level DEBUG = 'debug' # the module names that have been exported out EXPORT_MODULE_NAMES = 'export_module_names' # the package names that have been exported out; not currently supported # by any part of the library, but reserved nonetheless. EXPORT_PACKAGE_NAMES = 'export_package_names' # the container for the export target; either a file or directory; this # should not be changed after the prepare step. EXPORT_TARGET = 'export_target' # specify that export target is safe to be overwritten. EXPORT_TARGET_OVERWRITE = 'export_target_overwrite' # for loaderplugin sourcepath dicts, where the maps are grouped by the # name of the plugin; an intermediate step for processing of loader # plugins LOADERPLUGIN_SOURCEPATH_MAPS = 'loaderplugin_sourcepath_maps' # if true, generate source map GENERATE_SOURCE_MAP = 'generate_source_map' # source module names; currently not supported by any part of the # library, but reserved nonetheless SOURCE_MODULE_NAMES = 'source_module_names' # source Python package names that have been specified for source file # extraction before the workflow, or the top level package(s); should # not include automatically derived required packages. SOURCE_PACKAGE_NAMES = 'source_package_names' # for testing # name of test modules TEST_MODULE_NAMES = 'test_module_names' # mapping of test module to their paths; i.e. sourcepath_map, but not # labled as one to prevent naming conflicts (and they ARE to be # standalone modules to be used directly by the toolchain and testing # integration layer. TEST_MODULE_PATHS_MAP = 'test_module_paths_map' # name of test package TEST_PACKAGE_NAMES = 'test_package_names' # the binary that the toolchain encapsulates. TOOLCHAIN_BIN_PATH = 'toolchain_bin_path' # the working directory WORKING_DIR = 'working_dir' def cls_to_name(cls): return '%s:%s' % (cls.__module__, cls.__name__) def _opener(*a): return codecs.open(*a, encoding='utf-8') def partial_open(*a): return partial(codecs.open, *a, encoding='utf-8') def _check_key_exists(spec, keys): for key in keys: if key not in spec: continue logger.error( "attempted to write '%s' to spec but key already exists; " "not overwriting, skipping", key ) return True return False def _deprecation_warning(msg): warnings.warn(msg, DeprecationWarning) logger.warning(msg) def dict_setget(d, key, value): value = d[key] = d.get(key, value) return value def dict_setget_dict(d, key): return dict_setget(d, key, {}) def dict_update_overwrite_check(base, fresh): """ For updating a base dict with a fresh one, returning a list of 3-tuples containing the key, previous value (base[key]) and the fresh value (fresh[key]) for all colliding changes (reassignment of identical values are omitted). """ result = [ (key, base[key], fresh[key]) for key in set(base.keys()) & set(fresh.keys()) if base[key] != fresh[key] ] base.update(fresh) return result def log_exc_reason( etype, value, tb, msg="{etype} raised at {filename}:{lineno}; reason: {value}", log=logger.debug): while tb.tb_next is not None: tb = tb.tb_next frame = tb.tb_frame log(msg.format( etype=etype.__name__, filename=frame.f_code.co_filename, lineno=frame.f_lineno, value=value )) # Spec functions for interfacing with loaderplugins # # The following functions (named in the format spec_*_loaderplugin_*) # are helpers for extracting and filtering the mappings for interfacing # with the loaderplugin helpers through the registry system. These # helpers are here as they are part of the toolchain, not as part of the # loaderplugin module due to that module being the part that couples # tightly with npm. def spec_update_loaderplugin_registry(spec, default=None): """ Resolve a BasePluginLoaderRegistry instance from spec, and update spec[CALMJS_LOADERPLUGIN_REGISTRY] with that value before returning it. """ registry = spec.get(CALMJS_LOADERPLUGIN_REGISTRY) if isinstance(registry, BaseLoaderPluginRegistry): logger.debug( "loaderplugin registry '%s' already assigned to spec", registry.registry_name) return registry elif not registry: # resolving registry registry = get_registry(spec.get(CALMJS_LOADERPLUGIN_REGISTRY_NAME)) if isinstance(registry, BaseLoaderPluginRegistry): logger.info( "using loaderplugin registry '%s'", registry.registry_name) spec[CALMJS_LOADERPLUGIN_REGISTRY] = registry return registry # acquire the real default instance, if possible. if not isinstance(default, BaseLoaderPluginRegistry): default = get_registry(default) if not isinstance(default, BaseLoaderPluginRegistry): logger.info( "provided default is not a valid loaderplugin registry") default = None if default is None: default = BaseLoaderPluginRegistry('<default_loaderplugins>') # TODO determine the best way to optionally warn about this for # toolchains that require this. if registry: logger.info( "object referenced in spec is not a valid loaderplugin registry; " "using default loaderplugin registry '%s'", default.registry_name) else: logger.info( "no loaderplugin registry referenced in spec; " "using default loaderplugin registry '%s'", default.registry_name) spec[CALMJS_LOADERPLUGIN_REGISTRY] = registry = default return registry def spec_update_sourcepath_filter_loaderplugins( spec, sourcepath_map, sourcepath_map_key, loaderplugin_sourcepath_map_key=LOADERPLUGIN_SOURCEPATH_MAPS): """ Take an existing spec and a sourcepath mapping (that could be produced via calmjs.dist.*_module_registry_dependencies functions) and split out the keys that does not contain loaderplugin syntax and assign it to the spec under sourcepath_key. For the parts with loader plugin syntax (i.e. modnames (keys) that contain a '!' character), they are instead stored under a different mapping under its own mapping identified by the plugin_name. The mapping under loaderplugin_sourcepath_map_key will contain all mappings of this type. The resolution for the handlers will be done through the loader plugin registry provided via spec[CALMJS_LOADERPLUGIN_REGISTRY] if available, otherwise the registry instance will be acquired through the main registry using spec[CALMJS_LOADERPLUGIN_REGISTRY_NAME]. For the example sourcepath_map input: sourcepath = { 'module': 'something', 'plugin!inner': 'inner', 'plugin!other': 'other', 'plugin?query!question': 'question', 'plugin!plugin2!target': 'target', } The following will be stored under the following keys in spec: spec[sourcepath_key] = { 'module': 'something', } spec[loaderplugin_sourcepath_map_key] = { 'plugin': { 'plugin!inner': 'inner', 'plugin!other': 'other', 'plugin?query!question': 'question', 'plugin!plugin2!target': 'target', }, } The goal of this function is to aid in processing each of the plugin types by batch, one level at a time. It is up to the handler itself to trigger further lookups as there are implementations of loader plugins that do not respect the chaining mechanism, thus a generic lookup done at once may not be suitable. Note that nested/chained loaderplugins are not immediately grouped as they must be individually handled given that the internal syntax are generally proprietary to the outer plugin. The handling will be dealt with at the Toolchain.compile_loaderplugin_entry method through the associated handler call method. Toolchain implementations may either invoke this directly as part of the prepare step on the required sourcepaths values stored in the spec, or implement this at a higher level before invocating the toolchain instance with the spec. """ default = dict_setget_dict(spec, sourcepath_map_key) registry = spec_update_loaderplugin_registry(spec) # it is more loaderplugin_sourcepath_maps plugins = dict_setget_dict(spec, loaderplugin_sourcepath_map_key) for modname, sourcepath in sourcepath_map.items(): parts = modname.split('!', 1) if len(parts) == 1: # default default[modname] = sourcepath continue # don't actually do any processing yet. plugin_name = registry.to_plugin_name(modname) plugin = dict_setget_dict(plugins, plugin_name) plugin[modname] = sourcepath def toolchain_spec_prepare_loaderplugins( toolchain, spec, loaderplugin_read_key, handler_sourcepath_key, loaderplugin_sourcepath_map_key=LOADERPLUGIN_SOURCEPATH_MAPS): """ A standard helper function for combining the filtered (e.g. using ``spec_update_sourcepath_filter_loaderplugins``) loaderplugin sourcepath mappings back into one that is usable with the standard ``toolchain_spec_compile_entries`` function. Arguments: toolchain The toolchain spec The spec loaderplugin_read_key The read_key associated with the loaderplugin process as set up for the Toolchain that implemented this. If the toolchain has this in its compile_entries: ToolchainSpecCompileEntry('loaderplugin', 'plugsrc', 'plugsink') The loaderplugin_read_key it must use will be 'plugsrc'. handler_sourcepath_key All found handlers will have their handler_sourcepath method be invoked, and the combined results will be a dict stored in the spec under that key. loaderplugin_sourcepath_map_key It must be the same key to the value produced by ``spec_update_sourcepath_filter_loaderplugins`` """ # ensure the registry is applied to the spec registry = spec_update_loaderplugin_registry( spec, default=toolchain.loaderplugin_registry) # this one is named like so for the compile entry method plugin_sourcepath = dict_setget_dict( spec, loaderplugin_read_key + '_sourcepath') # the key is supplied by the toolchain that might make use of this if handler_sourcepath_key: handler_sourcepath = dict_setget_dict(spec, handler_sourcepath_key) else: # provide a null value for this. handler_sourcepath = {} for key, value in spec.get(loaderplugin_sourcepath_map_key, {}).items(): handler = registry.get(key) if handler: # assume handler will do the job. logger.debug("found handler for '%s' loader plugin", key) plugin_sourcepath.update(value) logger.debug( "plugin_sourcepath updated with %d keys", len(value)) # TODO figure out how to address the case where the actual # JavaScript module for the handling wasn't found. handler_sourcepath.update( handler.generate_handler_sourcepath(toolchain, spec, value)) else: logger.warning( "loaderplugin handler for '%s' not found in loaderplugin " "registry '%s'; as arguments associated with loader plugins " "are specific, processing is disabled for this group; the " "sources referenced by the following names will not be " "compiled into the build target: %s", key, registry.registry_name, sorted(value.keys()), ) def toolchain_spec_compile_entries( toolchain, spec, entries, process_name, overwrite_log=None): """ The standardized Toolchain Spec Entries compile function This function accepts a toolchain instance, the spec to be operated with and the entries provided for the process name. The standard flow is to deferr the actual processing to the toolchain method `compile_{process_name}_entry` for each entry in the entries list. The generic compile entries method for the compile process. Arguments: toolchain The toolchain to be used for the operation. spec The spec to be operated with. entries The entries for the source. process_name The name of the specific compile process of the provided toolchain. overwrite_log A callable that will accept a 4-tuple of suffix, key, original and new value, if monitoring of overwritten values are required. suffix is derived from the modpath_suffix or targetpath_suffix of the toolchain instance, key is the key on any of the keys on either of those mappings, original and new are the original and the replacement value. """ processor = getattr(toolchain, 'compile_%s_entry' % process_name) modpath_logger = ( partial(overwrite_log, toolchain.modpath_suffix) if callable(overwrite_log) else None) targetpath_logger = ( partial(overwrite_log, toolchain.targetpath_suffix) if callable(overwrite_log) else None) return process_compile_entries( processor, spec, entries, modpath_logger, targetpath_logger) def process_compile_entries( processor, spec, entries, modpath_logger=None, targetpath_logger=None): """ The generalized raw spec entry process invocation loop. """ # Contains a mapping of the module name to the compiled file's # relative path starting from the base build_dir. all_modpaths = {} all_targets = {} # List of exported module names, should be equal to all keys of # the compiled and bundled sources. all_export_module_names = [] def update(base, fresh, logger): if callable(logger): for dupes in dict_update_overwrite_check(base, fresh): logger(*dupes) else: base.update(fresh) for entry in entries: modpaths, targetpaths, export_module_names = processor(spec, entry) update(all_modpaths, modpaths, modpath_logger) update(all_targets, targetpaths, targetpath_logger) all_export_module_names.extend(export_module_names) return all_modpaths, all_targets, all_export_module_names ToolchainSpecCompileEntry = namedtuple('ToolchainSpecCompileEntry', [ 'process_name', 'read_key', 'store_key', 'logger', 'log_level']) ToolchainSpecCompileEntry.__new__.__defaults__ = (None, None) def debugger(spec, extras): if not spec.get(DEBUG): return for key in extras: if not key.startswith('debug_'): continue name = key.split('_', 1)[1] logger.debug("debugger advised at '%s'", name) spec.advise(name, pdb_set_trace) def null_transpiler(spec, reader, writer): writer.write(reader.read()) class Spec(dict): """ Instances of these will track the progress through a Toolchain instance. """ def __init__(self, *a, **kw): self._deprecation_match_4_0 = [(re.compile(p), r) for p, r in ( ('^((?!generate)(.*))_source_map$', '\\1_sourcepath'), ('_targets$', '_targetpaths'), )] clean_kw = { self.__process_deprecated_key(k): v for k, v in kw.items()} super(Spec, self).__init__(*a, **clean_kw) self._advices = {} self._frames = {} self._called = set() def __process_deprecated_key(self, key): for patt, repl in self._deprecation_match_4_0: if patt.search(key): break else: return key new_key = patt.sub(repl, key) _deprecation_warning( "Spec key '%s' has been remapped to '%s' in calmjs-3.0.0; this " "automatic remap will be removed by calmjs-4.0.0" % (key, new_key) ) return new_key def get(self, key, default=NotImplemented): key = self.__process_deprecated_key(key) if default is NotImplemented: return dict.get(self, key) else: return dict.get(self, key, default) def __getitem__(self, key): return dict.__getitem__( self, self.__process_deprecated_key(key)) def __setitem__(self, key, value): return dict.__setitem__( self, self.__process_deprecated_key(key), value) def __repr__(self): debug = self.get(DEBUG) if not isinstance(debug, int) or debug < 2: return object.__repr__(self) # for a repr, helpers by pprint module and even json doesn't # deal with circular references for this, so just use the trusty # parent class and be done with it (even though I wanted a # sorted output, this is fine for now as I don't want to spam # logs without debugging enabled). return dict.__repr__(self) def update_selected(self, other, selected): """ Like update, however a list of selected keys must be provided. """ self.update({k: other[k] for k in selected}) def __advice_stack_frame_protection(self, frame): """ Overriding of this is only permitted if and only if your name is Megumin and you have a pet/familiar named Chomusuke. """ if frame is None: logger.debug( 'currentframe() returned None; frame protection disabled') return f_back = frame.f_back while f_back: if f_back.f_code is self.handle.__code__: raise RuntimeError( "indirect invocation of '%s' by 'handle' is forbidden" % frame.f_code.co_name, ) f_back = f_back.f_back def advise(self, name, f, *a, **kw): """ Add an advice that will be handled later by the handle method. Arguments: name The name of the advice group f A callable method or function. The rest of the arguments will be passed as arguments and keyword arguments to f when it's invoked. """ if name is None: return advice = (f, a, kw) debug = self.get(DEBUG) frame = currentframe() if frame is None: logger.debug('currentframe() failed to return frame') else: if name in self._called: self.__advice_stack_frame_protection(frame) if debug: logger.debug( "advise '%s' invoked by %s:%d", name, frame.f_back.f_code.co_filename, frame.f_back.f_lineno, ) if debug > 1: # use the memory address of the tuple which should # be stable self._frames[id(advice)] = ''.join( format_stack(frame.f_back)) self._advices[name] = self._advices.get(name, []) self._advices[name].append(advice) def handle(self, name): """ Call all advices at the provided name. This has an analogue in the join point in aspected oriented programming, but the analogy is a weak one as we don't have the proper metaobject protocol to support this. Implementation that make use of this system should make it clear that they will call this method with name associated with its group before and after its execution, or that the method at hand that want this invoked be called by this other conductor method. For the Toolchain standard steps (prepare, compile, assemble, link and finalize), this handle method will only be called by invoking the toolchain as a callable. Calling those methods piecemeal will not trigger the invocation, even though it probably should. Modules, classes and methods that desire to call their own handler should instead follow the convention where the handle be called before and after with the appropriate names. For instance: def test(self, spec): spec.handle(BEFORE_TEST) # do the things spec.handle(AFTER_TEST) This arrangement will need to be revisited when a proper system is written at the metaclass level. Arguments: name The name of the advices group. All the callables registered to this group will be invoked, last-in-first-out. """ if name in self._called: logger.warning( "advice group '%s' has been called for this spec %r", name, self, ) # only now ensure checking self.__advice_stack_frame_protection(currentframe()) else: self._called.add(name) # Get a complete clone, so indirect manipulation done to the # reference that others have access to will not have an effect # within the scope of this execution. Please refer to the # test_toolchain, test_spec_advice_no_infinite_pop test case. advices = [] advices.extend(self._advices.get(name, [])) if advices and self.get('debug'): logger.debug( "handling %d advices in group '%s' ", len(advices), name) while advices: try: # advice processing is done lifo (last in first out) values = advices.pop() advice, a, kw = values if not ((callable(advice)) and isinstance(a, tuple) and isinstance(kw, dict)): raise TypeError except ValueError: logger.info('Spec advice extraction error: got %s', values) except TypeError: logger.info('Spec advice malformed: got %s', values) else: try: try: advice(*a, **kw) except Exception as e: # get that back by the id. frame = self._frames.get(id(values)) if frame: logger.info('Spec advice exception: %r', e) logger.info( 'Traceback for original advice:\n%s', frame) # continue on for the normal exception raise except AdviceCancel as e: logger.info( "advice %s in group '%s' signaled its cancellation " "during its execution: %s", advice, name, e ) if self.get(DEBUG): logger.debug( 'showing traceback for cancellation', exc_info=1, ) except AdviceAbort as e: # this is a signaled error with a planned abortion logger.warning( "advice %s in group '%s' encountered a known error " "during its execution: %s; continuing with toolchain " "execution", advice, name, e ) if self.get(DEBUG): logger.warning( 'showing traceback for error', exc_info=1, ) except ToolchainCancel: # this is the safe cancel raise except ToolchainAbort as e: logger.critical( "an advice in group '%s' triggered an abort: %s", name, str(e) ) raise except KeyboardInterrupt: raise ToolchainCancel('interrupted') except Exception as e: # a completely unplanned failure logger.critical( "advice %s in group '%s' terminated due to an " "unexpected exception: %s", advice, name, e ) if self.get(DEBUG): logger.critical( 'showing traceback for error', exc_info=1, ) class AdviceRegistry(BaseRegistry): """ Registry for Spec.advise application functions. The declaration is specific to one given toolchain, and they are declared as EntryPoints by packages. Once defined, the package may be refernced as an Advice Package and it may be specified by the spec key ADVICE_PACKAGES. For example, if 'example.package' declares the following:: [calmjs.toolchain.advice] example.package.toolchain:Toolchain = example.package.spec:apply And if that toolchain was invoked with a Spec that has the following definition: Spec({ADVICE_PACKAGES: ['example.package[extra1,extra2]']}) Then the target specified by that entry_point will then be invoked with the spec and the extras passed as an unordered list. For the implementation to function as expected, it requires the Toolchain to invoke process_toolchain_spec_package of instances of this registry. """ def _init(self): for entry_point in self.raw_entry_points: key = entry_point.dist.project_name records = self.records[key] = self.records.get(key, {}) records[entry_point.name] = entry_point def _to_requirement(self, value): try: return Requirement.parse(value) except ValueError as e: logger.error( "the specified value '%s' for advice setup is not valid for " "a package/requirement: %s", value, e, ) raise def get_record(self, name): return self.records.get(name) def applied_requirements_map_from_spec(self, toolchain, spec): # it may be good to warn about requirements that have been # replaced by the standalone method. return { req.name: req for req in spec.get(ADVICE_PACKAGES_APPLIED_REQUIREMENTS, []) } def apply_toolchain_spec(self, toolchain, spec): """ Apply the advice packages as defined by this registry to the provided toolchain and spec. This implementation will first apply whatever ADVICE_PACKAGES are provided by the spec, before applying whatever else that may be applied by this registry instance. Before application of the advice packages, the ADVICE_PACKAGES_APPLIED_REQUIREMENTS key from the spec will also be checked first to prevent the application of advice with the same requirement name. As the ADVICE_PACKAGES feature was originally implemented as a part of the SETUP advice applied by the Runtime class, and the implementation allowed multiple copies of the same requirement be applied, this feature will be maintained as this method implements a fully contained version of that along with the version that applies the ones recorded by the registry. However, multiple execution of this method will not reapply the ones that have been recorded as applied. """ # first step: apply all the advice packages as found in the # provided spec, as these are specified to be necessary which # may override whatever other requirements might be specified # in the accompanied apply registry. spec_advice_packages = spec.get(ADVICE_PACKAGES) or [] # construct a mapping based on the list of applied requirements # that have been also recorded on this spec by the common apply # standalone method. applied_req_map = self.applied_requirements_map_from_spec( toolchain, spec) newly_applied_req_map = {} logger.debug( "invoking apply_toolchain_spec using instance of %s named '%s'", cls_to_name(type(self)), self.registry_name, ) for value in spec_advice_packages: try: req = self._to_requirement(value) except ValueError: # error log entry already generated by the above method. continue if req.name in applied_req_map: logger.warning( "advice package '%s' already applied as '%s'; skipping", req, applied_req_map[req.name] ) continue if req.name in newly_applied_req_map: logger.warning( "advice package '%s' was previously applied as '%s'; " "the recommended usage manner is to only specify any " "given advice package once complete with all the required " "extras, and that underlying implementation be structured " "in a manner that support this one-shot invocation " "format", req, newly_applied_req_map[req.name] ) logger.debug("applying advice package '%s'", value) self._process_toolchain_spec_requirement(toolchain, spec, req) # still going to warn newly_applied_req_map[req.name] = req # finally, find the accompanied apply registry for the package # definitions advice_apply_registry_key = ( self.registry_name + CALMJS_TOOLCHAIN_ADVICE_APPLY_SUFFIX) advice_apply_registry = get_registry(advice_apply_registry_key) if not isinstance(advice_apply_registry, AdviceApplyRegistry): logger.warning( "registry key '%s' resulted in %r which is not a valid advice " "apply registry; no package level advice apply steps will be " "applied", advice_apply_registry_key, advice_apply_registry ) return # combine the newly applied ones with existing ones. applied_req_map.update(newly_applied_req_map) for pkg_name in spec.get(SOURCE_PACKAGE_NAMES, []): requirements = advice_apply_registry.get_record(pkg_name) if not requirements: continue logger.info( "source package '%s' specified %d advice package(s) to be " "applied", pkg_name, len(requirements) ) for req in requirements: if req.name in applied_req_map: logger.debug( "skipping specified advice package '%s' as '%s' was " "already applied", req, applied_req_map[req.name] ) continue logger.debug("apply advice package '%s'", req) self._process_toolchain_spec_requirement(toolchain, spec, req) applied_req_map[req.name] = req def process_toolchain_spec_package(self, toolchain, spec, value): # the original one-shot method. try: req = self._to_requirement(value) except ValueError: pass else: return self._process_toolchain_spec_requirement( toolchain, spec, req) def _process_toolchain_spec_requirement(self, toolchain, spec, req): if not isinstance(toolchain, Toolchain): logger.debug( 'apply_toolchain_spec or process_toolchain_spec_package ' 'must be invoked with a toolchain instance, not %s', toolchain, ) return pkg_name = req.project_name toolchain_cls = type(toolchain) toolchain_advices = self.get_record(pkg_name) if toolchain_advices is None and not default_working_set.find(req): logger.warning( "advice setup steps required from package/requirement " "'%s', however it is not found or not installed in this " "environment; build may continue; if there are errors or " "unexpected behavior that occur, it may be corrected by " "providing the missing requirement into this environment " "by installing the relevant package", req ) return elif not toolchain_advices: logger.debug( "no advice setup steps registered for package/requirement " "'%s'", req) return logger.debug( "found advice setup steps registered for package/requirement " "'%s'; checking for compatibility with toolchain '%s'", req, cls_to_name(toolchain_cls) ) entry_points = [ toolchain_advices.get(cls_name) for cls_name in ( cls_to_name(cls) for cls in toolchain_cls.__mro__ if issubclass(cls, Toolchain) ) if cls_name in toolchain_advices ] if not entry_points: logger.debug("no compatible advice setup steps found") for entry_point in entry_points: try: f = entry_point.load() except ImportError: logger.error( "ImportError: entry_point '%s' in group '%s' while " "processing toolchain spec advice setup step " "registered under advice package '%s'", entry_point, self.registry_name, req ) continue try: f(spec, sorted(req.extras)) except Exception: logger.exception( "failure encountered while setting up advices through " "entry_point '%s' in group '%s' " "registered under advice package '%s'", entry_point, self.registry_name, req ) else: logger.debug( "entry_point '%s' registered by advice package '%s' " "applied as an advice setup step by %s '%s'", entry_point, req, cls_to_name(type(self)), self.registry_name, ) # will just simply be applied regardless. spec.setdefault(ADVICE_PACKAGES_APPLIED_REQUIREMENTS, []) spec[ADVICE_PACKAGES_APPLIED_REQUIREMENTS].append(req) class AdviceApplyRegistry(BaseRegistry): """ Registry to automatically set up the the list of ADVICE_PACKAGES for a given Toolchain execution. If a package has entries declared in this registry, the default Toolchain will apply those declarations onto the list of ADVICE_PACKAGES whenever it appears as a member of SOURCE_PACKAGE_NAMES. For example (continuing on from the example in AdviceRegistry), if 'example.demo' declares the following: [calmjs.toolchain.advice.apply] example.demo = example.package[extra3,extra4] Whenever 'example.demo' appears as an entry in SOURCE_PACKAGE_NAMES in a spec, the following additional flags will be applied to the spec. {ADVICE_PACKAGES: ['example.package[extra1,extra2]']} Naturally, this would not be carried across dependents - if dependents also need that exact advice setup applied, it needs to also declare the same entry in this registry. Also, in order for the implementation to function as expected, the standard Toolchain must be used, and the relevant functionality should be invoked and not be overridden. Note that the key is ignored under the current implementation. """ def _init(self): # since the record keys are package names self.records = PackageKeyMapping() for entry_point in self.raw_entry_points: self._init_entry_point(entry_point) def _init_entry_point(self, entry_point): if not entry_point.dist: logger.warning( 'entry_points passed to %s for registration must provide a ' 'distribution with a project name; registration of %s skipped', cls_to_name(type(self)), entry_point, ) return key = entry_point.dist.project_name self.records.setdefault(key, []) # have to cast the entry point into try: requirement = Requirement.parse(str(entry_point).split('=', 1)[1]) except ValueError as e: logger.warning( "entry_point '%s' cannot be registered to %s due to the " "following error: %s", entry_point, cls_to_name(type(self)), e ) else: self.records[key].append(requirement) def get_record(self, name): return self.records.get(name) class Toolchain(BaseDriver): """ For shared methods between all toolchains. The objective of this class is to provide a consistent interface from calmjs to the various cli Node.js tools, this class inherits from the BaseDriver class. This means having the same foundation and also the ability to reuse a number of useful utility methods for talking to those scripts and binaries. This also involves some standardized processes within the calmjs framework, naming the definition of the following items that every subclass and implementation must support. BUILD_DIR The build directory. This can be manually specified, or be a temporary directory automatically created and destroyed. CALMJS_MODULE_REGISTRY_NAMES For recording the list of calmjs modules it has used, so other parts of the framework can make use of this, such as inferring which test modules it should use. EXPORT_TARGET A path on the filesystem that this toolchain will ultimately generate its output to. EXPORT_TARGET_OVERWRITE Signifies that the export target can be safely overwritten. WORKING_DIR The working directory where the relative paths will be based from. """ # subclasses may assign an identifier or instance of a compatible # loaderplugin registry for use with the encapsulated framework. loaderplugin_registry = None def __init__(self, *a, **kw): """ Refer to parent for exact arguments. """ super(Toolchain, self).__init__(*a, **kw) self.opener = _opener self.setup_filename_suffix() self.setup_transpiler() self.setup_prefix_suffix() self.setup_compile_entries() # Helpers def realpath(self, spec, key): """ Resolve and update the path key in the spec with its realpath, based on the working directory. """ if key not in spec: # do nothing for now return if not spec[key]: logger.warning( "cannot resolve realpath of '%s' as it is not defined", key) return check = realpath(join(spec.get(WORKING_DIR, ''), spec[key])) if check != spec[key]: spec[key] = check logger.warning( "realpath of '%s' resolved to '%s', spec is updated", key, check ) return check # Setup related methods def setup_filename_suffix(self): """ Set up the filename suffix for the sources and targets. """ self.filename_suffix = '.js' def setup_transpiler(self): """ Subclasses will need to implement this to setup the transpiler attribute, which the compile method will invoke. """ self.parser = NotImplemented self.transpiler = NotImplemented def setup_prefix_suffix(self): """ Set up the compile prefix, sourcepath and the targetpath suffix attributes, which are the prefix to the function name and the suffixes to retrieve the values from for creating the generator function. """ self.compile_prefix = 'compile_' self.sourcepath_suffix = '_sourcepath' self.modpath_suffix = '_modpaths' self.targetpath_suffix = '_targetpaths' # TODO BBB backward compat fixes @property def sourcemap_suffix(self): _deprecation_warning( 'sourcemap_suffix has been renamed to sourcepath_suffix; ' 'Toolchain attribute will be removed by calmjs-4.0.0', ) return self.sourcepath_suffix @sourcemap_suffix.setter def sourcemap_suffix(self, value): _deprecation_warning( 'sourcemap_suffix has been renamed to sourcepath_suffix; ' 'Toolchain attribute will be removed by calmjs-4.0.0', ) self.sourcepath_suffix = value @property def target_suffix(self): _deprecation_warning( 'target_suffix has been renamed to targetpath_suffix; ' 'Toolchain attribute will be removed by calmjs-4.0.0' ) return self.targetpath_suffix @target_suffix.setter def target_suffix(self, value): _deprecation_warning( 'target_suffix has been renamed to targetpath_suffix; ' 'Toolchain attribute will be removed by calmjs-4.0.0' ) self.targetpath_suffix = value def setup_compile_entries(self): """ The method that sets up the map that maps the compile methods stored in this class instance to the spec key that the generated maps should be stored at. """ self.compile_entries = self.build_compile_entries() def build_compile_entries(self): """ Build the entries that will be used to acquire the methods for the compile step. This is to be a list of 3-tuples. first element being the method name, which is a name that will be prefixed with the compile_prefix, default being `compile_`; alternatively a callable could be provided. This method must return a 2-tuple. second element being the read key for the sourcepath dict, which is the name to be read from the spec and it will be suffixed with the sourcepath_suffix, default being `_sourcepath`. third element being the write key for first return value of the method, it will be suffixed with the modpath_suffix, defaults to `_modpaths`, and the targetpath_suffix, default to `_targetpaths`. The method referenced SHOULD NOT assign values to the spec, and it must produce and return a 2-tuple: first element should be the map from the module to the written targets, the key being the module name (modname) and the value being the relative path of the final file to the build_dir the second element must be a list of module names that it exported. """ return ( # compile_*, *_sourcepath, (*_modpaths, *_targetpaths) ToolchainSpecCompileEntry('transpile', 'transpile', 'transpiled'), ToolchainSpecCompileEntry('bundle', 'bundle', 'bundled'), ) # Default built-in methods referenced by methods that will be # executed, as constructed by build_compile_entries. # Following are used for the transpile and bundle compile processes. def _validate_build_target(self, spec, target): """ Essentially validate that the target is inside the build_dir. """ if not realpath(target).startswith(spec[BUILD_DIR]): raise ValueError('build_target %s is outside build_dir' % target) # note that in the following methods, a shorthand notation is used # for some of the arguments: nearly all occurrences of source means # sourcepath, and target means targetpath. def _generate_transpile_target(self, spec, target): # ensure that the target is fully normalized. bd_target = join(spec[BUILD_DIR], normpath(target)) self._validate_build_target(spec, bd_target) if not exists(dirname(bd_target)): logger.debug("creating dir '%s'", dirname(bd_target)) makedirs(dirname(bd_target)) return bd_target def transpile_modname_source_target(self, spec, modname, source, target): """ The function that gets called by compile_transpile_entry for processing the provided JavaScript source file provided by some Python package through the transpiler instance. """ if not isinstance(self.transpiler, BaseUnparser): _deprecation_warning( 'transpiler callable assigned to %r must be an instance of ' 'calmjs.parse.unparsers.base.BaseUnparser by calmjs-4.0.0; ' 'if the original transpile behavior is to be retained, the ' 'subclass may instead override this method to call ' '`simple_transpile_modname_source_target` directly, as ' 'this fallback behavior will be removed by calmjs-4.0.0' % ( self, ) ) return self.simple_transpile_modname_source_target( spec, modname, source, target) # do the new thing here. return self._transpile_modname_source_target( spec, modname, source, target) def _transpile_modname_source_target(self, spec, modname, source, target): bd_target = self._generate_transpile_target(spec, target) logger.info('Transpiling %s to %s', source, bd_target) reader = partial_open(source, 'r') writer_main = partial_open(bd_target, 'w') writer_map = ( partial_open(bd_target + '.map', 'w') if spec.get(GENERATE_SOURCE_MAP) else None ) write(self.transpiler, [ read(self.parser, reader)], writer_main, writer_map) def simple_transpile_modname_source_target( self, spec, modname, source, target): """ The original simple transpile method called by compile_transpile on each target. """ opener = self.opener bd_target = self._generate_transpile_target(spec, target) logger.info('Transpiling %s to %s', source, bd_target) with opener(source, 'r') as reader, opener(bd_target, 'w') as _writer: writer = SourceWriter(_writer) self.transpiler(spec, reader, writer) if writer.mappings and spec.get(GENERATE_SOURCE_MAP): source_map_path = bd_target + '.map' with open(source_map_path, 'w') as sm_fd: self.dump(encode_sourcemap( filename=bd_target, mappings=writer.mappings, sources=[source], ), sm_fd) # just use basename source_map_url = basename(source_map_path) _writer.write('\n//# sourceMappingURL=') _writer.write(source_map_url) _writer.write('\n') def compile_transpile_entry(self, spec, entry): """ Handler for each entry for the transpile method of the compile process. This invokes the transpiler that was set up to transpile the input files into the build directory. """ modname, source, target, modpath = entry transpiled_modpath = {modname: modpath} transpiled_target = {modname: target} export_module_name = [modname] self.transpile_modname_source_target(spec, modname, source, target) return transpiled_modpath, transpiled_target, export_module_name def compile_bundle_entry(self, spec, entry): """ Handler for each entry for the bundle method of the compile process. This copies the source file or directory into the build directory. """ modname, source, target, modpath = entry bundled_modpath = {modname: modpath} bundled_target = {modname: target} export_module_name = [] if isfile(source): export_module_name.append(modname) copy_target = join(spec[BUILD_DIR], target) if not exists(dirname(copy_target)): makedirs(dirname(copy_target)) shutil.copy(source, copy_target) elif isdir(source): copy_target = join(spec[BUILD_DIR], modname) shutil.copytree(source, copy_target) return bundled_modpath, bundled_target, export_module_name def compile_loaderplugin_entry(self, spec, entry): """ Generic loader plugin entry handler. The default implementation assumes that everything up to the first '!' symbol resolves to some known loader plugin within the registry. The registry instance responsible for the resolution of the loader plugin handlers must be available in the spec under CALMJS_LOADERPLUGIN_REGISTRY """ modname, source, target, modpath = entry handler = spec[CALMJS_LOADERPLUGIN_REGISTRY].get(modname) if handler: return handler(self, spec, modname, source, target, modpath) logger.warning( "no loaderplugin handler found for plugin entry '%s'", modname) return {}, {}, [] # The naming methods, which are needed by certain toolchains that # need to generate specific names to maintain compatibility. The # intended use case for this set of methods is to provide a rigidly # defined name handling ruleset for a given implementation of a # toolchain, but for toolchains that have its own custom naming # schemes per whatever value combination, further handling can be # done within each of the compile_* and/or compile_*_entry methods # that are enabled or registered for use for that particular # toolchain implementation. # Also note that 'source' and 'target' refer to 'sourcepath' and # 'targetpath' respectively in argument and method names. def modname_source_to_modname(self, spec, modname, source): """ Method to get a modname. Should really return the modname, but subclass has the option to override this. Called by generator method `_gen_modname_source_target_modpath`. """ return modname def modname_source_to_source(self, spec, modname, source): """ Method to get a source file name. Should really return itself, but subclass has the option to override this. Called by generator method `_gen_modname_source_target_modpath`. """ return source def modname_source_to_target(self, spec, modname, source): """ Create a target file name from the input module name and its source file name. The result should be a path relative to the build_dir, and this is derived directly from the modname with NO implicit convers of path separators (i.e. '/' or any other) into a system or OS specific form (e.g. '\\'). The rationale for this choice is that there exists Node.js/JavaScript tools that handle this internally and/or these paths and values are directly exposed on the web and thus these separators must be preserved. If the specific implementation requires this to be done, implementations may override by wrapping the result of this using os.path.normpath. For the generation of transpile write targets, this will be done in _generate_transpile_target. Default is to append the module name with the filename_suffix assigned to this instance (setup by setup_filename_suffix), iff the provided source also end with this filename suffix. However, certain tools have issues dealing with loader plugin syntaxes showing up on the filesystem (and certain filesystems definitely do not like some of the characters), so the usage of the loaderplugin registry assigned to the spec may be used for lookup if available. Called by generator method `_gen_modname_source_target_modpath`. """ loaderplugin_registry = spec.get(CALMJS_LOADERPLUGIN_REGISTRY) if '!' in modname and loaderplugin_registry: handler = loaderplugin_registry.get(modname) if handler: return handler.modname_source_to_target( self, spec, modname, source) if (source.endswith(self.filename_suffix) and not modname.endswith(self.filename_suffix)): return modname + self.filename_suffix else: # assume that modname IS the filename return modname def modname_source_target_to_modpath(self, spec, modname, source, target): """ Typical JavaScript tools will get confused if '.js' is added, so by default the same modname is returned as path rather than the target file for the module path to be written to the output file for linkage by tools. Some other tools may desire the target to be returned instead, or construct some other string that is more suitable for the tool that will do the assemble and link step. The modname and source argument provided to aid pedantic tools, but really though this provides more consistency to method signatures. Called by generator method `_gen_modname_source_target_modpath`. """ return modname def modname_source_target_modnamesource_to_modpath( self, spec, modname, source, target, modname_source): """ Typical JavaScript tools will get confused if '.js' is added, so by default the same modname is returned as path rather than the target file for the module path to be written to the output file for linkage by tools. Some other tools may desire the target to be returned instead, or construct some other string that is more suitable for the tool that will do the assemble and link step. The modname and source argument provided to aid pedantic tools, but really though this provides more consistency to method signatures. Same as `self.modname_source_target_to_modpath`, but includes the original raw key-value as a 2-tuple. Called by generator method `_gen_modname_source_target_modpath`. """ return self.modname_source_target_to_modpath( spec, modname, source, target) # Generator methods def _gen_modname_source_target_modpath(self, spec, d): """ Private generator that will consume those above functions. This should NOT be overridden. Produces the following 4-tuple on iteration with the input dict; the definition is written at the module level documention for calmjs.toolchain, but in brief: modname The JavaScript module name. source Stands for sourcepath - path to some JavaScript source file. target Stands for targetpath - the target path relative to spec[BUILD_DIR] where the source file will be written to using the method that genearted this entry. modpath The module path that is compatible with tool referencing the target. While this is typically identical with modname, some tools require certain modifications or markers in additional to what is presented (e.g. such as the addition of a '?' symbol to ensure absolute lookup). """ for modname_source in d.items(): try: modname = self.modname_source_to_modname(spec, *modname_source) source = self.modname_source_to_source(spec, *modname_source) target = self.modname_source_to_target(spec, *modname_source) modpath = self.modname_source_target_modnamesource_to_modpath( spec, modname, source, target, modname_source) except ValueError as e: # figure out which of the above 3 functions failed by # acquiring the name from one frame down. f_name = sys.exc_info()[2].tb_next.tb_frame.f_code.co_name if isinstance(e, ValueSkip): # a purposely benign failure. log = partial( logger.info, "toolchain purposely skipping on '%s', " "reason: %s, where modname='%s', source='%s'", ) else: log = partial( logger.warning, "toolchain failed to acquire name with '%s', " "reason: %s, where modname='%s', source='%s'; " "skipping", ) log(f_name, e, *modname_source) continue yield modname, source, target, modpath # The core functions to be implemented for the toolchain. def prepare(self, spec): """ Optional preparation step for handling the spec. Implementation can make use of this to do pre-compilation checking and/or other validation steps in order to result in a successful compilation run. """ def compile(self, spec): """ Generic step that compiles from a spec to build the specified things into the build directory `build_dir`, by gathering all the files and feed them through the transpilation process or by simple copying. """ spec[EXPORT_MODULE_NAMES] = export_module_names = spec.get( EXPORT_MODULE_NAMES, []) if not isinstance(export_module_names, list): raise TypeError( "spec provided a '%s' but it is not of type list " "(got %r instead)" % (EXPORT_MODULE_NAMES, export_module_names) ) def compile_entry(method, read_key, store_key): spec_read_key = read_key + self.sourcepath_suffix spec_modpath_key = store_key + self.modpath_suffix spec_target_key = store_key + self.targetpath_suffix if _check_key_exists(spec, [spec_modpath_key, spec_target_key]): logger.error( "aborting compile step %r due to existing key", entry, ) return sourcepath_dict = spec.get(spec_read_key, {}) entries = self._gen_modname_source_target_modpath( spec, sourcepath_dict) (spec[spec_modpath_key], spec[spec_target_key], new_module_names) = method(spec, entries) logger.debug( "entry %r " "wrote %d entries to spec[%r], " "wrote %d entries to spec[%r], " "added %d export_module_names", entry, len(spec[spec_modpath_key]), spec_modpath_key, len(spec[spec_target_key]), spec_target_key, len(new_module_names), ) export_module_names.extend(new_module_names) for entry in self.compile_entries: if isinstance(entry, ToolchainSpecCompileEntry): log = partial( logging.getLogger(entry.logger).log, entry.log_level, ( entry.store_key + "%s['%s'] is being rewritten from " "'%s' to '%s'; configuration may now be invalid" ), ) if entry.logger else None compile_entry(partial( toolchain_spec_compile_entries, self, process_name=entry.process_name, overwrite_log=log, ), entry.read_key, entry.store_key) continue m, read_key, store_key = entry if callable(m): method = m else: method = getattr(self, self.compile_prefix + m, None) if not callable(method): logger.error( "'%s' not a callable attribute for %r from " "compile_entries entry %r; skipping", m, self, entry ) continue compile_entry(method, read_key, store_key) def assemble(self, spec): """ Assemble all the compiled files. This was intended to be the function that provides the aggregation of all compiled files in the build directory into a form that can then be linked. Typically this is for the generation of an actual specification or instruction file that will be passed to the linker, which is some binary that is installed on the system. """ raise NotImplementedError def link(self, spec): """ Should pass in the manifest path to the final JS linker, which is typically the bundler. """ raise NotImplementedError def finalize(self, spec): """ Optional finalizing step, where further usage of the build_dir, scripts and/or results are needed. This can be used to run some specific scripts through node's import system directly on the pre-linked assembled files, for instance. This step is optional. """ def _calf(self, spec): """ The main call, assuming the base spec is prepared. Also, no advices will be triggered. """ self.prepare(spec) self.compile(spec) self.assemble(spec) self.link(spec) self.finalize(spec) def setup_apply_advice_packages( self, spec, default_advice_registry=CALMJS_TOOLCHAIN_ADVICE): """ This method sets up the advices that have been specified in the ADVICE_PACKAGES key, and apply the advices to the spec. """ advice_registry_key = spec.get( CALMJS_TOOLCHAIN_ADVICE_REGISTRY, default_advice_registry) advice_registry = get_registry(advice_registry_key) if not isinstance(advice_registry, AdviceRegistry): logger.warning( "registry key '%s' resulted in %r which is not a valid advice " "registry; all package advice steps will be skipped", advice_registry_key, advice_registry ) return logger.debug( "setting up advices using %s '%s'", cls_to_name(type(advice_registry)), advice_registry_key ) advice_registry.apply_toolchain_spec(self, spec) def calf(self, spec): """ Typical safe usage is this, which sets everything that could be problematic up. Requires the filename which everything will be produced to. """ if not isinstance(spec, Spec): raise TypeError('spec must be of type Spec') # The following ensure steps really should be formalised into # some form of setup. This may be a version 4+ item to consider # for integration, where the current SETUP advice is changed to # BEFORE_SETUP and have the following ensure step be part of the # default setup. # ensure build directory is defined and sane. if not spec.get(BUILD_DIR): tempdir = realpath(mkdtemp()) spec.advise(CLEANUP, shutil.rmtree, tempdir) build_dir = join(tempdir, 'build') mkdir(build_dir) spec[BUILD_DIR] = build_dir else: build_dir = self.realpath(spec, BUILD_DIR) if not isdir(build_dir): logger.error("build_dir '%s' is not a directory", build_dir) raise_os_error(errno.ENOTDIR, build_dir) # ensure export target is sane self.realpath(spec, EXPORT_TARGET) # ensure advices specific to packages are applied, and applied # using the advice to maintain the feature as it was when # initially implemented as part of the runtime. This also allow # advice setup exceptions be handled as expected. spec.advise(SETUP, self.setup_apply_advice_packages, spec) try: # Finally, handle setup which may set up the deferred # advices, as all the toolchain (and its runtime and/or its # parent runtime and related toolchains) spec advises should # have been done. spec.handle(SETUP) process = ('prepare', 'compile', 'assemble', 'link', 'finalize') for p in process: spec.handle('before_' + p) getattr(self, p)(spec) spec.handle('after_' + p) spec.handle(SUCCESS) except ToolchainCancel: if spec.get(DEBUG): log_exc_reason(*sys.exc_info()) except ToolchainAbort: if spec.get(DEBUG): log_exc_reason(*sys.exc_info()) raise finally: spec.handle(CLEANUP) def __call__(self, spec): """ Alias, also make this callable directly. """ self.calf(spec) class NullToolchain(Toolchain): """ A null toolchain that does nothing except maybe move some files around. """ def __init__(self): super(NullToolchain, self).__init__() def setup_transpiler(self): self.transpiler = null_transpiler def transpile_modname_source_target(self, spec, modname, source, target): """ Calls the original version. """ return self.simple_transpile_modname_source_target( spec, modname, source, target) def prepare(self, spec): """ Does absolutely nothing """ spec['prepare'] = 'prepared' def assemble(self, spec): """ Does absolutely nothing """ spec['assemble'] = 'assembled' def link(self, spec): """ Does absolutely nothing """ spec['link'] = 'linked' class ES5Toolchain(Toolchain): """ A null toolchain that does nothing except maybe move some files around, using the es5 Unparser, pretty printer version. """ def __init__(self, *a, **kw): super(ES5Toolchain, self).__init__(*a, **kw) def setup_transpiler(self): self.transpiler = pretty_printer() self.parser = parse