Python luigi.Parameter() Examples

def get_global_parameters(config_names=("core", "scheduler", "worker", "retcode")):
    """
    Returns a list of global, luigi-internal configuration parameters. Each list item is a 4-tuple
    containing the configuration class, the parameter instance, the parameter name, and the full
    parameter name in the cli. When *config_names* is set, it should be a list of configuration
    class names that are exclusively taken into account.
    """
    params = []
    for cls in luigi.task.Config.__subclasses__():
        if config_names and cls.__name__ not in config_names:
            continue

        for attr in dir(cls):
            param = getattr(cls, attr)
            if not isinstance(param, luigi.Parameter):
                continue

            full_name = attr.replace("_", "-")
            if getattr(cls, "use_cmdline_section", True):
                full_name = "{}-{}".format(cls.__name__.replace("_", "-"), full_name)

            params.append((cls, param, attr, full_name))

    return params 

def run(self):
        with betterboto_client.ClientContextManager('ssm', region_name=self.region) as ssm:
            try:
                p = ssm.get_parameter(
                    Name=self.name,
                )
                self.write_output({
                    'Name': self.name,
                    'Region': self.region,
                    'Value': p.get('Parameter').get('Value')
                })
            except ssm.exceptions.ParameterNotFound as e:
                raise e 

def __init__(self, *args, **kwargs):
        luigi.Parameter.__init__(self, *args, **kwargs) 

def _parser_kwargs(self, *args, **kwargs):
        return luigi.Parameter._parser_kwargs(*args, *kwargs) 

def _make_representation(self, param_obj: luigi.Parameter, param_value):
        if isinstance(param_obj, TaskInstanceParameter):
            return f'{param_value.get_task_family()}({param_value.make_unique_id()})'
        if isinstance(param_obj, ListTaskInstanceParameter):
            return f"[{', '.join(f'{v.get_task_family()}({v.make_unique_id()})' for v in param_value)}]"
        return param_obj.serialize(param_value) 

def test_task(self,cleanup):

        e = d6tflow.pipes.FlowExport('utest-flowexport',tasks=Task1All(),write_dir=cfg_write_dir)
        e.generate()

        code = readfile(e.write_dir/e.write_filename_tasks)
        assert code == '''
import d6tflow
import luigi
import datetime

class Task1All(d6tflow.tasks.TaskCache):
    external=True
    persist=['data']
    idx=luigi.parameter.IntParameter(default=1)
    idx2=luigi.parameter.Parameter(default='test')
    idx3=luigi.parameter.Parameter(default='test3')

'''

        code = readfile(e.write_dir/e.write_filename_run)
        assert code == '''
# shared d6tflow workflow, see https://d6tflow.readthedocs.io/en/latest/collaborate.html
import d6tflow.pipes
import tasks_d6tpipe
import datetime

d6tflow.pipes.init('utest-flowexport',profile='default') # to customize see https://d6tflow.readthedocs.io/en/latest/d6tflow.html#d6tflow.pipes.init
d6tflow.pipes.get_pipe('utest-flowexport').pull()

# task output is loaded below, for more details see https://d6tflow.readthedocs.io/en/latest/tasks.html#load-output-data
df_task1all = tasks_d6tpipe.Task1All(idx=1, idx2='test', idx3='test3', ).outputLoad()
''' 

def test_task2(self,cleanup):

        e = d6tflow.pipes.FlowExport('utest-flowexport',tasks=[Task1A(),Task1All()],write_dir=cfg_write_dir)
        e.generate()

        code = readfile(e.write_dir/e.write_filename_tasks)
        assert code == '''
import d6tflow
import luigi
import datetime

class Task1All(d6tflow.tasks.TaskCache):
    external=True
    persist=['data']
    idx=luigi.parameter.IntParameter(default=1)
    idx2=luigi.parameter.Parameter(default='test')
    idx3=luigi.parameter.Parameter(default='test3')

class Task1A(d6tflow.tasks.TaskCache):
    external=True
    persist=['df', 'df2']
    idx=luigi.parameter.IntParameter(default=1)
    idx2=luigi.parameter.Parameter(default='test')

'''

        code = readfile(e.write_dir/e.write_filename_run)
        assert code == '''
# shared d6tflow workflow, see https://d6tflow.readthedocs.io/en/latest/collaborate.html
import d6tflow.pipes
import tasks_d6tpipe
import datetime

d6tflow.pipes.init('utest-flowexport',profile='default') # to customize see https://d6tflow.readthedocs.io/en/latest/d6tflow.html#d6tflow.pipes.init
d6tflow.pipes.get_pipe('utest-flowexport').pull()

# task output is loaded below, for more details see https://d6tflow.readthedocs.io/en/latest/tasks.html#load-output-data
df_task1all = tasks_d6tpipe.Task1All(idx=1, idx2='test', idx3='test3', ).outputLoad()
df_task1a_df, df_task1a_df2, = tasks_d6tpipe.Task1A(idx=1, idx2='test', ).outputLoad()
''' 

def __init__(self, *args, **kwargs):
        """ __init__(*args, cls=luigi.Parameter, **kwargs) """
        cls = kwargs.pop("cls", luigi.Parameter)

        # ensure that the default value is a tuple
        if "default" in kwargs:
            kwargs["default"] = make_tuple(kwargs["default"])

        super(CSVParameter, self).__init__(*args, **kwargs)

        self._inst = cls() 

def __init__(self, *args, **kwargs):
        super(SandboxTask, self).__init__(*args, **kwargs)

        # when we are already in a sandbox, this task is placed inside it, i.e., there is no nesting
        if _sandbox_switched:
            self.effective_sandbox = _current_sandbox[0]

        # when the sandbox is set via a parameter and not hard-coded,
        # check if the value is among the valid sandboxes, otherwise determine the fallback
        elif isinstance(self.__class__.sandbox, luigi.Parameter):
            if multi_match(self.sandbox, self.valid_sandboxes, mode=any):
                self.effective_sandbox = self.sandbox
            else:
                self.effective_sandbox = self.fallback_sandbox(self.sandbox)

        # just set the effective sandbox
        else:
            self.effective_sandbox = self.sandbox

        # at this point, the sandbox must be set unless it is explicitely allowed to be empty
        if self.effective_sandbox in (None, NO_STR):
            if not self.allow_empty_sandbox:
                raise Exception("task {!r} requires the sandbox parameter to be set".format(self))
            self.effective_sandbox = NO_STR

        # create the sandbox proxy when required
        if not self.is_sandboxed():
            self.sandbox_inst = Sandbox.new(self.effective_sandbox, self)
            self.sandbox_proxy = SandboxProxy(task=self)
            logger.debug("created sandbox proxy instance of type '{}'".format(
                self.effective_sandbox))
        else:
            self.sandbox_inst = None
            self.sandbox_proxy = None 

def new_task(name, cls, workflow_task, **kwargs):
    '''
    Instantiate a new task. Not supposed to be used by the end-user
    (use WorkflowTask.new_task() instead).
    '''
    slurminfo = None
    for key, val in [(key, val) for key, val in iteritems(kwargs)]:
        # Handle non-string keys
        if not isinstance(key, string_types):
            raise Exception("Key in kwargs to new_task is not string. Must be string: %s" % key)
        # Handle non-string values
        if isinstance(val, sciluigi.slurm.SlurmInfo):
            slurminfo = val
            kwargs[key] = val
        elif not isinstance(val, string_types):
            try:
                kwargs[key] = json.dumps(val) # Force conversion into string
            except TypeError:
                kwargs[key] = str(val)
    kwargs['instance_name'] = name
    kwargs['workflow_task'] = workflow_task
    kwargs['slurminfo'] = slurminfo
    with warnings.catch_warnings():
        # We are deliberately hacking Luigi's parameter system to use for
        # storing upstream tasks, thus this warning is not really helpful.
        warnings.filterwarnings('ignore',
                category=UserWarning,
                message='Parameter "workflow_task".*is not of type string')
        newtask = cls.from_str_params(kwargs)
        if slurminfo is not None:
            newtask.slurminfo = slurminfo
        return newtask 

def new_task(name, cls, workflow_task, **kwargs):
    '''
    Instantiate a new task. Not supposed to be used by the end-user
    (use WorkflowTask.new_task() instead).
    '''
    slurminfo = None
    for key, val in [(key, val) for key, val in iteritems(kwargs)]:
        # Handle non-string keys
        if not isinstance(key, string_types):
            raise Exception("Key in kwargs to new_task is not string. Must be string: %s" % key)
        # Handle non-string values
        if isinstance(val, sciluigi.slurm.SlurmInfo):
            slurminfo = val
            kwargs[key] = val
        elif not isinstance(val, string_types):
            try:
                kwargs[key] = json.dumps(val) # Force conversion into string
            except TypeError:
                kwargs[key] = str(val)
    kwargs['instance_name'] = name
    kwargs['workflow_task'] = workflow_task
    kwargs['slurminfo'] = slurminfo
    with warnings.catch_warnings():
        # We are deliberately hacking Luigi's parameter system to use for
        # storing upstream tasks, thus this warning is not really helpful.
        warnings.filterwarnings('ignore',
                category=UserWarning,
                message='Parameter "workflow_task".*is not of type string')
        newtask = cls.from_str_params(kwargs)
        if slurminfo is not None:
            newtask.slurminfo = slurminfo
        return newtask 

def test_flow(self,cleanup):

        e = d6tflow.pipes.FlowExport('utest-flowexport',flows=Task1All(),write_dir=cfg_write_dir)
        e.generate()

        code = readfile(e.write_dir/e.write_filename_tasks)
        assert code == '''
import d6tflow
import luigi
import datetime

class Task1A(d6tflow.tasks.TaskCache):
    external=True
    persist=['df', 'df2']
    idx=luigi.parameter.IntParameter(default=1)
    idx2=luigi.parameter.Parameter(default='test')

class Task1B(d6tflow.tasks.TaskCache):
    external=True
    persist=['df', 'df2']
    idx3=luigi.parameter.Parameter(default='test3')

class Task1All(d6tflow.tasks.TaskCache):
    external=True
    persist=['data']
    idx=luigi.parameter.IntParameter(default=1)
    idx2=luigi.parameter.Parameter(default='test')
    idx3=luigi.parameter.Parameter(default='test3')

'''

        code = readfile(e.write_dir/e.write_filename_run)
        assert code == '''
# shared d6tflow workflow, see https://d6tflow.readthedocs.io/en/latest/collaborate.html
import d6tflow.pipes
import tasks_d6tpipe
import datetime

d6tflow.pipes.init('utest-flowexport',profile='default') # to customize see https://d6tflow.readthedocs.io/en/latest/d6tflow.html#d6tflow.pipes.init
d6tflow.pipes.get_pipe('utest-flowexport').pull()

# task output is loaded below, for more details see https://d6tflow.readthedocs.io/en/latest/tasks.html#load-output-data
df_task1a_df, df_task1a_df2, = tasks_d6tpipe.Task1A(idx=1, idx2='test', ).outputLoad()
df_task1b_df, df_task1b_df2, = tasks_d6tpipe.Task1B(idx3='test3', ).outputLoad()
df_task1all = tasks_d6tpipe.Task1All(idx=1, idx2='test', idx3='test3', ).outputLoad()
''' 

def append_parameters(_app, _what, _name, obj, _options, lines):
    """
    Sphinx extension for appending a luigi.Task class's luigi.Parameter attributes
    to the class documentation as "parameters".

    * Uses the luigi.Parameter.description field to describe the attribute.
    * Marks parameters with default values as `optional`, and displays the default value
      (unless there is mention of a default already in the description).
    * Marks `insignificant` parameters.

    """
    default_re = re.compile(r'default', flags=re.IGNORECASE)
    if inspect.isclass(obj) and issubclass(obj, luigi.Task):
        members = inspect.getmembers(obj)
        for (membername, membervalue) in members:
            if isinstance(membervalue, luigi.Parameter):
                param = {
                    'name': membername,
                    'type': membervalue.__class__.__name__,
                    'description': '',
                }
                if membervalue.description is not None:
                    param['description'] = membervalue.description

                    # Append a full stop, for consistency.
                    if not param['description'].endswith('.'):
                        param['description'] = '{description}.'.format(description=param['description'])

                # Mark configured parameters (requires protected-access)
                # pylint: disable=W0212
                if hasattr(membervalue, '_config_path') and membervalue._config_path is not None:
                    param['default'] = 'pulled from ``{section}.{name}``'.format(**membervalue._config_path)
                    param['type'] = u'{type}, configurable'.format(**param)

                # Mark optional parameters
                elif hasattr(membervalue, '_default') and membervalue._default != _no_value:
                    param['default'] = membervalue._default
                    param['type'] = u'{type}, optional'.format(**param)

                if 'default' in param:
                    # Show default value, if not already in the description.
                    # NB: This test is useful to avoid redundant descriptions,
                    # and for dynamically determined defaults like date.today()
                    if not default_re.search(param['description']):
                        param['description'] = u'{description} Default is {default}.'.format(**param)

                # Mark insignificant parameters
                if not membervalue.significant:
                    param['type'] = u'{type}, insignificant'.format(**param)

                # Append the param description and type
                lines.append(u':param {name}: {description}'.format(**param))
                lines.append(u':type {name}: {type}'.format(**param))

        # Append blank line to avoid warning
        lines.append(u'')

    return lines