Python luigi.Target() Examples

def _get_input_args(self):
        # TODO(brianm): this doesn't work when subclass yields from `requires`
        job_input = self.input()
        if isinstance(job_input, luigi.Target):
            job_input = {"input": job_input}
        if len(job_input) == 0:  # default requires()
            return []
        if not isinstance(job_input, dict):
            raise ValueError("Input (requires()) must be dict type")
        input_args = []
        for (name, targets) in job_input.items():
            uris = [get_uri(target) for target in luigi.task.flatten(targets)]
            if isinstance(targets, dict):
                # If targets is a dict that means it had multiple outputs. In this case make the
                # input args "<input key>-<task output key>"
                names = ["%s-%s" % (name, key) for key in targets.keys()]
            else:
                names = [name] * len(uris)
            for (arg_name, uri) in zip(names, uris):
                input_args.append("--%s=%s" % (arg_name, uri))

        return input_args 

def test_methods(self):
        wf = sl.WorkflowTask()
        touta = wf.new_task('tout', MultiOutTask,
            an_id='a')
        toutb = wf.new_task('tout', MultiOutTask,
            an_id='b')
        toutc = wf.new_task('tout', MultiOutTask,
            an_id='c')
        tin = wf.new_task('tout', MultiInTask)

        tin.in_multi = [touta.out_multi, {'a': toutb.out_multi, 'b': toutc.out_multi()}]

        # Assert outputs returns luigi targets, or list of luigi targets
        outs = touta.output()
        self.assertIsInstance(outs, list)
        for out in outs:
            self.assertIsInstance(out, luigi.Target)

        reqs = tin.requires()
        self.assertIsInstance(reqs, list)
        for req in reqs:
            self.assertIsInstance(req, luigi.Task) 

def __init__(self, *args, **kwargs):
        super(PythonDataflowTask, self).__init__(*args, **kwargs)
        self._output = self.output()
        if isinstance(self._output, luigi.Target):
            self._output = {"output": self._output}
        if self.job_name is None:
            # job_name must consist of only the characters [-a-z0-9]
            cls_name = self.__class__.__name__.replace("_", "-").lower()
            self.job_name = "{cls_name}-{timestamp}".format(cls_name=cls_name,
                                                            timestamp=str(int(time.time()))) 

def _get_input_args(self):
        """
        Collects outputs from requires() and converts them to input arguments.
        file_pattern() is called to construct input file path glob with default value "part-*"
        """
        job_input = self.input()
        if isinstance(job_input, luigi.Target):
            job_input = {"input": job_input}
        if not isinstance(job_input, dict):
            raise ValueError("Input (requires()) must be dict type")

        input_args = []
        file_pattern_dict = self._get_file_pattern()
        for (name, targets) in job_input.items():
            uri_targets = luigi.task.flatten(targets)
            pattern = file_pattern_dict.get(name, "part-*")
            uris = [self._get_input_uri(pattern, uri_target) for uri_target in uri_targets]
            if isinstance(targets, dict):
                # If targets is a dict that means it had multiple outputs.
                #  Make the input args in that case "<input key>-<task output key>"
                names = ["%s-%s" % (name, key) for key in targets.keys()]
            else:
                names = [name] * len(uris)
            for (arg_name, uri) in zip(names, uris):
                input_args.append("--%s=%s" % (arg_name, uri))

        return input_args 

def output(self):
        # `make_target` makes an instance of `luigi.Target`.
        # This infers the output format and the destination of an output objects.
        # The target file path is
        #     '{TaskOnKart.workspace_directory}/output_of_task_b_{self.make_unique_id()}.pkl'.
        return self.make_target('output_of_task_b.pkl') 

def get_object_storage_target(path: str, format: Format) -> luigi.Target:
        if path.startswith('s3://'):
            return luigi.contrib.s3.S3Target(path, client=S3Config().get_s3_client(), format=format)
        elif path.startswith('gs://'):
            return luigi.contrib.gcs.GCSTarget(path, client=GCSConfig().get_gcs_client(), format=format)
        else:
            raise