Python toolz.pipe() Examples

def _valid_log_files(log_dir):
    def _valid_or_warn(log_file):
        if log_file.has_valid_filename():
            return True

        logging.warning("Invalid filename: %s", log_file.filename())
        return False

    def _to_paths(triple):
        root, _, files_in_dir = triple
        return [os.path.join(root, file_in_dir) for file_in_dir in files_in_dir]

    return pipe(os.walk(log_dir),
                mapcatz(_to_paths),
                mapz(LogFile),
                filterz(_valid_or_warn)) 

def plot(self, gpu_measurement='sm', num_gpus=1, plot_width=600, plot_height=400, y_range=(0, 110)):
        """ Plot the specified GPU measurement

        Parameters
        ----------
        gpu_measurement: GPU measurement to plot possible values
        num_gpus: Number of GPUs to plot ['pwr', 'temp', 'sm', 'mem', 'enc', 'dec', 'mclk', 'pclk']
        plot_width:
        plot_height:
        y_range:

        Returns
        -------
        Bokeh Figure
        """
        df = pipe(self._log_file,
                  parse_log,
                  extract(gpu_measurement))
        return plot(df,
                    num_gpus=num_gpus,
                    plot_width=plot_width,
                    plot_height=plot_height,
                    y_range=y_range) 

def test_sample():
    """Test the sample data transformer."""
    data = _create_dataframe(20)
    result = pipe(data, sample(n=10))
    assert len(result) == 10
    assert isinstance(result, pd.DataFrame)
    data = _create_data_with_values(20)
    result = sample(data, n=10)
    assert isinstance(result, dict)
    assert "values" in result
    assert len(result["values"]) == 10
    data = _create_dataframe(20)
    result = pipe(data, sample(frac=0.5))
    assert len(result) == 10
    assert isinstance(result, pd.DataFrame)
    data = _create_data_with_values(20)
    result = sample(data, frac=0.5)
    assert isinstance(result, dict)
    assert "values" in result
    assert len(result["values"]) == 10 

def test_dataframe_to_json():
    """Test to_json
    - make certain the filename is deterministic
    - make certain the file contents match the data
    """
    data = _create_dataframe(10)
    try:
        result1 = pipe(data, to_json)
        result2 = pipe(data, to_json)
        filename = result1["url"]
        output = pd.read_json(filename)
    finally:
        os.remove(filename)

    assert result1 == result2
    assert output.equals(data) 

def test_dataframe_to_csv():
    """Test to_csv with dataframe input
    - make certain the filename is deterministic
    - make certain the file contents match the data
    """
    data = _create_dataframe(10)
    try:
        result1 = pipe(data, to_csv)
        result2 = pipe(data, to_csv)
        filename = result1["url"]
        output = pd.read_csv(filename)
    finally:
        os.remove(filename)

    assert result1 == result2
    assert output.equals(data) 

def test_dict_to_csv():
    """Test to_csv with dict input
    - make certain the filename is deterministic
    - make certain the file contents match the data
    """
    data = _create_data_with_values(10)
    try:
        result1 = pipe(data, to_csv)
        result2 = pipe(data, to_csv)
        filename = result1["url"]
        output = pd.read_csv(filename).to_dict(orient="records")
    finally:
        os.remove(filename)

    assert result1 == result2
    assert data == {"values": output} 

def __call__(self, coords):
        return tz.pipe(coords, *self) 

def get_comment(identifier):
    with suppress(PostDoesNotExist):
        return pipe(
            Post(identifier).export(),
            strip_dot_from_keys,
            safe_json_metadata
        ) 

def img_url_to_json(url):
    img_data = toolz.pipe(url,
                          to_img,
                          to_base64)
    return json.dumps({'input':'[\"{0}\"]'.format(img_data)}) 

def to_img(img_url):
    return toolz.pipe(img_url,
                      read_image_from,
                      to_rgb,
                      resize(new_size=(224,224))) 

def read_image_from(url):
    return toolz.pipe(url,
                      urllib.request.urlopen,
                      lambda x: x.read(),
                      BytesIO) 

def _preprocess_images(filename):
    return pipe(filename, tf.read_file) 

def known_iam_actions(prefix):
    """Return known IAM actions for a prefix, e.g. all ec2 actions"""
    # This could be memoized for performance improvements
    knowledge = pipe(all_known_iam_permissions(),
                     mapz(_parse_action),
                     groupbyz(lambda x: x.prefix))

    return knowledge.get(prefix, []) 

def generate_policy(selected_records):
    """Generates a policy from a set of records"""
    statements = pipe(selected_records,
                      mapz(Record.to_statement),
                      filterz(lambda statement: statement is not None),
                      _combine_statements_by(lambda statement: statement.Resource),
                      _combine_statements_by(lambda statement: statement.Action),
                      sortedz())

    return PolicyDocument(
        Version="2012-10-17",
        Statement=statements,
    ) 

def filter_records(records,
                   arns_to_filter_for=None,
                   from_date=datetime.datetime(1970, 1, 1, tzinfo=pytz.utc),
                   to_date=datetime.datetime.now(tz=pytz.utc)):
    """Filter records so they match the given condition"""
    result = list(pipe(records, filterz(_by_timeframe(from_date, to_date)), filterz(_by_role_arns(arns_to_filter_for))))
    if not result and records:
        logging.warning(ALL_RECORDS_FILTERED)

    return result 

def last_event_timestamp_in_dir(log_dir):
    """Return the timestamp of the most recent event in the given directory"""
    most_recent_file = pipe(_valid_log_files(log_dir),
                            sortedz(key=LogFile.timestamp),
                            lastz,
                            LogFile.records,
                            sortedz(key=lambda record: record.event_time),
                            lastz)

    return most_recent_file.event_time 

def simplified(self) -> 'Transform':
        """Return the composite of the transforms inside the transform chain."""
        if len(self) == 0:
            return None
        if len(self) == 1:
            return self[0]
        else:
            return tz.pipe(self[0], *[tf.compose for tf in self[1:]]) 

def get_contract_address(task_uuid):
    await_tr = partial(await_blockchain_success_evil, timeout=timeout)
    return pipe(task_uuid, await_tr, lambda r: r.get('contract_address')) 

def test_type_error():
    """Ensure that TypeError is raised for types other than dict/DataFrame."""
    for f in (sample, limit_rows, to_values):
        with pytest.raises(TypeError):
            pipe(0, f) 

def test_to_values():
    """Test the to_values data transformer."""
    data = _create_dataframe(10)
    result = pipe(data, to_values)
    assert result == {"values": data.to_dict(orient="records")} 

def test_limit_rows():
    """Test the limit_rows data transformer."""
    data = _create_dataframe(10)
    result = limit_rows(data, max_rows=20)
    assert data is result
    with pytest.raises(MaxRowsError):
        pipe(data, limit_rows(max_rows=5))
    data = _create_data_with_values(10)
    result = pipe(data, limit_rows(max_rows=20))
    assert data is result
    with pytest.raises(MaxRowsError):
        limit_rows(data, max_rows=5) 

def _open_image(self, image_path):
        return pipe(image_path, _open_to_array, _rescale) 

def _open_mask(self, mask_path):
        return pipe(mask_path, _open_to_array) 

def _open_image(self, image_path):
        return pipe(image_path, _open_to_array, _rescale) 

def _patch_label_2d(
    model, img, pre_processing, output_processing, patch_size, stride, batch_size, device, num_classes, split, debug
):
    """Processes a whole section
    """
    img = torch.squeeze(img)
    h, w = img.shape[-2], img.shape[-1]  # height and width

    # Pad image with patch_size/2:
    ps = int(np.floor(patch_size / 2))  # pad size
    img_p = F.pad(img, pad=(ps, ps, ps, ps), mode="constant", value=0)
    output_p = torch.zeros([1, num_classes, h + 2 * ps, w + 2 * ps])

    # generate output:
    for batch_indexes in _generate_batches(h, w, ps, patch_size, stride, batch_size=batch_size):
        batch = torch.stack(
            [pipe(img_p, _extract_patch(hdx, wdx, ps, patch_size), pre_processing,) for hdx, wdx in batch_indexes],
            dim=0,
        )

        model_output = model(batch.to(device))

        for (hdx, wdx), output in zip(batch_indexes, model_output.detach().cpu()):
            output = output_processing(output)
            output_p[:, :, hdx + ps : hdx + ps + patch_size, wdx + ps : wdx + ps + patch_size,] += output

        # dump the data right before it's being put into the model and after scoring
        if debug:
            outdir = f"debug/batch_{split}"
            generate_path(outdir)
            for i in range(batch.shape[0]):
                path_prefix = f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}"
                model_output = model_output.detach().cpu()
                # save image:
                image_to_disk(np.array(batch[i, 0, :, :]), path_prefix + "_img.png")
                # dump model prediction:
                mask_to_disk(model_output[i, :, :, :].argmax(dim=0).numpy(), path_prefix + "_pred.png", num_classes)
                # dump model confidence values
                for nclass in range(num_classes):
                    image_to_disk(
                        model_output[i, nclass, :, :].numpy(), path_prefix + f"_class_{nclass}_conf.png",
                    )

    # crop the output_p in the middle
    output = output_p[:, :, ps:-ps, ps:-ps]
    return output 

def __init__(
        self,
        root,
        patch_size,
        stride,
        split="train",
        transforms=None,
        exclude_files=None,
        max_inlines=None,
        n_channels=3,
        complete_patches_only=True,
    ):
        """Initialise Penobscot Dataset

        Args:
           root (str): root directory to load data from
           patch_size (int): the size of the patch in pixels
           stride (int): the stride applied when extracting patches
           split (str, optional): what split to load, (train, val, test). Defaults to `train`
           transforms (albumentations.augmentations.transforms, optional): albumentation transforms
                                                                           to apply to patches.
                                                                           Defaults to None
           exclude_files (list[str], optional): list of files to exclude. Defaults to None
           max_inlines (int, optional): maximum number of inlines to load. Defaults to None
           n_channels (int, optional): number of channels that the output should contain. Defaults to 3
           complete_patches_only (bool, optional): whether to load incomplete patches
                                                   that are padded to patch_size. Defaults to True
        """

        assert n_channels == 3, (
            f"For the Section Depth based dataset the number of channels can only be 3."
            f"Currently n_channels={n_channels}"
        )
        super(PenobscotInlinePatchSectionDepthDataset, self).__init__(
            root,
            patch_size,
            stride,
            split=split,
            transforms=transforms,
            exclude_files=exclude_files,
            max_inlines=max_inlines,
            n_channels=n_channels,
            complete_patches_only=complete_patches_only,
        )

        def _open_image(self, image_path):
            return pipe(image_path, _open_to_array, _rescale, add_depth_channels)

        def _add_extra_channels(self, image):
            return image