Python lzma.FILTER_LZMA2 Examples

def CompressLZMA(string):
    """
    CompressBrotli(string) -> str
    Returns without delimiters the given string compressed \
using the lzstring compression method.

    """
    compressed = lzma.compress(
        string.encode("ascii"),
        format=lzma.FORMAT_RAW,
        filters=[{'id': lzma.FILTER_LZMA2, 'preset': 9 | lzma.PRESET_EXTREME}]
    )
    number = 1
    for c in compressed:
        number = number * 256 + c
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[LZMA_ENCODING] + result 

def DecompressLZMA(string):
    """
    DecompressBrotli(string) -> str
    Returns the original form of the given string compressed \
using Google's brotli compression method., passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    compressed = []
    while number > 1:
        compressed = [number % 256] + compressed
        number //= 256
    return lzma.decompress(
        bytes(compressed),
        format=lzma.FORMAT_RAW,
        filters=[{'id': lzma.FILTER_LZMA2, 'preset': 9 | lzma.PRESET_EXTREME}]
    ).decode("ascii") 

def __init__(self, filters=None, password=None):
        if filters is None:
            self.filters = [{"id": lzma.FILTER_LZMA2, "preset": 7 | lzma.PRESET_EXTREME}]
        else:
            self.filters = filters
        if len(self.filters) > 4:
            raise UnsupportedCompressionMethodError('Maximum cascade of filters is 4 but got {}.'.format(len(self.filters)))
        self.methods_map = [SupportedMethods.is_native_filter(filter) for filter in self.filters]
        self.coders = []
        self.cchain = CompressorChain(self.methods_map)
        if all(self.methods_map) and SupportedMethods.is_compressor(self.filters[-1]):  # all native
            self._set_native_compressors_coders(self.filters)
        elif not any(self.methods_map):  # all alternative
            for filter in filters:
                self._set_alternate_compressors_coders(filter, password)
        elif SupportedMethods.is_crypto(self.filters[-1]) and all(self.methods_map[:-1]):  # Crypto + native compression
            self._set_native_compressors_coders(self.filters[:-1])
            self._set_alternate_compressors_coders(self.filters[-1], password)
        else:
            raise UnsupportedCompressionMethodError 

def test_compress_with_custom_filter(tmp_path):
    my_filters = [
        {"id": py7zr.FILTER_DELTA, "dist": 5},
        {"id": py7zr.FILTER_LZMA2, "preset": 7 | py7zr.PRESET_EXTREME},
    ]
    target = tmp_path.joinpath('target.7z')
    archive = py7zr.SevenZipFile(target, 'w', filters=my_filters)
    archive.set_encoded_header_mode(True)
    archive.writeall(os.path.join(testdata_path, "src"), "src")
    archive.close()
    #
    with py7zr.SevenZipFile(target, 'r') as arc:
        assert arc.test()
    #
    if shutil.which('7z'):
        result = subprocess.run(['7z', 't', target.as_posix()], stdout=subprocess.PIPE)
        if result.returncode != 0:
            print(result.stdout)
            pytest.fail('7z command report error') 

def test_compress_arm64(tmp_path):
    my_filters = [{"id": py7zr.FILTER_ARM}, {"id": py7zr.FILTER_LZMA2, "preset": 7}]
    target = tmp_path.joinpath('target.7z')
    archive = py7zr.SevenZipFile(target, 'w', filters=my_filters)
    archive.write(os.path.join(testdata_path, "lib"), "lib")
    archive.writeall(os.path.join(testdata_path, "lib", "aarch64-linux-gnu"), "lib/aarch64-linux-gnu")
    archive.close()
    #
    with py7zr.SevenZipFile(target, 'r') as archive:
        archive.extractall(path=tmp_path / 'tgt')
    #
    if shutil.which('7z'):
        result = subprocess.run(['7z', 't', (tmp_path / 'target.7z').as_posix()], stdout=subprocess.PIPE)
        if result.returncode != 0:
            print(result.stdout)
            pytest.fail('7z command report error') 

def test_compress_ppc(tmp_path):
    my_filters = [{"id": py7zr.FILTER_POWERPC}, {"id": py7zr.FILTER_LZMA2, "preset": 7}]
    target = tmp_path.joinpath('target.7z')
    archive = py7zr.SevenZipFile(target, 'w', filters=my_filters)
    archive.write(os.path.join(testdata_path, "lib"), "lib")
    archive.writeall(os.path.join(testdata_path, "lib", "powerpc64le-linux-gnu"), "lib/powerpc64le-linux-gnu")
    archive.close()
    #
    with py7zr.SevenZipFile(target, 'r') as archive:
        archive.extractall(path=tmp_path / 'tgt')
    #
    if shutil.which('7z'):
        result = subprocess.run(['7z', 't', (tmp_path / 'target.7z').as_posix()], stdout=subprocess.PIPE)
        if result.returncode != 0:
            print(result.stdout)
            pytest.fail('7z command report error') 

def test_compress_sparc(tmp_path):
    my_filters = [{"id": py7zr.FILTER_SPARC}, {"id": py7zr.FILTER_LZMA2, "preset": 7}]
    target = tmp_path.joinpath('target.7z')
    archive = py7zr.SevenZipFile(target, 'w', filters=my_filters)
    archive.write(os.path.join(testdata_path, "lib"), "lib")
    archive.writeall(os.path.join(testdata_path, "lib", "powerpc64le-linux-gnu"), "lib/powerpc64le-linux-gnu")
    archive.close()
    #
    with py7zr.SevenZipFile(target, 'r') as archive:
        archive.extractall(path=tmp_path / 'tgt')
    #
    if shutil.which('7z'):
        result = subprocess.run(['7z', 't', (tmp_path / 'target.7z').as_posix()], stdout=subprocess.PIPE)
        if result.returncode != 0:
            print(result.stdout)
            pytest.fail('7z command report error') 

def test_simple_compress_and_decompress():
    filters = [{"id": lzma.FILTER_LZMA2, "preset": 7 | lzma.PRESET_DEFAULT}, ]
    sevenzip_compressor = py7zr.compressor.SevenZipCompressor(filters=filters)
    lzc = sevenzip_compressor.cchain
    out1 = lzc.compress(b"Some data\n")
    out2 = lzc.compress(b"Another piece of data\n")
    out3 = lzc.compress(b"Even more data\n")
    out4 = lzc.flush()
    result = b"".join([out1, out2, out3, out4])
    size = len(result)
    #
    filters = sevenzip_compressor.filters
    decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=filters)
    out5 = decompressor.decompress(result)
    assert out5 == b'Some data\nAnother piece of data\nEven more data\n'
    #
    coders = sevenzip_compressor.coders
    crc = py7zr.helpers.calculate_crc32(result)
    decompressor = py7zr.compressor.SevenZipDecompressor(coders, size, [len(out5)], crc)
    out6 = decompressor.decompress(result)
    assert out6 == b'Some data\nAnother piece of data\nEven more data\n' 

def test_compressor_lzma2bcj(tmp_path):
    my_filters = [{"id": py7zr.FILTER_X86},
                  {"id": py7zr.FILTER_LZMA2, "preset": 7}]
    plain_data = b"\x00*\x1a\t'd\x19\xb08s\xca\x8b\x13 \xaf:\x1b\x8d\x97\xf8|#M\xe9\xe1W\xd4\xe4\x97BB\xd2"
    plain_data += plain_data
    compressor = py7zr.compressor.SevenZipCompressor(filters=my_filters)
    outdata = compressor.compress(plain_data)
    outdata += compressor.flush()
    unpacksizes = compressor.unpacksizes
    assert len(outdata) > 1
    coders = [{'method': b'!', 'properties': b'\x18', 'numinstreams': 1, 'numoutstreams': 1},
              {'method': b'\x03\x03\x01\x03', 'numinstreams': 1, 'numoutstreams': 1}]
    decompressor = py7zr.compressor.SevenZipDecompressor(coders=coders, packsize=len(plain_data), unpacksizes=unpacksizes,
                                                         crc=None)
    revert_data = decompressor.decompress(outdata, max_length=len(plain_data))
    assert revert_data == plain_data 

def init_compressions():
    ident = lambda x: x
    compressions = {'': Compression(ident, ident)}
    for name in ('bz2', 'zlib'):
        try:
            m = __import__(name)
        except ImportError:
            warnings.warn('%s is not available' % name)
        else:
            compressions[name] = Compression(
                lambda x: m.compress(x, 9), m.decompress)

    try:
        import lzma
    except ImportError:
        warnings.warn('lzma is not available')
    else:
        filters = [{'id': lzma.FILTER_LZMA2}]
        compress = lambda s: lzma.compress(s,
                                           format=lzma.FORMAT_RAW,
                                           filters=filters)
        decompress = lambda s: lzma.decompress(s,
                                               format=lzma.FORMAT_RAW,
                                               filters=filters)
        compressions['lzma2'] = Compression(compress, decompress)
    return compressions 

def write_masks_to_zarr(ds_mask, output_path):
    import lzma
    lzma_filters = [dict(id=lzma.FILTER_DELTA, dist=1),
                    dict(id=lzma.FILTER_LZMA2, preset=1)]
    from numcodecs import LZMA
    compressor = LZMA(filters=lzma_filters, format=lzma.FORMAT_RAW)
    encoding = {vname: {'compressor': compressor} for vname in ds_mask.data_vars}
    return ds_mask.to_zarr(output_path, encoding=encoding) 

def get_coder(cls, filter) -> Dict[str, Any]:
        method = cls.get_method_id(filter)
        if filter['id'] in [lzma.FILTER_LZMA1, lzma.FILTER_LZMA2, lzma.FILTER_DELTA]:
            properties = lzma._encode_filter_properties(filter)  # type: Optional[bytes] # type: ignore  # noqa
        else:
            properties = None
        return {'method': method, 'properties': properties, 'numinstreams': 1, 'numoutstreams': 1} 

def test_compress_with_simple_filter(tmp_path):
    my_filters = [{"id": py7zr.FILTER_LZMA2, "preset": py7zr.PRESET_DEFAULT}, ]
    target = tmp_path.joinpath('target.7z')
    archive = py7zr.SevenZipFile(target, 'w', filters=my_filters)
    archive.writeall(os.path.join(testdata_path, "src"), "src")
    archive.close()
    #
    if shutil.which('7z'):
        result = subprocess.run(['7z', 't', (tmp_path / 'target.7z').as_posix()], stdout=subprocess.PIPE)
        if result.returncode != 0:
            print(result.stdout)
            pytest.fail('7z command report error') 

def test_compress_ia64(tmp_path):
    my_filters = [{"id": py7zr.FILTER_IA64}, {"id": py7zr.FILTER_LZMA2, "preset": 7}]
    target = tmp_path.joinpath('target.7z')
    archive = py7zr.SevenZipFile(target, 'w', filters=my_filters)
    archive.write(os.path.join(testdata_path, "lib"), "lib")
    archive.writeall(os.path.join(testdata_path, "lib", "powerpc64le-linux-gnu"), "lib/powerpc64le-linux-gnu")
    archive.close()
    #
    with py7zr.SevenZipFile(target, 'r') as archive:
        archive.extractall(path=tmp_path / 'tgt')
    #
    if shutil.which('7z'):
        result = subprocess.run(['7z', 't', (tmp_path / 'target.7z').as_posix()], stdout=subprocess.PIPE)
        if result.returncode != 0:
            print(result.stdout)
            pytest.fail('7z command report error') 

def test_compress_symlink(tmp_path):
    tmp_path.joinpath('src').mkdir()
    tmp_path.joinpath('tgt').mkdir()
    py7zr.unpack_7zarchive(os.path.join(testdata_path, 'symlink.7z'), path=tmp_path.joinpath('src'))
    target = tmp_path.joinpath('target.7z')
    os.chdir(tmp_path.joinpath('src'))
    filters = [{"id": lzma.FILTER_LZMA2, "preset": 7 | lzma.PRESET_EXTREME}, ]
    archive = py7zr.SevenZipFile(target, 'w', filters=filters)
    archive.set_encoded_header_mode(False)
    archive.writeall('.')
    archive._write_archive()
    assert len(archive.header.files_info.files) == 6
    assert archive.header.main_streams.substreamsinfo.num_unpackstreams_folders == [5]
    assert len(archive.files) == 6
    assert len(archive.header.files_info.files) == 6
    expected = [True, False, False, False, False, False]
    for i, f in enumerate(archive.header.files_info.files):
        f['emptystream'] = expected[i]
    assert archive.header.files_info.files[5]['maxsize'] == 1543
    assert archive.header.main_streams.packinfo.packsizes == [1543]
    assert archive.header.files_info.files[4]['uncompressed'] == 6536
    assert archive.header.files_info.files[1]['filename'] == 'lib/libabc.so'
    assert archive.header.files_info.files[2]['filename'] == 'lib/libabc.so.1'
    if os.name == 'nt':
        assert check_bit(archive.header.files_info.files[2]['attributes'], stat.FILE_ATTRIBUTE_REPARSE_POINT)
    else:
        assert check_bit(archive.header.files_info.files[2]['attributes'], FILE_ATTRIBUTE_UNIX_EXTENSION)
        assert stat.S_ISLNK(archive.header.files_info.files[2]['attributes'] >> 16)
    assert archive.header.main_streams.packinfo.numstreams == 1
    assert archive.header.main_streams.substreamsinfo.digestsdefined == [True, True, True, True, True]
    assert archive.header.main_streams.substreamsinfo.unpacksizes == [11, 13, 15, 6536, 3]
    assert archive.header.main_streams.substreamsinfo.digests == [4262439050, 2607345479,
                                                                  2055456646, 437637236, 2836347852]
    assert archive.header.main_streams.substreamsinfo.num_unpackstreams_folders == [5]
    assert len(archive.header.main_streams.unpackinfo.folders) == 1
    assert len(archive.header.main_streams.unpackinfo.folders[0].coders) == 1
    assert archive.header.main_streams.unpackinfo.numfolders == 1
    assert archive.header.main_streams.unpackinfo.folders[0].coders[0]['numinstreams'] == 1
    assert archive.header.main_streams.unpackinfo.folders[0].coders[0]['numoutstreams'] == 1
    assert archive.header.main_streams.unpackinfo.folders[0].solid
    assert archive.header.main_streams.unpackinfo.folders[0].bindpairs == []
    assert archive.header.main_streams.unpackinfo.folders[0].solid is True
    assert archive.header.main_streams.unpackinfo.folders[0].unpacksizes == [6578]
    assert archive.header.main_streams.unpackinfo.folders[0].digestdefined is False
    assert archive.header.main_streams.unpackinfo.folders[0].crc is None
    archive._fpclose()
    # split archive.close() into _write_archive() and _fpclose()
    reader = py7zr.SevenZipFile(target, 'r')
    reader.extractall(path=tmp_path.joinpath('tgt'))
    reader.close() 

def test_compress_multi_filter_delta(tmp_path):
    my_filters = [{"id": py7zr.FILTER_DELTA},
                  {"id": py7zr.FILTER_LZMA2, "preset": 7}]
    target = tmp_path.joinpath('target.7z')
    archive = py7zr.SevenZipFile(target, 'w', filters=my_filters)
    archive.writeall(os.path.join(testdata_path, "src"), "src")
    assert archive.files is not None
    assert len(archive.files) == 2
    for f in archive.files:
        assert f.filename in ('src', pathlib.Path('src').joinpath('bra.txt').as_posix())
    archive.set_encoded_header_mode(False)
    archive._write_archive()
    assert len(archive.header.files_info.files) == 2
    assert archive.header.main_streams.substreamsinfo.num_unpackstreams_folders == [1]
    assert len(archive.header.files_info.files) == 2
    assert archive.header.main_streams.unpackinfo.folders[0].coders[0]['numinstreams'] == 1
    assert archive.header.main_streams.unpackinfo.folders[0].coders[0]['numoutstreams'] == 1
    assert archive.header.main_streams.unpackinfo.folders[0].coders[1]['numinstreams'] == 1
    assert archive.header.main_streams.unpackinfo.folders[0].coders[1]['numoutstreams'] == 1
    assert archive.header.main_streams.unpackinfo.folders[0].solid
    assert archive.header.main_streams.unpackinfo.folders[0].bindpairs[0].incoder == 1
    assert archive.header.main_streams.unpackinfo.folders[0].bindpairs[0].outcoder == 0
    assert archive.header.main_streams.unpackinfo.folders[0].digestdefined is False
    assert archive.header.main_streams.unpackinfo.folders[0].crc is None
    expected = [True, False]
    for i, f in enumerate(archive.header.files_info.files):
        f['emptystream'] = expected[i]
    archive._fpclose()
    #
    with py7zr.SevenZipFile(target, 'r') as archive:
        assert len(archive.header.files_info.files) == 2
        assert archive.header.main_streams.substreamsinfo.num_unpackstreams_folders == [1]
        assert archive.header.main_streams.unpackinfo.folders[0].coders[0]['numinstreams'] == 1
        assert archive.header.main_streams.unpackinfo.folders[0].coders[0]['numoutstreams'] == 1
        assert archive.header.main_streams.unpackinfo.folders[0].coders[1]['numinstreams'] == 1
        assert archive.header.main_streams.unpackinfo.folders[0].coders[1]['numoutstreams'] == 1
        assert archive.header.main_streams.unpackinfo.folders[0].bindpairs[0].incoder == 1
        assert archive.header.main_streams.unpackinfo.folders[0].bindpairs[0].outcoder == 0
        assert archive.header.main_streams.unpackinfo.folders[0].digestdefined is False
        assert archive.header.main_streams.unpackinfo.folders[0].crc is None
        archive.extractall(path=tmp_path / 'tgt')
    #
    if shutil.which('7z'):
        result = subprocess.run(['7z', 't', (tmp_path / 'target.7z').as_posix()], stdout=subprocess.PIPE)
        if result.returncode != 0:
            print(result.stdout)
            pytest.fail('7z command report error')