Java Code Examples for org.apache.hadoop.io.compress.lz4.Lz4Decompressor

The following examples show how to use org.apache.hadoop.io.compress.lz4.Lz4Decompressor. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestCompressorDecompressor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressorDecompressor() {
  // no more for this data
  int SIZE = 44 * 1024;
  
  byte[] rawData = generate(SIZE);
  try {
    CompressDecompressTester.of(rawData)
        .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor())
        .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor())
        .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater())
        .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
        .test();

  } catch (Exception ex) {
    fail("testCompressorDecompressor error !!!" + ex);
  }
}
 
Example 2
Source Project: hadoop   Source File: TestCompressorDecompressor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressorDecompressorWithExeedBufferLimit() {
  int BYTE_SIZE = 100 * 1024;
  byte[] rawData = generate(BYTE_SIZE);
  try {
    CompressDecompressTester.of(rawData)
        .withCompressDecompressPair(
            new SnappyCompressor(BYTE_SIZE + BYTE_SIZE / 2),
            new SnappyDecompressor(BYTE_SIZE + BYTE_SIZE / 2))
        .withCompressDecompressPair(new Lz4Compressor(BYTE_SIZE),
            new Lz4Decompressor(BYTE_SIZE))
        .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
        .test();

  } catch (Exception ex) {
    fail("testCompressorDecompressorWithExeedBufferLimit error !!!" + ex);
  }
}
 
Example 3
Source Project: big-c   Source File: TestCompressorDecompressor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressorDecompressor() {
  // no more for this data
  int SIZE = 44 * 1024;
  
  byte[] rawData = generate(SIZE);
  try {
    CompressDecompressTester.of(rawData)
        .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor())
        .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor())
        .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater())
        .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
        .test();

  } catch (Exception ex) {
    fail("testCompressorDecompressor error !!!" + ex);
  }
}
 
Example 4
Source Project: big-c   Source File: TestCompressorDecompressor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressorDecompressorWithExeedBufferLimit() {
  int BYTE_SIZE = 100 * 1024;
  byte[] rawData = generate(BYTE_SIZE);
  try {
    CompressDecompressTester.of(rawData)
        .withCompressDecompressPair(
            new SnappyCompressor(BYTE_SIZE + BYTE_SIZE / 2),
            new SnappyDecompressor(BYTE_SIZE + BYTE_SIZE / 2))
        .withCompressDecompressPair(new Lz4Compressor(BYTE_SIZE),
            new Lz4Decompressor(BYTE_SIZE))
        .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
        .test();

  } catch (Exception ex) {
    fail("testCompressorDecompressorWithExeedBufferLimit error !!!" + ex);
  }
}
 
Example 5
Source Project: hadoop   Source File: Lz4Codec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
 *
 * @return the type of decompressor needed by this codec.
 */
@Override
public Class<? extends Decompressor> getDecompressorType() {
  if (!isNativeCodeLoaded()) {
    throw new RuntimeException("native lz4 library not available");
  }

  return Lz4Decompressor.class;
}
 
Example 6
Source Project: hadoop   Source File: Lz4Codec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
 *
 * @return a new decompressor for use by this codec
 */
@Override
public Decompressor createDecompressor() {
  if (!isNativeCodeLoaded()) {
    throw new RuntimeException("native lz4 library not available");
  }
  int bufferSize = conf.getInt(
      CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
      CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
  return new Lz4Decompressor(bufferSize);
}
 
Example 7
Source Project: hadoop   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecompressorSetInputNullPointerException() {
  try {
    Lz4Decompressor decompressor = new Lz4Decompressor();
    decompressor.setInput(null, 0, 10);
    fail("testDecompressorSetInputNullPointerException error !!!");
  } catch (NullPointerException ex) {
    // expected
  } catch (Exception e) {
    fail("testDecompressorSetInputNullPointerException ex error !!!");
  }
}
 
Example 8
Source Project: hadoop   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecompressorSetInputAIOUBException() {
  try {
    Lz4Decompressor decompressor = new Lz4Decompressor();
    decompressor.setInput(new byte[] {}, -5, 10);
    fail("testDecompressorSetInputAIOBException error !!!");
  } catch (ArrayIndexOutOfBoundsException ex) {
    // expected
  } catch (Exception e) {
    fail("testDecompressorSetInputAIOBException ex error !!!");
  }
}
 
Example 9
Source Project: hadoop   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecompressorCompressNullPointerException() {
  try {
    Lz4Decompressor decompressor = new Lz4Decompressor();
    byte[] bytes = generate(1024 * 6);
    decompressor.setInput(bytes, 0, bytes.length);
    decompressor.decompress(null, 0, 0);
    fail("testDecompressorCompressNullPointerException error !!!");
  } catch (NullPointerException ex) {
    // expected
  } catch (Exception e) {
    fail("testDecompressorCompressNullPointerException ex error !!!");
  }
}
 
Example 10
Source Project: hadoop   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecompressorCompressAIOBException() {
  try {
    Lz4Decompressor decompressor = new Lz4Decompressor();
    byte[] bytes = generate(1024 * 6);
    decompressor.setInput(bytes, 0, bytes.length);
    decompressor.decompress(new byte[] {}, 0, -1);
    fail("testDecompressorCompressAIOBException error !!!");
  } catch (ArrayIndexOutOfBoundsException ex) {
    // expected
  } catch (Exception e) {
    fail("testDecompressorCompressAIOBException ex error !!!");
  }
}
 
Example 11
Source Project: hadoop   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCompressDecompress() {
  int BYTE_SIZE = 1024 * 54;
  byte[] bytes = generate(BYTE_SIZE);
  Lz4Compressor compressor = new Lz4Compressor();
  try {
    compressor.setInput(bytes, 0, bytes.length);
    assertTrue("Lz4CompressDecompress getBytesRead error !!!",
        compressor.getBytesRead() > 0);
    assertTrue(
        "Lz4CompressDecompress getBytesWritten before compress error !!!",
        compressor.getBytesWritten() == 0);

    byte[] compressed = new byte[BYTE_SIZE];
    int cSize = compressor.compress(compressed, 0, compressed.length);
    assertTrue(
        "Lz4CompressDecompress getBytesWritten after compress error !!!",
        compressor.getBytesWritten() > 0);
    Lz4Decompressor decompressor = new Lz4Decompressor();
    // set as input for decompressor only compressed data indicated with cSize
    decompressor.setInput(compressed, 0, cSize);
    byte[] decompressed = new byte[BYTE_SIZE];
    decompressor.decompress(decompressed, 0, decompressed.length);

    assertTrue("testLz4CompressDecompress finished error !!!", decompressor.finished());      
    assertArrayEquals(bytes, decompressed);
    compressor.reset();
    decompressor.reset();
    assertTrue("decompressor getRemaining error !!!",decompressor.getRemaining() == 0);
  } catch (Exception e) {
    fail("testLz4CompressDecompress ex error!!!");
  }
}
 
Example 12
Source Project: big-c   Source File: Lz4Codec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
 *
 * @return the type of decompressor needed by this codec.
 */
@Override
public Class<? extends Decompressor> getDecompressorType() {
  if (!isNativeCodeLoaded()) {
    throw new RuntimeException("native lz4 library not available");
  }

  return Lz4Decompressor.class;
}
 
Example 13
Source Project: big-c   Source File: Lz4Codec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
 *
 * @return a new decompressor for use by this codec
 */
@Override
public Decompressor createDecompressor() {
  if (!isNativeCodeLoaded()) {
    throw new RuntimeException("native lz4 library not available");
  }
  int bufferSize = conf.getInt(
      CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
      CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
  return new Lz4Decompressor(bufferSize);
}
 
Example 14
Source Project: big-c   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecompressorSetInputNullPointerException() {
  try {
    Lz4Decompressor decompressor = new Lz4Decompressor();
    decompressor.setInput(null, 0, 10);
    fail("testDecompressorSetInputNullPointerException error !!!");
  } catch (NullPointerException ex) {
    // expected
  } catch (Exception e) {
    fail("testDecompressorSetInputNullPointerException ex error !!!");
  }
}
 
Example 15
Source Project: big-c   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecompressorSetInputAIOUBException() {
  try {
    Lz4Decompressor decompressor = new Lz4Decompressor();
    decompressor.setInput(new byte[] {}, -5, 10);
    fail("testDecompressorSetInputAIOBException error !!!");
  } catch (ArrayIndexOutOfBoundsException ex) {
    // expected
  } catch (Exception e) {
    fail("testDecompressorSetInputAIOBException ex error !!!");
  }
}
 
Example 16
Source Project: big-c   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecompressorCompressNullPointerException() {
  try {
    Lz4Decompressor decompressor = new Lz4Decompressor();
    byte[] bytes = generate(1024 * 6);
    decompressor.setInput(bytes, 0, bytes.length);
    decompressor.decompress(null, 0, 0);
    fail("testDecompressorCompressNullPointerException error !!!");
  } catch (NullPointerException ex) {
    // expected
  } catch (Exception e) {
    fail("testDecompressorCompressNullPointerException ex error !!!");
  }
}
 
Example 17
Source Project: big-c   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecompressorCompressAIOBException() {
  try {
    Lz4Decompressor decompressor = new Lz4Decompressor();
    byte[] bytes = generate(1024 * 6);
    decompressor.setInput(bytes, 0, bytes.length);
    decompressor.decompress(new byte[] {}, 0, -1);
    fail("testDecompressorCompressAIOBException error !!!");
  } catch (ArrayIndexOutOfBoundsException ex) {
    // expected
  } catch (Exception e) {
    fail("testDecompressorCompressAIOBException ex error !!!");
  }
}
 
Example 18
Source Project: big-c   Source File: TestLz4CompressorDecompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCompressDecompress() {
  int BYTE_SIZE = 1024 * 54;
  byte[] bytes = generate(BYTE_SIZE);
  Lz4Compressor compressor = new Lz4Compressor();
  try {
    compressor.setInput(bytes, 0, bytes.length);
    assertTrue("Lz4CompressDecompress getBytesRead error !!!",
        compressor.getBytesRead() > 0);
    assertTrue(
        "Lz4CompressDecompress getBytesWritten before compress error !!!",
        compressor.getBytesWritten() == 0);

    byte[] compressed = new byte[BYTE_SIZE];
    int cSize = compressor.compress(compressed, 0, compressed.length);
    assertTrue(
        "Lz4CompressDecompress getBytesWritten after compress error !!!",
        compressor.getBytesWritten() > 0);
    Lz4Decompressor decompressor = new Lz4Decompressor();
    // set as input for decompressor only compressed data indicated with cSize
    decompressor.setInput(compressed, 0, cSize);
    byte[] decompressed = new byte[BYTE_SIZE];
    decompressor.decompress(decompressed, 0, decompressed.length);

    assertTrue("testLz4CompressDecompress finished error !!!", decompressor.finished());      
    assertArrayEquals(bytes, decompressed);
    compressor.reset();
    decompressor.reset();
    assertTrue("decompressor getRemaining error !!!",decompressor.getRemaining() == 0);
  } catch (Exception e) {
    fail("testLz4CompressDecompress ex error!!!");
  }
}
 
Example 19
Source Project: datacollector   Source File: Lz4Codec.java    License: Apache License 2.0 4 votes vote down vote up
public Lz4Codec() {
  super(Lz4Compressor.class, Lz4Decompressor.class, ".lz4");
}
 
Example 20
Source Project: datacollector   Source File: Lz4Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Decompressor createDecompressor() {
  return new Lz4Decompressor(unsafe, getBufferSize());
}