Java Code Examples for org.apache.hadoop.io.compress.snappy.SnappyCompressor

The following examples show how to use org.apache.hadoop.io.compress.snappy.SnappyCompressor. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: SnappyCodec.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Are the native snappy libraries loaded & initialized?
 */
public static void checkNativeCodeLoaded() {
    if (!NativeCodeLoader.isNativeCodeLoaded() ||
        !NativeCodeLoader.buildSupportsSnappy()) {
      throw new RuntimeException("native snappy library not available: " +
          "this version of libhadoop was built without " +
          "snappy support.");
    }
    if (!SnappyCompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyCompressor has not been loaded.");
    }
    if (!SnappyDecompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyDecompressor has not been loaded.");
    }
}
 
Example 2
Source Project: hadoop   Source File: CompressDecompressTester.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Method for compressor availability check
 */
private static <T extends Compressor, E extends Decompressor> boolean isAvailable(TesterPair<T, E> pair) {
  Compressor compressor = pair.compressor;

  if (compressor.getClass().isAssignableFrom(Lz4Compressor.class)
          && (NativeCodeLoader.isNativeCodeLoaded()))
    return true;

  else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class)
          && NativeCodeLoader.isNativeCodeLoaded())
    return true;

  else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) {
    return ZlibFactory.isNativeZlibLoaded(new Configuration());
  }              
  else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)
          && isNativeSnappyLoadable())
    return true;
  
  return false;      
}
 
Example 3
Source Project: hadoop   Source File: TestCompressorDecompressor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressorDecompressor() {
  // no more for this data
  int SIZE = 44 * 1024;
  
  byte[] rawData = generate(SIZE);
  try {
    CompressDecompressTester.of(rawData)
        .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor())
        .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor())
        .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater())
        .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
        .test();

  } catch (Exception ex) {
    fail("testCompressorDecompressor error !!!" + ex);
  }
}
 
Example 4
Source Project: hadoop   Source File: TestCompressorDecompressor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressorDecompressorWithExeedBufferLimit() {
  int BYTE_SIZE = 100 * 1024;
  byte[] rawData = generate(BYTE_SIZE);
  try {
    CompressDecompressTester.of(rawData)
        .withCompressDecompressPair(
            new SnappyCompressor(BYTE_SIZE + BYTE_SIZE / 2),
            new SnappyDecompressor(BYTE_SIZE + BYTE_SIZE / 2))
        .withCompressDecompressPair(new Lz4Compressor(BYTE_SIZE),
            new Lz4Decompressor(BYTE_SIZE))
        .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
        .test();

  } catch (Exception ex) {
    fail("testCompressorDecompressorWithExeedBufferLimit error !!!" + ex);
  }
}
 
Example 5
Source Project: big-c   Source File: SnappyCodec.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Are the native snappy libraries loaded & initialized?
 */
public static void checkNativeCodeLoaded() {
    if (!NativeCodeLoader.isNativeCodeLoaded() ||
        !NativeCodeLoader.buildSupportsSnappy()) {
      throw new RuntimeException("native snappy library not available: " +
          "this version of libhadoop was built without " +
          "snappy support.");
    }
    if (!SnappyCompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyCompressor has not been loaded.");
    }
    if (!SnappyDecompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyDecompressor has not been loaded.");
    }
}
 
Example 6
Source Project: big-c   Source File: CompressDecompressTester.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Method for compressor availability check
 */
private static <T extends Compressor, E extends Decompressor> boolean isAvailable(TesterPair<T, E> pair) {
  Compressor compressor = pair.compressor;

  if (compressor.getClass().isAssignableFrom(Lz4Compressor.class)
          && (NativeCodeLoader.isNativeCodeLoaded()))
    return true;

  else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class)
          && NativeCodeLoader.isNativeCodeLoaded())
    return true;

  else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) {
    return ZlibFactory.isNativeZlibLoaded(new Configuration());
  }              
  else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)
          && isNativeSnappyLoadable())
    return true;
  
  return false;      
}
 
Example 7
Source Project: big-c   Source File: TestCompressorDecompressor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressorDecompressor() {
  // no more for this data
  int SIZE = 44 * 1024;
  
  byte[] rawData = generate(SIZE);
  try {
    CompressDecompressTester.of(rawData)
        .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor())
        .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor())
        .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater())
        .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
        .test();

  } catch (Exception ex) {
    fail("testCompressorDecompressor error !!!" + ex);
  }
}
 
Example 8
Source Project: big-c   Source File: TestCompressorDecompressor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressorDecompressorWithExeedBufferLimit() {
  int BYTE_SIZE = 100 * 1024;
  byte[] rawData = generate(BYTE_SIZE);
  try {
    CompressDecompressTester.of(rawData)
        .withCompressDecompressPair(
            new SnappyCompressor(BYTE_SIZE + BYTE_SIZE / 2),
            new SnappyDecompressor(BYTE_SIZE + BYTE_SIZE / 2))
        .withCompressDecompressPair(new Lz4Compressor(BYTE_SIZE),
            new Lz4Decompressor(BYTE_SIZE))
        .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                    CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
        .test();

  } catch (Exception ex) {
    fail("testCompressorDecompressorWithExeedBufferLimit error !!!" + ex);
  }
}
 
Example 9
Source Project: hadoop   Source File: SnappyCodec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
 *
 * @return a new compressor for use by this codec
 */
@Override
public Compressor createCompressor() {
  checkNativeCodeLoaded();
  int bufferSize = conf.getInt(
      CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
      CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
  return new SnappyCompressor(bufferSize);
}
 
Example 10
Source Project: big-c   Source File: SnappyCodec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
 *
 * @return a new compressor for use by this codec
 */
@Override
public Compressor createCompressor() {
  checkNativeCodeLoaded();
  int bufferSize = conf.getInt(
      CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
      CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
  return new SnappyCompressor(bufferSize);
}
 
Example 11
Source Project: attic-apex-malhar   Source File: FilterStreamCodec.java    License: Apache License 2.0 5 votes vote down vote up
public SnappyFilterStreamContext(OutputStream outputStream) throws IOException
{
  SnappyCodec codec = new SnappyCodec();
  codec.setConf(new Configuration());
  try {
    filterStream = new SnappyFilterStream(
        codec.createOutputStream(outputStream, new SnappyCompressor(bufferSize)));
  } catch (IOException e) {
    throw e;
  }
}
 
Example 12
Source Project: RDFS   Source File: SnappyCodec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the type of {@link Compressor} needed by this
 * {@link CompressionCodec}.
 *
 * @return the type of compressor needed by this codec.
 */
@Override
public Class<? extends Compressor> getCompressorType() {
  if (!isNativeSnappyLoaded(conf)) {
    throw new RuntimeException("native snappy library not available");
  }

  return SnappyCompressor.class;
}
 
Example 13
Source Project: RDFS   Source File: SnappyCodec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
 *
 * @return a new compressor for use by this codec
 */
@Override
public Compressor createCompressor() {
  if (!isNativeSnappyLoaded(conf)) {
    throw new CodecUnavailableException("native snappy library not available");
  }
  int bufferSize = conf.getInt(
      IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
      IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
  return new SnappyCompressor(bufferSize);
}
 
Example 14
Source Project: hadoop   Source File: SnappyCodec.java    License: Apache License 2.0 4 votes vote down vote up
public static boolean isNativeCodeLoaded() {
  return SnappyCompressor.isNativeCodeLoaded() && 
      SnappyDecompressor.isNativeCodeLoaded();
}
 
Example 15
Source Project: hadoop   Source File: SnappyCodec.java    License: Apache License 2.0 4 votes vote down vote up
public static String getLibraryName() {
  return SnappyCompressor.getLibraryName();
}
 
Example 16
Source Project: hadoop   Source File: SnappyCodec.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
 *
 * @return the type of compressor needed by this codec.
 */
@Override
public Class<? extends Compressor> getCompressorType() {
  checkNativeCodeLoaded();
  return SnappyCompressor.class;
}
 
Example 17
Source Project: big-c   Source File: SnappyCodec.java    License: Apache License 2.0 4 votes vote down vote up
public static boolean isNativeCodeLoaded() {
  return SnappyCompressor.isNativeCodeLoaded() && 
      SnappyDecompressor.isNativeCodeLoaded();
}
 
Example 18
Source Project: big-c   Source File: SnappyCodec.java    License: Apache License 2.0 4 votes vote down vote up
public static String getLibraryName() {
  return SnappyCompressor.getLibraryName();
}
 
Example 19
Source Project: big-c   Source File: SnappyCodec.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
 *
 * @return the type of compressor needed by this codec.
 */
@Override
public Class<? extends Compressor> getCompressorType() {
  checkNativeCodeLoaded();
  return SnappyCompressor.class;
}
 
Example 20
Source Project: datacollector   Source File: SnappyCodec.java    License: Apache License 2.0 4 votes vote down vote up
public SnappyCodec() {
  super(SnappyCompressor.class, SnappyDecompressor.class, ".snappy");
}
 
Example 21
Source Project: datacollector   Source File: SnappyCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Compressor createCompressor() {
  return new SnappyCompressor(getBufferSize());
}