Java Code Examples for org.apache.hadoop.util.NativeCodeLoader

The following examples show how to use org.apache.hadoop.util.NativeCodeLoader. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestFileJournalManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that internal renames are done using native code on platforms that
 * have it.  The native rename includes more detailed information about the
 * failure, which can be useful for troubleshooting.
 */
@Test
public void testDoPreUpgradeIOError() throws IOException {
  File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror");
  List<URI> editUris = Collections.singletonList(storageDir.toURI());
  NNStorage storage = setupEdits(editUris, 5);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
  assertNotNull(sd);
  // Change storage directory so that renaming current to previous.tmp fails.
  FileUtil.setWritable(storageDir, false);
  FileJournalManager jm = null;
  try {
    jm = new FileJournalManager(conf, sd, storage);
    exception.expect(IOException.class);
    if (NativeCodeLoader.isNativeCodeLoaded()) {
      exception.expectMessage("failure in native rename");
    }
    jm.doPreUpgrade();
  } finally {
    IOUtils.cleanup(LOG, jm);
    // Restore permissions on storage directory and make sure we can delete.
    FileUtil.setWritable(storageDir, true);
    FileUtil.fullyDelete(storageDir);
  }
}
 
Example 2
Source Project: hadoop   Source File: SnappyCodec.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Are the native snappy libraries loaded & initialized?
 */
public static void checkNativeCodeLoaded() {
    if (!NativeCodeLoader.isNativeCodeLoaded() ||
        !NativeCodeLoader.buildSupportsSnappy()) {
      throw new RuntimeException("native snappy library not available: " +
          "this version of libhadoop was built without " +
          "snappy support.");
    }
    if (!SnappyCompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyCompressor has not been loaded.");
    }
    if (!SnappyDecompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyDecompressor has not been loaded.");
    }
}
 
Example 3
Source Project: hadoop   Source File: TestCryptoCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=120000)
public void testJceAesCtrCryptoCodec() throws Exception {
  if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
    LOG.warn("Skipping since test was not run with -Pnative flag");
    Assume.assumeTrue(false);
  }
  if (!NativeCodeLoader.buildSupportsOpenssl()) {
    LOG.warn("Skipping test since openSSL library not loaded");
    Assume.assumeTrue(false);
  }
  Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
  cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
  // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
  for(int i = 0; i < 8; i++) {
    iv[8 + i] = (byte) 0xff;
  }
  cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
}
 
Example 4
Source Project: hadoop   Source File: TestCryptoCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=120000)
public void testOpensslAesCtrCryptoCodec() throws Exception {
  if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
    LOG.warn("Skipping since test was not run with -Pnative flag");
    Assume.assumeTrue(false);
  }
  if (!NativeCodeLoader.buildSupportsOpenssl()) {
    LOG.warn("Skipping test since openSSL library not loaded");
    Assume.assumeTrue(false);
  }
  Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
  cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
  // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
  for(int i = 0; i < 8; i++) {
    iv[8 + i] = (byte) 0xff;
  }
  cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
}
 
Example 5
Source Project: hadoop-gpu   Source File: SequenceFile.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * @param conf The configuration.
 * @param out The stream on top which the writer is to be constructed.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param compressionType The compression type.
 * @param codec The compression codec.
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer
  createWriter(Configuration conf, FSDataOutputStream out, 
               Class keyClass, Class valClass, CompressionType compressionType,
               CompressionCodec codec, Metadata metadata)
  throws IOException {
  if ((codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (compressionType == CompressionType.NONE) {
    writer = new Writer(conf, out, keyClass, valClass, metadata);
  } else if (compressionType == CompressionType.RECORD) {
    writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  } else if (compressionType == CompressionType.BLOCK){
    writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  }
  
  return writer;
}
 
Example 6
Source Project: hadoop   Source File: CompressDecompressTester.java    License: Apache License 2.0 6 votes vote down vote up
private static boolean isNativeSnappyLoadable() {
  boolean snappyAvailable = false;
  boolean loaded = false;
  try {
    System.loadLibrary("snappy");
    logger.warn("Snappy native library is available");
    snappyAvailable = true;
    boolean hadoopNativeAvailable = NativeCodeLoader.isNativeCodeLoaded();
    loaded = snappyAvailable && hadoopNativeAvailable;
    if (loaded) {
      logger.info("Snappy native library loaded");
    } else {
      logger.warn("Snappy native library not loaded");
    }
  } catch (Throwable t) {
    logger.warn("Failed to load snappy: ", t);
    return false;
  }
  return loaded;
}
 
Example 7
Source Project: hadoop   Source File: CompressDecompressTester.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Method for compressor availability check
 */
private static <T extends Compressor, E extends Decompressor> boolean isAvailable(TesterPair<T, E> pair) {
  Compressor compressor = pair.compressor;

  if (compressor.getClass().isAssignableFrom(Lz4Compressor.class)
          && (NativeCodeLoader.isNativeCodeLoaded()))
    return true;

  else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class)
          && NativeCodeLoader.isNativeCodeLoaded())
    return true;

  else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) {
    return ZlibFactory.isNativeZlibLoaded(new Configuration());
  }              
  else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)
          && isNativeSnappyLoadable())
    return true;
  
  return false;      
}
 
Example 8
Source Project: hadoop   Source File: TestCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testBZip2NativeCodec() throws IOException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
      conf.set("io.compression.codec.bzip2.library", "java-builtin");
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 
Example 9
Source Project: hadoop   Source File: TestCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLz4Codec() throws IOException {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Lz4Codec.isNativeCodeLoaded()) {
      conf.setBoolean(
          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
          false);
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.Lz4Codec");
      conf.setBoolean(
          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
          true);
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.Lz4Codec");
    } else {
      Assert.fail("Native hadoop library available but lz4 not");
    }
  }
}
 
Example 10
Source Project: hadoop   Source File: TestCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testSequenceFileBZip2NativeCodec() throws IOException, 
                      ClassNotFoundException, InstantiationException, 
                      IllegalAccessException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      sequenceFileCodecTest(conf, 0, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 100, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 200000, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 
                            1000000);
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 
Example 11
Source Project: big-c   Source File: TestFileJournalManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that internal renames are done using native code on platforms that
 * have it.  The native rename includes more detailed information about the
 * failure, which can be useful for troubleshooting.
 */
@Test
public void testDoPreUpgradeIOError() throws IOException {
  File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror");
  List<URI> editUris = Collections.singletonList(storageDir.toURI());
  NNStorage storage = setupEdits(editUris, 5);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
  assertNotNull(sd);
  // Change storage directory so that renaming current to previous.tmp fails.
  FileUtil.setWritable(storageDir, false);
  FileJournalManager jm = null;
  try {
    jm = new FileJournalManager(conf, sd, storage);
    exception.expect(IOException.class);
    if (NativeCodeLoader.isNativeCodeLoaded()) {
      exception.expectMessage("failure in native rename");
    }
    jm.doPreUpgrade();
  } finally {
    IOUtils.cleanup(LOG, jm);
    // Restore permissions on storage directory and make sure we can delete.
    FileUtil.setWritable(storageDir, true);
    FileUtil.fullyDelete(storageDir);
  }
}
 
Example 12
Source Project: big-c   Source File: SnappyCodec.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Are the native snappy libraries loaded & initialized?
 */
public static void checkNativeCodeLoaded() {
    if (!NativeCodeLoader.isNativeCodeLoaded() ||
        !NativeCodeLoader.buildSupportsSnappy()) {
      throw new RuntimeException("native snappy library not available: " +
          "this version of libhadoop was built without " +
          "snappy support.");
    }
    if (!SnappyCompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyCompressor has not been loaded.");
    }
    if (!SnappyDecompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyDecompressor has not been loaded.");
    }
}
 
Example 13
Source Project: big-c   Source File: TestCryptoCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=120000)
public void testJceAesCtrCryptoCodec() throws Exception {
  if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
    LOG.warn("Skipping since test was not run with -Pnative flag");
    Assume.assumeTrue(false);
  }
  if (!NativeCodeLoader.buildSupportsOpenssl()) {
    LOG.warn("Skipping test since openSSL library not loaded");
    Assume.assumeTrue(false);
  }
  Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
  cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
  // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
  for(int i = 0; i < 8; i++) {
    iv[8 + i] = (byte) 0xff;
  }
  cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
}
 
Example 14
Source Project: big-c   Source File: TestCryptoCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=120000)
public void testOpensslAesCtrCryptoCodec() throws Exception {
  if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
    LOG.warn("Skipping since test was not run with -Pnative flag");
    Assume.assumeTrue(false);
  }
  if (!NativeCodeLoader.buildSupportsOpenssl()) {
    LOG.warn("Skipping test since openSSL library not loaded");
    Assume.assumeTrue(false);
  }
  Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
  cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
  // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
  for(int i = 0; i < 8; i++) {
    iv[8 + i] = (byte) 0xff;
  }
  cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
}
 
Example 15
Source Project: big-c   Source File: TestNativeCodeLoader.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNativeCodeLoaded() {
  if (requireTestJni() == false) {
    LOG.info("TestNativeCodeLoader: libhadoop.so testing is not required.");
    return;
  }
  if (!NativeCodeLoader.isNativeCodeLoaded()) {
    fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
        "libhadoop.so was not loaded.");
  }
  assertFalse(NativeCodeLoader.getLibraryName().isEmpty());
  // library names are depended on platform and build envs
  // so just check names are available
  assertFalse(ZlibFactory.getLibraryName().isEmpty());
  if (NativeCodeLoader.buildSupportsSnappy()) {
    assertFalse(SnappyCodec.getLibraryName().isEmpty());
  }
  if (NativeCodeLoader.buildSupportsOpenssl()) {
    assertFalse(OpensslCipher.getLibraryName().isEmpty());
  }
  assertFalse(Lz4Codec.getLibraryName().isEmpty());
  LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
}
 
Example 16
Source Project: big-c   Source File: CompressDecompressTester.java    License: Apache License 2.0 6 votes vote down vote up
private static boolean isNativeSnappyLoadable() {
  boolean snappyAvailable = false;
  boolean loaded = false;
  try {
    System.loadLibrary("snappy");
    logger.warn("Snappy native library is available");
    snappyAvailable = true;
    boolean hadoopNativeAvailable = NativeCodeLoader.isNativeCodeLoaded();
    loaded = snappyAvailable && hadoopNativeAvailable;
    if (loaded) {
      logger.info("Snappy native library loaded");
    } else {
      logger.warn("Snappy native library not loaded");
    }
  } catch (Throwable t) {
    logger.warn("Failed to load snappy: ", t);
    return false;
  }
  return loaded;
}
 
Example 17
Source Project: big-c   Source File: CompressDecompressTester.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Method for compressor availability check
 */
private static <T extends Compressor, E extends Decompressor> boolean isAvailable(TesterPair<T, E> pair) {
  Compressor compressor = pair.compressor;

  if (compressor.getClass().isAssignableFrom(Lz4Compressor.class)
          && (NativeCodeLoader.isNativeCodeLoaded()))
    return true;

  else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class)
          && NativeCodeLoader.isNativeCodeLoaded())
    return true;

  else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) {
    return ZlibFactory.isNativeZlibLoaded(new Configuration());
  }              
  else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)
          && isNativeSnappyLoadable())
    return true;
  
  return false;      
}
 
Example 18
Source Project: big-c   Source File: TestCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testBZip2NativeCodec() throws IOException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
      conf.set("io.compression.codec.bzip2.library", "java-builtin");
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 
Example 19
Source Project: big-c   Source File: TestCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLz4Codec() throws IOException {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Lz4Codec.isNativeCodeLoaded()) {
      conf.setBoolean(
          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
          false);
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.Lz4Codec");
      conf.setBoolean(
          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
          true);
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.Lz4Codec");
    } else {
      Assert.fail("Native hadoop library available but lz4 not");
    }
  }
}
 
Example 20
Source Project: big-c   Source File: TestCodec.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testSequenceFileBZip2NativeCodec() throws IOException, 
                      ClassNotFoundException, InstantiationException, 
                      IllegalAccessException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      sequenceFileCodecTest(conf, 0, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 100, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 200000, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 
                            1000000);
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 
Example 21
Source Project: hbase   Source File: TestCompressionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTestCompression() {
  assertTrue(CompressionTest.testCompression("NONE"));
  assertTrue(CompressionTest.testCompression("GZ"));

  if (NativeCodeLoader.isNativeCodeLoaded()) {
    nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
    nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
    nativeCodecTest("SNAPPY", "snappy", "org.apache.hadoop.io.compress.SnappyCodec");
    nativeCodecTest("BZIP2", "bzip2", "org.apache.hadoop.io.compress.BZip2Codec");
    nativeCodecTest("ZSTD", "zstd", "org.apache.hadoop.io.compress.ZStandardCodec");
  } else {
    // Hadoop nativelib is not available
    LOG.debug("Native code not loaded");
    assertFalse(CompressionTest.testCompression("LZO"));
    assertFalse(CompressionTest.testCompression("LZ4"));
    assertFalse(CompressionTest.testCompression("SNAPPY"));
    assertFalse(CompressionTest.testCompression("BZIP2"));
    assertFalse(CompressionTest.testCompression("ZSTD"));
  }
}
 
Example 22
Source Project: RDFS   Source File: SequenceFile.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * @param out The stream on top which the writer is to be constructed.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param compress Compress data?
 * @param blockCompress Compress blocks?
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
private static Writer
  createWriter(Configuration conf, FSDataOutputStream out, 
               Class keyClass, Class valClass, boolean compress, boolean blockCompress,
               CompressionCodec codec, Metadata metadata)
  throws IOException {
  if (codec != null && (codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (!compress) {
    writer = new Writer(conf, out, keyClass, valClass, metadata);
  } else if (compress && !blockCompress) {
    writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  } else {
    writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  }
  
  return writer;
}
 
Example 23
Source Project: RDFS   Source File: SequenceFile.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * @param conf The configuration.
 * @param out The stream on top which the writer is to be constructed.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param compressionType The compression type.
 * @param codec The compression codec.
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer
  createWriter(Configuration conf, FSDataOutputStream out, 
               Class keyClass, Class valClass, CompressionType compressionType,
               CompressionCodec codec, Metadata metadata)
  throws IOException {
  if ((codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (compressionType == CompressionType.NONE) {
    writer = new Writer(conf, out, keyClass, valClass, metadata);
  } else if (compressionType == CompressionType.RECORD) {
    writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  } else if (compressionType == CompressionType.BLOCK){
    writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  }
  
  return writer;
}
 
Example 24
Source Project: hadoop-gpu   Source File: SequenceFile.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * @param out The stream on top which the writer is to be constructed.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param compress Compress data?
 * @param blockCompress Compress blocks?
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
private static Writer
  createWriter(Configuration conf, FSDataOutputStream out, 
               Class keyClass, Class valClass, boolean compress, boolean blockCompress,
               CompressionCodec codec, Metadata metadata)
  throws IOException {
  if (codec != null && (codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (!compress) {
    writer = new Writer(conf, out, keyClass, valClass, metadata);
  } else if (compress && !blockCompress) {
    writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  } else {
    writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  }
  
  return writer;
}
 
Example 25
Source Project: hadoop-ozone   Source File: OzoneAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Main for the Ozone Admin shell Command handling.
 *
 * @param argv - System Args Strings[]
 * @throws Exception
 */
public static void main(String[] argv) throws Exception {
  LogManager.resetConfiguration();
  Logger.getRootLogger().setLevel(Level.INFO);
  Logger.getRootLogger()
      .addAppender(new ConsoleAppender(new PatternLayout("%m%n")));
  Logger.getLogger(NativeCodeLoader.class).setLevel(Level.ERROR);

  new OzoneAdmin().run(argv);
}
 
Example 26
Source Project: hadoop   Source File: TestHdfsNativeCodeLoader.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@Ignore
public void testNativeCodeLoaded() {
  if (requireTestJni() == false) {
    LOG.info("TestNativeCodeLoader: libhadoop.so testing is not required.");
    return;
  }
  if (!NativeCodeLoader.isNativeCodeLoaded()) {
    String LD_LIBRARY_PATH = System.getenv().get("LD_LIBRARY_PATH");
    if (LD_LIBRARY_PATH == null) LD_LIBRARY_PATH = "";
    fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
        "libhadoop.so was not loaded.  LD_LIBRARY_PATH = " + LD_LIBRARY_PATH);
  }
  LOG.info("TestHdfsNativeCodeLoader: libhadoop.so is loaded.");
}
 
Example 27
Source Project: hadoop   Source File: TestTracingShortCircuitLocalRead.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testShortCircuitTraceHooks() throws IOException {
  assumeTrue(NativeCodeLoader.isNativeCodeLoaded() && !Path.WINDOWS);
  conf = new Configuration();
  conf.set(DFSConfigKeys.DFS_CLIENT_HTRACE_PREFIX +
      SpanReceiverHost.SPAN_RECEIVERS_CONF_SUFFIX,
      TestTracing.SetSpanReceiver.class.getName());
  conf.setLong("dfs.blocksize", 100 * 1024);
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, false);
  conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
      "testShortCircuitTraceHooks._PORT");
  conf.set(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, "CRC32C");
  cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(1)
      .build();
  dfs = cluster.getFileSystem();

  try {
    DFSTestUtil.createFile(dfs, TEST_PATH, TEST_LENGTH, (short)1, 5678L);

    TraceScope ts = Trace.startSpan("testShortCircuitTraceHooks", Sampler.ALWAYS);
    FSDataInputStream stream = dfs.open(TEST_PATH);
    byte buf[] = new byte[TEST_LENGTH];
    IOUtils.readFully(stream, buf, 0, TEST_LENGTH);
    stream.close();
    ts.close();

    String[] expectedSpanNames = {
      "OpRequestShortCircuitAccessProto",
      "ShortCircuitShmRequestProto"
    };
    TestTracing.assertSpanNamesFound(expectedSpanNames);
  } finally {
    dfs.close();
    cluster.shutdown();
  }
}
 
Example 28
public JniBasedUnixGroupsMappingWithFallback() {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    this.impl = new JniBasedUnixGroupsMapping();
  } else {
    PerformanceAdvisory.LOG.debug("Falling back to shell based");
    this.impl = new ShellBasedUnixGroupsMapping();
  }
  if (LOG.isDebugEnabled()){
    LOG.debug("Group mapping impl=" + impl.getClass().getName());
  }
}
 
Example 29
public JniBasedUnixGroupsNetgroupMappingWithFallback() {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    this.impl = new JniBasedUnixGroupsNetgroupMapping();
  } else {
    LOG.info("Falling back to shell based");
    this.impl = new ShellBasedUnixGroupsNetgroupMapping();
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Group mapping impl=" + impl.getClass().getName());
  }
}
 
Example 30
Source Project: hadoop   Source File: Bzip2Factory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Check if native-bzip2 code is loaded & initialized correctly and 
 * can be loaded for this job.
 * 
 * @param conf configuration
 * @return <code>true</code> if native-bzip2 is loaded & initialized 
 *         and can be loaded for this job, else <code>false</code>
 */
public static boolean isNativeBzip2Loaded(Configuration conf) {
  String libname = conf.get("io.compression.codec.bzip2.library", 
                            "system-native");
  if (!bzip2LibraryName.equals(libname)) {
    nativeBzip2Loaded = false;
    bzip2LibraryName = libname;
    if (libname.equals("java-builtin")) {
      LOG.info("Using pure-Java version of bzip2 library");
    } else if (conf.getBoolean(
              CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, 
              CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
        NativeCodeLoader.isNativeCodeLoaded()) {
      try {
        // Initialize the native library.
        Bzip2Compressor.initSymbols(libname);
        Bzip2Decompressor.initSymbols(libname);
        nativeBzip2Loaded = true;
        LOG.info("Successfully loaded & initialized native-bzip2 library " +
                 libname);
      } catch (Throwable t) {
        LOG.warn("Failed to load/initialize native-bzip2 library " + 
                 libname + ", will use pure-Java version");
      }
    }
  }
  return nativeBzip2Loaded;
}