Java Code Examples for org.apache.hadoop.hdfs.DFSTestUtil#formatNameNode()

The following examples show how to use org.apache.hadoop.hdfs.DFSTestUtil#formatNameNode() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestFSNamesystem.java From big-c with Apache License 2.0

6 votes

/**
 * Test that FSNamesystem#clear clears all leases.
 */
@Test
public void testFSNamespaceClearLeases() throws Exception {
  Configuration conf = new HdfsConfiguration();
  File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
  conf.set(DFS_NAMENODE_NAME_DIR_KEY, nameDir.getAbsolutePath());

  NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
  LeaseManager leaseMan = fsn.getLeaseManager();
  leaseMan.addLease("client1", "importantFile");
  assertEquals(1, leaseMan.countLease());
  fsn.clear();
  leaseMan = fsn.getLeaseManager();
  assertEquals(0, leaseMan.countLease());
}

Example 2

Source File: TestValidateConfigurationSettings.java From big-c with Apache License 2.0

6 votes

/**
 * Tests setting the rpc port to the same as the web port to test that 
 * an exception
 * is thrown when trying to re-use the same port
 */
@Test(expected = BindException.class, timeout = 300000)
public void testThatMatchingRPCandHttpPortsThrowException() 
    throws IOException {

  NameNode nameNode = null;
  try {
    Configuration conf = new HdfsConfiguration();
    File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
        nameDir.getAbsolutePath());

    Random rand = new Random();
    final int port = 30000 + rand.nextInt(30000);

    // set both of these to the same port. It should fail.
    FileSystem.setDefaultUri(conf, "hdfs://localhost:" + port);
    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:" + port);
    DFSTestUtil.formatNameNode(conf);
    nameNode = new NameNode(conf);
  } finally {
    if (nameNode != null) {
      nameNode.stop();
    }
  }
}

Example 3

Source File: TestSaveNamespace.java From big-c with Apache License 2.0

5 votes

@Test (timeout=30000)
public void testSaveWhileEditsRolled() throws Exception {
  Configuration conf = getConf();
  NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);

  try {
    doAnEdit(fsn, 1);
    CheckpointSignature sig = fsn.rollEditLog();
    LOG.warn("Checkpoint signature: " + sig);
    // Do another edit
    doAnEdit(fsn, 2);

    // Save namespace
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fsn.saveNamespace();

    // Now shut down and restart the NN
    fsn.close();
    fsn = null;

    // Start a new namesystem, which should be able to recover
    // the namespace from the previous incarnation.
    fsn = FSNamesystem.loadFromDisk(conf);

    // Make sure the image loaded including our edits.
    checkEditExists(fsn, 1);
    checkEditExists(fsn, 2);
  } finally {
    if (fsn != null) {
      fsn.close();
    }
  }
}

Example 4

Source File: TestCreateEditsLog.java From big-c with Apache License 2.0

5 votes

/**
 * Tests that an edits log created using CreateEditsLog is valid and can be
 * loaded successfully by a namenode.
 */
@Test(timeout=60000)
public void testCanLoadCreatedEditsLog() throws Exception {
  // Format namenode.
  HdfsConfiguration conf = new HdfsConfiguration();
  File nameDir = new File(HDFS_DIR, "name");
  conf.set(DFS_NAMENODE_NAME_DIR_KEY, Util.fileAsURI(nameDir).toString());
  DFSTestUtil.formatNameNode(conf);

  // Call CreateEditsLog and move the resulting edits to the name dir.
  CreateEditsLog.main(new String[] { "-f", "1000", "0", "1", "-d",
    TEST_DIR.getAbsolutePath() });
  Path editsWildcard = new Path(TEST_DIR.getAbsolutePath(), "*");
  FileContext localFc = FileContext.getLocalFSFileContext();
  for (FileStatus edits: localFc.util().globStatus(editsWildcard)) {
    Path src = edits.getPath();
    Path dst = new Path(new File(nameDir, "current").getAbsolutePath(),
      src.getName());
    localFc.rename(src, dst);
  }

  // Start a namenode to try to load the edits.
  cluster = new MiniDFSCluster.Builder(conf)
    .format(false)
    .manageNameDfsDirs(false)
    .waitSafeMode(false)
    .build();
  cluster.waitClusterUp();

  // Test successful, because no exception thrown.
}

Example 5

Source File: TestNNThroughputBenchmark.java From big-c with Apache License 2.0

5 votes

/**
 * This test runs all benchmarks defined in {@link NNThroughputBenchmark}.
 */
@Test
public void testNNThroughput() throws Exception {
  Configuration conf = new HdfsConfiguration();
  File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
  conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
      nameDir.getAbsolutePath());
  FileSystem.setDefaultUri(conf, "hdfs://localhost:" + 0);
  conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
  DFSTestUtil.formatNameNode(conf);
  String[] args = new String[] {"-op", "all"};
  NNThroughputBenchmark.runBenchmark(conf, Arrays.asList(args));
}

Example 6

Source File: TestValidateConfigurationSettings.java From hadoop with Apache License 2.0

5 votes

/**
 * HDFS-3013: NameNode format command doesn't pick up
 * dfs.namenode.name.dir.NameServiceId configuration.
 */
@Test(timeout = 300000)
public void testGenericKeysForNameNodeFormat()
    throws IOException {
  Configuration conf = new HdfsConfiguration();

  // Set ephemeral ports 
  conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY,
      "127.0.0.1:0");
  conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
      "127.0.0.1:0");
  
  conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns1");
  
  // Set a nameservice-specific configuration for name dir
  File dir = new File(MiniDFSCluster.getBaseDirectory(),
      "testGenericKeysForNameNodeFormat");
  if (dir.exists()) {
    FileUtil.fullyDelete(dir);
  }
  conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + ".ns1",
      dir.getAbsolutePath());
  
  // Format and verify the right dir is formatted.
  DFSTestUtil.formatNameNode(conf);
  GenericTestUtils.assertExists(dir);

  // Ensure that the same dir is picked up by the running NN
  NameNode nameNode = new NameNode(conf);
  nameNode.stop();
}

Example 7

Source File: TestValidateConfigurationSettings.java From hadoop with Apache License 2.0

5 votes

/**
 * Tests setting the rpc port to a different as the web port that an 
 * exception is NOT thrown 
 */
@Test(timeout = 300000)
public void testThatDifferentRPCandHttpPortsAreOK() 
    throws IOException {

  Configuration conf = new HdfsConfiguration();
  File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
  conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
      nameDir.getAbsolutePath());

  Random rand = new Random();

  // A few retries in case the ports we choose are in use.
  for (int i = 0; i < 5; ++i) {
    final int port1 = 30000 + rand.nextInt(10000);
    final int port2 = port1 + 1 + rand.nextInt(10000);

    FileSystem.setDefaultUri(conf, "hdfs://localhost:" + port1);
    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:" + port2);
    DFSTestUtil.formatNameNode(conf);
    NameNode nameNode = null;

    try {
      nameNode = new NameNode(conf); // should be OK!
      break;
    } catch(BindException be) {
      continue;     // Port in use? Try another.
    } finally {
      if (nameNode != null) {
        nameNode.stop();
      }
    }
  }
}

Example 8

Source File: TestValidateConfigurationSettings.java From big-c with Apache License 2.0

5 votes

/**
 * Tests setting the rpc port to a different as the web port that an 
 * exception is NOT thrown 
 */
@Test(timeout = 300000)
public void testThatDifferentRPCandHttpPortsAreOK() 
    throws IOException {

  Configuration conf = new HdfsConfiguration();
  File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
  conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
      nameDir.getAbsolutePath());

  Random rand = new Random();

  // A few retries in case the ports we choose are in use.
  for (int i = 0; i < 5; ++i) {
    final int port1 = 30000 + rand.nextInt(10000);
    final int port2 = port1 + 1 + rand.nextInt(10000);

    FileSystem.setDefaultUri(conf, "hdfs://localhost:" + port1);
    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:" + port2);
    DFSTestUtil.formatNameNode(conf);
    NameNode nameNode = null;

    try {
      nameNode = new NameNode(conf); // should be OK!
      break;
    } catch(BindException be) {
      continue;     // Port in use? Try another.
    } finally {
      if (nameNode != null) {
        nameNode.stop();
      }
    }
  }
}

Example 9