Java Code Examples for org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil#waitForStandbyToCatchUp()

The following examples show how to use org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil#waitForStandbyToCatchUp() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestEncryptionZonesWithHA.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Test that encryption zones are properly tracked by the standby.
 */
@Test(timeout = 60000)
public void testEncryptionZonesTrackedOnStandby() throws Exception {
  final int len = 8196;
  final Path dir = new Path("/enc");
  final Path dirChild = new Path(dir, "child");
  final Path dirFile = new Path(dir, "file");
  fs.mkdir(dir, FsPermission.getDirDefault());
  dfsAdmin0.createEncryptionZone(dir, TEST_KEY);
  fs.mkdir(dirChild, FsPermission.getDirDefault());
  DFSTestUtil.createFile(fs, dirFile, len, (short) 1, 0xFEED);
  String contents = DFSTestUtil.readFile(fs, dirFile);

  // Failover the current standby to active.
  HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
  cluster.shutdownNameNode(0);
  cluster.transitionToActive(1);

  Assert.assertEquals("Got unexpected ez path", dir.toString(),
      dfsAdmin1.getEncryptionZoneForPath(dir).getPath().toString());
  Assert.assertEquals("Got unexpected ez path", dir.toString(),
      dfsAdmin1.getEncryptionZoneForPath(dirChild).getPath().toString());
  Assert.assertEquals("File contents after failover were changed",
      contents, DFSTestUtil.readFile(fs, dirFile));
}
Example 2
Source File: TestEncryptionZonesWithHA.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Test that encryption zones are properly tracked by the standby.
 */
@Test(timeout = 60000)
public void testEncryptionZonesTrackedOnStandby() throws Exception {
  final int len = 8196;
  final Path dir = new Path("/enc");
  final Path dirChild = new Path(dir, "child");
  final Path dirFile = new Path(dir, "file");
  fs.mkdir(dir, FsPermission.getDirDefault());
  dfsAdmin0.createEncryptionZone(dir, TEST_KEY);
  fs.mkdir(dirChild, FsPermission.getDirDefault());
  DFSTestUtil.createFile(fs, dirFile, len, (short) 1, 0xFEED);
  String contents = DFSTestUtil.readFile(fs, dirFile);

  // Failover the current standby to active.
  HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
  cluster.shutdownNameNode(0);
  cluster.transitionToActive(1);

  Assert.assertEquals("Got unexpected ez path", dir.toString(),
      dfsAdmin1.getEncryptionZoneForPath(dir).getPath().toString());
  Assert.assertEquals("Got unexpected ez path", dir.toString(),
      dfsAdmin1.getEncryptionZoneForPath(dirChild).getPath().toString());
  Assert.assertEquals("File contents after failover were changed",
      contents, DFSTestUtil.readFile(fs, dirFile));
}
Example 3
Source File: TestBookKeeperAsHASharedDir.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void assertCanStartHANameNodes(MiniDFSCluster cluster,
    Configuration conf, String path) throws ServiceFailedException,
    IOException, URISyntaxException, InterruptedException {
  // Now should be able to start both NNs. Pass "false" here so that we don't
  // try to waitActive on all NNs, since the second NN doesn't exist yet.
  cluster.restartNameNode(0, false);
  cluster.restartNameNode(1, true);

  // Make sure HA is working.
  cluster
      .getNameNode(0)
      .getRpcServer()
      .transitionToActive(
          new StateChangeRequestInfo(RequestSource.REQUEST_BY_USER));
  FileSystem fs = null;
  try {
    Path newPath = new Path(path);
    fs = HATestUtil.configureFailoverFs(cluster, conf);
    assertTrue(fs.mkdirs(newPath));
    HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0),
        cluster.getNameNode(1));
    assertTrue(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
        newPath.toString(), false).isDir());
  } finally {
    if (fs != null) {
      fs.close();
    }
  }
}
Example 4
Source File: TestBookKeeperAsHASharedDir.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void assertCanStartHANameNodes(MiniDFSCluster cluster,
    Configuration conf, String path) throws ServiceFailedException,
    IOException, URISyntaxException, InterruptedException {
  // Now should be able to start both NNs. Pass "false" here so that we don't
  // try to waitActive on all NNs, since the second NN doesn't exist yet.
  cluster.restartNameNode(0, false);
  cluster.restartNameNode(1, true);

  // Make sure HA is working.
  cluster
      .getNameNode(0)
      .getRpcServer()
      .transitionToActive(
          new StateChangeRequestInfo(RequestSource.REQUEST_BY_USER));
  FileSystem fs = null;
  try {
    Path newPath = new Path(path);
    fs = HATestUtil.configureFailoverFs(cluster, conf);
    assertTrue(fs.mkdirs(newPath));
    HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0),
        cluster.getNameNode(1));
    assertTrue(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
        newPath.toString(), false).isDir());
  } finally {
    if (fs != null) {
      fs.close();
    }
  }
}
Example 5
Source File: TestBootstrapStandbyWithBKJM.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * While boostrapping, in_progress transaction entries should be skipped.
 * Bootstrap usage for BKJM : "-force", "-nonInteractive", "-skipSharedEditsCheck"
 */
@Test
public void testBootstrapStandbyWithActiveNN() throws Exception {
  // make nn0 active
  cluster.transitionToActive(0);
 
  // do ops and generate in-progress edit log data
  Configuration confNN1 = cluster.getConfiguration(1);
  DistributedFileSystem dfs = (DistributedFileSystem) HATestUtil
      .configureFailoverFs(cluster, confNN1);
  for (int i = 1; i <= 10; i++) {
    dfs.mkdirs(new Path("/test" + i));
  }
  dfs.close();

  // shutdown nn1 and delete its edit log files
  cluster.shutdownNameNode(1);
  deleteEditLogIfExists(confNN1);
  cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_ENTER, true);
  cluster.getNameNodeRpc(0).saveNamespace();
  cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_LEAVE, true);

  // check without -skipSharedEditsCheck, Bootstrap should fail for BKJM
  // immediately after saveNamespace
  int rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive" },
    confNN1);
  Assert.assertEquals("Mismatches return code", 6, rc);

  // check with -skipSharedEditsCheck
  rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive",
      "-skipSharedEditsCheck" }, confNN1);
  Assert.assertEquals("Mismatches return code", 0, rc);

  // Checkpoint as fast as we can, in a tight loop.
  confNN1.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
  cluster.restartNameNode(1);
  cluster.transitionToStandby(1);
 
  NameNode nn0 = cluster.getNameNode(0);
  HATestUtil.waitForStandbyToCatchUp(nn0, cluster.getNameNode(1));
  long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0)
      .getFSImage().getMostRecentCheckpointTxId();
  HATestUtil.waitForCheckpoint(cluster, 1,
      ImmutableList.of((int) expectedCheckpointTxId));

  // Should have copied over the namespace
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of((int) expectedCheckpointTxId));
  FSImageTestUtil.assertNNFilesMatch(cluster);
}
Example 6
Source File: TestBootstrapStandbyWithBKJM.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * While boostrapping, in_progress transaction entries should be skipped.
 * Bootstrap usage for BKJM : "-force", "-nonInteractive", "-skipSharedEditsCheck"
 */
@Test
public void testBootstrapStandbyWithActiveNN() throws Exception {
  // make nn0 active
  cluster.transitionToActive(0);
 
  // do ops and generate in-progress edit log data
  Configuration confNN1 = cluster.getConfiguration(1);
  DistributedFileSystem dfs = (DistributedFileSystem) HATestUtil
      .configureFailoverFs(cluster, confNN1);
  for (int i = 1; i <= 10; i++) {
    dfs.mkdirs(new Path("/test" + i));
  }
  dfs.close();

  // shutdown nn1 and delete its edit log files
  cluster.shutdownNameNode(1);
  deleteEditLogIfExists(confNN1);
  cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_ENTER, true);
  cluster.getNameNodeRpc(0).saveNamespace();
  cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_LEAVE, true);

  // check without -skipSharedEditsCheck, Bootstrap should fail for BKJM
  // immediately after saveNamespace
  int rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive" },
    confNN1);
  Assert.assertEquals("Mismatches return code", 6, rc);

  // check with -skipSharedEditsCheck
  rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive",
      "-skipSharedEditsCheck" }, confNN1);
  Assert.assertEquals("Mismatches return code", 0, rc);

  // Checkpoint as fast as we can, in a tight loop.
  confNN1.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
  cluster.restartNameNode(1);
  cluster.transitionToStandby(1);
 
  NameNode nn0 = cluster.getNameNode(0);
  HATestUtil.waitForStandbyToCatchUp(nn0, cluster.getNameNode(1));
  long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0)
      .getFSImage().getMostRecentCheckpointTxId();
  HATestUtil.waitForCheckpoint(cluster, 1,
      ImmutableList.of((int) expectedCheckpointTxId));

  // Should have copied over the namespace
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of((int) expectedCheckpointTxId));
  FSImageTestUtil.assertNNFilesMatch(cluster);
}