Java Code Examples for org.apache.hadoop.hdfs.MiniDFSCluster#DataNodeProperties

The following examples show how to use org.apache.hadoop.hdfs.MiniDFSCluster#DataNodeProperties . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestDataNodeRollingUpgrade.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void rollbackRollingUpgrade() throws Exception {
  // Shutdown datanodes and namenodes
  // Restart the namenode with rolling upgrade rollback
  LOG.info("Starting rollback of the rolling upgrade");
  MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0);
  dnprop.setDnArgs("-rollback");
  cluster.shutdownNameNodes();
  cluster.restartNameNode("-rollingupgrade", "rollback");
  cluster.restartDataNode(dnprop);
  cluster.waitActive();
  nn = cluster.getNameNode(0);
  dn0 = cluster.getDataNodes().get(0);
  triggerHeartBeats();
  LOG.info("The cluster is active after rollback");
}
Example 2
Source File: TestDataNodeRollingUpgrade.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void rollbackRollingUpgrade() throws Exception {
  // Shutdown datanodes and namenodes
  // Restart the namenode with rolling upgrade rollback
  LOG.info("Starting rollback of the rolling upgrade");
  MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0);
  dnprop.setDnArgs("-rollback");
  cluster.shutdownNameNodes();
  cluster.restartNameNode("-rollingupgrade", "rollback");
  cluster.restartDataNode(dnprop);
  cluster.waitActive();
  nn = cluster.getNameNode(0);
  dn0 = cluster.getDataNodes().get(0);
  triggerHeartBeats();
  LOG.info("The cluster is active after rollback");
}
Example 3
Source File: TestDataNodeRollingUpgrade.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Support for layout version change with rolling upgrade was
 * added by HDFS-6800 and HDFS-6981.
 */
@Test(timeout=300000)
public void testWithLayoutChangeAndFinalize() throws Exception {
  final long seed = 0x600DF00D;
  try {
    startCluster();

    Path[] paths = new Path[3];
    File[] blockFiles = new File[3];

    // Create two files in DFS.
    for (int i = 0; i < 2; ++i) {
      paths[i] = new Path("/" + GenericTestUtils.getMethodName() + "." + i + ".dat");
      DFSTestUtil.createFile(fs, paths[i], BLOCK_SIZE, (short) 2, seed);
    }

    startRollingUpgrade();

    // Delete the first file. The DN will save its block files in trash.
    blockFiles[0] = getBlockForFile(paths[0], true);
    File trashFile0 = getTrashFileForBlock(blockFiles[0], false);
    deleteAndEnsureInTrash(paths[0], blockFiles[0], trashFile0);

    // Restart the DN with a new layout version to trigger layout upgrade.
    LOG.info("Shutting down the Datanode");
    MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0);
    DFSTestUtil.addDataNodeLayoutVersion(
        DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1,
        "Test Layout for TestDataNodeRollingUpgrade");
    LOG.info("Restarting the DataNode");
    cluster.restartDataNode(dnprop, true);
    cluster.waitActive();

    dn0 = cluster.getDataNodes().get(0);
    LOG.info("The DN has been restarted");
    assertFalse(trashFile0.exists());
    assertFalse(dn0.getStorage().getBPStorage(blockPoolId).isTrashAllowed(blockFiles[0]));

    // Ensure that the block file for the first file was moved from 'trash' to 'previous'.
    assertTrue(isBlockFileInPrevious(blockFiles[0]));
    assertFalse(isTrashRootPresent());

    // Delete the second file. Ensure that its block file is in previous.
    blockFiles[1] = getBlockForFile(paths[1], true);
    fs.delete(paths[1], false);
    assertTrue(isBlockFileInPrevious(blockFiles[1]));
    assertFalse(isTrashRootPresent());

    // Finalize and ensure that neither block file exists in trash or previous.
    finalizeRollingUpgrade();
    assertFalse(isTrashRootPresent());
    assertFalse(isBlockFileInPrevious(blockFiles[0]));
    assertFalse(isBlockFileInPrevious(blockFiles[1]));
  } finally {
    shutdownCluster();
  }
}
Example 4
Source File: TestDataNodeRollingUpgrade.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Support for layout version change with rolling upgrade was
 * added by HDFS-6800 and HDFS-6981.
 */
@Test(timeout=300000)
public void testWithLayoutChangeAndRollback() throws Exception {
  final long seed = 0x600DF00D;
  try {
    startCluster();

    Path[] paths = new Path[3];
    File[] blockFiles = new File[3];

    // Create two files in DFS.
    for (int i = 0; i < 2; ++i) {
      paths[i] = new Path("/" + GenericTestUtils.getMethodName() + "." + i + ".dat");
      DFSTestUtil.createFile(fs, paths[i], BLOCK_SIZE, (short) 1, seed);
    }

    startRollingUpgrade();

    // Delete the first file. The DN will save its block files in trash.
    blockFiles[0] = getBlockForFile(paths[0], true);
    File trashFile0 = getTrashFileForBlock(blockFiles[0], false);
    deleteAndEnsureInTrash(paths[0], blockFiles[0], trashFile0);

    // Restart the DN with a new layout version to trigger layout upgrade.
    LOG.info("Shutting down the Datanode");
    MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0);
    DFSTestUtil.addDataNodeLayoutVersion(
        DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1,
        "Test Layout for TestDataNodeRollingUpgrade");
    LOG.info("Restarting the DataNode");
    cluster.restartDataNode(dnprop, true);
    cluster.waitActive();

    dn0 = cluster.getDataNodes().get(0);
    LOG.info("The DN has been restarted");
    assertFalse(trashFile0.exists());
    assertFalse(dn0.getStorage().getBPStorage(blockPoolId).isTrashAllowed(blockFiles[0]));

    // Ensure that the block file for the first file was moved from 'trash' to 'previous'.
    assertTrue(isBlockFileInPrevious(blockFiles[0]));
    assertFalse(isTrashRootPresent());

    // Delete the second file. Ensure that its block file is in previous.
    blockFiles[1] = getBlockForFile(paths[1], true);
    fs.delete(paths[1], false);
    assertTrue(isBlockFileInPrevious(blockFiles[1]));
    assertFalse(isTrashRootPresent());

    // Create and delete a third file. Its block file should not be
    // in either trash or previous after deletion.
    paths[2] = new Path("/" + GenericTestUtils.getMethodName() + ".2.dat");
    DFSTestUtil.createFile(fs, paths[2], BLOCK_SIZE, (short) 1, seed);
    blockFiles[2] = getBlockForFile(paths[2], true);
    fs.delete(paths[2], false);
    assertFalse(isBlockFileInPrevious(blockFiles[2]));
    assertFalse(isTrashRootPresent());

    // Rollback and ensure that the first two file contents were restored.
    rollbackRollingUpgrade();
    for (int i = 0; i < 2; ++i) {
      byte[] actual = DFSTestUtil.readFileBuffer(fs, paths[i]);
      byte[] calculated = DFSTestUtil.calculateFileContentsFromSeed(seed, BLOCK_SIZE);
      assertArrayEquals(actual, calculated);
    }

    // And none of the block files must be in previous or trash.
    assertFalse(isTrashRootPresent());
    for (int i = 0; i < 3; ++i) {
      assertFalse(isBlockFileInPrevious(blockFiles[i]));
    }
  } finally {
    shutdownCluster();
  }
}
Example 5
Source File: TestDataNodeRollingUpgrade.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Support for layout version change with rolling upgrade was
 * added by HDFS-6800 and HDFS-6981.
 */
@Test(timeout=300000)
public void testWithLayoutChangeAndFinalize() throws Exception {
  final long seed = 0x600DF00D;
  try {
    startCluster();

    Path[] paths = new Path[3];
    File[] blockFiles = new File[3];

    // Create two files in DFS.
    for (int i = 0; i < 2; ++i) {
      paths[i] = new Path("/" + GenericTestUtils.getMethodName() + "." + i + ".dat");
      DFSTestUtil.createFile(fs, paths[i], BLOCK_SIZE, (short) 2, seed);
    }

    startRollingUpgrade();

    // Delete the first file. The DN will save its block files in trash.
    blockFiles[0] = getBlockForFile(paths[0], true);
    File trashFile0 = getTrashFileForBlock(blockFiles[0], false);
    deleteAndEnsureInTrash(paths[0], blockFiles[0], trashFile0);

    // Restart the DN with a new layout version to trigger layout upgrade.
    LOG.info("Shutting down the Datanode");
    MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0);
    DFSTestUtil.addDataNodeLayoutVersion(
        DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1,
        "Test Layout for TestDataNodeRollingUpgrade");
    LOG.info("Restarting the DataNode");
    cluster.restartDataNode(dnprop, true);
    cluster.waitActive();

    dn0 = cluster.getDataNodes().get(0);
    LOG.info("The DN has been restarted");
    assertFalse(trashFile0.exists());
    assertFalse(dn0.getStorage().getBPStorage(blockPoolId).isTrashAllowed(blockFiles[0]));

    // Ensure that the block file for the first file was moved from 'trash' to 'previous'.
    assertTrue(isBlockFileInPrevious(blockFiles[0]));
    assertFalse(isTrashRootPresent());

    // Delete the second file. Ensure that its block file is in previous.
    blockFiles[1] = getBlockForFile(paths[1], true);
    fs.delete(paths[1], false);
    assertTrue(isBlockFileInPrevious(blockFiles[1]));
    assertFalse(isTrashRootPresent());

    // Finalize and ensure that neither block file exists in trash or previous.
    finalizeRollingUpgrade();
    assertFalse(isTrashRootPresent());
    assertFalse(isBlockFileInPrevious(blockFiles[0]));
    assertFalse(isBlockFileInPrevious(blockFiles[1]));
  } finally {
    shutdownCluster();
  }
}
Example 6
Source File: TestDataNodeRollingUpgrade.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Support for layout version change with rolling upgrade was
 * added by HDFS-6800 and HDFS-6981.
 */
@Test(timeout=300000)
public void testWithLayoutChangeAndRollback() throws Exception {
  final long seed = 0x600DF00D;
  try {
    startCluster();

    Path[] paths = new Path[3];
    File[] blockFiles = new File[3];

    // Create two files in DFS.
    for (int i = 0; i < 2; ++i) {
      paths[i] = new Path("/" + GenericTestUtils.getMethodName() + "." + i + ".dat");
      DFSTestUtil.createFile(fs, paths[i], BLOCK_SIZE, (short) 1, seed);
    }

    startRollingUpgrade();

    // Delete the first file. The DN will save its block files in trash.
    blockFiles[0] = getBlockForFile(paths[0], true);
    File trashFile0 = getTrashFileForBlock(blockFiles[0], false);
    deleteAndEnsureInTrash(paths[0], blockFiles[0], trashFile0);

    // Restart the DN with a new layout version to trigger layout upgrade.
    LOG.info("Shutting down the Datanode");
    MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0);
    DFSTestUtil.addDataNodeLayoutVersion(
        DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1,
        "Test Layout for TestDataNodeRollingUpgrade");
    LOG.info("Restarting the DataNode");
    cluster.restartDataNode(dnprop, true);
    cluster.waitActive();

    dn0 = cluster.getDataNodes().get(0);
    LOG.info("The DN has been restarted");
    assertFalse(trashFile0.exists());
    assertFalse(dn0.getStorage().getBPStorage(blockPoolId).isTrashAllowed(blockFiles[0]));

    // Ensure that the block file for the first file was moved from 'trash' to 'previous'.
    assertTrue(isBlockFileInPrevious(blockFiles[0]));
    assertFalse(isTrashRootPresent());

    // Delete the second file. Ensure that its block file is in previous.
    blockFiles[1] = getBlockForFile(paths[1], true);
    fs.delete(paths[1], false);
    assertTrue(isBlockFileInPrevious(blockFiles[1]));
    assertFalse(isTrashRootPresent());

    // Create and delete a third file. Its block file should not be
    // in either trash or previous after deletion.
    paths[2] = new Path("/" + GenericTestUtils.getMethodName() + ".2.dat");
    DFSTestUtil.createFile(fs, paths[2], BLOCK_SIZE, (short) 1, seed);
    blockFiles[2] = getBlockForFile(paths[2], true);
    fs.delete(paths[2], false);
    assertFalse(isBlockFileInPrevious(blockFiles[2]));
    assertFalse(isTrashRootPresent());

    // Rollback and ensure that the first two file contents were restored.
    rollbackRollingUpgrade();
    for (int i = 0; i < 2; ++i) {
      byte[] actual = DFSTestUtil.readFileBuffer(fs, paths[i]);
      byte[] calculated = DFSTestUtil.calculateFileContentsFromSeed(seed, BLOCK_SIZE);
      assertArrayEquals(actual, calculated);
    }

    // And none of the block files must be in previous or trash.
    assertFalse(isTrashRootPresent());
    for (int i = 0; i < 3; ++i) {
      assertFalse(isBlockFileInPrevious(blockFiles[i]));
    }
  } finally {
    shutdownCluster();
  }
}