org.apache.hadoop.conf.ReconfigurationException Java Examples

The following examples show how to use org.apache.hadoop.conf.ReconfigurationException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDataNodeVolumeFailureReporting.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Reconfigure a DataNode by setting a new list of volumes.
 *
 * @param dn DataNode to reconfigure
 * @param newVols new volumes to configure
 * @throws Exception if there is any failure
 */
private static void reconfigureDataNode(DataNode dn, File... newVols)
    throws Exception {
  StringBuilder dnNewDataDirs = new StringBuilder();
  for (File newVol: newVols) {
    if (dnNewDataDirs.length() > 0) {
      dnNewDataDirs.append(',');
    }
    dnNewDataDirs.append(newVol.getAbsolutePath());
  }
  try {
    dn.reconfigurePropertyImpl(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
        dnNewDataDirs.toString());
  } catch (ReconfigurationException e) {
    // This can be thrown if reconfiguration tries to use a failed volume.
    // We need to swallow the exception, because some of our tests want to
    // cover this case.
    LOG.warn("Could not reconfigure DataNode.", e);
  }
}
 
Example #2
Source File: DataNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void reconfigurePropertyImpl(String property, String newVal) 
  throws ReconfigurationException {
  if (property.equals("dfs.data.dir")) {
    try {
      LOG.info("Reconfigure " + property + " to " + newVal);
      this.refreshVolumes(newVal);
    } catch (Exception e) {
      throw new ReconfigurationException(property, 
      newVal, getConf().get(property), e);
    }
  } else {
    throw new ReconfigurationException(property, newVal,
                                      getConf().get(property));
  }
}
 
Example #3
Source File: FairScheduler.java    From RDFS with Apache License 2.0 6 votes vote down vote up
@Override
public final String reconfigureProperty(String property, String newVal)
  throws ReconfigurationException {
  if (isPropertyReconfigurable(property)) {
    String oldVal;
    synchronized(getConf()) {
      oldVal = getConf().get(property);
      reconfigurePropertyImpl(property, newVal);
      if (newVal != null) {
        getConf().set(property, newVal);
      } else {
        getConf().unset(property);
      }
    }
    return oldVal;
  } else {
    throw new ReconfigurationException(property, newVal,
                                           getConf().get(property));
  }
}
 
Example #4
Source File: JobTrackerReconfigurable.java    From RDFS with Apache License 2.0 6 votes vote down vote up
@Override
protected void reconfigurePropertyImpl(String property, String newVal)
  throws ReconfigurationException {
  try {
    if (property.equals(MAX_TRACKER_BLACKLISTS_PROPERTY)) {
      LOG.info("changing maxTrackerBlacklists to " + newVal +
          " from " + initialJobRefreshTimeoutMs);
      maxTrackerBlacklists = Integer.parseInt(newVal);
    } else if (property.equals(MAX_UNIQUE_COUNTER_NAMES)) {
      LOG.info("changing maxUniqueCounterNames to " + newVal +
          " from " + initialJobRefreshTimeoutMs);
      maxUniqueCounterNames = Integer.parseInt(newVal);
    } else if (property.equals(INITIAL_JOB_REFRESH_TIMEOUT_MS_PROPERTY)) {
      LOG.info("changing initialJobRefreshTimeoutMs to " + newVal +
          " from " + initialJobRefreshTimeoutMs);
      initialJobRefreshTimeoutMs = Long.parseLong(newVal);
    }
  } catch (NumberFormatException e) {
    LOG.warn("reconfigurePropertyImpl: Invalid property " + property +
        " or newVal " + newVal);
  }
}
 
Example #5
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test(timeout=60000)
public void testReplicatingAfterRemoveVolume()
    throws InterruptedException, TimeoutException, IOException,
    ReconfigurationException {
  startDFSCluster(1, 2);

  final FileSystem fs = cluster.getFileSystem();
  final short replFactor = 2;
  Path testFile = new Path("/test");
  createFile(testFile, 4, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
      new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));

  triggerDeleteReport(dn);

  waitReplication(fs, testFile, 1, 1);
  DFSTestUtil.waitReplication(fs, testFile, replFactor);
}
 
Example #6
Source File: TestDataNodeVolumeFailureReporting.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Reconfigure a DataNode by setting a new list of volumes.
 *
 * @param dn DataNode to reconfigure
 * @param newVols new volumes to configure
 * @throws Exception if there is any failure
 */
private static void reconfigureDataNode(DataNode dn, File... newVols)
    throws Exception {
  StringBuilder dnNewDataDirs = new StringBuilder();
  for (File newVol: newVols) {
    if (dnNewDataDirs.length() > 0) {
      dnNewDataDirs.append(',');
    }
    dnNewDataDirs.append(newVol.getAbsolutePath());
  }
  try {
    dn.reconfigurePropertyImpl(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
        dnNewDataDirs.toString());
  } catch (ReconfigurationException e) {
    // This can be thrown if reconfiguration tries to use a failed volume.
    // We need to swallow the exception, because some of our tests want to
    // cover this case.
    LOG.warn("Could not reconfigure DataNode.", e);
  }
}
 
Example #7
Source File: DataNode.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void reconfigurePropertyImpl(String property, String newVal)
    throws ReconfigurationException {
  if (property.equals(DFS_DATANODE_DATA_DIR_KEY)) {
    try {
      LOG.info("Reconfiguring " + property + " to " + newVal);
      this.refreshVolumes(newVal);
    } catch (IOException e) {
      throw new ReconfigurationException(property, newVal,
          getConf().get(property), e);
    }
  } else {
    throw new ReconfigurationException(
        property, newVal, getConf().get(property));
  }
}
 
Example #8
Source File: DataNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void reconfigurePropertyImpl(String property, String newVal)
    throws ReconfigurationException {
  if (property.equals(DFS_DATANODE_DATA_DIR_KEY)) {
    try {
      LOG.info("Reconfiguring " + property + " to " + newVal);
      this.refreshVolumes(newVal);
    } catch (IOException e) {
      throw new ReconfigurationException(property, newVal,
          getConf().get(property), e);
    }
  } else {
    throw new ReconfigurationException(
        property, newVal, getConf().get(property));
  }
}
 
Example #9
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout=60000)
public void testReplicatingAfterRemoveVolume()
    throws InterruptedException, TimeoutException, IOException,
    ReconfigurationException {
  startDFSCluster(1, 2);

  final FileSystem fs = cluster.getFileSystem();
  final short replFactor = 2;
  Path testFile = new Path("/test");
  createFile(testFile, 4, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
      new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));

  triggerDeleteReport(dn);

  waitReplication(fs, testFile, 1, 1);
  DFSTestUtil.waitReplication(fs, testFile, replFactor);
}
 
Example #10
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAddVolumesToFederationNN()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  // Starts a Cluster with 2 NameNode and 3 DataNodes. Each DataNode has 2
  // volumes.
  final int numNameNodes = 2;
  final int numDataNodes = 1;
  startDFSCluster(numNameNodes, numDataNodes);
  Path testFile = new Path("/test");
  // Create a file on the first namespace with 4 blocks.
  createFile(0, testFile, 4);
  // Create a file on the second namespace with 4 blocks.
  createFile(1, testFile, 4);

  // Add 2 volumes to the first DataNode.
  final int numNewVolumes = 2;
  addVolumes(numNewVolumes);

  // Append to the file on the first namespace.
  DFSTestUtil.appendFile(cluster.getFileSystem(0), testFile, BLOCK_SIZE * 8);

  List<List<Integer>> actualNumBlocks = getNumBlocksReport(0);
  assertEquals(cluster.getDataNodes().size(), actualNumBlocks.size());
  List<Integer> blocksOnFirstDN = actualNumBlocks.get(0);
  Collections.sort(blocksOnFirstDN);
  assertEquals(Arrays.asList(2, 2, 4, 4), blocksOnFirstDN);

  // Verify the second namespace also has the new volumes and they are empty.
  actualNumBlocks = getNumBlocksReport(1);
  assertEquals(4, actualNumBlocks.get(0).size());
  assertEquals(numNewVolumes,
      Collections.frequency(actualNumBlocks.get(0), 0));
}
 
Example #11
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Test adding one volume on a running MiniDFSCluster with only one NameNode.
 */
@Test(timeout=60000)
public void testAddOneNewVolume()
    throws IOException, ReconfigurationException,
    InterruptedException, TimeoutException {
  startDFSCluster(1, 1);
  String bpid = cluster.getNamesystem().getBlockPoolId();
  final int numBlocks = 10;

  addVolumes(1);

  Path testFile = new Path("/test");
  createFile(testFile, numBlocks);

  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals(1, blockReports.size());  // 1 DataNode
  assertEquals(3, blockReports.get(0).size());  // 3 volumes

  // FSVolumeList uses Round-Robin block chooser by default. Thus the new
  // blocks should be evenly located in all volumes.
  int minNumBlocks = Integer.MAX_VALUE;
  int maxNumBlocks = Integer.MIN_VALUE;
  for (BlockListAsLongs blockList : blockReports.get(0).values()) {
    minNumBlocks = Math.min(minNumBlocks, blockList.getNumberOfBlocks());
    maxNumBlocks = Math.max(maxNumBlocks, blockList.getNumberOfBlocks());
  }
  assertTrue(Math.abs(maxNumBlocks - maxNumBlocks) <= 1);
  verifyFileLength(cluster.getFileSystem(), testFile, numBlocks);
}
 
Example #12
Source File: TaskTracker.java    From RDFS with Apache License 2.0 5 votes vote down vote up
@Override
protected void reconfigurePropertyImpl(String property, String newVal)
    throws ReconfigurationException {
  if (property.equals(TT_FAST_FETCH)) {
    this.fastFetch = Boolean.valueOf(newVal);
  } else if (property.equals(TT_OUTOFBAND_HEARBEAT)) {
    this.oobHeartbeatOnTaskCompletion = Boolean.valueOf(newVal);
  } else if (property.equals(TT_PROFILE_ALL_TASKS)) {
    this.profileAllTasks = Boolean.valueOf(newVal);
  }
}
 
Example #13
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAddVolumesDuringWrite()
    throws IOException, InterruptedException, TimeoutException,
    ReconfigurationException {
  startDFSCluster(1, 1);
  String bpid = cluster.getNamesystem().getBlockPoolId();
  Path testFile = new Path("/test");
  createFile(testFile, 4);  // Each volume has 2 blocks.

  addVolumes(2);

  // Continue to write the same file, thus the new volumes will have blocks.
  DFSTestUtil.appendFile(cluster.getFileSystem(), testFile, BLOCK_SIZE * 8);
  verifyFileLength(cluster.getFileSystem(), testFile, 8 + 4);
  // After appending data, there should be [2, 2, 4, 4] blocks in each volume
  // respectively.
  List<Integer> expectedNumBlocks = Arrays.asList(2, 2, 4, 4);

  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals(1, blockReports.size());  // 1 DataNode
  assertEquals(4, blockReports.get(0).size());  // 4 volumes
  Map<DatanodeStorage, BlockListAsLongs> dnReport =
      blockReports.get(0);
  List<Integer> actualNumBlocks = new ArrayList<Integer>();
  for (BlockListAsLongs blockList : dnReport.values()) {
    actualNumBlocks.add(blockList.getNumberOfBlocks());
  }
  Collections.sort(actualNumBlocks);
  assertEquals(expectedNumBlocks, actualNumBlocks);
}
 
Example #14
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Test that a full block report is sent after hot swapping volumes */
@Test(timeout=100000)
public void testFullBlockReportAfterRemovingVolumes()
    throws IOException, ReconfigurationException {

  Configuration conf = new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);

  // Similar to TestTriggerBlockReport, set a really long value for
  // dfs.heartbeat.interval, so that incremental block reports and heartbeats
  // won't be sent during this test unless they're triggered
  // manually.
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10800000L);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1080L);

  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  cluster.waitActive();

  final DataNode dn = cluster.getDataNodes().get(0);
  DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, cluster.getNameNode());

  // Remove a data dir from datanode
  File dataDirToKeep = new File(cluster.getDataDirectory(), "data1");
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, dataDirToKeep.toString());

  // We should get 1 full report
  Mockito.verify(spy, timeout(60000).times(1)).blockReport(
      any(DatanodeRegistration.class),
      anyString(),
      any(StorageBlockReport[].class),
      any(BlockReportContext.class));
}
 
Example #15
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Verify that {@link DataNode#checkDiskErrors()} removes all metadata in
 * DataNode upon a volume failure. Thus we can run reconfig on the same
 * configuration to reload the new volume on the same directory as the failed one.
 */
@Test(timeout=60000)
public void testDirectlyReloadAfterCheckDiskError()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  startDFSCluster(1, 2);
  createFile(new Path("/test"), 32, (short)2);

  DataNode dn = cluster.getDataNodes().get(0);
  final String oldDataDir = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY);
  File dirToFail = new File(cluster.getDataDirectory(), "data1");

  FsVolumeImpl failedVolume = getVolume(dn, dirToFail);
  assertTrue("No FsVolume was found for " + dirToFail,
      failedVolume != null);
  long used = failedVolume.getDfsUsed();

  DataNodeTestUtils.injectDataDirFailure(dirToFail);
  // Call and wait DataNode to detect disk failure.
  long lastDiskErrorCheck = dn.getLastDiskErrorCheck();
  dn.checkDiskErrorAsync();
  while (dn.getLastDiskErrorCheck() == lastDiskErrorCheck) {
    Thread.sleep(100);
  }

  createFile(new Path("/test1"), 32, (short)2);
  assertEquals(used, failedVolume.getDfsUsed());

  DataNodeTestUtils.restoreDataDirFromFailure(dirToFail);
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);

  createFile(new Path("/test2"), 32, (short)2);
  FsVolumeImpl restoredVolume = getVolume(dn, dirToFail);
  assertTrue(restoredVolume != null);
  assertTrue(restoredVolume != failedVolume);
  // More data has been written to this volume.
  assertTrue(restoredVolume.getDfsUsed() > used);
}
 
Example #16
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAddBackRemovedVolume()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  startDFSCluster(1, 2);
  // Create some data on every volume.
  createFile(new Path("/test"), 32);

  DataNode dn = cluster.getDataNodes().get(0);
  Configuration conf = dn.getConf();
  String oldDataDir = conf.get(DFS_DATANODE_DATA_DIR_KEY);
  String keepDataDir = oldDataDir.split(",")[0];
  String removeDataDir = oldDataDir.split(",")[1];

  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, keepDataDir);
  for (int i = 0; i < cluster.getNumNameNodes(); i++) {
    String bpid = cluster.getNamesystem(i).getBlockPoolId();
    BlockPoolSliceStorage bpsStorage =
        dn.getStorage().getBPStorage(bpid);
    // Make sure that there is no block pool level storage under removeDataDir.
    for (int j = 0; j < bpsStorage.getNumStorageDirs(); j++) {
      Storage.StorageDirectory sd = bpsStorage.getStorageDir(j);
      assertFalse(sd.getRoot().getAbsolutePath().startsWith(
          new File(removeDataDir).getAbsolutePath()
      ));
    }
    assertEquals(dn.getStorage().getBPStorage(bpid).getNumStorageDirs(), 1);
  }

  // Bring the removed directory back. It only successes if all metadata about
  // this directory were removed from the previous step.
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);
}
 
Example #17
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=180000)
public void testRemoveVolumeBeingWritten()
    throws InterruptedException, TimeoutException, ReconfigurationException,
    IOException, BrokenBarrierException {
  // test against removing volumes on the different DataNode on the pipeline.
  for (int i = 0; i < 3; i++) {
    testRemoveVolumeBeingWrittenForDatanode(i);
  }
}
 
Example #18
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testRemoveOneVolume()
    throws ReconfigurationException, InterruptedException, TimeoutException,
    IOException {
  startDFSCluster(1, 1);
  final short replFactor = 1;
  Path testFile = new Path("/test");
  createFile(testFile, 10, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
    new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));
  dn.scheduleAllBlockReport(0);

  try {
    DFSTestUtil.readFile(cluster.getFileSystem(), testFile);
    fail("Expect to throw BlockMissingException.");
  } catch (BlockMissingException e) {
    GenericTestUtils.assertExceptionContains("Could not obtain block", e);
  }

  Path newFile = new Path("/newFile");
  createFile(newFile, 6);

  String bpid = cluster.getNamesystem().getBlockPoolId();
  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals((int)replFactor, blockReports.size());

  BlockListAsLongs blocksForVolume1 =
      blockReports.get(0).values().iterator().next();
  // The first volume has half of the testFile and full of newFile.
  assertEquals(10 / 2 + 6, blocksForVolume1.getNumberOfBlocks());
}
 
Example #19
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAddVolumesToFederationNN()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  // Starts a Cluster with 2 NameNode and 3 DataNodes. Each DataNode has 2
  // volumes.
  final int numNameNodes = 2;
  final int numDataNodes = 1;
  startDFSCluster(numNameNodes, numDataNodes);
  Path testFile = new Path("/test");
  // Create a file on the first namespace with 4 blocks.
  createFile(0, testFile, 4);
  // Create a file on the second namespace with 4 blocks.
  createFile(1, testFile, 4);

  // Add 2 volumes to the first DataNode.
  final int numNewVolumes = 2;
  addVolumes(numNewVolumes);

  // Append to the file on the first namespace.
  DFSTestUtil.appendFile(cluster.getFileSystem(0), testFile, BLOCK_SIZE * 8);

  List<List<Integer>> actualNumBlocks = getNumBlocksReport(0);
  assertEquals(cluster.getDataNodes().size(), actualNumBlocks.size());
  List<Integer> blocksOnFirstDN = actualNumBlocks.get(0);
  Collections.sort(blocksOnFirstDN);
  assertEquals(Arrays.asList(2, 2, 4, 4), blocksOnFirstDN);

  // Verify the second namespace also has the new volumes and they are empty.
  actualNumBlocks = getNumBlocksReport(1);
  assertEquals(4, actualNumBlocks.get(0).size());
  assertEquals(numNewVolumes,
      Collections.frequency(actualNumBlocks.get(0), 0));
}
 
Example #20
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAddVolumesDuringWrite()
    throws IOException, InterruptedException, TimeoutException,
    ReconfigurationException {
  startDFSCluster(1, 1);
  String bpid = cluster.getNamesystem().getBlockPoolId();
  Path testFile = new Path("/test");
  createFile(testFile, 4);  // Each volume has 2 blocks.

  addVolumes(2);

  // Continue to write the same file, thus the new volumes will have blocks.
  DFSTestUtil.appendFile(cluster.getFileSystem(), testFile, BLOCK_SIZE * 8);
  verifyFileLength(cluster.getFileSystem(), testFile, 8 + 4);
  // After appending data, there should be [2, 2, 4, 4] blocks in each volume
  // respectively.
  List<Integer> expectedNumBlocks = Arrays.asList(2, 2, 4, 4);

  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals(1, blockReports.size());  // 1 DataNode
  assertEquals(4, blockReports.get(0).size());  // 4 volumes
  Map<DatanodeStorage, BlockListAsLongs> dnReport =
      blockReports.get(0);
  List<Integer> actualNumBlocks = new ArrayList<Integer>();
  for (BlockListAsLongs blockList : dnReport.values()) {
    actualNumBlocks.add(blockList.getNumberOfBlocks());
  }
  Collections.sort(actualNumBlocks);
  assertEquals(expectedNumBlocks, actualNumBlocks);
}
 
Example #21
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Test adding one volume on a running MiniDFSCluster with only one NameNode.
 */
@Test(timeout=60000)
public void testAddOneNewVolume()
    throws IOException, ReconfigurationException,
    InterruptedException, TimeoutException {
  startDFSCluster(1, 1);
  String bpid = cluster.getNamesystem().getBlockPoolId();
  final int numBlocks = 10;

  addVolumes(1);

  Path testFile = new Path("/test");
  createFile(testFile, numBlocks);

  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals(1, blockReports.size());  // 1 DataNode
  assertEquals(3, blockReports.get(0).size());  // 3 volumes

  // FSVolumeList uses Round-Robin block chooser by default. Thus the new
  // blocks should be evenly located in all volumes.
  int minNumBlocks = Integer.MAX_VALUE;
  int maxNumBlocks = Integer.MIN_VALUE;
  for (BlockListAsLongs blockList : blockReports.get(0).values()) {
    minNumBlocks = Math.min(minNumBlocks, blockList.getNumberOfBlocks());
    maxNumBlocks = Math.max(maxNumBlocks, blockList.getNumberOfBlocks());
  }
  assertTrue(Math.abs(maxNumBlocks - maxNumBlocks) <= 1);
  verifyFileLength(cluster.getFileSystem(), testFile, numBlocks);
}
 
Example #22
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testRemoveOneVolume()
    throws ReconfigurationException, InterruptedException, TimeoutException,
    IOException {
  startDFSCluster(1, 1);
  final short replFactor = 1;
  Path testFile = new Path("/test");
  createFile(testFile, 10, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
    new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));
  dn.scheduleAllBlockReport(0);

  try {
    DFSTestUtil.readFile(cluster.getFileSystem(), testFile);
    fail("Expect to throw BlockMissingException.");
  } catch (BlockMissingException e) {
    GenericTestUtils.assertExceptionContains("Could not obtain block", e);
  }

  Path newFile = new Path("/newFile");
  createFile(newFile, 6);

  String bpid = cluster.getNamesystem().getBlockPoolId();
  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals((int)replFactor, blockReports.size());

  BlockListAsLongs blocksForVolume1 =
      blockReports.get(0).values().iterator().next();
  // The first volume has half of the testFile and full of newFile.
  assertEquals(10 / 2 + 6, blocksForVolume1.getNumberOfBlocks());
}
 
Example #23
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Test that a full block report is sent after hot swapping volumes */
@Test(timeout=100000)
public void testFullBlockReportAfterRemovingVolumes()
    throws IOException, ReconfigurationException {

  Configuration conf = new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);

  // Similar to TestTriggerBlockReport, set a really long value for
  // dfs.heartbeat.interval, so that incremental block reports and heartbeats
  // won't be sent during this test unless they're triggered
  // manually.
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10800000L);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1080L);

  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  cluster.waitActive();

  final DataNode dn = cluster.getDataNodes().get(0);
  DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, cluster.getNameNode());

  // Remove a data dir from datanode
  File dataDirToKeep = new File(cluster.getDataDirectory(), "data1");
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, dataDirToKeep.toString());

  // We should get 1 full report
  Mockito.verify(spy, timeout(60000).times(1)).blockReport(
      any(DatanodeRegistration.class),
      anyString(),
      any(StorageBlockReport[].class),
      any(BlockReportContext.class));
}
 
Example #24
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Verify that {@link DataNode#checkDiskErrors()} removes all metadata in
 * DataNode upon a volume failure. Thus we can run reconfig on the same
 * configuration to reload the new volume on the same directory as the failed one.
 */
@Test(timeout=60000)
public void testDirectlyReloadAfterCheckDiskError()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  startDFSCluster(1, 2);
  createFile(new Path("/test"), 32, (short)2);

  DataNode dn = cluster.getDataNodes().get(0);
  final String oldDataDir = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY);
  File dirToFail = new File(cluster.getDataDirectory(), "data1");

  FsVolumeImpl failedVolume = getVolume(dn, dirToFail);
  assertTrue("No FsVolume was found for " + dirToFail,
      failedVolume != null);
  long used = failedVolume.getDfsUsed();

  DataNodeTestUtils.injectDataDirFailure(dirToFail);
  // Call and wait DataNode to detect disk failure.
  long lastDiskErrorCheck = dn.getLastDiskErrorCheck();
  dn.checkDiskErrorAsync();
  while (dn.getLastDiskErrorCheck() == lastDiskErrorCheck) {
    Thread.sleep(100);
  }

  createFile(new Path("/test1"), 32, (short)2);
  assertEquals(used, failedVolume.getDfsUsed());

  DataNodeTestUtils.restoreDataDirFromFailure(dirToFail);
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);

  createFile(new Path("/test2"), 32, (short)2);
  FsVolumeImpl restoredVolume = getVolume(dn, dirToFail);
  assertTrue(restoredVolume != null);
  assertTrue(restoredVolume != failedVolume);
  // More data has been written to this volume.
  assertTrue(restoredVolume.getDfsUsed() > used);
}
 
Example #25
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAddBackRemovedVolume()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  startDFSCluster(1, 2);
  // Create some data on every volume.
  createFile(new Path("/test"), 32);

  DataNode dn = cluster.getDataNodes().get(0);
  Configuration conf = dn.getConf();
  String oldDataDir = conf.get(DFS_DATANODE_DATA_DIR_KEY);
  String keepDataDir = oldDataDir.split(",")[0];
  String removeDataDir = oldDataDir.split(",")[1];

  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, keepDataDir);
  for (int i = 0; i < cluster.getNumNameNodes(); i++) {
    String bpid = cluster.getNamesystem(i).getBlockPoolId();
    BlockPoolSliceStorage bpsStorage =
        dn.getStorage().getBPStorage(bpid);
    // Make sure that there is no block pool level storage under removeDataDir.
    for (int j = 0; j < bpsStorage.getNumStorageDirs(); j++) {
      Storage.StorageDirectory sd = bpsStorage.getStorageDir(j);
      assertFalse(sd.getRoot().getAbsolutePath().startsWith(
          new File(removeDataDir).getAbsolutePath()
      ));
    }
    assertEquals(dn.getStorage().getBPStorage(bpid).getNumStorageDirs(), 1);
  }

  // Bring the removed directory back. It only successes if all metadata about
  // this directory were removed from the previous step.
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);
}
 
Example #26
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=180000)
public void testRemoveVolumeBeingWritten()
    throws InterruptedException, TimeoutException, ReconfigurationException,
    IOException, BrokenBarrierException {
  // test against removing volumes on the different DataNode on the pipeline.
  for (int i = 0; i < 3; i++) {
    testRemoveVolumeBeingWrittenForDatanode(i);
  }
}
 
Example #27
Source File: TestDataNodeHotSwapVolumes.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testAddVolumeFailures() throws IOException {
  startDFSCluster(1, 1);
  final String dataDir = cluster.getDataDirectory();

  DataNode dn = cluster.getDataNodes().get(0);
  List<String> newDirs = Lists.newArrayList();
  final int NUM_NEW_DIRS = 4;
  for (int i = 0; i < NUM_NEW_DIRS; i++) {
    File newVolume = new File(dataDir, "new_vol" + i);
    newDirs.add(newVolume.toString());
    if (i % 2 == 0) {
      // Make addVolume() fail.
      newVolume.createNewFile();
    }
  }

  String newValue = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY) + "," +
      Joiner.on(",").join(newDirs);
  try {
    dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, newValue);
    fail("Expect to throw IOException.");
  } catch (ReconfigurationException e) {
    String errorMessage = e.getCause().getMessage();
    String messages[] = errorMessage.split("\\r?\\n");
    assertEquals(2, messages.length);
    assertThat(messages[0], containsString("new_vol0"));
    assertThat(messages[1], containsString("new_vol2"));
  }

  // Make sure that vol0 and vol2's metadata are not left in memory.
  FsDatasetSpi<?> dataset = dn.getFSDataset();
  for (FsVolumeSpi volume : dataset.getVolumes()) {
    assertThat(volume.getBasePath(), is(not(anyOf(
        is(newDirs.get(0)), is(newDirs.get(2))))));
  }
  DataStorage storage = dn.getStorage();
  for (int i = 0; i < storage.getNumStorageDirs(); i++) {
    Storage.StorageDirectory sd = storage.getStorageDir(i);
    assertThat(sd.getRoot().toString(),
        is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
  }

  // The newly effective conf does not have vol0 and vol2.
  String[] effectiveVolumes =
      dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY).split(",");
  assertEquals(4, effectiveVolumes.length);
  for (String ev : effectiveVolumes) {
    assertThat(StorageLocation.parse(ev).getFile().getCanonicalPath(),
        is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
  }
}
 
Example #28
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testAddVolumeFailures() throws IOException {
  startDFSCluster(1, 1);
  final String dataDir = cluster.getDataDirectory();

  DataNode dn = cluster.getDataNodes().get(0);
  List<String> newDirs = Lists.newArrayList();
  final int NUM_NEW_DIRS = 4;
  for (int i = 0; i < NUM_NEW_DIRS; i++) {
    File newVolume = new File(dataDir, "new_vol" + i);
    newDirs.add(newVolume.toString());
    if (i % 2 == 0) {
      // Make addVolume() fail.
      newVolume.createNewFile();
    }
  }

  String newValue = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY) + "," +
      Joiner.on(",").join(newDirs);
  try {
    dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, newValue);
    fail("Expect to throw IOException.");
  } catch (ReconfigurationException e) {
    String errorMessage = e.getCause().getMessage();
    String messages[] = errorMessage.split("\\r?\\n");
    assertEquals(2, messages.length);
    assertThat(messages[0], containsString("new_vol0"));
    assertThat(messages[1], containsString("new_vol2"));
  }

  // Make sure that vol0 and vol2's metadata are not left in memory.
  FsDatasetSpi<?> dataset = dn.getFSDataset();
  for (FsVolumeSpi volume : dataset.getVolumes()) {
    assertThat(volume.getBasePath(), is(not(anyOf(
        is(newDirs.get(0)), is(newDirs.get(2))))));
  }
  DataStorage storage = dn.getStorage();
  for (int i = 0; i < storage.getNumStorageDirs(); i++) {
    Storage.StorageDirectory sd = storage.getStorageDir(i);
    assertThat(sd.getRoot().toString(),
        is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
  }

  // The newly effective conf does not have vol0 and vol2.
  String[] effectiveVolumes =
      dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY).split(",");
  assertEquals(4, effectiveVolumes.length);
  for (String ev : effectiveVolumes) {
    assertThat(StorageLocation.parse(ev).getFile().getCanonicalPath(),
        is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
  }
}
 
Example #29
Source File: TestDataNodeHotSwapVolumes.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Test the case that remove a data volume on a particular DataNode when the
 * volume is actively being written.
 * @param dataNodeIdx the index of the DataNode to remove a volume.
 */
private void testRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
    throws IOException, ReconfigurationException, TimeoutException,
    InterruptedException, BrokenBarrierException {
  // Starts DFS cluster with 3 DataNodes to form a pipeline.
  startDFSCluster(1, 3);

  final short REPLICATION = 3;
  final DataNode dn = cluster.getDataNodes().get(dataNodeIdx);
  final FileSystem fs = cluster.getFileSystem();
  final Path testFile = new Path("/test");
  final long lastTimeDiskErrorCheck = dn.getLastDiskErrorCheck();

  FSDataOutputStream out = fs.create(testFile, REPLICATION);

  Random rb = new Random(0);
  byte[] writeBuf = new byte[BLOCK_SIZE / 2];  // half of the block.
  rb.nextBytes(writeBuf);
  out.write(writeBuf);
  out.hflush();

  // Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
  // BlockReceiver releases volume reference before finalizeBlock(), the blocks
  // on the volume will be removed, and finalizeBlock() throws IOE.
  final FsDatasetSpi<? extends FsVolumeSpi> data = dn.data;
  dn.data = Mockito.spy(data);
  doAnswer(new Answer<Object>() {
        public Object answer(InvocationOnMock invocation)
            throws IOException, InterruptedException {
          Thread.sleep(1000);
          // Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
          // the block is not removed, since the volume reference should not
          // be released at this point.
          data.finalizeBlock((ExtendedBlock) invocation.getArguments()[0]);
          return null;
        }
      }).when(dn.data).finalizeBlock(any(ExtendedBlock.class));

  final CyclicBarrier barrier = new CyclicBarrier(2);

  List<String> oldDirs = getDataDirs(dn);
  final String newDirs = oldDirs.get(1);  // Remove the first volume.
  final List<Exception> exceptions = new ArrayList<>();
  Thread reconfigThread = new Thread() {
    public void run() {
      try {
        barrier.await();
        dn.reconfigurePropertyImpl(
            DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
      } catch (ReconfigurationException |
          InterruptedException |
          BrokenBarrierException e) {
        exceptions.add(e);
      }
    }
  };
  reconfigThread.start();

  barrier.await();
  rb.nextBytes(writeBuf);
  out.write(writeBuf);
  out.hflush();
  out.close();

  reconfigThread.join();

  // Verify the file has sufficient replications.
  DFSTestUtil.waitReplication(fs, testFile, REPLICATION);
  // Read the content back
  byte[] content = DFSTestUtil.readFileBuffer(fs, testFile);
  assertEquals(BLOCK_SIZE, content.length);

  // If an IOException thrown from BlockReceiver#run, it triggers
  // DataNode#checkDiskError(). So we can test whether checkDiskError() is called,
  // to see whether there is IOException in BlockReceiver#run().
  assertEquals(lastTimeDiskErrorCheck, dn.getLastDiskErrorCheck());

  if (!exceptions.isEmpty()) {
    throw new IOException(exceptions.get(0).getCause());
  }
}
 
Example #30
Source File: TestNameNodeReconfigure.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Test that we can change the block placement policy through the
 * reconfigurable API.
 */
@Test
public void testChangeBlockPlacementPolicy()
  throws IOException, ReconfigurationException {
  AtomicInteger callCounter = new AtomicInteger(0);
  MockPlacementPolicy.setCallCounter(callCounter);

  DFSTestUtil util = new DFSTestUtil("", 2, 1, 512);

  // write some files with the default block placement policy
  util.createFiles(fs, "/reconfdat1", (short) 3);
  util.waitReplication(fs, "/reconfdat1", (short) 3);

  assertTrue("calls already made to MockPlacementPolicy",
             callCounter.get() == 0);

  // switch over to the mock placement policy
  cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname",
                                            "org.apache.hadoop.hdfs.server." +
                                            "namenode." +
                                            "TestNameNodeReconfigure$" +
                                            "MockPlacementPolicy");

  // write some files with the mock placement policy
  util.createFiles(fs, "/reconfdat2", (short) 3);
  util.waitReplication(fs, "/reconfdat2", (short) 3);

  int callsMade1 = callCounter.get();
  
  // check that calls were made to mock placement policy
  assertTrue("no calls made to MockPlacementPolicy",
             callsMade1 > 0);
  LOG.info("" + callsMade1 + " calls made to MockPlacementPolicy");

  // now try to change it to a non-existent class
  try {
    cluster.getNameNode().
      reconfigureProperty("dfs.block.replicator.classname",
                          "does.not.exist");
    fail("ReconfigurationException expected");
  } catch (RuntimeException expected) {
    assertTrue("exception should have cause", expected.getCause() != null);
    assertTrue("exception's cause should have cause",
               expected.getCause().getCause() != null);
    assertTrue("ClassNotFoundException expected but got " +
               expected.getCause().getCause().getClass().getCanonicalName(),
               expected.getCause().getCause() instanceof
               ClassNotFoundException);
  }

  // write some files, they should still go to the mock placemeny policy
  util.createFiles(fs, "/reconfdat3", (short) 3);
  util.waitReplication(fs, "/reconfdat3", (short) 3);

  int callsMade2 = callCounter.get();

  // check that more calls were made to mock placement policy
  assertTrue("no calls made to MockPlacementPolicy",
             callsMade2 > callsMade1);
  LOG.info("" + (callsMade2 - callsMade1) +
    " calls made to MockPlacementPolicy");

  // now revert back to the default policy
  cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname",
                                            null);

  // write some files with the default block placement policy
  util.createFiles(fs, "/reconfdat4", (short) 3);
  util.waitReplication(fs, "/reconfdat4", (short) 3);

  // make sure that no more calls were made to mock placement policy
  assertTrue("more calls made to MockPlacementPolicy",
             callCounter.get() == callsMade2);

  util.cleanup(fs, "/reconfdat1");
  util.cleanup(fs, "/reconfdat2");
  util.cleanup(fs, "/reconfdat3");
  util.cleanup(fs, "/reconfdat4");
}