Java Code Examples for org.apache.hadoop.hdfs.util.MD5FileUtils

The following examples show how to use org.apache.hadoop.hdfs.util.MD5FileUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: FSImage.java    License: Apache License 2.0 6 votes vote down vote up
private void renameImageFileInDir(StorageDirectory sd, NameNodeFile fromNnf,
    NameNodeFile toNnf, long txid, boolean renameMD5) throws IOException {
  final File fromFile = NNStorage.getStorageFile(sd, fromNnf, txid);
  final File toFile = NNStorage.getStorageFile(sd, toNnf, txid);
  // renameTo fails on Windows if the destination file already exists.
  if(LOG.isDebugEnabled()) {
    LOG.debug("renaming  " + fromFile.getAbsolutePath() 
              + " to " + toFile.getAbsolutePath());
  }
  if (!fromFile.renameTo(toFile)) {
    if (!toFile.delete() || !fromFile.renameTo(toFile)) {
      throw new IOException("renaming  " + fromFile.getAbsolutePath() + " to "  + 
          toFile.getAbsolutePath() + " FAILED");
    }
  }
  if (renameMD5) {
    MD5FileUtils.renameMD5File(fromFile, toFile);
  }
}
 
Example 2
Source Project: hadoop   Source File: TestStartup.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Corrupts the MD5 sum of the fsimage.
 * 
 * @param corruptAll
 *          whether to corrupt one or all of the MD5 sums in the configured
 *          namedirs
 * @throws IOException
 */
private void corruptFSImageMD5(boolean corruptAll) throws IOException {
  List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config);
  // Corrupt the md5 files in all the namedirs
  for (URI uri: nameDirs) {
    // Directory layout looks like:
    // test/data/dfs/nameN/current/{fsimage,edits,...}
    File nameDir = new File(uri.getPath());
    File dfsDir = nameDir.getParentFile();
    assertEquals(dfsDir.getName(), "dfs"); // make sure we got right dir
    // Set the md5 file to all zeros
    File imageFile = new File(nameDir,
        Storage.STORAGE_DIR_CURRENT + "/"
        + NNStorage.getImageFileName(0));
    MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16]));
    // Only need to corrupt one if !corruptAll
    if (!corruptAll) {
      break;
    }
  }
}
 
Example 3
Source Project: hadoop   Source File: TestFSImage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that the digest written by the saver equals to the digest of the
 * file.
 */
@Test
public void testDigest() throws IOException {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    DistributedFileSystem fs = cluster.getFileSystem();
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get(
        0);
    File fsimage = FSImageTestUtil.findNewestImageFile(currentDir
        .getAbsolutePath());
    assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage),
        MD5FileUtils.computeMd5ForFile(fsimage));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 4
Source Project: big-c   Source File: FSImage.java    License: Apache License 2.0 6 votes vote down vote up
private void renameImageFileInDir(StorageDirectory sd, NameNodeFile fromNnf,
    NameNodeFile toNnf, long txid, boolean renameMD5) throws IOException {
  final File fromFile = NNStorage.getStorageFile(sd, fromNnf, txid);
  final File toFile = NNStorage.getStorageFile(sd, toNnf, txid);
  // renameTo fails on Windows if the destination file already exists.
  if(LOG.isDebugEnabled()) {
    LOG.debug("renaming  " + fromFile.getAbsolutePath() 
              + " to " + toFile.getAbsolutePath());
  }
  if (!fromFile.renameTo(toFile)) {
    if (!toFile.delete() || !fromFile.renameTo(toFile)) {
      throw new IOException("renaming  " + fromFile.getAbsolutePath() + " to "  + 
          toFile.getAbsolutePath() + " FAILED");
    }
  }
  if (renameMD5) {
    MD5FileUtils.renameMD5File(fromFile, toFile);
  }
}
 
Example 5
Source Project: big-c   Source File: TestStartup.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Corrupts the MD5 sum of the fsimage.
 * 
 * @param corruptAll
 *          whether to corrupt one or all of the MD5 sums in the configured
 *          namedirs
 * @throws IOException
 */
private void corruptFSImageMD5(boolean corruptAll) throws IOException {
  List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config);
  // Corrupt the md5 files in all the namedirs
  for (URI uri: nameDirs) {
    // Directory layout looks like:
    // test/data/dfs/nameN/current/{fsimage,edits,...}
    File nameDir = new File(uri.getPath());
    File dfsDir = nameDir.getParentFile();
    assertEquals(dfsDir.getName(), "dfs"); // make sure we got right dir
    // Set the md5 file to all zeros
    File imageFile = new File(nameDir,
        Storage.STORAGE_DIR_CURRENT + "/"
        + NNStorage.getImageFileName(0));
    MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16]));
    // Only need to corrupt one if !corruptAll
    if (!corruptAll) {
      break;
    }
  }
}
 
Example 6
Source Project: big-c   Source File: TestFSImage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that the digest written by the saver equals to the digest of the
 * file.
 */
@Test
public void testDigest() throws IOException {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    DistributedFileSystem fs = cluster.getFileSystem();
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get(
        0);
    File fsimage = FSImageTestUtil.findNewestImageFile(currentDir
        .getAbsolutePath());
    assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage),
        MD5FileUtils.computeMd5ForFile(fsimage));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 7
Source Project: hadoop   Source File: FSImageFormatProtobuf.java    License: Apache License 2.0 5 votes vote down vote up
void load(File file) throws IOException {
  long start = Time.monotonicNow();
  imgDigest = MD5FileUtils.computeMd5ForFile(file);
  RandomAccessFile raFile = new RandomAccessFile(file, "r");
  FileInputStream fin = new FileInputStream(file);
  try {
    loadInternal(raFile, fin);
    long end = Time.monotonicNow();
    LOG.info("Loaded FSImage in " + (end - start) / 1000 + " seconds.");
  } finally {
    fin.close();
    raFile.close();
  }
}
 
Example 8
Source Project: hadoop   Source File: FSImage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Load the image namespace from the given image file, verifying
 * it against the MD5 sum stored in its associated .md5 file.
 */
private void loadFSImage(File imageFile, FSNamesystem target,
    MetaRecoveryContext recovery, boolean requireSameLayoutVersion)
    throws IOException {
  MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
  if (expectedMD5 == null) {
    throw new IOException("No MD5 file found corresponding to image file "
        + imageFile);
  }
  loadFSImage(imageFile, expectedMD5, target, recovery,
      requireSameLayoutVersion);
}
 
Example 9
Source Project: hadoop   Source File: FSImage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Save the contents of the FS image to the file.
 */
void saveFSImage(SaveNamespaceContext context, StorageDirectory sd,
    NameNodeFile dstType) throws IOException {
  long txid = context.getTxId();
  File newFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE_NEW, txid);
  File dstFile = NNStorage.getStorageFile(sd, dstType, txid);
  
  FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context);
  FSImageCompression compression = FSImageCompression.createCompression(conf);
  saver.save(newFile, compression);
  
  MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest());
  storage.setMostRecentCheckpointInfo(txid, Time.now());
}
 
Example 10
Source Project: hadoop   Source File: FSImage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This is called by the 2NN after having downloaded an image, and by
 * the NN after having received a new image from the 2NN. It
 * renames the image from fsimage_N.ckpt to fsimage_N and also
 * saves the related .md5 file into place.
 */
public synchronized void saveDigestAndRenameCheckpointImage(NameNodeFile nnf,
    long txid, MD5Hash digest) throws IOException {
  // Write and rename MD5 file
  List<StorageDirectory> badSds = Lists.newArrayList();
  
  for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.IMAGE)) {
    File imageFile = NNStorage.getImageFile(sd, nnf, txid);
    try {
      MD5FileUtils.saveMD5File(imageFile, digest);
    } catch (IOException ioe) {
      badSds.add(sd);
    }
  }
  storage.reportErrorsOnDirectories(badSds);
  
  CheckpointFaultInjector.getInstance().afterMD5Rename();
  
  // Rename image from tmp file
  renameCheckpoint(txid, NameNodeFile.IMAGE_NEW, nnf, false);
  // So long as this is the newest image available,
  // advertise it as such to other checkpointers
  // from now on
  if (txid > storage.getMostRecentCheckpointTxId()) {
    storage.setMostRecentCheckpointInfo(txid, Time.now());
  }
}
 
Example 11
Source Project: hadoop   Source File: ImageServlet.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Set headers for content length, and, if available, md5.
 * @throws IOException 
 */
public static void setVerificationHeadersForGet(HttpServletResponse response,
    File file) throws IOException {
  response.setHeader(TransferFsImage.CONTENT_LENGTH,
      String.valueOf(file.length()));
  MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
  if (hash != null) {
    response.setHeader(TransferFsImage.MD5_HEADER, hash.toString());
  }
}
 
Example 12
Source Project: hadoop   Source File: ImageServlet.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Set headers for image length and if available, md5.
 * 
 * @throws IOException
 */
static void setVerificationHeadersForPut(HttpURLConnection connection,
    File file) throws IOException {
  connection.setRequestProperty(TransferFsImage.CONTENT_LENGTH,
      String.valueOf(file.length()));
  MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
  if (hash != null) {
    connection
        .setRequestProperty(TransferFsImage.MD5_HEADER, hash.toString());
  }
}
 
Example 13
Source Project: hadoop   Source File: TestFetchImage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Run `hdfs dfsadmin -fetchImage ...' and verify that the downloaded image is
 * correct.
 */
private static void runFetchImage(DFSAdmin dfsAdmin, MiniDFSCluster cluster)
    throws Exception {
  int retVal = dfsAdmin.run(new String[]{"-fetchImage",
      FETCHED_IMAGE_FILE.getPath() });
  
  assertEquals(0, retVal);
  
  File highestImageOnNn = getHighestFsImageOnCluster(cluster);
  MD5Hash expected = MD5FileUtils.computeMd5ForFile(highestImageOnNn);
  MD5Hash actual = MD5FileUtils.computeMd5ForFile(
      new File(FETCHED_IMAGE_FILE, highestImageOnNn.getName()));
  
  assertEquals(expected, actual);
}
 
Example 14
Source Project: big-c   Source File: FSImageFormatProtobuf.java    License: Apache License 2.0 5 votes vote down vote up
void load(File file) throws IOException {
  long start = Time.monotonicNow();
  imgDigest = MD5FileUtils.computeMd5ForFile(file);
  RandomAccessFile raFile = new RandomAccessFile(file, "r");
  FileInputStream fin = new FileInputStream(file);
  try {
    loadInternal(raFile, fin);
    long end = Time.monotonicNow();
    LOG.info("Loaded FSImage in " + (end - start) / 1000 + " seconds.");
  } finally {
    fin.close();
    raFile.close();
  }
}
 
Example 15
Source Project: big-c   Source File: FSImage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Load the image namespace from the given image file, verifying
 * it against the MD5 sum stored in its associated .md5 file.
 */
private void loadFSImage(File imageFile, FSNamesystem target,
    MetaRecoveryContext recovery, boolean requireSameLayoutVersion)
    throws IOException {
  MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
  if (expectedMD5 == null) {
    throw new IOException("No MD5 file found corresponding to image file "
        + imageFile);
  }
  loadFSImage(imageFile, expectedMD5, target, recovery,
      requireSameLayoutVersion);
}
 
Example 16
Source Project: big-c   Source File: FSImage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Save the contents of the FS image to the file.
 */
void saveFSImage(SaveNamespaceContext context, StorageDirectory sd,
    NameNodeFile dstType) throws IOException {
  long txid = context.getTxId();
  File newFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE_NEW, txid);
  File dstFile = NNStorage.getStorageFile(sd, dstType, txid);
  
  FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context);
  FSImageCompression compression = FSImageCompression.createCompression(conf);
  saver.save(newFile, compression);
  
  MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest());
  storage.setMostRecentCheckpointInfo(txid, Time.now());
}
 
Example 17
Source Project: big-c   Source File: FSImage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This is called by the 2NN after having downloaded an image, and by
 * the NN after having received a new image from the 2NN. It
 * renames the image from fsimage_N.ckpt to fsimage_N and also
 * saves the related .md5 file into place.
 */
public synchronized void saveDigestAndRenameCheckpointImage(NameNodeFile nnf,
    long txid, MD5Hash digest) throws IOException {
  // Write and rename MD5 file
  List<StorageDirectory> badSds = Lists.newArrayList();
  
  for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.IMAGE)) {
    File imageFile = NNStorage.getImageFile(sd, nnf, txid);
    try {
      MD5FileUtils.saveMD5File(imageFile, digest);
    } catch (IOException ioe) {
      badSds.add(sd);
    }
  }
  storage.reportErrorsOnDirectories(badSds);
  
  CheckpointFaultInjector.getInstance().afterMD5Rename();
  
  // Rename image from tmp file
  renameCheckpoint(txid, NameNodeFile.IMAGE_NEW, nnf, false);
  // So long as this is the newest image available,
  // advertise it as such to other checkpointers
  // from now on
  if (txid > storage.getMostRecentCheckpointTxId()) {
    storage.setMostRecentCheckpointInfo(txid, Time.now());
  }
}
 
Example 18
Source Project: big-c   Source File: ImageServlet.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Set headers for content length, and, if available, md5.
 * @throws IOException 
 */
public static void setVerificationHeadersForGet(HttpServletResponse response,
    File file) throws IOException {
  response.setHeader(TransferFsImage.CONTENT_LENGTH,
      String.valueOf(file.length()));
  MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
  if (hash != null) {
    response.setHeader(TransferFsImage.MD5_HEADER, hash.toString());
  }
}
 
Example 19
Source Project: big-c   Source File: ImageServlet.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Set headers for image length and if available, md5.
 * 
 * @throws IOException
 */
static void setVerificationHeadersForPut(HttpURLConnection connection,
    File file) throws IOException {
  connection.setRequestProperty(TransferFsImage.CONTENT_LENGTH,
      String.valueOf(file.length()));
  MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
  if (hash != null) {
    connection
        .setRequestProperty(TransferFsImage.MD5_HEADER, hash.toString());
  }
}
 
Example 20
Source Project: big-c   Source File: TestFetchImage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Run `hdfs dfsadmin -fetchImage ...' and verify that the downloaded image is
 * correct.
 */
private static void runFetchImage(DFSAdmin dfsAdmin, MiniDFSCluster cluster)
    throws Exception {
  int retVal = dfsAdmin.run(new String[]{"-fetchImage",
      FETCHED_IMAGE_FILE.getPath() });
  
  assertEquals(0, retVal);
  
  File highestImageOnNn = getHighestFsImageOnCluster(cluster);
  MD5Hash expected = MD5FileUtils.computeMd5ForFile(highestImageOnNn);
  MD5Hash actual = MD5FileUtils.computeMd5ForFile(
      new File(FETCHED_IMAGE_FILE, highestImageOnNn.getName()));
  
  assertEquals(expected, actual);
}
 
Example 21
Source Project: NNAnalytics   Source File: TransferFsImageWrapper.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This is meant to download the latest FSImage without relying on FSNamesystem or other running
 * HDFS classes within NameNodeLoader.
 *
 * @throws IOException if FileSystem can not be initialized
 */
public void downloadMostRecentImage() throws IOException {
  FileSystem fileSystem = nameNodeLoader.getFileSystem();
  Configuration conf = nameNodeLoader.getConfiguration();
  String namespaceDirPath = conf.get(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
  File namespaceDir = new File(namespaceDirPath, "current");
  SecurityUtil.login(
      conf,
      DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
      DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY,
      InetAddress.getLocalHost().getCanonicalHostName());
  InetSocketAddress addressOfActive = HAUtil.getAddressOfActive(fileSystem);
  URL infoServer =
      DFSUtil.getInfoServer(addressOfActive, conf, DFSUtil.getHttpClientScheme(conf)).toURL();
  SecurityUtil.doAsLoginUser(
      () -> {
        NamenodeProtocol nnProtocolProxy =
            NameNodeProxies.createNonHAProxy(
                    conf,
                    addressOfActive,
                    NamenodeProtocol.class,
                    UserGroupInformation.getLoginUser(),
                    true)
                .getProxy();
        NamespaceInfo namespaceInfo = nnProtocolProxy.versionRequest();
        String fileId = ImageServlet.getParamStringForMostRecentImage();
        NNStorage storage =
            new NNStorage(
                conf,
                FSNamesystem.getNamespaceDirs(conf),
                FSNamesystem.getNamespaceEditsDirs(conf));
        storage.format(namespaceInfo);
        MD5Hash md5 =
            TransferFsImage.getFileClient(
                infoServer, fileId, Lists.newArrayList(namespaceDir), storage, true);
        FSImageTransactionalStorageInspector inspector =
            new FSImageTransactionalStorageInspector(EnumSet.of(NNStorage.NameNodeFile.IMAGE));
        storage.inspectStorageDirs(inspector);
        File imageFile = inspector.getLatestImages().get(0).getFile();
        MD5FileUtils.saveMD5File(imageFile, md5);
        return null;
      });
}
 
Example 22
Source Project: hadoop   Source File: NNStorageRetentionManager.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void purgeImage(FSImageFile image) {
  LOG.info("Purging old image " + image);
  deleteOrWarn(image.getFile());
  deleteOrWarn(MD5FileUtils.getDigestFileForFile(image.getFile()));
}
 
Example 23
Source Project: hadoop   Source File: TestSaveNamespace.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=20000)
public void testCancelSaveNamespace() throws Exception {
  Configuration conf = getConf();
  NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);

  // Replace the FSImage with a spy
  final FSImage image = fsn.getFSImage();
  NNStorage storage = image.getStorage();
  storage.close(); // unlock any directories that FSNamesystem's initialization may have locked
  storage.setStorageDirectories(
      FSNamesystem.getNamespaceDirs(conf), 
      FSNamesystem.getNamespaceEditsDirs(conf));

  FSNamesystem spyFsn = spy(fsn);
  final FSNamesystem finalFsn = spyFsn;
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
  BlockIdManager bid = spy(spyFsn.getBlockIdManager());
  Whitebox.setInternalState(finalFsn, "blockIdManager", bid);
  doAnswer(delayer).when(bid).getGenerationStampV2();

  ExecutorService pool = Executors.newFixedThreadPool(2);
  
  try {
    doAnEdit(fsn, 1);
    final Canceler canceler = new Canceler();
    
    // Save namespace
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    try {
      Future<Void> saverFuture = pool.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
          image.saveNamespace(finalFsn, NameNodeFile.IMAGE, canceler);
          return null;
        }
      });

      // Wait until saveNamespace calls getGenerationStamp
      delayer.waitForCall();
      // then cancel the saveNamespace
      Future<Void> cancelFuture = pool.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
          canceler.cancel("cancelled");
          return null;
        }
      });
      // give the cancel call time to run
      Thread.sleep(500);
      
      // allow saveNamespace to proceed - it should check the cancel flag after
      // this point and throw an exception
      delayer.proceed();

      cancelFuture.get();
      saverFuture.get();
      fail("saveNamespace did not fail even though cancelled!");
    } catch (Throwable t) {
      GenericTestUtils.assertExceptionContains(
          "SaveNamespaceCancelledException", t);
    }
    LOG.info("Successfully cancelled a saveNamespace");


    // Check that we have only the original image and not any
    // cruft left over from half-finished images
    FSImageTestUtil.logStorageContents(LOG, storage);
    for (StorageDirectory sd : storage.dirIterable(null)) {
      File curDir = sd.getCurrentDir();
      GenericTestUtils.assertGlobEquals(curDir, "fsimage_.*",
          NNStorage.getImageFileName(0),
          NNStorage.getImageFileName(0) + MD5FileUtils.MD5_SUFFIX);
    }      
  } finally {
    fsn.close();
  }
}
 
Example 24
Source Project: big-c   Source File: NNStorageRetentionManager.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void purgeImage(FSImageFile image) {
  LOG.info("Purging old image " + image);
  deleteOrWarn(image.getFile());
  deleteOrWarn(MD5FileUtils.getDigestFileForFile(image.getFile()));
}
 
Example 25
Source Project: big-c   Source File: TestSaveNamespace.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=20000)
public void testCancelSaveNamespace() throws Exception {
  Configuration conf = getConf();
  NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);

  // Replace the FSImage with a spy
  final FSImage image = fsn.getFSImage();
  NNStorage storage = image.getStorage();
  storage.close(); // unlock any directories that FSNamesystem's initialization may have locked
  storage.setStorageDirectories(
      FSNamesystem.getNamespaceDirs(conf), 
      FSNamesystem.getNamespaceEditsDirs(conf));

  FSNamesystem spyFsn = spy(fsn);
  final FSNamesystem finalFsn = spyFsn;
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
  BlockIdManager bid = spy(spyFsn.getBlockIdManager());
  Whitebox.setInternalState(finalFsn, "blockIdManager", bid);
  doAnswer(delayer).when(bid).getGenerationStampV2();

  ExecutorService pool = Executors.newFixedThreadPool(2);
  
  try {
    doAnEdit(fsn, 1);
    final Canceler canceler = new Canceler();
    
    // Save namespace
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    try {
      Future<Void> saverFuture = pool.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
          image.saveNamespace(finalFsn, NameNodeFile.IMAGE, canceler);
          return null;
        }
      });

      // Wait until saveNamespace calls getGenerationStamp
      delayer.waitForCall();
      // then cancel the saveNamespace
      Future<Void> cancelFuture = pool.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
          canceler.cancel("cancelled");
          return null;
        }
      });
      // give the cancel call time to run
      Thread.sleep(500);
      
      // allow saveNamespace to proceed - it should check the cancel flag after
      // this point and throw an exception
      delayer.proceed();

      cancelFuture.get();
      saverFuture.get();
      fail("saveNamespace did not fail even though cancelled!");
    } catch (Throwable t) {
      GenericTestUtils.assertExceptionContains(
          "SaveNamespaceCancelledException", t);
    }
    LOG.info("Successfully cancelled a saveNamespace");


    // Check that we have only the original image and not any
    // cruft left over from half-finished images
    FSImageTestUtil.logStorageContents(LOG, storage);
    for (StorageDirectory sd : storage.dirIterable(null)) {
      File curDir = sd.getCurrentDir();
      GenericTestUtils.assertGlobEquals(curDir, "fsimage_.*",
          NNStorage.getImageFileName(0),
          NNStorage.getImageFileName(0) + MD5FileUtils.MD5_SUFFIX);
    }      
  } finally {
    fsn.close();
  }
}
 
Example 26
Source Project: hadoop   Source File: FSImageTestUtil.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * This function returns a md5 hash of a file.
 * 
 * @param file input file
 * @return The md5 string
 */
public static String getFileMD5(File file) throws IOException {
  return MD5FileUtils.computeMd5ForFile(file).toString();
}
 
Example 27
Source Project: big-c   Source File: FSImageTestUtil.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * This function returns a md5 hash of a file.
 * 
 * @param file input file
 * @return The md5 string
 */
public static String getFileMD5(File file) throws IOException {
  return MD5FileUtils.computeMd5ForFile(file).toString();
}