org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction Java Examples
The following examples show how to use
org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestSaveNamespace.java From big-c with Apache License 2.0 | 6 votes |
/** * Test for save namespace should succeed when parent directory renamed with * open lease and destination directory exist. * This test is a regression for HDFS-2827 */ @Test public void testSaveNamespaceWithRenamedLease() throws Exception { MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration()) .numDataNodes(1).build(); cluster.waitActive(); DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); OutputStream out = null; try { fs.mkdirs(new Path("/test-target")); out = fs.create(new Path("/test-source/foo")); // don't close fs.rename(new Path("/test-source/"), new Path("/test-target/")); fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); cluster.getNameNodeRpc().saveNamespace(); fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); } finally { IOUtils.cleanup(LOG, out, fs); if (cluster != null) { cluster.shutdown(); } } }
Example #2
Source File: TestRollingUpgrade.java From big-c with Apache License 2.0 | 6 votes |
private static void startRollingUpgrade(Path foo, Path bar, Path file, byte[] data, MiniDFSCluster cluster) throws IOException { final DistributedFileSystem dfs = cluster.getFileSystem(); //start rolling upgrade dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); dfs.mkdirs(bar); Assert.assertTrue(dfs.exists(foo)); Assert.assertTrue(dfs.exists(bar)); //truncate a file final int newLength = DFSUtil.getRandom().nextInt(data.length - 1) + 1; dfs.truncate(file, newLength); TestFileTruncate.checkBlockRecovery(file, dfs); AppendTestUtil.checkFullFile(dfs, file, newLength, data); }
Example #3
Source File: TestRollingUpgrade.java From hadoop with Apache License 2.0 | 6 votes |
@Test (timeout = 300000) public void testQueryAfterRestart() throws IOException, InterruptedException { final Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); cluster.waitActive(); DistributedFileSystem dfs = cluster.getFileSystem(); dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); // start rolling upgrade dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); queryForPreparation(dfs); dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); dfs.saveNamespace(); dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNodes(); dfs.rollingUpgrade(RollingUpgradeAction.QUERY); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #4
Source File: TestRollingUpgrade.java From hadoop with Apache License 2.0 | 6 votes |
private static void startRollingUpgrade(Path foo, Path bar, Path file, byte[] data, MiniDFSCluster cluster) throws IOException { final DistributedFileSystem dfs = cluster.getFileSystem(); //start rolling upgrade dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); dfs.mkdirs(bar); Assert.assertTrue(dfs.exists(foo)); Assert.assertTrue(dfs.exists(bar)); //truncate a file final int newLength = DFSUtil.getRandom().nextInt(data.length - 1) + 1; dfs.truncate(file, newLength); TestFileTruncate.checkBlockRecovery(file, dfs); AppendTestUtil.checkFullFile(dfs, file, newLength, data); }
Example #5
Source File: TestSnapshotDeletion.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testCorrectNumberOfBlocksAfterRestart() throws IOException { final Path foo = new Path("/foo"); final Path bar = new Path(foo, "bar"); final Path file = new Path(foo, "file"); final String snapshotName = "ss0"; DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, seed); hdfs.mkdirs(bar); hdfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1); hdfs.setQuota(bar, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1); hdfs.allowSnapshot(foo); hdfs.createSnapshot(foo, snapshotName); hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); hdfs.saveNamespace(); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); hdfs.deleteSnapshot(foo, snapshotName); hdfs.delete(bar, true); hdfs.delete(foo, true); long numberOfBlocks = cluster.getNamesystem().getBlocksTotal(); cluster.restartNameNode(0); assertEquals(numberOfBlocks, cluster.getNamesystem().getBlocksTotal()); }
Example #6
Source File: TestFSImage.java From big-c with Apache License 2.0 | 6 votes |
/** * Ensure that the digest written by the saver equals to the digest of the * file. */ @Test public void testDigest() throws IOException { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); DistributedFileSystem fs = cluster.getFileSystem(); fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fs.saveNamespace(); fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get( 0); File fsimage = FSImageTestUtil.findNewestImageFile(currentDir .getAbsolutePath()); assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage), MD5FileUtils.computeMd5ForFile(fsimage)); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #7
Source File: TestRollingUpgradeDowngrade.java From hadoop with Apache License 2.0 | 6 votes |
/** * Ensure that during downgrade the NN fails to load a fsimage with newer * format. */ @Test(expected = IncorrectVersionException.class) public void testRejectNewFsImage() throws IOException { final Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fs.saveNamespace(); fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); NNStorage storage = spy(cluster.getNameNode().getFSImage().getStorage()); int futureVersion = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1; doReturn(futureVersion).when(storage).getServiceLayoutVersion(); storage.writeAll(); cluster.restartNameNode(0, true, "-rollingUpgrade", "downgrade"); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #8
Source File: TestSaveNamespace.java From big-c with Apache License 2.0 | 6 votes |
@Test (timeout=30000) public void testSaveNamespaceWithDanglingLease() throws Exception { MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration()) .numDataNodes(1).build(); cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); try { cluster.getNamesystem().leaseManager.addLease("me", "/non-existent"); fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); cluster.getNameNodeRpc().saveNamespace(); fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #9
Source File: TestSaveNamespace.java From hadoop with Apache License 2.0 | 6 votes |
/** * Test for save namespace should succeed when parent directory renamed with * open lease and destination directory exist. * This test is a regression for HDFS-2827 */ @Test public void testSaveNamespaceWithRenamedLease() throws Exception { MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration()) .numDataNodes(1).build(); cluster.waitActive(); DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); OutputStream out = null; try { fs.mkdirs(new Path("/test-target")); out = fs.create(new Path("/test-source/foo")); // don't close fs.rename(new Path("/test-source/"), new Path("/test-target/")); fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); cluster.getNameNodeRpc().saveNamespace(); fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); } finally { IOUtils.cleanup(LOG, out, fs); if (cluster != null) { cluster.shutdown(); } } }
Example #10
Source File: TestRollingUpgradeDowngrade.java From big-c with Apache License 2.0 | 6 votes |
/** * Ensure that during downgrade the NN fails to load a fsimage with newer * format. */ @Test(expected = IncorrectVersionException.class) public void testRejectNewFsImage() throws IOException { final Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fs.saveNamespace(); fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); NNStorage storage = spy(cluster.getNameNode().getFSImage().getStorage()); int futureVersion = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1; doReturn(futureVersion).when(storage).getServiceLayoutVersion(); storage.writeAll(); cluster.restartNameNode(0, true, "-rollingUpgrade", "downgrade"); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #11
Source File: TestWithMiniClusterBase.java From NNAnalytics with Apache License 2.0 | 6 votes |
@Test //(timeout = 120000L) public void testRestartFetchNamespace() throws Exception { // Shutdown NNA. long currentTxid = nna.getLoader().getCurrentTxId(); nna.shutdown(); // Trigger file system updates. addFiles(100, 0L); DistributedFileSystem fileSystem = (DistributedFileSystem) FileSystem.get(CONF); fileSystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fileSystem.saveNamespace(); fileSystem.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); nnaConf.set("nna.bootstrap.auto.fetch.namespace", "true"); nna.init(nnaConf, null, CONF); long restartedTxid = nna.getLoader().getCurrentTxId(); assertThat(restartedTxid, is(greaterThan(currentTxid + 99))); }
Example #12
Source File: TestDataNodeRollingUpgrade.java From hadoop with Apache License 2.0 | 5 votes |
private void startRollingUpgrade() throws Exception { LOG.info("Starting rolling upgrade"); fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); final DFSAdmin dfsadmin = new DFSAdmin(conf); TestRollingUpgrade.runCmd(dfsadmin, true, "-rollingUpgrade", "prepare"); triggerHeartBeats(); // Ensure datanode rolling upgrade is started assertTrue(dn0.getFSDataset().trashEnabled(blockPoolId)); }
Example #13
Source File: TestCheckpoint.java From hadoop with Apache License 2.0 | 5 votes |
/** * Regression test for HDFS-3835 - "Long-lived 2NN cannot perform a * checkpoint if security is enabled and the NN restarts without outstanding * delegation tokens" */ @Test public void testSecondaryNameNodeWithDelegationTokens() throws IOException { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; Configuration conf = new HdfsConfiguration(); conf.setBoolean( DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) .format(true).build(); assertNotNull(cluster.getNamesystem().getDelegationToken(new Text("atm"))); secondary = startSecondaryNameNode(conf); // Checkpoint once, so the 2NN loads the DT into its in-memory sate. secondary.doCheckpoint(); // Perform a saveNamespace, so that the NN has a new fsimage, and the 2NN // therefore needs to download a new fsimage the next time it performs a // checkpoint. cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); cluster.getNameNodeRpc().saveNamespace(); cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_LEAVE, false); // Ensure that the 2NN can still perform a checkpoint. secondary.doCheckpoint(); } finally { cleanup(secondary); secondary = null; cleanup(cluster); cluster = null; } }
Example #14
Source File: TestFSImageWithSnapshot.java From big-c with Apache License 2.0 | 5 votes |
/** * Test fsimage loading when 1) there is an empty file loaded from fsimage, * and 2) there is later an append operation to be applied from edit log. */ @Test (timeout=60000) public void testLoadImageWithEmptyFile() throws Exception { // create an empty file Path file = new Path(dir, "file"); FSDataOutputStream out = hdfs.create(file); out.close(); // save namespace hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); hdfs.saveNamespace(); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); // append to the empty file out = hdfs.append(file); out.write(1); out.close(); // restart cluster cluster.shutdown(); cluster = new MiniDFSCluster.Builder(conf).format(false) .numDataNodes(REPLICATION).build(); cluster.waitActive(); hdfs = cluster.getFileSystem(); FileStatus status = hdfs.getFileStatus(file); assertEquals(1, status.getLen()); }
Example #15
Source File: PBHelper.java From big-c with Apache License 2.0 | 5 votes |
public static SafeModeActionProto convert( SafeModeAction a) { switch (a) { case SAFEMODE_LEAVE: return SafeModeActionProto.SAFEMODE_LEAVE; case SAFEMODE_ENTER: return SafeModeActionProto.SAFEMODE_ENTER; case SAFEMODE_GET: return SafeModeActionProto.SAFEMODE_GET; default: throw new IllegalArgumentException("Unexpected SafeModeAction :" + a); } }
Example #16
Source File: TestHASafeMode.java From big-c with Apache License 2.0 | 5 votes |
/** * Regression test for HDFS-2804: standby should not populate replication * queues when exiting safe mode. */ @Test public void testNoPopulatingReplQueuesWhenExitingSafemode() throws Exception { DFSTestUtil.createFile(fs, new Path("/test"), 15*BLOCK_SIZE, (short)3, 1L); HATestUtil.waitForStandbyToCatchUp(nn0, nn1); // get some blocks in the SBN's image nn1.getRpcServer().setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); NameNodeAdapter.saveNamespace(nn1); nn1.getRpcServer().setSafeMode(SafeModeAction.SAFEMODE_LEAVE, false); // and some blocks in the edit logs DFSTestUtil.createFile(fs, new Path("/test2"), 15*BLOCK_SIZE, (short)3, 1L); nn0.getRpcServer().rollEditLog(); cluster.stopDataNode(1); cluster.shutdownNameNode(1); //Configuration sbConf = cluster.getConfiguration(1); //sbConf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 1); cluster.restartNameNode(1, false); nn1 = cluster.getNameNode(1); GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { return !nn1.isInSafeMode(); } }, 100, 10000); BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager()); assertEquals(0L, nn1.getNamesystem().getUnderReplicatedBlocks()); assertEquals(0L, nn1.getNamesystem().getPendingReplicationBlocks()); }
Example #17
Source File: ClientNamenodeProtocolTranslatorPB.java From big-c with Apache License 2.0 | 5 votes |
@Override public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { SetSafeModeRequestProto req = SetSafeModeRequestProto.newBuilder() .setAction(PBHelper.convert(action)).setChecked(isChecked).build(); try { return rpcProxy.setSafeMode(null, req).getResult(); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
Example #18
Source File: TestStartup.java From big-c with Apache License 2.0 | 5 votes |
private void checkNameSpace(Configuration conf) throws IOException { NameNode namenode = new NameNode(conf); NamenodeProtocols nnRpc = namenode.getRpcServer(); assertTrue(nnRpc.getFileInfo("/test").isDir()); nnRpc.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); nnRpc.saveNamespace(); namenode.stop(); namenode.join(); }
Example #19
Source File: DFSAdmin.java From big-c with Apache License 2.0 | 5 votes |
private boolean waitExitSafeMode(ClientProtocol nn, boolean inSafeMode) throws IOException { while (inSafeMode) { try { Thread.sleep(5000); } catch (java.lang.InterruptedException e) { throw new IOException("Wait Interrupted"); } inSafeMode = nn.setSafeMode(SafeModeAction.SAFEMODE_GET, false); } return inSafeMode; }
Example #20
Source File: NameNodeRpcServer.java From big-c with Apache License 2.0 | 5 votes |
@Override // ClientProtocol public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { checkNNStartup(); OperationCategory opCategory = OperationCategory.UNCHECKED; if (isChecked) { if (action == SafeModeAction.SAFEMODE_GET) { opCategory = OperationCategory.READ; } else { opCategory = OperationCategory.WRITE; } } namesystem.checkOperation(opCategory); return namesystem.setSafeMode(action); }
Example #21
Source File: TestSaveNamespace.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout=30000) public void testSaveWhileEditsRolled() throws Exception { Configuration conf = getConf(); NameNode.initMetrics(conf, NamenodeRole.NAMENODE); DFSTestUtil.formatNameNode(conf); FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); try { doAnEdit(fsn, 1); CheckpointSignature sig = fsn.rollEditLog(); LOG.warn("Checkpoint signature: " + sig); // Do another edit doAnEdit(fsn, 2); // Save namespace fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fsn.saveNamespace(); // Now shut down and restart the NN fsn.close(); fsn = null; // Start a new namesystem, which should be able to recover // the namespace from the previous incarnation. fsn = FSNamesystem.loadFromDisk(conf); // Make sure the image loaded including our edits. checkEditExists(fsn, 1); checkEditExists(fsn, 2); } finally { if (fsn != null) { fsn.close(); } } }
Example #22
Source File: TestSnapshotDeletion.java From big-c with Apache License 2.0 | 5 votes |
/** * Test applying editlog of operation which deletes a snapshottable directory * without snapshots. The snapshottable dir list in snapshot manager should be * updated. */ @Test (timeout=300000) public void testApplyEditLogForDeletion() throws Exception { final Path foo = new Path("/foo"); final Path bar1 = new Path(foo, "bar1"); final Path bar2 = new Path(foo, "bar2"); hdfs.mkdirs(bar1); hdfs.mkdirs(bar2); // allow snapshots on bar1 and bar2 hdfs.allowSnapshot(bar1); hdfs.allowSnapshot(bar2); assertEquals(2, cluster.getNamesystem().getSnapshotManager() .getNumSnapshottableDirs()); assertEquals(2, cluster.getNamesystem().getSnapshotManager() .getSnapshottableDirs().length); // delete /foo hdfs.delete(foo, true); cluster.restartNameNode(0); // the snapshottable dir list in snapshot manager should be empty assertEquals(0, cluster.getNamesystem().getSnapshotManager() .getNumSnapshottableDirs()); assertEquals(0, cluster.getNamesystem().getSnapshotManager() .getSnapshottableDirs().length); hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); hdfs.saveNamespace(); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNode(0); }
Example #23
Source File: TestSafeMode.java From big-c with Apache License 2.0 | 5 votes |
public void testSafeModeUtils() throws IOException { dfs = cluster.getFileSystem(); // Enter safemode. dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); assertTrue("State was expected to be in safemode.", dfs.isInSafeMode()); // Exit safemode. dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); assertFalse("State was expected to be out of safemode.", dfs.isInSafeMode()); }
Example #24
Source File: TestSnapshot.java From hadoop with Apache License 2.0 | 5 votes |
/** * Restart the cluster to check edit log applying and fsimage saving/loading */ private void checkFSImage() throws Exception { File fsnBefore = getDumpTreeFile(testDir, "before"); File fsnMiddle = getDumpTreeFile(testDir, "middle"); File fsnAfter = getDumpTreeFile(testDir, "after"); SnapshotTestHelper.dumpTree2File(fsdir, fsnBefore); cluster.shutdown(); cluster = new MiniDFSCluster.Builder(conf).format(false) .numDataNodes(REPLICATION).build(); cluster.waitActive(); fsn = cluster.getNamesystem(); hdfs = cluster.getFileSystem(); // later check fsnMiddle to see if the edit log is applied correctly SnapshotTestHelper.dumpTree2File(fsdir, fsnMiddle); // save namespace and restart cluster hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); hdfs.saveNamespace(); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); cluster.shutdown(); cluster = new MiniDFSCluster.Builder(conf).format(false) .numDataNodes(REPLICATION).build(); cluster.waitActive(); fsn = cluster.getNamesystem(); hdfs = cluster.getFileSystem(); // dump the namespace loaded from fsimage SnapshotTestHelper.dumpTree2File(fsdir, fsnAfter); SnapshotTestHelper.compareDumpedTreeInFile(fsnBefore, fsnMiddle, true); SnapshotTestHelper.compareDumpedTreeInFile(fsnBefore, fsnAfter, true); }
Example #25
Source File: TestSaveNamespace.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout=30000) public void testTxIdPersistence() throws Exception { Configuration conf = getConf(); NameNode.initMetrics(conf, NamenodeRole.NAMENODE); DFSTestUtil.formatNameNode(conf); FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); try { // We have a BEGIN_LOG_SEGMENT txn to start assertEquals(1, fsn.getEditLog().getLastWrittenTxId()); doAnEdit(fsn, 1); assertEquals(2, fsn.getEditLog().getLastWrittenTxId()); fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fsn.saveNamespace(); // 2 more txns: END the first segment, BEGIN a new one assertEquals(4, fsn.getEditLog().getLastWrittenTxId()); // Shut down and restart fsn.getFSImage().close(); fsn.close(); // 1 more txn to END that segment assertEquals(5, fsn.getEditLog().getLastWrittenTxId()); fsn = null; fsn = FSNamesystem.loadFromDisk(conf); // 1 more txn to start new segment on restart assertEquals(6, fsn.getEditLog().getLastWrittenTxId()); } finally { if (fsn != null) { fsn.close(); } } }
Example #26
Source File: TestRenameWithSnapshots.java From hadoop with Apache License 2.0 | 5 votes |
private void restartClusterAndCheckImage(boolean compareQuota) throws IOException { File fsnBefore = new File(testDir, "dumptree_before"); File fsnMiddle = new File(testDir, "dumptree_middle"); File fsnAfter = new File(testDir, "dumptree_after"); SnapshotTestHelper.dumpTree2File(fsdir, fsnBefore); cluster.shutdown(); cluster = new MiniDFSCluster.Builder(conf).format(false) .numDataNodes(REPL).build(); cluster.waitActive(); fsn = cluster.getNamesystem(); fsdir = fsn.getFSDirectory(); hdfs = cluster.getFileSystem(); // later check fsnMiddle to see if the edit log is applied correctly SnapshotTestHelper.dumpTree2File(fsdir, fsnMiddle); // save namespace and restart cluster hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); hdfs.saveNamespace(); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); cluster.shutdown(); cluster = new MiniDFSCluster.Builder(conf).format(false) .numDataNodes(REPL).build(); cluster.waitActive(); fsn = cluster.getNamesystem(); fsdir = fsn.getFSDirectory(); hdfs = cluster.getFileSystem(); // dump the namespace loaded from fsimage SnapshotTestHelper.dumpTree2File(fsdir, fsnAfter); SnapshotTestHelper.compareDumpedTreeInFile(fsnBefore, fsnMiddle, compareQuota); SnapshotTestHelper.compareDumpedTreeInFile(fsnBefore, fsnAfter, compareQuota); }
Example #27
Source File: TestSnapshotDeletion.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testFsImageCorruption() throws Exception { final Path st = new Path("/st"); final Path nonst = new Path("/nonst"); final Path stY = new Path(st, "y"); final Path nonstTrash = new Path(nonst, "trash"); hdfs.mkdirs(stY); hdfs.allowSnapshot(st); hdfs.createSnapshot(st, "s0"); Path f = new Path(stY, "nn.log"); hdfs.createNewFile(f); hdfs.createSnapshot(st, "s1"); Path f2 = new Path(stY, "nn2.log"); hdfs.rename(f, f2); hdfs.createSnapshot(st, "s2"); Path trashSt = new Path(nonstTrash, "st"); hdfs.mkdirs(trashSt); hdfs.rename(stY, trashSt); hdfs.delete(nonstTrash, true); hdfs.deleteSnapshot(st, "s1"); hdfs.deleteSnapshot(st, "s2"); hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); hdfs.saveNamespace(); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNodes(); }
Example #28
Source File: TestCheckpoint.java From hadoop with Apache License 2.0 | 5 votes |
/** * Test case where the secondary does a checkpoint, then stops for a while. * In the meantime, the NN saves its image several times, so that the * logs that connect the 2NN's old checkpoint to the current txid * get archived. Then, the 2NN tries to checkpoint again. */ @Test public void testSecondaryHasVeryOutOfDateImage() throws IOException { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; Configuration conf = new HdfsConfiguration(); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) .format(true).build(); secondary = startSecondaryNameNode(conf); // Checkpoint once secondary.doCheckpoint(); // Now primary NN saves namespace 3 times NamenodeProtocols nn = cluster.getNameNodeRpc(); nn.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); for (int i = 0; i < 3; i++) { nn.saveNamespace(); } nn.setSafeMode(SafeModeAction.SAFEMODE_LEAVE, false); // Now the secondary tries to checkpoint again with its // old image in memory. secondary.doCheckpoint(); } finally { cleanup(secondary); secondary = null; cleanup(cluster); cluster = null; } }
Example #29
Source File: TestSnapshotDeletion.java From hadoop with Apache License 2.0 | 5 votes |
/** * Test applying editlog of operation which deletes a snapshottable directory * without snapshots. The snapshottable dir list in snapshot manager should be * updated. */ @Test (timeout=300000) public void testApplyEditLogForDeletion() throws Exception { final Path foo = new Path("/foo"); final Path bar1 = new Path(foo, "bar1"); final Path bar2 = new Path(foo, "bar2"); hdfs.mkdirs(bar1); hdfs.mkdirs(bar2); // allow snapshots on bar1 and bar2 hdfs.allowSnapshot(bar1); hdfs.allowSnapshot(bar2); assertEquals(2, cluster.getNamesystem().getSnapshotManager() .getNumSnapshottableDirs()); assertEquals(2, cluster.getNamesystem().getSnapshotManager() .getSnapshottableDirs().length); // delete /foo hdfs.delete(foo, true); cluster.restartNameNode(0); // the snapshottable dir list in snapshot manager should be empty assertEquals(0, cluster.getNamesystem().getSnapshotManager() .getNumSnapshottableDirs()); assertEquals(0, cluster.getNamesystem().getSnapshotManager() .getSnapshottableDirs().length); hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); hdfs.saveNamespace(); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNode(0); }
Example #30
Source File: TestSnapshotBlocksMap.java From big-c with Apache License 2.0 | 5 votes |
/** * Make sure that a delete of a non-zero-length file which results in a * zero-length file in a snapshot works. */ @Test public void testDeletionOfLaterBlocksWithZeroSizeFirstBlock() throws Exception { final Path foo = new Path("/foo"); final Path bar = new Path(foo, "bar"); final byte[] testData = "foo bar baz".getBytes(); // Create a zero-length file. DFSTestUtil.createFile(hdfs, bar, 0, REPLICATION, 0L); assertEquals(0, fsdir.getINode4Write(bar.toString()).asFile().getBlocks().length); // Create a snapshot that includes that file. SnapshotTestHelper.createSnapshot(hdfs, foo, "s0"); // Extend that file. FSDataOutputStream out = hdfs.append(bar); out.write(testData); out.close(); INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile(); BlockInfoContiguous[] blks = barNode.getBlocks(); assertEquals(1, blks.length); assertEquals(testData.length, blks[0].getNumBytes()); // Delete the file. hdfs.delete(bar, true); // Now make sure that the NN can still save an fsimage successfully. cluster.getNameNode().getRpcServer().setSafeMode( SafeModeAction.SAFEMODE_ENTER, false); cluster.getNameNode().getRpcServer().saveNamespace(); }