Java Code Examples for org.apache.hadoop.hdfs.DistributedFileSystem#saveNamespace()

The following examples show how to use org.apache.hadoop.hdfs.DistributedFileSystem#saveNamespace() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DFSAdmin.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Command to ask the namenode to save the namespace.
 * Usage: java DFSAdmin -saveNamespace [force] [uncompressed]
 *
 * @param argv List of of command line parameters.
 * @param idx The index of the command that is being processed.
 * @exception IOException 
 */
public int saveNamespace(String[] argv, int idx) throws IOException {
  int exitCode = -1;

  DistributedFileSystem dfs = getDFS();
  if (dfs == null) {
    System.err.println("FileSystem is " + getFS().getUri());
    return exitCode;
  }

  boolean force = false;
  boolean uncompressed = false;
  for( ; idx < argv.length; idx++) {
    if (argv[idx].equals("force")) {
      force = true;
    } else if (argv[idx].equals("uncompressed")) {
      uncompressed = true;
    } else {
      printUsage("saveNamespace");
      return exitCode;
    }
  }
  dfs.saveNamespace(force, uncompressed);
  return 0;
}
 
Example 2
Source File: TestFSImage.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that the digest written by the saver equals to the digest of the
 * file.
 */
@Test
public void testDigest() throws IOException {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    DistributedFileSystem fs = cluster.getFileSystem();
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get(
        0);
    File fsimage = FSImageTestUtil.findNewestImageFile(currentDir
        .getAbsolutePath());
    assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage),
        MD5FileUtils.computeMd5ForFile(fsimage));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 3
Source File: TestFSImage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that the digest written by the saver equals to the digest of the
 * file.
 */
@Test
public void testDigest() throws IOException {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    DistributedFileSystem fs = cluster.getFileSystem();
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get(
        0);
    File fsimage = FSImageTestUtil.findNewestImageFile(currentDir
        .getAbsolutePath());
    assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage),
        MD5FileUtils.computeMd5ForFile(fsimage));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 4
Source File: TestWithMiniClusterBase.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up
@Test //(timeout = 120000L)
public void testRestartFetchNamespace() throws Exception {
  // Shutdown NNA.
  long currentTxid = nna.getLoader().getCurrentTxId();
  nna.shutdown();

  // Trigger file system updates.
  addFiles(100, 0L);
  DistributedFileSystem fileSystem = (DistributedFileSystem) FileSystem.get(CONF);
  fileSystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
  fileSystem.saveNamespace();
  fileSystem.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);

  nnaConf.set("nna.bootstrap.auto.fetch.namespace", "true");
  nna.init(nnaConf, null, CONF);
  long restartedTxid = nna.getLoader().getCurrentTxId();
  assertThat(restartedTxid, is(greaterThan(currentTxid + 99)));
}
 
Example 5
Source File: DFSAdmin.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Command to ask the namenode to save the namespace.
 * Usage: hdfs dfsadmin -saveNamespace
 * @exception IOException 
 * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace()
 */
public int saveNamespace() throws IOException {
  int exitCode = -1;

  DistributedFileSystem dfs = getDFS();
  Configuration dfsConf = dfs.getConf();
  URI dfsUri = dfs.getUri();
  boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri);

  if (isHaEnabled) {
    String nsId = dfsUri.getHost();
    List<ProxyAndInfo<ClientProtocol>> proxies =
        HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf,
        nsId, ClientProtocol.class);
    for (ProxyAndInfo<ClientProtocol> proxy : proxies) {
      proxy.getProxy().saveNamespace();
      System.out.println("Save namespace successful for " +
          proxy.getAddress());
    }
  } else {
    dfs.saveNamespace();
    System.out.println("Save namespace successful");
  }
  exitCode = 0;
 
  return exitCode;
}
 
Example 6
Source File: DFSAdmin.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/**
 * Command to ask the namenode to save the namespace.
 * Usage: java DFSAdmin -saveNamespace
 * @exception IOException 
 * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace()
 */
public int saveNamespace() throws IOException {
  int exitCode = -1;

  if (!(fs instanceof DistributedFileSystem)) {
    System.err.println("FileSystem is " + fs.getUri());
    return exitCode;
  }

  DistributedFileSystem dfs = (DistributedFileSystem) fs;
  dfs.saveNamespace();
  exitCode = 0;
 
  return exitCode;
}
 
Example 7
Source File: TestFSImageWithXAttr.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Restart the NameNode, optionally saving a new checkpoint.
 *
 * @param fs DistributedFileSystem used for saving namespace
 * @param persistNamespace boolean true to save a new checkpoint
 * @throws IOException if restart fails
 */
private void restart(DistributedFileSystem fs, boolean persistNamespace)
    throws IOException {
  if (persistNamespace) {
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  }

  cluster.restartNameNode();
  cluster.waitActive();
}
 
Example 8
Source File: TestFSImageWithAcl.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Restart the NameNode, optionally saving a new checkpoint.
 *
 * @param fs DistributedFileSystem used for saving namespace
 * @param persistNamespace boolean true to save a new checkpoint
 * @throws IOException if restart fails
 */
private void restart(DistributedFileSystem fs, boolean persistNamespace)
    throws IOException {
  if (persistNamespace) {
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  }

  cluster.restartNameNode();
  cluster.waitActive();
}
 
Example 9
Source File: TestFSImageWithXAttr.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Restart the NameNode, optionally saving a new checkpoint.
 *
 * @param fs DistributedFileSystem used for saving namespace
 * @param persistNamespace boolean true to save a new checkpoint
 * @throws IOException if restart fails
 */
private void restart(DistributedFileSystem fs, boolean persistNamespace)
    throws IOException {
  if (persistNamespace) {
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  }

  cluster.restartNameNode();
  cluster.waitActive();
}
 
Example 10
Source File: TestFSImageWithAcl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Restart the NameNode, optionally saving a new checkpoint.
 *
 * @param fs DistributedFileSystem used for saving namespace
 * @param persistNamespace boolean true to save a new checkpoint
 * @throws IOException if restart fails
 */
private void restart(DistributedFileSystem fs, boolean persistNamespace)
    throws IOException {
  if (persistNamespace) {
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  }

  cluster.restartNameNode();
  cluster.waitActive();
}
 
Example 11
Source File: DFSAdmin.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Command to ask the namenode to save the namespace.
 * Usage: hdfs dfsadmin -saveNamespace
 * @exception IOException 
 * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace()
 */
public int saveNamespace() throws IOException {
  int exitCode = -1;

  DistributedFileSystem dfs = getDFS();
  Configuration dfsConf = dfs.getConf();
  URI dfsUri = dfs.getUri();
  boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri);

  if (isHaEnabled) {
    String nsId = dfsUri.getHost();
    List<ProxyAndInfo<ClientProtocol>> proxies =
        HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf,
        nsId, ClientProtocol.class);
    for (ProxyAndInfo<ClientProtocol> proxy : proxies) {
      proxy.getProxy().saveNamespace();
      System.out.println("Save namespace successful for " +
          proxy.getAddress());
    }
  } else {
    dfs.saveNamespace();
    System.out.println("Save namespace successful");
  }
  exitCode = 0;
 
  return exitCode;
}
 
Example 12
Source File: TestOfflineImageViewer.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void createOriginalFSImage() throws IOException {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    conf.setLong(
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
    conf.setLong(
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
    conf.setBoolean(
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL,
        "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem hdfs = cluster.getFileSystem();

    // Create a reasonable namespace
    for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
      Path dir = new Path("/dir" + i);
      hdfs.mkdirs(dir);
      writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
      for (int j = 0; j < FILES_PER_DIR; j++) {
        Path file = new Path(dir, "file" + j);
        FSDataOutputStream o = hdfs.create(file);
        o.write(23);
        o.close();

        writtenFiles.put(file.toString(),
            pathToFileEntry(hdfs, file.toString()));
      }
    }

    // Create an empty directory
    Path emptydir = new Path("/emptydir");
    hdfs.mkdirs(emptydir);
    dirCount++;
    writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir));

    //Create a directory whose name should be escaped in XML
    Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here");
    hdfs.mkdirs(invalidXMLDir);
    dirCount++;

    // Get delegation tokens so we log the delegation token op
    Token<?>[] delegationTokens = hdfs
        .addDelegationTokens(TEST_RENEWER, null);
    for (Token<?> t : delegationTokens) {
      LOG.debug("got token " + t);
    }

    // Create INodeReference
    final Path src = new Path("/src");
    hdfs.mkdirs(src);
    dirCount++;
    writtenFiles.put(src.toString(), hdfs.getFileStatus(src));
    final Path orig = new Path("/src/orig");
    hdfs.mkdirs(orig);
    hdfs.allowSnapshot(src);
    hdfs.createSnapshot(src, "snapshot");
    final Path dst = new Path("/dst");
    hdfs.rename(orig, dst);
    dirCount++;
    writtenFiles.put(dst.toString(), hdfs.getFileStatus(dst));

    // Set XAttrs so the fsimage contains XAttr ops
    final Path xattr = new Path("/xattr");
    hdfs.mkdirs(xattr);
    dirCount++;
    hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 });
    hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 });
    // OIV should be able to handle empty value XAttrs
    hdfs.setXAttr(xattr, "user.a3", null);
    writtenFiles.put(xattr.toString(), hdfs.getFileStatus(xattr));

    // Write results to the fsimage file
    hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
    hdfs.saveNamespace();

    // Determine location of fsimage file
    originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
        .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
    if (originalFsimage == null) {
      throw new RuntimeException("Didn't generate or can't find fsimage");
    }
    LOG.debug("original FS image file is " + originalFsimage);
  } finally {
    if (cluster != null)
      cluster.shutdown();
  }
}
 
Example 13
Source File: TestFSImage.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Ensure mtime and atime can be loaded from fsimage.
 */
@Test(timeout=60000)
public void testLoadMtimeAtime() throws Exception {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem hdfs = cluster.getFileSystem();
    String userDir = hdfs.getHomeDirectory().toUri().getPath().toString();
    Path file = new Path(userDir, "file");
    Path dir = new Path(userDir, "/dir");
    Path link = new Path(userDir, "/link");
    hdfs.createNewFile(file);
    hdfs.mkdirs(dir);
    hdfs.createSymlink(file, link, false);

    long mtimeFile = hdfs.getFileStatus(file).getModificationTime();
    long atimeFile = hdfs.getFileStatus(file).getAccessTime();
    long mtimeDir = hdfs.getFileStatus(dir).getModificationTime();
    long mtimeLink = hdfs.getFileLinkStatus(link).getModificationTime();
    long atimeLink = hdfs.getFileLinkStatus(link).getAccessTime();

    // save namespace and restart cluster
    hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
    hdfs.saveNamespace();
    hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
    cluster.shutdown();
    cluster = new MiniDFSCluster.Builder(conf).format(false)
        .numDataNodes(1).build();
    cluster.waitActive();
    hdfs = cluster.getFileSystem();
    
    assertEquals(mtimeFile, hdfs.getFileStatus(file).getModificationTime());
    assertEquals(atimeFile, hdfs.getFileStatus(file).getAccessTime());
    assertEquals(mtimeDir, hdfs.getFileStatus(dir).getModificationTime());
    assertEquals(mtimeLink, hdfs.getFileLinkStatus(link).getModificationTime());
    assertEquals(atimeLink, hdfs.getFileLinkStatus(link).getAccessTime());
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 14
Source File: TestFSImageWithAcl.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void testAcl(boolean persistNamespace) throws IOException {
  Path p = new Path("/p");
  DistributedFileSystem fs = cluster.getFileSystem();
  fs.create(p).close();
  fs.mkdirs(new Path("/23"));

  AclEntry e = new AclEntry.Builder().setName("foo")
      .setPermission(READ_EXECUTE).setScope(ACCESS).setType(USER).build();
  fs.modifyAclEntries(p, Lists.newArrayList(e));

  restart(fs, persistNamespace);

  AclStatus s = cluster.getNamesystem().getAclStatus(p.toString());
  AclEntry[] returned = Lists.newArrayList(s.getEntries()).toArray(
      new AclEntry[0]);
  Assert.assertArrayEquals(new AclEntry[] {
      aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
      aclEntry(ACCESS, GROUP, READ) }, returned);

  fs.removeAcl(p);

  if (persistNamespace) {
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  }

  cluster.restartNameNode();
  cluster.waitActive();

  s = cluster.getNamesystem().getAclStatus(p.toString());
  returned = Lists.newArrayList(s.getEntries()).toArray(new AclEntry[0]);
  Assert.assertArrayEquals(new AclEntry[] { }, returned);

  fs.modifyAclEntries(p, Lists.newArrayList(e));
  s = cluster.getNamesystem().getAclStatus(p.toString());
  returned = Lists.newArrayList(s.getEntries()).toArray(new AclEntry[0]);
  Assert.assertArrayEquals(new AclEntry[] {
      aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
      aclEntry(ACCESS, GROUP, READ) }, returned);
}
 
Example 15
Source File: TestFSImage.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void testPersistHelper(Configuration conf) throws IOException {
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    FSNamesystem fsn = cluster.getNamesystem();
    DistributedFileSystem fs = cluster.getFileSystem();

    final Path dir = new Path("/abc/def");
    final Path file1 = new Path(dir, "f1");
    final Path file2 = new Path(dir, "f2");

    // create an empty file f1
    fs.create(file1).close();

    // create an under-construction file f2
    FSDataOutputStream out = fs.create(file2);
    out.writeBytes("hello");
    ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet
        .of(SyncFlag.UPDATE_LENGTH));

    // checkpoint
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);

    cluster.restartNameNode();
    cluster.waitActive();
    fs = cluster.getFileSystem();

    assertTrue(fs.isDirectory(dir));
    assertTrue(fs.exists(file1));
    assertTrue(fs.exists(file2));

    // check internals of file2
    INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile();
    assertEquals("hello".length(), file2Node.computeFileSize());
    assertTrue(file2Node.isUnderConstruction());
    BlockInfoContiguous[] blks = file2Node.getBlocks();
    assertEquals(1, blks.length);
    assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
    // check lease manager
    Lease lease = fsn.leaseManager.getLeaseByPath(file2.toString());
    Assert.assertNotNull(lease);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 16
Source File: TestFSImage.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void testPersistHelper(Configuration conf) throws IOException {
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    FSNamesystem fsn = cluster.getNamesystem();
    DistributedFileSystem fs = cluster.getFileSystem();

    final Path dir = new Path("/abc/def");
    final Path file1 = new Path(dir, "f1");
    final Path file2 = new Path(dir, "f2");

    // create an empty file f1
    fs.create(file1).close();

    // create an under-construction file f2
    FSDataOutputStream out = fs.create(file2);
    out.writeBytes("hello");
    ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet
        .of(SyncFlag.UPDATE_LENGTH));

    // checkpoint
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);

    cluster.restartNameNode();
    cluster.waitActive();
    fs = cluster.getFileSystem();

    assertTrue(fs.isDirectory(dir));
    assertTrue(fs.exists(file1));
    assertTrue(fs.exists(file2));

    // check internals of file2
    INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile();
    assertEquals("hello".length(), file2Node.computeFileSize());
    assertTrue(file2Node.isUnderConstruction());
    BlockInfoContiguous[] blks = file2Node.getBlocks();
    assertEquals(1, blks.length);
    assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
    // check lease manager
    Lease lease = fsn.leaseManager.getLeaseByPath(file2.toString());
    Assert.assertNotNull(lease);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 17
Source File: TestFSImage.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Ensure mtime and atime can be loaded from fsimage.
 */
@Test(timeout=60000)
public void testLoadMtimeAtime() throws Exception {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = null;
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem hdfs = cluster.getFileSystem();
    String userDir = hdfs.getHomeDirectory().toUri().getPath().toString();
    Path file = new Path(userDir, "file");
    Path dir = new Path(userDir, "/dir");
    Path link = new Path(userDir, "/link");
    hdfs.createNewFile(file);
    hdfs.mkdirs(dir);
    hdfs.createSymlink(file, link, false);

    long mtimeFile = hdfs.getFileStatus(file).getModificationTime();
    long atimeFile = hdfs.getFileStatus(file).getAccessTime();
    long mtimeDir = hdfs.getFileStatus(dir).getModificationTime();
    long mtimeLink = hdfs.getFileLinkStatus(link).getModificationTime();
    long atimeLink = hdfs.getFileLinkStatus(link).getAccessTime();

    // save namespace and restart cluster
    hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
    hdfs.saveNamespace();
    hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
    cluster.shutdown();
    cluster = new MiniDFSCluster.Builder(conf).format(false)
        .numDataNodes(1).build();
    cluster.waitActive();
    hdfs = cluster.getFileSystem();
    
    assertEquals(mtimeFile, hdfs.getFileStatus(file).getModificationTime());
    assertEquals(atimeFile, hdfs.getFileStatus(file).getAccessTime());
    assertEquals(mtimeDir, hdfs.getFileStatus(dir).getModificationTime());
    assertEquals(mtimeLink, hdfs.getFileLinkStatus(link).getModificationTime());
    assertEquals(atimeLink, hdfs.getFileLinkStatus(link).getAccessTime());
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 18
Source File: TestOfflineImageViewer.java    From big-c with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void createOriginalFSImage() throws IOException {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    conf.setLong(
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
    conf.setLong(
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
    conf.setBoolean(
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL,
        "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem hdfs = cluster.getFileSystem();

    // Create a reasonable namespace
    for (int i = 0; i < NUM_DIRS; i++) {
      Path dir = new Path("/dir" + i);
      hdfs.mkdirs(dir);
      writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
      for (int j = 0; j < FILES_PER_DIR; j++) {
        Path file = new Path(dir, "file" + j);
        FSDataOutputStream o = hdfs.create(file);
        o.write(23);
        o.close();

        writtenFiles.put(file.toString(),
            pathToFileEntry(hdfs, file.toString()));
      }
    }

    // Create an empty directory
    Path emptydir = new Path("/emptydir");
    hdfs.mkdirs(emptydir);
    writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir));

    //Create a directory whose name should be escaped in XML
    Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here");
    hdfs.mkdirs(invalidXMLDir);

    // Get delegation tokens so we log the delegation token op
    Token<?>[] delegationTokens = hdfs
        .addDelegationTokens(TEST_RENEWER, null);
    for (Token<?> t : delegationTokens) {
      LOG.debug("got token " + t);
    }

    final Path snapshot = new Path("/snapshot");
    hdfs.mkdirs(snapshot);
    hdfs.allowSnapshot(snapshot);
    hdfs.mkdirs(new Path("/snapshot/1"));
    hdfs.delete(snapshot, true);

    // Set XAttrs so the fsimage contains XAttr ops
    final Path xattr = new Path("/xattr");
    hdfs.mkdirs(xattr);
    hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 });
    hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 });
    // OIV should be able to handle empty value XAttrs
    hdfs.setXAttr(xattr, "user.a3", null);
    writtenFiles.put(xattr.toString(), hdfs.getFileStatus(xattr));

    // Write results to the fsimage file
    hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
    hdfs.saveNamespace();

    // Determine location of fsimage file
    originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
        .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
    if (originalFsimage == null) {
      throw new RuntimeException("Didn't generate or can't find fsimage");
    }
    LOG.debug("original FS image file is " + originalFsimage);
  } finally {
    if (cluster != null)
      cluster.shutdown();
  }
}
 
Example 19
Source File: TestFSImageWithAcl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void testAcl(boolean persistNamespace) throws IOException {
  Path p = new Path("/p");
  DistributedFileSystem fs = cluster.getFileSystem();
  fs.create(p).close();
  fs.mkdirs(new Path("/23"));

  AclEntry e = new AclEntry.Builder().setName("foo")
      .setPermission(READ_EXECUTE).setScope(ACCESS).setType(USER).build();
  fs.modifyAclEntries(p, Lists.newArrayList(e));

  restart(fs, persistNamespace);

  AclStatus s = cluster.getNamesystem().getAclStatus(p.toString());
  AclEntry[] returned = Lists.newArrayList(s.getEntries()).toArray(
      new AclEntry[0]);
  Assert.assertArrayEquals(new AclEntry[] {
      aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
      aclEntry(ACCESS, GROUP, READ) }, returned);

  fs.removeAcl(p);

  if (persistNamespace) {
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  }

  cluster.restartNameNode();
  cluster.waitActive();

  s = cluster.getNamesystem().getAclStatus(p.toString());
  returned = Lists.newArrayList(s.getEntries()).toArray(new AclEntry[0]);
  Assert.assertArrayEquals(new AclEntry[] { }, returned);

  fs.modifyAclEntries(p, Lists.newArrayList(e));
  s = cluster.getNamesystem().getAclStatus(p.toString());
  returned = Lists.newArrayList(s.getEntries()).toArray(new AclEntry[0]);
  Assert.assertArrayEquals(new AclEntry[] {
      aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
      aclEntry(ACCESS, GROUP, READ) }, returned);
}