Java Code Examples for org.apache.hadoop.fs.FileSystem#setPermission()

The following examples show how to use org.apache.hadoop.fs.FileSystem#setPermission() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestDFSPermission.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAccessOthers() throws IOException, InterruptedException {
  FileSystem rootFs = FileSystem.get(conf);
  Path p3 = new Path("/p3");
  rootFs.mkdirs(p3);
  rootFs.setPermission(p3, new FsPermission((short) 0774));
  fs = USER1.doAs(new PrivilegedExceptionAction<FileSystem>() {
    @Override
    public FileSystem run() throws Exception {
      return FileSystem.get(conf);
    }
  });
  fs.access(p3, FsAction.READ);
  try {
    fs.access(p3, FsAction.READ_WRITE);
    fail("The access call should have failed.");
  } catch (AccessControlException e) {
    assertTrue("Permission denied messages must carry the username",
            e.getMessage().contains(USER1_NAME));
    assertTrue("Permission denied messages must carry the path parent",
            e.getMessage().contains(
                p3.getParent().toUri().getPath()));
  }
}
 
Example 2
Source File: HadoopIgfsSecondaryFileSystemDelegateImpl.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
@Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) {
    HadoopIgfsProperties props0 = new HadoopIgfsProperties(props);

    final FileSystem fileSys = fileSystemForUser();

    Path hadoopPath = convert(path);

    try {
        if (!fileSys.exists(hadoopPath))
            return null;

        if (props0.userName() != null || props0.groupName() != null)
            fileSys.setOwner(hadoopPath, props0.userName(), props0.groupName());

        if (props0.permission() != null)
            fileSys.setPermission(hadoopPath, props0.permission());
    }
    catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to update file properties [path=" + path + "]");
    }

    return info(path);
}
 
Example 3
Source File: TestDistCpUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static String createTestSetup(String baseDir,
                                     FileSystem fs,
                                     FsPermission perm) throws IOException {
  String base = getBase(baseDir);
  fs.mkdirs(new Path(base + "/newTest/hello/world1"));
  fs.mkdirs(new Path(base + "/newTest/hello/world2/newworld"));
  fs.mkdirs(new Path(base + "/newTest/hello/world3/oldworld"));
  fs.setPermission(new Path(base + "/newTest"), perm);
  fs.setPermission(new Path(base + "/newTest/hello"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world1"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world2"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world2/newworld"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world3"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world3/oldworld"), perm);
  createFile(fs, new Path(base, "/newTest/1"));
  createFile(fs, new Path(base, "/newTest/hello/2"));
  createFile(fs, new Path(base, "/newTest/hello/world3/oldworld/3"));
  createFile(fs, new Path(base, "/newTest/hello/world2/4"));
  return base;
}
 
Example 4
Source File: TestDistCpUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static String createTestSetup(String baseDir,
                                     FileSystem fs,
                                     FsPermission perm) throws IOException {
  String base = getBase(baseDir);
  fs.mkdirs(new Path(base + "/newTest/hello/world1"));
  fs.mkdirs(new Path(base + "/newTest/hello/world2/newworld"));
  fs.mkdirs(new Path(base + "/newTest/hello/world3/oldworld"));
  fs.setPermission(new Path(base + "/newTest"), perm);
  fs.setPermission(new Path(base + "/newTest/hello"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world1"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world2"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world2/newworld"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world3"), perm);
  fs.setPermission(new Path(base + "/newTest/hello/world3/oldworld"), perm);
  createFile(fs, new Path(base, "/newTest/1"));
  createFile(fs, new Path(base, "/newTest/hello/2"));
  createFile(fs, new Path(base, "/newTest/hello/world3/oldworld/3"));
  createFile(fs, new Path(base, "/newTest/hello/world2/4"));
  return base;
}
 
Example 5
Source File: Util.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Create a directory. */
static boolean createNonexistingDirectory(FileSystem fs, Path dir) throws IOException {
  if (fs.exists(dir)) {
    Util.err.println("dir (= " + dir + ") already exists.");
    return false;
  } else if (!fs.mkdirs(dir)) {
    throw new IOException("Cannot create working directory " + dir);
  }
  fs.setPermission(dir, new FsPermission((short)0777));
  return true;
}
 
Example 6
Source File: TestDistCpUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveGroupOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 7
Source File: TestDistCpUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveTimestampOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 8
Source File: TestDistCpUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testPreservePermissionOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
}
 
Example 9
Source File: JobPreparationHelper.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
public static void setDirPermission(FileSystem fileSystem, Path dirPath, String defaultPermissionsMask)
    throws IOException {
  if (defaultPermissionsMask != null) {
    FsPermission permission = FsPermission.getDirDefault().applyUMask(new FsPermission(defaultPermissionsMask));
    _logger.info("Setting permission: {} to directory: {}", permission, dirPath);
    fileSystem.setPermission(dirPath, permission);
  }
}
 
Example 10
Source File: TestMiniMRWithDFSWithDistinctUsers.java    From hadoop with Apache License 2.0 5 votes vote down vote up
static void mkdir(FileSystem fs, String dir,
                  String user, String group, short mode) throws IOException {
  Path p = new Path(dir);
  fs.mkdirs(p);
  fs.setPermission(p, new FsPermission(mode));
  fs.setOwner(p, user, group);
}
 
Example 11
Source File: DistCp.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private static void updateDestStatus(FileStatus src, FileStatus dst,
    EnumSet<FileAttribute> preserved, FileSystem destFileSys
    ) throws IOException {
  String owner = null;
  String group = null;
  if (preserved.contains(FileAttribute.USER)
      && !src.getOwner().equals(dst.getOwner())) {
    owner = src.getOwner();
  }
  if (preserved.contains(FileAttribute.GROUP)
      && !src.getGroup().equals(dst.getGroup())) {
    group = src.getGroup();
  }
  if (owner != null || group != null) {
    destFileSys.setOwner(dst.getPath(), owner, group);
  }
  if (preserved.contains(FileAttribute.PERMISSION)
      && !src.getPermission().equals(dst.getPermission())) {
    destFileSys.setPermission(dst.getPath(), src.getPermission());
  }
  if (preserved.contains(FileAttribute.TIMES)) {
    try {
      destFileSys.setTimes(dst.getPath(), src.getModificationTime(), src.getAccessTime());
    } catch (IOException exc) {
      if (!dst.isDir()) { //hadoop 0.20 doesn't allow setTimes on dirs
        throw exc;
      }
    }
  }
}
 
Example 12
Source File: LogAggregationService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private boolean checkExists(FileSystem fs, Path path, FsPermission fsPerm)
    throws IOException {
  boolean exists = true;
  try {
    FileStatus appDirStatus = fs.getFileStatus(path);
    if (!APP_DIR_PERMISSIONS.equals(appDirStatus.getPermission())) {
      fs.setPermission(path, APP_DIR_PERMISSIONS);
    }
  } catch (FileNotFoundException fnfe) {
    exists = false;
  }
  return exists;
}
 
Example 13
Source File: LogAggregationService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void createDir(FileSystem fs, Path path, FsPermission fsPerm)
    throws IOException {
  FsPermission dirPerm = new FsPermission(fsPerm);
  fs.mkdirs(path, dirPerm);
  FsPermission umask = FsPermission.getUMask(fs.getConf());
  if (!dirPerm.equals(dirPerm.applyUMask(umask))) {
    fs.setPermission(path, new FsPermission(fsPerm));
  }
}
 
Example 14
Source File: TestDistCpUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveTimestampOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
}
 
Example 15
Source File: IIBulkLoadJob.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    try {
        options.addOption(OPTION_INPUT_PATH);
        options.addOption(OPTION_HTABLE_NAME);
        options.addOption(OPTION_II_NAME);
        parseOptions(options, args);

        String tableName = getOptionValue(OPTION_HTABLE_NAME);
        String input = getOptionValue(OPTION_INPUT_PATH);
        String iiname = getOptionValue(OPTION_II_NAME);

        FileSystem fs = FileSystem.get(getConf());
        FsPermission permission = new FsPermission((short) 0777);
        fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission);

        int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName });

        IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
        IIInstance ii = mgr.getII(iiname);
        IISegment seg = ii.getFirstSegment();
        seg.setStorageLocationIdentifier(tableName);
        seg.setStatus(SegmentStatusEnum.READY);
        mgr.updateII(ii);

        return hbaseExitCode;

    } catch (Exception e) {
        printUsage(options);
        throw e;
    }
}
 
Example 16
Source File: JobControlCompiler.java    From spork with Apache License 2.0 5 votes vote down vote up
private static Path getCacheStagingDir(Configuration conf) throws IOException {
    String pigTempDir = conf.get(PigConfiguration.PIG_USER_CACHE_LOCATION,
            conf.get(PigConfiguration.PIG_TEMP_DIR, "/tmp"));
    String currentUser = System.getProperty("user.name");
    Path stagingDir = new Path(pigTempDir + "/" + currentUser + "/", ".pigcache");
    FileSystem fs = FileSystem.get(conf);
    fs.mkdirs(stagingDir);
    fs.setPermission(stagingDir, FileLocalizer.OWNER_ONLY_PERMS);
    return stagingDir;
}
 
Example 17
Source File: TestSaveNamespace.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Verify that a saveNamespace command brings faulty directories
 * in fs.name.dir and fs.edit.dir back online.
 */
@Test (timeout=30000)
public void testReinsertnamedirsInSavenamespace() throws Exception {
  // create a configuration with the key to restore error
  // directories in fs.name.dir
  Configuration conf = getConf();
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, true);

  NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);

  // Replace the FSImage with a spy
  FSImage originalImage = fsn.getFSImage();
  NNStorage storage = originalImage.getStorage();

  FSImage spyImage = spy(originalImage);
  Whitebox.setInternalState(fsn, "fsImage", spyImage);
  
  FileSystem fs = FileSystem.getLocal(conf);
  File rootDir = storage.getStorageDir(0).getRoot();
  Path rootPath = new Path(rootDir.getPath(), "current");
  final FsPermission permissionNone = new FsPermission((short) 0);
  final FsPermission permissionAll = new FsPermission(
      FsAction.ALL, FsAction.READ_EXECUTE, FsAction.READ_EXECUTE);
  fs.setPermission(rootPath, permissionNone);

  try {
    doAnEdit(fsn, 1);
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);

    // Save namespace - should mark the first storage dir as faulty
    // since it's not traversable.
    LOG.info("Doing the first savenamespace.");
    fsn.saveNamespace();
    LOG.info("First savenamespace sucessful.");      
    
    assertTrue("Savenamespace should have marked one directory as bad." +
               " But found " + storage.getRemovedStorageDirs().size() +
               " bad directories.", 
                 storage.getRemovedStorageDirs().size() == 1);

    fs.setPermission(rootPath, permissionAll);

    // The next call to savenamespace should try inserting the
    // erroneous directory back to fs.name.dir. This command should
    // be successful.
    LOG.info("Doing the second savenamespace.");
    fsn.saveNamespace();
    LOG.warn("Second savenamespace sucessful.");
    assertTrue("Savenamespace should have been successful in removing " +
               " bad directories from Image."  +
               " But found " + storage.getRemovedStorageDirs().size() +
               " bad directories.", 
               storage.getRemovedStorageDirs().size() == 0);

    // Now shut down and restart the namesystem
    LOG.info("Shutting down fsimage.");
    originalImage.close();
    fsn.close();      
    fsn = null;

    // Start a new namesystem, which should be able to recover
    // the namespace from the previous incarnation.
    LOG.info("Loading new FSmage from disk.");
    fsn = FSNamesystem.loadFromDisk(conf);

    // Make sure the image loaded including our edit.
    LOG.info("Checking reloaded image.");
    checkEditExists(fsn, 1);
    LOG.info("Reloaded image is good.");
  } finally {
    if (rootDir.exists()) {
      fs.setPermission(rootPath, permissionAll);
    }

    if (fsn != null) {
      try {
        fsn.close();
      } catch (Throwable t) {
        LOG.fatal("Failed to shut down", t);
      }
    }
  }
}
 
Example 18
Source File: TezSpillRecord.java    From tez with Apache License 2.0 4 votes vote down vote up
public static void ensureSpillFilePermissions(Path loc, FileSystem rfs) throws IOException {
  if (!SPILL_FILE_PERMS.equals(SPILL_FILE_PERMS.applyUMask(FsPermission.getUMask(rfs.getConf())))) {
    rfs.setPermission(loc, SPILL_FILE_PERMS);
  }
}
 
Example 19
Source File: TestMiniMRWithDFSWithDistinctUsers.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
static void mkdir(FileSystem fs, String dir) throws IOException {
  Path p = new Path(dir);
  fs.mkdirs(p);
  fs.setPermission(p, new FsPermission((short)0777));
}
 
Example 20
Source File: FSUtil.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up
/**
   * Copied from FileUtil to transfer ownership
   *
   * @param srcFS
   * @param srcStatus
   * @param dstFS
   * @param dst
   * @param deleteSource
   * @param overwrite
   * @param conf
   * @return
   * @throws IOException
   */
  public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
      FileSystem dstFS, Path dst,
      boolean deleteSource,
      boolean overwrite,
      Configuration conf) throws IOException
  {
    Path src = srcStatus.getPath();
    //dst = checkDest(src.getName(), dstFS, dst, overwrite);
    if (srcStatus.isDirectory()) {
      //checkDependencies(srcFS, src, dstFS, dst);
      if (!mkdirs(dstFS, dst)) {
        return false;
      }

      FileStatus[] contents = srcFS.listStatus(src);
      for (int i = 0; i < contents.length; i++) {
        copy(srcFS, contents[i], dstFS,
            new Path(dst, contents[i].getPath().getName()),
            deleteSource, overwrite, conf);
      }
    } else {
      try (InputStream in = srcFS.open(src);
          OutputStream out = dstFS.create(dst, overwrite)) {
        org.apache.hadoop.io.IOUtils.copyBytes(in, out, conf, true);
      }
    }

    // TODO: change group and limit write to group
    if (srcStatus.isDirectory()) {
      dstFS.setPermission(dst, new FsPermission((short)0777));
    } else {
      dstFS.setPermission(dst, new FsPermission((short)0777)/*"ugo+w"*/);
    }
    //dstFS.setOwner(dst, null, srcStatus.getGroup());

/*
    try {
      // transfer owner
      // DOES NOT WORK only super user can change file owner
      dstFS.setOwner(dst, srcStatus.getOwner(), srcStatus.getGroup());
    } catch (IOException e) {
      LOG.warn("Failed to change owner on {} to {}", dst, srcStatus.getOwner(), e);
      throw e;
    }
*/
    if (deleteSource) {
      return srcFS.delete(src, true);
    } else {
      return true;
    }

  }