Java Code Examples for org.apache.hadoop.fs.Path#SEPARATOR_CHAR

The following examples show how to use org.apache.hadoop.fs.Path#SEPARATOR_CHAR . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: LeaseManager.java    License: Apache License 2.0 6 votes vote down vote up
static private Map<String, Lease> findLeaseWithPrefixPath(
    String prefix, SortedMap<String, Lease> path2lease) {
  if (LOG.isDebugEnabled()) {
    LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
  }

  final Map<String, Lease> entries = new HashMap<String, Lease>();
  int srclen = prefix.length();
  
  // prefix may ended with '/'
  if (prefix.charAt(srclen - 1) == Path.SEPARATOR_CHAR) {
    srclen -= 1;
  }

  for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
    final String p = entry.getKey();
    if (!p.startsWith(prefix)) {
      return entries;
    }
    if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
      entries.put(entry.getKey(), entry.getValue());
    }
  }
  return entries;
}
 
Example 2
Source Project: hadoop   File: PathData.java    License: Apache License 2.0 6 votes vote down vote up
private static int findLongestDirPrefix(String cwd, String path, boolean isDir) {
  // add the path separator to dirs to simplify finding the longest match
  if (!cwd.endsWith(Path.SEPARATOR)) {
    cwd += Path.SEPARATOR;
  }
  if (isDir && !path.endsWith(Path.SEPARATOR)) {
    path += Path.SEPARATOR;
  }

  // find longest directory prefix 
  int len = Math.min(cwd.length(), path.length());
  int lastSep = -1;
  for (int i=0; i < len; i++) {
    if (cwd.charAt(i) != path.charAt(i)) break;
    if (cwd.charAt(i) == Path.SEPARATOR_CHAR) lastSep = i;
  }
  return lastSep;
}
 
Example 3
Source Project: RDFS   File: MRAsyncDiskService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get the relative path name with respect to the root of the volume.
 * @param absolutePathName The absolute path name
 * @param volume Root of the volume.
 * @return null if the absolute path name is outside of the volume.
 */
private static String getRelativePathName(String absolutePathName,
    String volume) {
  
  absolutePathName = normalizePath(absolutePathName);
  // Get the file names
  if (!absolutePathName.startsWith(volume)) {
    return null;
  }
  // Get rid of the volume prefix
  String fileName = absolutePathName.substring(volume.length());
  if (fileName.charAt(0) == Path.SEPARATOR_CHAR) {
    fileName = fileName.substring(1);
  }
  return fileName;
}
 
Example 4
Source Project: hadoop-gpu   File: LeaseManager.java    License: Apache License 2.0 6 votes vote down vote up
static private List<Map.Entry<String, Lease>> findLeaseWithPrefixPath(
    String prefix, SortedMap<String, Lease> path2lease) {
  if (LOG.isDebugEnabled()) {
    LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
  }

  List<Map.Entry<String, Lease>> entries = new ArrayList<Map.Entry<String, Lease>>();
  final int srclen = prefix.length();

  for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
    final String p = entry.getKey();
    if (!p.startsWith(prefix)) {
      return entries;
    }
    if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
      entries.add(entry);
    }
  }
  return entries;
}
 
Example 5
Source Project: big-c   File: MRAsyncDiskService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get the relative path name with respect to the root of the volume.
 * @param absolutePathName The absolute path name
 * @param volume Root of the volume.
 * @return null if the absolute path name is outside of the volume.
 */
private String getRelativePathName(String absolutePathName,
    String volume) {
  
  absolutePathName = normalizePath(absolutePathName);
  // Get the file names
  if (!absolutePathName.startsWith(volume)) {
    return null;
  }
  // Get rid of the volume prefix
  String fileName = absolutePathName.substring(volume.length());
  if (fileName.charAt(0) == Path.SEPARATOR_CHAR) {
    fileName = fileName.substring(1);
  }
  return fileName;
}
 
Example 6
Source Project: big-c   File: LeaseManager.java    License: Apache License 2.0 6 votes vote down vote up
static private Map<String, Lease> findLeaseWithPrefixPath(
    String prefix, SortedMap<String, Lease> path2lease) {
  if (LOG.isDebugEnabled()) {
    LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
  }

  final Map<String, Lease> entries = new HashMap<String, Lease>();
  int srclen = prefix.length();
  
  // prefix may ended with '/'
  if (prefix.charAt(srclen - 1) == Path.SEPARATOR_CHAR) {
    srclen -= 1;
  }

  for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
    final String p = entry.getKey();
    if (!p.startsWith(prefix)) {
      return entries;
    }
    if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
      entries.put(entry.getKey(), entry.getValue());
    }
  }
  return entries;
}
 
Example 7
Source Project: big-c   File: PathData.java    License: Apache License 2.0 6 votes vote down vote up
private static int findLongestDirPrefix(String cwd, String path, boolean isDir) {
  // add the path separator to dirs to simplify finding the longest match
  if (!cwd.endsWith(Path.SEPARATOR)) {
    cwd += Path.SEPARATOR;
  }
  if (isDir && !path.endsWith(Path.SEPARATOR)) {
    path += Path.SEPARATOR;
  }

  // find longest directory prefix 
  int len = Math.min(cwd.length(), path.length());
  int lastSep = -1;
  for (int i=0; i < len; i++) {
    if (cwd.charAt(i) != path.charAt(i)) break;
    if (cwd.charAt(i) == Path.SEPARATOR_CHAR) lastSep = i;
  }
  return lastSep;
}
 
Example 8
Source Project: hadoop   File: TestMRAsyncDiskService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test creates some directories inside the toBeDeleted directory and
 * then start the asyncDiskService.
 * AsyncDiskService will create tasks to delete the content inside the
 * toBeDeleted directories.
 */
@Test
public void testMRAsyncDiskServiceStartupCleaning() throws Throwable {
  FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
  String[] vols = new String[]{TEST_ROOT_DIR + "/0",
      TEST_ROOT_DIR + "/1"};

  String a = "a";
  String b = "b";
  String c = "b/c";
  String d = "d";
  
  // Create directories inside SUBDIR
  String suffix = Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED;
  File fa = new File(vols[0] + suffix, a);
  File fb = new File(vols[1] + suffix, b);
  File fc = new File(vols[1] + suffix, c);
  File fd = new File(vols[1] + suffix, d);
  
  // Create the directories
  fa.mkdirs();
  fb.mkdirs();
  fc.mkdirs();
  fd.mkdirs();

  assertTrue(fa.exists());
  assertTrue(fb.exists());
  assertTrue(fc.exists());
  assertTrue(fd.exists());
  
  // Create the asyncDiskService which will delete all contents inside SUBDIR
  MRAsyncDiskService service = new MRAsyncDiskService(
      localFileSystem, vols);
  
  // Make sure everything is cleaned up
  makeSureCleanedUp(vols, service);
}
 
Example 9
Source Project: RDFS   File: TestMRAsyncDiskService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test creates some directories inside the toBeDeleted directory and
 * then start the asyncDiskService.
 * AsyncDiskService will create tasks to delete the content inside the
 * toBeDeleted directories.
 */
public void testMRAsyncDiskServiceStartupCleaning() throws Throwable {
  FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
  String[] vols = new String[]{TEST_ROOT_DIR + "/0",
      TEST_ROOT_DIR + "/1"};

  String a = "a";
  String b = "b";
  String c = "b/c";
  String d = "d";
  
  // Create directories inside SUBDIR
  File fa = new File(vols[0] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, a);
  File fb = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, b);
  File fc = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, c);
  File fd = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, d);
  
  // Create the directories
  fa.mkdirs();
  fb.mkdirs();
  fc.mkdirs();
  fd.mkdirs();

  assertTrue(fa.exists());
  assertTrue(fb.exists());
  assertTrue(fc.exists());
  assertTrue(fd.exists());
  
  // Create the asyncDiskService which will delete all contents inside SUBDIR
  MRAsyncDiskService service = new MRAsyncDiskService(
      localFileSystem, vols);
  
  // Make sure everything is cleaned up
  makeSureCleanedUp(vols, service);
}
 
Example 10
Source Project: hadoop   File: Snapshot.java    License: Apache License 2.0 5 votes vote down vote up
public static String getSnapshotPath(String snapshottableDir,
    String snapshotRelativePath) {
  final StringBuilder b = new StringBuilder(snapshottableDir);
  if (b.charAt(b.length() - 1) != Path.SEPARATOR_CHAR) {
    b.append(Path.SEPARATOR);
  }
  return b.append(HdfsConstants.DOT_SNAPSHOT_DIR)
      .append(Path.SEPARATOR)
      .append(snapshotRelativePath)
      .toString();
}
 
Example 11
Source Project: hadoop   File: INodeAttributeProvider.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
String[] getPathElements(String path) {
  path = path.trim();
  if (path.charAt(0) != Path.SEPARATOR_CHAR) {
    throw new IllegalArgumentException("It must be an absolute path: " +
        path);
  }
  int numOfElements = StringUtils.countMatches(path, Path.SEPARATOR);
  if (path.length() > 1 && path.endsWith(Path.SEPARATOR)) {
    numOfElements--;
  }
  String[] pathElements = new String[numOfElements];
  int elementIdx = 0;
  int idx = 0;
  int found = path.indexOf(Path.SEPARATOR_CHAR, idx);
  while (found > -1) {
    if (found > idx) {
      pathElements[elementIdx++] = path.substring(idx, found);
    }
    idx = found + 1;
    found = path.indexOf(Path.SEPARATOR_CHAR, idx);
  }
  if (idx < path.length()) {
    pathElements[elementIdx] = path.substring(idx);
  }
  return pathElements;
}
 
Example 12
Source Project: big-c   File: FSDirRenameOp.java    License: Apache License 2.0 5 votes vote down vote up
private static void validateDestination(
    String src, String dst, INode srcInode)
    throws IOException {
  String error;
  if (srcInode.isSymlink() &&
      dst.equals(srcInode.asSymlink().getSymlinkString())) {
    throw new FileAlreadyExistsException("Cannot rename symlink " + src
        + " to its target " + dst);
  }
  // dst cannot be a directory or a file under src
  if (dst.startsWith(src)
      && dst.charAt(src.length()) == Path.SEPARATOR_CHAR) {
    error = "Rename destination " + dst
        + " is a directory or file under source " + src;
    NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
        + error);
    throw new IOException(error);
  }
}
 
Example 13
Source Project: big-c   File: Snapshot.java    License: Apache License 2.0 5 votes vote down vote up
public static String getSnapshotPath(String snapshottableDir,
    String snapshotRelativePath) {
  final StringBuilder b = new StringBuilder(snapshottableDir);
  if (b.charAt(b.length() - 1) != Path.SEPARATOR_CHAR) {
    b.append(Path.SEPARATOR);
  }
  return b.append(HdfsConstants.DOT_SNAPSHOT_DIR)
      .append(Path.SEPARATOR)
      .append(snapshotRelativePath)
      .toString();
}
 
Example 14
Source Project: hadoop   File: ClientProtocolService.java    License: Apache License 2.0 4 votes vote down vote up
private String getCacheEntryFilePath(String checksum, String filename) {
  return SharedCacheUtil.getCacheEntryPath(this.cacheDepth,
      this.cacheRoot, checksum) + Path.SEPARATOR_CHAR + filename;
}
 
Example 15
Source Project: hadoop   File: MRAsyncDiskService.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Move the path name on one volume to a temporary location and then 
 * delete them.
 * 
 * This functions returns when the moves are done, but not necessarily all
 * deletions are done. This is usually good enough because applications 
 * won't see the path name under the old name anyway after the move. 
 * 
 * @param volume       The disk volume
 * @param pathName     The path name relative to volume root.
 * @throws IOException If the move failed 
 * @return   false     if the file is not found
 */
public boolean moveAndDeleteRelativePath(String volume, String pathName)
    throws IOException {
  
  volume = normalizePath(volume);
  
  // Move the file right now, so that it can be deleted later
  String newPathName = 
      format.format(new Date()) + "_" + uniqueId.getAndIncrement();
  newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
  
  Path source = new Path(volume, pathName);
  Path target = new Path(volume, newPathName);
  try {
    if (!localFileSystem.rename(source, target)) {
      // If the source does not exists, return false.
      // This is necessary because rename can return false if the source  
      // does not exists.
      if (!localFileSystem.exists(source)) {
        return false;
      }
      // Try to recreate the parent directory just in case it gets deleted.
      if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
        throw new IOException("Cannot create " + TOBEDELETED + " under "
            + volume);
      }
      // Try rename again. If it fails, return false.
      if (!localFileSystem.rename(source, target)) {
        throw new IOException("Cannot rename " + source + " to "
            + target);
      }
    }
  } catch (FileNotFoundException e) {
    // Return false in case that the file is not found.  
    return false;
  }

  DeleteTask task = new DeleteTask(volume, pathName, newPathName);
  execute(volume, task);
  return true;
}
 
Example 16
Source Project: RDFS   File: DFSUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static byte[][] splitAndGetPathComponents(String str) {
  try {
    final int len;
    if (str == null 
        || ((len = str.length()) == 0)
        || str.charAt(0) != Path.SEPARATOR_CHAR) {
      return null;
    }

    char[] charArray = UTF8.getCharArray(len);
    str.getChars(0, len, charArray, 0);
    
    // allocate list for components
    List<byte[]> componentByteList = new ArrayList<byte[]>(20);

    // for each component to know if it has only ascii chars
    boolean canFastConvert = true;
    int startIndex = 0;

    for (int i = 0; i < len; i++) {
      if (charArray[i] == Path.SEPARATOR_CHAR) {
        componentByteList.add(extractBytes(str, startIndex, i, charArray,
            canFastConvert));
        startIndex = i + 1;
        // assume only ASCII chars
        canFastConvert = true;
      } else if (charArray[i] > UTF8.MAX_ASCII_CODE) {
        // found non-ASCII char, we will do 
        // full conversion for this component
        canFastConvert = false;
      }
    }
    // the case when the last component is a filename ("/" not at the end)
    if (charArray[len - 1] != Path.SEPARATOR_CHAR) {
      componentByteList.add(extractBytes(str, startIndex, len, charArray,
          canFastConvert));
    }
    int last = componentByteList.size(); // last split
    while (--last>=1 && componentByteList.get(last).length == 0) {
      componentByteList.remove(last);
    }
    return componentByteList.toArray(new byte[last+1][]);
  } catch (UnsupportedEncodingException e) {
    return null;
  }
}
 
Example 17
Source Project: jstorm   File: DirLock.java    License: Apache License 2.0 4 votes vote down vote up
private static Path getDirLockFile(Path dir) {
  return new Path(dir.toString() + Path.SEPARATOR_CHAR + DIR_LOCK_FILE );
}
 
Example 18
Source Project: big-c   File: MRAsyncDiskService.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Move the path name on one volume to a temporary location and then 
 * delete them.
 * 
 * This functions returns when the moves are done, but not necessarily all
 * deletions are done. This is usually good enough because applications 
 * won't see the path name under the old name anyway after the move. 
 * 
 * @param volume       The disk volume
 * @param pathName     The path name relative to volume root.
 * @throws IOException If the move failed 
 * @return   false     if the file is not found
 */
public boolean moveAndDeleteRelativePath(String volume, String pathName)
    throws IOException {
  
  volume = normalizePath(volume);
  
  // Move the file right now, so that it can be deleted later
  String newPathName = 
      format.format(new Date()) + "_" + uniqueId.getAndIncrement();
  newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
  
  Path source = new Path(volume, pathName);
  Path target = new Path(volume, newPathName);
  try {
    if (!localFileSystem.rename(source, target)) {
      // If the source does not exists, return false.
      // This is necessary because rename can return false if the source  
      // does not exists.
      if (!localFileSystem.exists(source)) {
        return false;
      }
      // Try to recreate the parent directory just in case it gets deleted.
      if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
        throw new IOException("Cannot create " + TOBEDELETED + " under "
            + volume);
      }
      // Try rename again. If it fails, return false.
      if (!localFileSystem.rename(source, target)) {
        throw new IOException("Cannot rename " + source + " to "
            + target);
      }
    }
  } catch (FileNotFoundException e) {
    // Return false in case that the file is not found.  
    return false;
  }

  DeleteTask task = new DeleteTask(volume, pathName, newPathName);
  execute(volume, task);
  return true;
}
 
Example 19
Source Project: ranger   File: RangerHdfsAuthorizer.java    License: Apache License 2.0 4 votes vote down vote up
private AuthzStatus isAccessAllowedForHierarchy(INode inode, INodeAttributes inodeAttribs, String path, FsAction access, String user, Set<String> groups, RangerHdfsPlugin plugin) {
	AuthzStatus ret   = null;
	String  pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null;

	if (pathOwner == null && inode != null) {
		pathOwner = inode.getUserName();
	}

	if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) {
		path = HDFS_ROOT_FOLDER_PATH;
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + ")");
	}

	if (path != null) {

		Set<String> accessTypes = access2ActionListMapper.get(access);

		if (accessTypes == null) {
			LOG.warn("RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): no Ranger accessType found for " + access);

			accessTypes = access2ActionListMapper.get(FsAction.NONE);
		}

		String subDirPath = path;
		if (subDirPath.charAt(subDirPath.length() - 1) != Path.SEPARATOR_CHAR) {
			subDirPath = subDirPath + Character.toString(Path.SEPARATOR_CHAR);
		}
		subDirPath = subDirPath + rangerPlugin.getRandomizedWildcardPathName();

		for (String accessType : accessTypes) {
			RangerHdfsAccessRequest request = new RangerHdfsAccessRequest(null, subDirPath, pathOwner, access, accessType, user, groups);

			RangerAccessResult result = plugin.isAccessAllowed(request, null);

			if (result == null || !result.getIsAccessDetermined()) {
				ret = AuthzStatus.NOT_DETERMINED;
				// don't break yet; subsequent accessType could be denied
			} else if(! result.getIsAllowed()) { // explicit deny
				ret = AuthzStatus.DENY;
				break;
			} else { // allowed
				if(!AuthzStatus.NOT_DETERMINED.equals(ret)) { // set to ALLOW only if there was no NOT_DETERMINED earlier
					ret = AuthzStatus.ALLOW;
				}
			}
		}
	}

	if(ret == null) {
		ret = AuthzStatus.NOT_DETERMINED;
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): " + ret);
	}

	return ret;
}
 
Example 20
Source Project: RDFS   File: MRAsyncDiskService.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Move the path name on one volume to a temporary location and then 
 * delete them.
 * 
 * This functions returns when the moves are done, but not necessarily all
 * deletions are done. This is usually good enough because applications 
 * won't see the path name under the old name anyway after the move. 
 * 
 * @param volume       The disk volume
 * @param pathName     The path name relative to volume root.
 * @throws IOException If the move failed 
 * @return   false     if the file is not found
 */
public boolean moveAndDeleteRelativePath(String volume, String pathName)
    throws IOException {
  
  volume = normalizePath(volume);
  
  // Move the file right now, so that it can be deleted later
  String newPathName = 
      format.format(new Date()) + "_" + uniqueId.getAndIncrement();
  newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
  
  Path source = new Path(volume, pathName);
  Path target = new Path(volume, newPathName);
  try {
    if (!localFileSystem.rename(source, target)) {
      // If the source does not exists, return false.
      // This is necessary because rename can return false if the source  
      // does not exists.
      if (!localFileSystem.exists(source)) {
        return false;
      }
      // Try to recreate the parent directory just in case it gets deleted.
      if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
        throw new IOException("Cannot create " + TOBEDELETED + " under "
            + volume);
      }
      // Try rename again. If it fails, return false.
      if (!localFileSystem.rename(source, target)) {
        throw new IOException("Cannot rename " + source + " to "
            + target);
      }
    }
  } catch (FileNotFoundException e) {
    // Return false in case that the file is not found.  
    return false;
  }

  DeleteTask task = new DeleteTask(volume, pathName, newPathName);
  execute(volume, task);
  return true;
}