Java Code Examples for org.apache.hadoop.fs.Path#SEPARATOR_CHAR

/**
 * Get the relative path name with respect to the root of the volume.
 * @param absolutePathName The absolute path name
 * @param volume Root of the volume.
 * @return null if the absolute path name is outside of the volume.
 */
private static String getRelativePathName(String absolutePathName,
    String volume) {
  
  absolutePathName = normalizePath(absolutePathName);
  // Get the file names
  if (!absolutePathName.startsWith(volume)) {
    return null;
  }
  // Get rid of the volume prefix
  String fileName = absolutePathName.substring(volume.length());
  if (fileName.charAt(0) == Path.SEPARATOR_CHAR) {
    fileName = fileName.substring(1);
  }
  return fileName;
}
 

private static int findLongestDirPrefix(String cwd, String path, boolean isDir) {
  // add the path separator to dirs to simplify finding the longest match
  if (!cwd.endsWith(Path.SEPARATOR)) {
    cwd += Path.SEPARATOR;
  }
  if (isDir && !path.endsWith(Path.SEPARATOR)) {
    path += Path.SEPARATOR;
  }

  // find longest directory prefix 
  int len = Math.min(cwd.length(), path.length());
  int lastSep = -1;
  for (int i=0; i < len; i++) {
    if (cwd.charAt(i) != path.charAt(i)) break;
    if (cwd.charAt(i) == Path.SEPARATOR_CHAR) lastSep = i;
  }
  return lastSep;
}
 

static private Map<String, Lease> findLeaseWithPrefixPath(
    String prefix, SortedMap<String, Lease> path2lease) {
  if (LOG.isDebugEnabled()) {
    LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
  }

  final Map<String, Lease> entries = new HashMap<String, Lease>();
  int srclen = prefix.length();
  
  // prefix may ended with '/'
  if (prefix.charAt(srclen - 1) == Path.SEPARATOR_CHAR) {
    srclen -= 1;
  }

  for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
    final String p = entry.getKey();
    if (!p.startsWith(prefix)) {
      return entries;
    }
    if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
      entries.put(entry.getKey(), entry.getValue());
    }
  }
  return entries;
}
 

/**
 * Get the relative path name with respect to the root of the volume.
 * @param absolutePathName The absolute path name
 * @param volume Root of the volume.
 * @return null if the absolute path name is outside of the volume.
 */
private String getRelativePathName(String absolutePathName,
    String volume) {
  
  absolutePathName = normalizePath(absolutePathName);
  // Get the file names
  if (!absolutePathName.startsWith(volume)) {
    return null;
  }
  // Get rid of the volume prefix
  String fileName = absolutePathName.substring(volume.length());
  if (fileName.charAt(0) == Path.SEPARATOR_CHAR) {
    fileName = fileName.substring(1);
  }
  return fileName;
}
 

static private List<Map.Entry<String, Lease>> findLeaseWithPrefixPath(
    String prefix, SortedMap<String, Lease> path2lease) {
  if (LOG.isDebugEnabled()) {
    LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
  }

  List<Map.Entry<String, Lease>> entries = new ArrayList<Map.Entry<String, Lease>>();
  final int srclen = prefix.length();

  for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
    final String p = entry.getKey();
    if (!p.startsWith(prefix)) {
      return entries;
    }
    if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
      entries.add(entry);
    }
  }
  return entries;
}
 

private static int findLongestDirPrefix(String cwd, String path, boolean isDir) {
  // add the path separator to dirs to simplify finding the longest match
  if (!cwd.endsWith(Path.SEPARATOR)) {
    cwd += Path.SEPARATOR;
  }
  if (isDir && !path.endsWith(Path.SEPARATOR)) {
    path += Path.SEPARATOR;
  }

  // find longest directory prefix 
  int len = Math.min(cwd.length(), path.length());
  int lastSep = -1;
  for (int i=0; i < len; i++) {
    if (cwd.charAt(i) != path.charAt(i)) break;
    if (cwd.charAt(i) == Path.SEPARATOR_CHAR) lastSep = i;
  }
  return lastSep;
}
 

static private Map<String, Lease> findLeaseWithPrefixPath(
    String prefix, SortedMap<String, Lease> path2lease) {
  if (LOG.isDebugEnabled()) {
    LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
  }

  final Map<String, Lease> entries = new HashMap<String, Lease>();
  int srclen = prefix.length();
  
  // prefix may ended with '/'
  if (prefix.charAt(srclen - 1) == Path.SEPARATOR_CHAR) {
    srclen -= 1;
  }

  for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
    final String p = entry.getKey();
    if (!p.startsWith(prefix)) {
      return entries;
    }
    if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
      entries.put(entry.getKey(), entry.getValue());
    }
  }
  return entries;
}
 

public static String getSnapshotPath(String snapshottableDir,
    String snapshotRelativePath) {
  final StringBuilder b = new StringBuilder(snapshottableDir);
  if (b.charAt(b.length() - 1) != Path.SEPARATOR_CHAR) {
    b.append(Path.SEPARATOR);
  }
  return b.append(HdfsConstants.DOT_SNAPSHOT_DIR)
      .append(Path.SEPARATOR)
      .append(snapshotRelativePath)
      .toString();
}
 

@VisibleForTesting
String[] getPathElements(String path) {
  path = path.trim();
  if (path.charAt(0) != Path.SEPARATOR_CHAR) {
    throw new IllegalArgumentException("It must be an absolute path: " +
        path);
  }
  int numOfElements = StringUtils.countMatches(path, Path.SEPARATOR);
  if (path.length() > 1 && path.endsWith(Path.SEPARATOR)) {
    numOfElements--;
  }
  String[] pathElements = new String[numOfElements];
  int elementIdx = 0;
  int idx = 0;
  int found = path.indexOf(Path.SEPARATOR_CHAR, idx);
  while (found > -1) {
    if (found > idx) {
      pathElements[elementIdx++] = path.substring(idx, found);
    }
    idx = found + 1;
    found = path.indexOf(Path.SEPARATOR_CHAR, idx);
  }
  if (idx < path.length()) {
    pathElements[elementIdx] = path.substring(idx);
  }
  return pathElements;
}
 

/**
 * This test creates some directories inside the toBeDeleted directory and
 * then start the asyncDiskService.
 * AsyncDiskService will create tasks to delete the content inside the
 * toBeDeleted directories.
 */
public void testMRAsyncDiskServiceStartupCleaning() throws Throwable {
  FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
  String[] vols = new String[]{TEST_ROOT_DIR + "/0",
      TEST_ROOT_DIR + "/1"};

  String a = "a";
  String b = "b";
  String c = "b/c";
  String d = "d";
  
  // Create directories inside SUBDIR
  File fa = new File(vols[0] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, a);
  File fb = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, b);
  File fc = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, c);
  File fd = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, d);
  
  // Create the directories
  fa.mkdirs();
  fb.mkdirs();
  fc.mkdirs();
  fd.mkdirs();

  assertTrue(fa.exists());
  assertTrue(fb.exists());
  assertTrue(fc.exists());
  assertTrue(fd.exists());
  
  // Create the asyncDiskService which will delete all contents inside SUBDIR
  MRAsyncDiskService service = new MRAsyncDiskService(
      localFileSystem, vols);
  
  // Make sure everything is cleaned up
  makeSureCleanedUp(vols, service);
}
 

private static void validateDestination(
    String src, String dst, INode srcInode)
    throws IOException {
  String error;
  if (srcInode.isSymlink() &&
      dst.equals(srcInode.asSymlink().getSymlinkString())) {
    throw new FileAlreadyExistsException("Cannot rename symlink " + src
        + " to its target " + dst);
  }
  // dst cannot be a directory or a file under src
  if (dst.startsWith(src)
      && dst.charAt(src.length()) == Path.SEPARATOR_CHAR) {
    error = "Rename destination " + dst
        + " is a directory or file under source " + src;
    NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
        + error);
    throw new IOException(error);
  }
}
 

public static String getSnapshotPath(String snapshottableDir,
    String snapshotRelativePath) {
  final StringBuilder b = new StringBuilder(snapshottableDir);
  if (b.charAt(b.length() - 1) != Path.SEPARATOR_CHAR) {
    b.append(Path.SEPARATOR);
  }
  return b.append(HdfsConstants.DOT_SNAPSHOT_DIR)
      .append(Path.SEPARATOR)
      .append(snapshotRelativePath)
      .toString();
}
 

/**
 * This test creates some directories inside the toBeDeleted directory and
 * then start the asyncDiskService.
 * AsyncDiskService will create tasks to delete the content inside the
 * toBeDeleted directories.
 */
@Test
public void testMRAsyncDiskServiceStartupCleaning() throws Throwable {
  FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
  String[] vols = new String[]{TEST_ROOT_DIR + "/0",
      TEST_ROOT_DIR + "/1"};

  String a = "a";
  String b = "b";
  String c = "b/c";
  String d = "d";
  
  // Create directories inside SUBDIR
  String suffix = Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED;
  File fa = new File(vols[0] + suffix, a);
  File fb = new File(vols[1] + suffix, b);
  File fc = new File(vols[1] + suffix, c);
  File fd = new File(vols[1] + suffix, d);
  
  // Create the directories
  fa.mkdirs();
  fb.mkdirs();
  fc.mkdirs();
  fd.mkdirs();

  assertTrue(fa.exists());
  assertTrue(fb.exists());
  assertTrue(fc.exists());
  assertTrue(fd.exists());
  
  // Create the asyncDiskService which will delete all contents inside SUBDIR
  MRAsyncDiskService service = new MRAsyncDiskService(
      localFileSystem, vols);
  
  // Make sure everything is cleaned up
  makeSureCleanedUp(vols, service);
}
 

private static Path getDirLockFile(Path dir) {
  return new Path(dir.toString() + Path.SEPARATOR_CHAR + DIR_LOCK_FILE );
}
 

private String getCacheEntryFilePath(String checksum, String filename) {
  return SharedCacheUtil.getCacheEntryPath(this.cacheDepth,
      this.cacheRoot, checksum) + Path.SEPARATOR_CHAR + filename;
}
 

/**
 * Move the path name on one volume to a temporary location and then 
 * delete them.
 * 
 * This functions returns when the moves are done, but not necessarily all
 * deletions are done. This is usually good enough because applications 
 * won't see the path name under the old name anyway after the move. 
 * 
 * @param volume       The disk volume
 * @param pathName     The path name relative to volume root.
 * @throws IOException If the move failed 
 * @return   false     if the file is not found
 */
public boolean moveAndDeleteRelativePath(String volume, String pathName)
    throws IOException {
  
  volume = normalizePath(volume);
  
  // Move the file right now, so that it can be deleted later
  String newPathName = 
      format.format(new Date()) + "_" + uniqueId.getAndIncrement();
  newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
  
  Path source = new Path(volume, pathName);
  Path target = new Path(volume, newPathName);
  try {
    if (!localFileSystem.rename(source, target)) {
      // If the source does not exists, return false.
      // This is necessary because rename can return false if the source  
      // does not exists.
      if (!localFileSystem.exists(source)) {
        return false;
      }
      // Try to recreate the parent directory just in case it gets deleted.
      if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
        throw new IOException("Cannot create " + TOBEDELETED + " under "
            + volume);
      }
      // Try rename again. If it fails, return false.
      if (!localFileSystem.rename(source, target)) {
        throw new IOException("Cannot rename " + source + " to "
            + target);
      }
    }
  } catch (FileNotFoundException e) {
    // Return false in case that the file is not found.  
    return false;
  }

  DeleteTask task = new DeleteTask(volume, pathName, newPathName);
  execute(volume, task);
  return true;
}
 

public static byte[][] splitAndGetPathComponents(String str) {
  try {
    final int len;
    if (str == null 
        || ((len = str.length()) == 0)
        || str.charAt(0) != Path.SEPARATOR_CHAR) {
      return null;
    }

    char[] charArray = UTF8.getCharArray(len);
    str.getChars(0, len, charArray, 0);
    
    // allocate list for components
    List<byte[]> componentByteList = new ArrayList<byte[]>(20);

    // for each component to know if it has only ascii chars
    boolean canFastConvert = true;
    int startIndex = 0;

    for (int i = 0; i < len; i++) {
      if (charArray[i] == Path.SEPARATOR_CHAR) {
        componentByteList.add(extractBytes(str, startIndex, i, charArray,
            canFastConvert));
        startIndex = i + 1;
        // assume only ASCII chars
        canFastConvert = true;
      } else if (charArray[i] > UTF8.MAX_ASCII_CODE) {
        // found non-ASCII char, we will do 
        // full conversion for this component
        canFastConvert = false;
      }
    }
    // the case when the last component is a filename ("/" not at the end)
    if (charArray[len - 1] != Path.SEPARATOR_CHAR) {
      componentByteList.add(extractBytes(str, startIndex, len, charArray,
          canFastConvert));
    }
    int last = componentByteList.size(); // last split
    while (--last>=1 && componentByteList.get(last).length == 0) {
      componentByteList.remove(last);
    }
    return componentByteList.toArray(new byte[last+1][]);
  } catch (UnsupportedEncodingException e) {
    return null;
  }
}
 

/**
 * Move the path name on one volume to a temporary location and then 
 * delete them.
 * 
 * This functions returns when the moves are done, but not necessarily all
 * deletions are done. This is usually good enough because applications 
 * won't see the path name under the old name anyway after the move. 
 * 
 * @param volume       The disk volume
 * @param pathName     The path name relative to volume root.
 * @throws IOException If the move failed 
 * @return   false     if the file is not found
 */
public boolean moveAndDeleteRelativePath(String volume, String pathName)
    throws IOException {
  
  volume = normalizePath(volume);
  
  // Move the file right now, so that it can be deleted later
  String newPathName = 
      format.format(new Date()) + "_" + uniqueId.getAndIncrement();
  newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
  
  Path source = new Path(volume, pathName);
  Path target = new Path(volume, newPathName);
  try {
    if (!localFileSystem.rename(source, target)) {
      // If the source does not exists, return false.
      // This is necessary because rename can return false if the source  
      // does not exists.
      if (!localFileSystem.exists(source)) {
        return false;
      }
      // Try to recreate the parent directory just in case it gets deleted.
      if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
        throw new IOException("Cannot create " + TOBEDELETED + " under "
            + volume);
      }
      // Try rename again. If it fails, return false.
      if (!localFileSystem.rename(source, target)) {
        throw new IOException("Cannot rename " + source + " to "
            + target);
      }
    }
  } catch (FileNotFoundException e) {
    // Return false in case that the file is not found.  
    return false;
  }

  DeleteTask task = new DeleteTask(volume, pathName, newPathName);
  execute(volume, task);
  return true;
}
 

private AuthzStatus isAccessAllowedForHierarchy(INode inode, INodeAttributes inodeAttribs, String path, FsAction access, String user, Set<String> groups, RangerHdfsPlugin plugin) {
	AuthzStatus ret   = null;
	String  pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null;

	if (pathOwner == null && inode != null) {
		pathOwner = inode.getUserName();
	}

	if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) {
		path = HDFS_ROOT_FOLDER_PATH;
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + ")");
	}

	if (path != null) {

		Set<String> accessTypes = access2ActionListMapper.get(access);

		if (accessTypes == null) {
			LOG.warn("RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): no Ranger accessType found for " + access);

			accessTypes = access2ActionListMapper.get(FsAction.NONE);
		}

		String subDirPath = path;
		if (subDirPath.charAt(subDirPath.length() - 1) != Path.SEPARATOR_CHAR) {
			subDirPath = subDirPath + Character.toString(Path.SEPARATOR_CHAR);
		}
		subDirPath = subDirPath + rangerPlugin.getRandomizedWildcardPathName();

		for (String accessType : accessTypes) {
			RangerHdfsAccessRequest request = new RangerHdfsAccessRequest(null, subDirPath, pathOwner, access, accessType, user, groups);

			RangerAccessResult result = plugin.isAccessAllowed(request, null);

			if (result == null || !result.getIsAccessDetermined()) {
				ret = AuthzStatus.NOT_DETERMINED;
				// don't break yet; subsequent accessType could be denied
			} else if(! result.getIsAllowed()) { // explicit deny
				ret = AuthzStatus.DENY;
				break;
			} else { // allowed
				if(!AuthzStatus.NOT_DETERMINED.equals(ret)) { // set to ALLOW only if there was no NOT_DETERMINED earlier
					ret = AuthzStatus.ALLOW;
				}
			}
		}
	}

	if(ret == null) {
		ret = AuthzStatus.NOT_DETERMINED;
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): " + ret);
	}

	return ret;
}
 

/**
 * Move the path name on one volume to a temporary location and then 
 * delete them.
 * 
 * This functions returns when the moves are done, but not necessarily all
 * deletions are done. This is usually good enough because applications 
 * won't see the path name under the old name anyway after the move. 
 * 
 * @param volume       The disk volume
 * @param pathName     The path name relative to volume root.
 * @throws IOException If the move failed 
 * @return   false     if the file is not found
 */
public boolean moveAndDeleteRelativePath(String volume, String pathName)
    throws IOException {
  
  volume = normalizePath(volume);
  
  // Move the file right now, so that it can be deleted later
  String newPathName = 
      format.format(new Date()) + "_" + uniqueId.getAndIncrement();
  newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
  
  Path source = new Path(volume, pathName);
  Path target = new Path(volume, newPathName);
  try {
    if (!localFileSystem.rename(source, target)) {
      // If the source does not exists, return false.
      // This is necessary because rename can return false if the source  
      // does not exists.
      if (!localFileSystem.exists(source)) {
        return false;
      }
      // Try to recreate the parent directory just in case it gets deleted.
      if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
        throw new IOException("Cannot create " + TOBEDELETED + " under "
            + volume);
      }
      // Try rename again. If it fails, return false.
      if (!localFileSystem.rename(source, target)) {
        throw new IOException("Cannot rename " + source + " to "
            + target);
      }
    }
  } catch (FileNotFoundException e) {
    // Return false in case that the file is not found.  
    return false;
  }

  DeleteTask task = new DeleteTask(volume, pathName, newPathName);
  execute(volume, task);
  return true;
}