Java Code Examples for org.apache.hadoop.fs.AbstractFileSystem

The following examples show how to use org.apache.hadoop.fs.AbstractFileSystem. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: ChRootedFs.java    License: Apache License 2.0 6 votes vote down vote up
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
  throws URISyntaxException {
  super(fs.getUri(), fs.getUri().getScheme(),
      fs.getUri().getAuthority() != null, fs.getUriDefaultPort());
  myFs = fs;
  myFs.checkPath(theRoot);
  chRootPathPart = new Path(myFs.getUriPath(theRoot));
  chRootPathPartString = chRootPathPart.toUri().getPath();
  /*
   * We are making URI include the chrootedPath: e.g. file:///chrootedPath.
   * This is questionable since Path#makeQualified(uri, path) ignores
   * the pathPart of a uri. Since this class is internal we can ignore
   * this issue but if we were to make it external then this needs
   * to be resolved.
   */
  // Handle the two cases:
  //              scheme:/// and scheme://authority/
  myUri = new URI(myFs.getUri().toString() + 
      (myFs.getUri().getAuthority() == null ? "" :  Path.SEPARATOR) +
        chRootPathPart.toUri().getPath().substring(1));
  super.checkPath(theRoot);
}
 
Example 2
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 
Example 3
Source Project: big-c   Source File: ViewFsBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetFileChecksum() throws AccessControlException
  , UnresolvedLinkException, IOException {
  AbstractFileSystem mockAFS = Mockito.mock(AbstractFileSystem.class);
  InodeTree.ResolveResult<AbstractFileSystem> res =
    new InodeTree.ResolveResult<AbstractFileSystem>(null, mockAFS , null,
      new Path("someFile"));
  @SuppressWarnings("unchecked")
  InodeTree<AbstractFileSystem> fsState = Mockito.mock(InodeTree.class);
  Mockito.when(fsState.resolve(Mockito.anyString()
    , Mockito.anyBoolean())).thenReturn(res);
  ViewFs vfs = Mockito.mock(ViewFs.class);
  vfs.fsState = fsState;

  Mockito.when(vfs.getFileChecksum(new Path("/tmp/someFile")))
    .thenCallRealMethod();
  vfs.getFileChecksum(new Path("/tmp/someFile"));

  Mockito.verify(mockAFS).getFileChecksum(new Path("someFile"));
}
 
Example 4
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus getFileStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);

  //  FileStatus#getPath is a fully qualified path relative to the root of 
  // target file system.
  // We need to change it to viewfs URI - relative to root of mount table.
  
  // The implementors of RawLocalFileSystem were trying to be very smart.
  // They implement FileStatus#getOwener lazily -- the object
  // returned is really a RawLocalFileSystem that expect the
  // FileStatus#getPath to be unchanged so that it can get owner when needed.
  // Hence we need to interpose a new ViewFsFileStatus that works around.
  
  
  FileStatus status =  res.targetFileSystem.getFileStatus(res.remainingPath);
  return new ViewFsFileStatus(status, this.makeQualified(f));
}
 
Example 5
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  
  FileStatus[] statusLst = res.targetFileSystem.listStatus(res.remainingPath);
  if (!res.isInternalDir()) {
    // We need to change the name in the FileStatus as described in
    // {@link #getFileStatus }
    ChRootedFs targetFs;
    targetFs = (ChRootedFs) res.targetFileSystem;
    int i = 0;
    for (FileStatus status : statusLst) {
        String suffix = targetFs.stripOutRoot(status.getPath());
        statusLst[i++] = new ViewFsFileStatus(status, this.makeQualified(
            suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  }
  return statusLst;
}
 
Example 6
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void createSymlink(final Path target, final Path link,
    final boolean createParent) throws IOException, UnresolvedLinkException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(link), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("createSymlink", link);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  res.targetFileSystem.createSymlink(target, res.remainingPath,
      createParent);  
}
 
Example 7
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
  List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
              fsState.getMountPoints();
  int initialListSize  = 0;
  for (InodeTree.MountPoint<AbstractFileSystem> im : mountPoints) {
    initialListSize += im.target.targetDirLinkList.length; 
  }
  List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
  for ( int i = 0; i < mountPoints.size(); ++i ) {
    List<Token<?>> tokens = 
      mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer);
    if (tokens != null) {
      result.addAll(tokens);
    }
  }
  return result;
}
 
Example 8
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
    primaryFsUri = new URI(primaryFileSystemUriPath());

    primaryFsCfg = new Configuration();

    primaryFsCfg.addResource(U.resolveIgniteUrl(primaryFileSystemConfigPath()));

    UserGroupInformation ugi = UserGroupInformation.getBestUGI(null, getClientFsUser());

    // Create Fs on behalf of the client user:
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
        @Override public Object run() throws Exception {
            fs = AbstractFileSystem.get(primaryFsUri, primaryFsCfg);

            return null;
        }
    });

    barrier = new CyclicBarrier(THREAD_CNT);
}
 
Example 9
Source Project: hadoop   Source File: ViewFsBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetFileChecksum() throws AccessControlException
  , UnresolvedLinkException, IOException {
  AbstractFileSystem mockAFS = Mockito.mock(AbstractFileSystem.class);
  InodeTree.ResolveResult<AbstractFileSystem> res =
    new InodeTree.ResolveResult<AbstractFileSystem>(null, mockAFS , null,
      new Path("someFile"));
  @SuppressWarnings("unchecked")
  InodeTree<AbstractFileSystem> fsState = Mockito.mock(InodeTree.class);
  Mockito.when(fsState.resolve(Mockito.anyString()
    , Mockito.anyBoolean())).thenReturn(res);
  ViewFs vfs = Mockito.mock(ViewFs.class);
  vfs.fsState = fsState;

  Mockito.when(vfs.getFileChecksum(new Path("/tmp/someFile")))
    .thenCallRealMethod();
  vfs.getFileChecksum(new Path("/tmp/someFile"));

  Mockito.verify(mockAFS).getFileChecksum(new Path("someFile"));
}
 
Example 10
Source Project: big-c   Source File: ChRootedFs.java    License: Apache License 2.0 6 votes vote down vote up
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
  throws URISyntaxException {
  super(fs.getUri(), fs.getUri().getScheme(),
      fs.getUri().getAuthority() != null, fs.getUriDefaultPort());
  myFs = fs;
  myFs.checkPath(theRoot);
  chRootPathPart = new Path(myFs.getUriPath(theRoot));
  chRootPathPartString = chRootPathPart.toUri().getPath();
  /*
   * We are making URI include the chrootedPath: e.g. file:///chrootedPath.
   * This is questionable since Path#makeQualified(uri, path) ignores
   * the pathPart of a uri. Since this class is internal we can ignore
   * this issue but if we were to make it external then this needs
   * to be resolved.
   */
  // Handle the two cases:
  //              scheme:/// and scheme://authority/
  myUri = new URI(myFs.getUri().toString() + 
      (myFs.getUri().getAuthority() == null ? "" :  Path.SEPARATOR) +
        chRootPathPart.toUri().getPath().substring(1));
  super.checkPath(theRoot);
}
 
Example 11
Source Project: big-c   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 
Example 12
Source Project: big-c   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  
  FileStatus[] statusLst = res.targetFileSystem.listStatus(res.remainingPath);
  if (!res.isInternalDir()) {
    // We need to change the name in the FileStatus as described in
    // {@link #getFileStatus }
    ChRootedFs targetFs;
    targetFs = (ChRootedFs) res.targetFileSystem;
    int i = 0;
    for (FileStatus status : statusLst) {
        String suffix = targetFs.stripOutRoot(status.getPath());
        statusLst[i++] = new ViewFsFileStatus(status, this.makeQualified(
            suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  }
  return statusLst;
}
 
Example 13
Source Project: big-c   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void createSymlink(final Path target, final Path link,
    final boolean createParent) throws IOException, UnresolvedLinkException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(link), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("createSymlink", link);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  res.targetFileSystem.createSymlink(target, res.remainingPath,
      createParent);  
}
 
Example 14
Source Project: big-c   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
  List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
              fsState.getMountPoints();
  int initialListSize  = 0;
  for (InodeTree.MountPoint<AbstractFileSystem> im : mountPoints) {
    initialListSize += im.target.targetDirLinkList.length; 
  }
  List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
  for ( int i = 0; i < mountPoints.size(); ++i ) {
    List<Token<?>> tokens = 
      mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer);
    if (tokens != null) {
      result.addAll(tokens);
    }
  }
  return result;
}
 
Example 15
Source Project: big-c   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setXAttr(Path path, String name, byte[] value,
                     EnumSet<XAttrSetFlag> flag) throws IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(path), true);
  res.targetFileSystem.setXAttr(res.remainingPath, name, value, flag);
}
 
Example 16
Source Project: hadoop   Source File: TestDelegationTokensWithHA.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 300000)
public void testHdfsGetCanonicalServiceName() throws Exception {
  Configuration conf = dfs.getConf();
  URI haUri = HATestUtil.getLogicalUri(cluster);
  AbstractFileSystem afs =  AbstractFileSystem.createFileSystem(haUri, conf);    
  String haService = HAUtil.buildTokenServiceForLogicalUri(haUri,
      HdfsConstants.HDFS_URI_SCHEME).toString();
  assertEquals(haService, afs.getCanonicalServiceName());
  Token<?> token = afs.getDelegationTokens(
      UserGroupInformation.getCurrentUser().getShortUserName()).get(0);
  assertEquals(haService, token.getService().toString());
  // make sure the logical uri is handled correctly
  token.renew(conf);
  token.cancel(conf);
}
 
Example 17
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This constructor has the signature needed by
 * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
 * 
 * @param theUri which must be that of ViewFs
 * @param conf
 * @throws IOException
 * @throws URISyntaxException 
 */
ViewFs(final URI theUri, final Configuration conf) throws IOException,
    URISyntaxException {
  super(theUri, FsConstants.VIEWFS_SCHEME, false, -1);
  creationTime = Time.now();
  ugi = UserGroupInformation.getCurrentUser();
  config = conf;
  // Now build  client side view (i.e. client side mount table) from config.
  String authority = theUri.getAuthority();
  fsState = new InodeTree<AbstractFileSystem>(conf, authority) {

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        String pathString = uri.getPath();
        if (pathString.isEmpty()) {
          pathString = "/";
        }
        return new ChRootedFs(
            AbstractFileSystem.createFileSystem(uri, config),
            new Path(pathString));
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(
        final INodeDir<AbstractFileSystem> dir) throws URISyntaxException {
      return new InternalDirOfViewFs(dir, creationTime, ugi, getUri());
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      throw new UnsupportedFileSystemException("mergefs not implemented yet");
      // return MergeFs.createMergeFs(mergeFsURIList, config);
    }
  };
}
 
Example 18
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Path resolvePath(final Path f) throws FileNotFoundException,
        AccessControlException, UnresolvedLinkException, IOException {
  final InodeTree.ResolveResult<AbstractFileSystem> res;
    res = fsState.resolve(getUriPath(f), true);
  if (res.isInternalDir()) {
    return f;
  }
  return res.targetFileSystem.resolvePath(res.remainingPath);

}
 
Example 19
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FileChecksum getFileChecksum(final Path f)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.getFileChecksum(res.remainingPath);
}
 
Example 20
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void access(Path path, FsAction mode) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(path), true);
  res.targetFileSystem.access(res.remainingPath, mode);
}
 
Example 21
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FileStatus getFileLinkStatus(final Path f)
   throws AccessControlException, FileNotFoundException,
   UnsupportedFileSystemException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), false); // do not follow mount link
  return res.targetFileSystem.getFileLinkStatus(res.remainingPath);
}
 
Example 22
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public RemoteIterator<FileStatus> listStatusIterator(final Path f)
  throws AccessControlException, FileNotFoundException,
  UnresolvedLinkException, IOException {
  final InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  final RemoteIterator<FileStatus> fsIter =
    res.targetFileSystem.listStatusIterator(res.remainingPath);
  if (res.isInternalDir()) {
    return fsIter;
  }
  
  return new RemoteIterator<FileStatus>() {
    final RemoteIterator<FileStatus> myIter;
    final ChRootedFs targetFs;
    { // Init
        myIter = fsIter;
        targetFs = (ChRootedFs) res.targetFileSystem;
    }
    
    @Override
    public boolean hasNext() throws IOException {
      return myIter.hasNext();
    }
    
    @Override
    public FileStatus next() throws IOException {
      FileStatus status =  myIter.next();
      String suffix = targetFs.stripOutRoot(status.getPath());
      return new ViewFsFileStatus(status, makeQualified(
          suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  };
}
 
Example 23
Source Project: big-c   Source File: TestChRootedFs.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIsValidNameInvalidInBaseFs() throws Exception {
  AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
  ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
  Mockito.doReturn(false).when(baseFs).isValidName(Mockito.anyString());
  Assert.assertFalse(chRootedFs.isValidName("/test"));
  Mockito.verify(baseFs).isValidName("/chroot/test");
}
 
Example 24
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FSDataInputStream open(final Path f, final int bufferSize)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
      fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.open(res.remainingPath, bufferSize);
}
 
Example 25
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean truncate(final Path f, final long newLength)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.truncate(res.remainingPath, newLength);
}
 
Example 26
Source Project: big-c   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Map<String, byte[]> getXAttrs(Path path, List<String> names)
    throws IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(path), true);
  return res.targetFileSystem.getXAttrs(res.remainingPath, names);
}
 
Example 27
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setPermission(final Path f, final FsPermission permission)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  res.targetFileSystem.setPermission(res.remainingPath, permission); 
  
}
 
Example 28
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean setReplication(final Path f, final short replication)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.setReplication(res.remainingPath, replication);
}
 
Example 29
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setTimes(final Path f, final long mtime, final long atime)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  res.targetFileSystem.setTimes(res.remainingPath, mtime, atime); 
}
 
Example 30
Source Project: big-c   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void removeAcl(Path path)
    throws IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(path), true);
  res.targetFileSystem.removeAcl(res.remainingPath);
}