org.apache.hadoop.fs.UnsupportedFileSystemException Java Examples

The following examples show how to use org.apache.hadoop.fs.UnsupportedFileSystemException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LoadGenerator.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
  out.println("Result of running LoadGenerator against fileSystem: " + 
  FileContext.getFileContext().getDefaultFileSystem().getUri());
  if (numOfOps[OPEN] != 0) {
    out.println("Average open execution time: " + 
        (double)executionTime[OPEN]/numOfOps[OPEN] + "ms");
  }
  if (numOfOps[LIST] != 0) {
    out.println("Average list execution time: " + 
        (double)executionTime[LIST]/numOfOps[LIST] + "ms");
  }
  if (numOfOps[DELETE] != 0) {
    out.println("Average deletion execution time: " + 
        (double)executionTime[DELETE]/numOfOps[DELETE] + "ms");
    out.println("Average create execution time: " + 
        (double)executionTime[CREATE]/numOfOps[CREATE] + "ms");
    out.println("Average write_close execution time: " + 
        (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms");
  }
  if (totalTime != 0) { 
    out.println("Average operations per second: " + 
        (double)totalOps/totalTime +"ops/s");
  }
  out.println();
}
 
Example #2
Source File: AppLogAggregatorImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void doAppLogAggregationPostCleanUp() {
  // Remove the local app-log-dirs
  List<Path> localAppLogDirs = new ArrayList<Path>();
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logPath = new Path(rootLogDir, applicationId);
    try {
      // check if log dir exists
      lfs.getFileStatus(logPath);
      localAppLogDirs.add(logPath);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue);
      continue;
    } catch (IOException fe) {
      continue;
    }
  }

  if (localAppLogDirs.size() > 0) {
    this.delService.delete(this.userUgi.getShortUserName(), null,
      localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
}
 
Example #3
Source File: LoadGenerator.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
  out.println("Result of running LoadGenerator against fileSystem: " + 
  FileContext.getFileContext().getDefaultFileSystem().getUri());
  if (numOfOps[OPEN] != 0) {
    out.println("Average open execution time: " + 
        (double)executionTime[OPEN]/numOfOps[OPEN] + "ms");
  }
  if (numOfOps[LIST] != 0) {
    out.println("Average list execution time: " + 
        (double)executionTime[LIST]/numOfOps[LIST] + "ms");
  }
  if (numOfOps[DELETE] != 0) {
    out.println("Average deletion execution time: " + 
        (double)executionTime[DELETE]/numOfOps[DELETE] + "ms");
    out.println("Average create execution time: " + 
        (double)executionTime[CREATE]/numOfOps[CREATE] + "ms");
    out.println("Average write_close execution time: " + 
        (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms");
  }
  if (totalTime != 0) { 
    out.println("Average operations per second: " + 
        (double)totalOps/totalTime +"ops/s");
  }
  out.println();
}
 
Example #4
Source File: FileContextLocationFactory.java    From twill with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new instance.
 *
 * @param configuration the hadoop configuration
 * @param pathBase base path for all non-absolute location created through this {@link LocationFactory}.
 */
public FileContextLocationFactory(final Configuration configuration, String pathBase) {
  this.configuration = configuration;
  this.pathBase = new Path(pathBase.startsWith("/") ? pathBase : "/" + pathBase);

  int maxCacheSize = configuration.getInt(Configs.Keys.FILE_CONTEXT_CACHE_MAX_SIZE,
                                          Configs.Defaults.FILE_CONTEXT_CACHE_MAX_SIZE);
  this.fileContextCache = CacheBuilder
    .newBuilder()
    .weakKeys()
    .weakValues()
    .maximumSize(maxCacheSize)
    .build(new CacheLoader<UserGroupInformation, FileContext>() {
      @Override
      public FileContext load(UserGroupInformation ugi) throws Exception {
        return ugi.doAs(new PrivilegedExceptionAction<FileContext>() {
          @Override
          public FileContext run() throws UnsupportedFileSystemException {
            return FileContext.getFileContext(configuration);
          }
        });
      }
    });
}
 
Example #5
Source File: ViewFs.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 
Example #6
Source File: ViewFs.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 
Example #7
Source File: AppLogAggregatorImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void doAppLogAggregationPostCleanUp() {
  // Remove the local app-log-dirs
  List<Path> localAppLogDirs = new ArrayList<Path>();
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logPath = new Path(rootLogDir, applicationId);
    try {
      // check if log dir exists
      lfs.getFileStatus(logPath);
      localAppLogDirs.add(logPath);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue);
      continue;
    } catch (IOException fe) {
      continue;
    }
  }

  if (localAppLogDirs.size() > 0) {
    this.delService.delete(this.userUgi.getShortUserName(), null,
      localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
}
 
Example #8
Source File: ContainersLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  try {
    //TODO Is this required?
    FileContext.getLocalFSFileContext(conf);
  } catch (UnsupportedFileSystemException e) {
    throw new YarnRuntimeException("Failed to start ContainersLauncher", e);
  }
  super.serviceInit(conf);
}
 
Example #9
Source File: YARNRunner.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
/**
 * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)}
 * but allowing injecting {@link ClientCache}. Enable mocking and testing.
 * @param conf the configuration object
 * @param resMgrDelegate the resource manager delegate
 * @param clientCache the client cache object.
 */
public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate,
    ClientCache clientCache) {
  this.conf = conf;
  this.tezConf = new TezConfiguration(conf);
  try {
    this.resMgrDelegate = resMgrDelegate;
    this.clientCache = clientCache;
    this.defaultFileContext = FileContext.getFileContext(this.conf);

  } catch (UnsupportedFileSystemException ufe) {
    throw new RuntimeException("Error in instantiating YarnClient", ufe);
  }
}
 
Example #10
Source File: DeletionService.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up
static final FileContext getLfs() {
  try {
    return FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example #11
Source File: DeletionService.java    From tajo with Apache License 2.0 5 votes vote down vote up
static final FileContext getLfs() {
  try {
    return FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example #12
Source File: FileContextUtils.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
public static FileContext getFileContext(@NotNull Path path, @Nullable Configuration conf)
  throws UnsupportedFileSystemException
{
  Preconditions.checkNotNull(path, "path");
  URI pathUri = path.toUri();

  if (pathUri.getScheme() != null) {
    return FileContext.getFileContext(pathUri, conf == null ? new Configuration() : conf);
  } else {
    return FileContext.getFileContext(conf == null ? new Configuration() : conf);
  }
}
 
Example #13
Source File: TestViewFsConfig.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(expected=FileAlreadyExistsException.class)
public void testInvalidConfig() throws IOException, URISyntaxException {
  Configuration conf = new Configuration();
  ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
      new Path("file:///dir2").toUri());
  ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3",
      new Path("file:///dir3").toUri());
  
  class Foo { };
  
   new InodeTree<Foo>(conf, null) {

    @Override
    protected
    Foo getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        return null;
    }

    @Override
    protected
    Foo getTargetFileSystem(
        org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo>
                                        dir)
      throws URISyntaxException {
      return null;
    }

    @Override
    protected
    Foo getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      return null;
    }
  };
}
 
Example #14
Source File: ViewFs.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  throw readOnlyMountTable("create", f);
}
 
Example #15
Source File: ViewFs.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public FileStatus getFileLinkStatus(final Path f)
   throws AccessControlException, FileNotFoundException,
   UnsupportedFileSystemException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), false); // do not follow mount link
  return res.targetFileSystem.getFileLinkStatus(res.remainingPath);
}
 
Example #16
Source File: DeletionService.java    From big-c with Apache License 2.0 5 votes vote down vote up
static final FileContext getLfs() {
  try {
    return FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example #17
Source File: YARNRunner.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)}
 * but allowing injecting {@link ClientCache}. Enable mocking and testing.
 * @param conf the configuration object
 * @param resMgrDelegate the resource manager delegate
 * @param clientCache the client cache object.
 */
public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate,
    ClientCache clientCache) {
  this.conf = conf;
  this.tezConf = new TezConfiguration(conf);
  try {
    this.resMgrDelegate = resMgrDelegate;
    this.clientCache = clientCache;
    this.defaultFileContext = FileContext.getFileContext(this.conf);

  } catch (UnsupportedFileSystemException ufe) {
    throw new RuntimeException("Error in instantiating YarnClient", ufe);
  }
}
 
Example #18
Source File: LocalContainerLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
public LocalContainerLauncher(AppContext context,
                              TaskUmbilicalProtocol umbilical) {
  super(LocalContainerLauncher.class.getName());
  this.context = context;
  this.umbilical = umbilical;
      // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us
      // (TODO/FIXME:  pointless to use RPC to talk to self; should create
      // LocalTaskAttemptListener or similar:  implement umbilical protocol
      // but skip RPC stuff)

  try {
    curFC = FileContext.getFileContext(curDir.toURI());
  } catch (UnsupportedFileSystemException ufse) {
    LOG.error("Local filesystem " + curDir.toURI().toString()
              + " is unsupported?? (should never happen)");
  }

  // Save list of files/dirs that are supposed to be present so can delete
  // any extras created by one task before starting subsequent task.  Note
  // that there's no protection against deleted or renamed localization;
  // users who do that get what they deserve (and will have to disable
  // uberization in order to run correctly).
  File[] curLocalFiles = curDir.listFiles();
  localizedFiles = new HashSet<File>(curLocalFiles.length);
  for (int j = 0; j < curLocalFiles.length; ++j) {
    localizedFiles.add(curLocalFiles[j]);
  }

  // Relocalization note/future FIXME (per chrisdo, 20110315):  At moment,
  // full localization info is in AppSubmissionContext passed from client to
  // RM and then to NM for AM-container launch:  no difference between AM-
  // localization and MapTask- or ReduceTask-localization, so can assume all
  // OK.  Longer-term, will need to override uber-AM container-localization
  // request ("needed resources") with union of regular-AM-resources + task-
  // resources (and, if maps and reduces ever differ, then union of all three
  // types), OR will need localizer service/API that uber-AM can request
  // after running (e.g., "localizeForTask()" or "localizeForMapTask()").
}
 
Example #19
Source File: TestNodeManagerShutdown.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  tmpDir.mkdirs();
  logsDir.mkdirs();
  remoteLogsDir.mkdirs();
  nmLocalDir.mkdirs();

  // Construct the Container-id
  cId = createContainerId();
}
 
Example #20
Source File: ViewFs.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * This constructor has the signature needed by
 * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
 * 
 * @param theUri which must be that of ViewFs
 * @param conf
 * @throws IOException
 * @throws URISyntaxException 
 */
ViewFs(final URI theUri, final Configuration conf) throws IOException,
    URISyntaxException {
  super(theUri, FsConstants.VIEWFS_SCHEME, false, -1);
  creationTime = Time.now();
  ugi = UserGroupInformation.getCurrentUser();
  config = conf;
  // Now build  client side view (i.e. client side mount table) from config.
  String authority = theUri.getAuthority();
  fsState = new InodeTree<AbstractFileSystem>(conf, authority) {

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        String pathString = uri.getPath();
        if (pathString.isEmpty()) {
          pathString = "/";
        }
        return new ChRootedFs(
            AbstractFileSystem.createFileSystem(uri, config),
            new Path(pathString));
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(
        final INodeDir<AbstractFileSystem> dir) throws URISyntaxException {
      return new InternalDirOfViewFs(dir, creationTime, ugi, getUri());
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      throw new UnsupportedFileSystemException("mergefs not implemented yet");
      // return MergeFs.createMergeFs(mergeFsURIList, config);
    }
  };
}
 
Example #21
Source File: BaseContainerManagerTest.java    From big-c with Apache License 2.0 5 votes vote down vote up
public BaseContainerManagerTest() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  localDir =
      new File("target", this.getClass().getSimpleName() + "-localDir")
          .getAbsoluteFile();
  localLogDir =
      new File("target", this.getClass().getSimpleName() + "-localLogDir")
          .getAbsoluteFile();
  remoteLogDir =
    new File("target", this.getClass().getSimpleName() + "-remoteLogDir")
        .getAbsoluteFile();
  tmpDir = new File("target", this.getClass().getSimpleName() + "-tmpDir");
}
 
Example #22
Source File: DefaultContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up
public DefaultContainerExecutor() {
  try {
    this.lfs = FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example #23
Source File: TestNodeManagerResync.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  tmpDir.mkdirs();
  logsDir.mkdirs();
  remoteLogsDir.mkdirs();
  nmLocalDir.mkdirs();
  syncBarrier = new CyclicBarrier(2);
}
 
Example #24
Source File: DockerContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up
public DockerContainerExecutor() {
  try {
    this.lfs = FileContext.getLocalFSFileContext();
    this.dockerImagePattern = Pattern.compile(DOCKER_IMAGE_PATTERN);
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example #25
Source File: NonAggregatingLogHandler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void run() {
  List<Path> localAppLogDirs = new ArrayList<Path>();
  FileContext lfs = getLocalFileContext(getConfig());
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logDir = new Path(rootLogDir, applicationId.toString());
    try {
      lfs.getFileStatus(logDir);
      localAppLogDirs.add(logDir);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Unsupported file system used for log dir " + logDir, ue);
      continue;
    } catch (IOException ie) {
      continue;
    }
  }

  // Inform the application before the actual delete itself, so that links
  // to logs will no longer be there on NM web-UI.
  NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle(
    new ApplicationEvent(this.applicationId,
      ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
  if (localAppLogDirs.size() > 0) {
    NonAggregatingLogHandler.this.delService.delete(user, null,
      (Path[]) localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
  try {
    NonAggregatingLogHandler.this.stateStore.removeLogDeleter(
        this.applicationId);
  } catch (IOException e) {
    LOG.error("Error removing log deletion state", e);
  }
}
 
Example #26
Source File: ResourceLocalizationService.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void submitDirForDeletion(String userName, Path dir) {
  try {
    lfs.getFileStatus(dir);
    delService.delete(userName, dir, new Path[] {});
  } catch (UnsupportedFileSystemException ue) {
    LOG.warn("Local dir " + dir + " is an unsupported filesystem", ue);
  } catch (IOException ie) {
    // ignore
    return;
  }
}
 
Example #27
Source File: TestDiskFailures.java    From big-c with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws AccessControlException,
    FileNotFoundException, UnsupportedFileSystemException, IOException {
  localFS = FileContext.getLocalFSFileContext();
  localFS.delete(new Path(localFSDirBase.getAbsolutePath()), true);
  localFSDirBase.mkdirs();
  // Do not start cluster here
}
 
Example #28
Source File: HadoopFileSystemWrapper.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Override
public FileStatus getFileLinkStatus(Path f) throws AccessControlException, FileNotFoundException,
    UnsupportedFileSystemException, IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    return underlyingFs.getFileLinkStatus(f);
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}
 
Example #29
Source File: HadoopFileSystemWrapper.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Override
public void createSymlink(Path target, Path link, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    underlyingFs.createSymlink(target, link, createParent);
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}
 
Example #30
Source File: TestViewFsConfig.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(expected=FileAlreadyExistsException.class)
public void testInvalidConfig() throws IOException, URISyntaxException {
  Configuration conf = new Configuration();
  ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
      new Path("file:///dir2").toUri());
  ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3",
      new Path("file:///dir3").toUri());
  
  class Foo { };
  
   new InodeTree<Foo>(conf, null) {

    @Override
    protected
    Foo getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        return null;
    }

    @Override
    protected
    Foo getTargetFileSystem(
        org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo>
                                        dir)
      throws URISyntaxException {
      return null;
    }

    @Override
    protected
    Foo getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      return null;
    }
  };
}