Java Code Examples for org.apache.hadoop.fs.UnsupportedFileSystemException

The following examples show how to use org.apache.hadoop.fs.UnsupportedFileSystemException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: AppLogAggregatorImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void doAppLogAggregationPostCleanUp() {
  // Remove the local app-log-dirs
  List<Path> localAppLogDirs = new ArrayList<Path>();
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logPath = new Path(rootLogDir, applicationId);
    try {
      // check if log dir exists
      lfs.getFileStatus(logPath);
      localAppLogDirs.add(logPath);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue);
      continue;
    } catch (IOException fe) {
      continue;
    }
  }

  if (localAppLogDirs.size() > 0) {
    this.delService.delete(this.userUgi.getShortUserName(), null,
      localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
}
 
Example 2
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 
Example 3
Source Project: hadoop   Source File: LoadGenerator.java    License: Apache License 2.0 6 votes vote down vote up
protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
  out.println("Result of running LoadGenerator against fileSystem: " + 
  FileContext.getFileContext().getDefaultFileSystem().getUri());
  if (numOfOps[OPEN] != 0) {
    out.println("Average open execution time: " + 
        (double)executionTime[OPEN]/numOfOps[OPEN] + "ms");
  }
  if (numOfOps[LIST] != 0) {
    out.println("Average list execution time: " + 
        (double)executionTime[LIST]/numOfOps[LIST] + "ms");
  }
  if (numOfOps[DELETE] != 0) {
    out.println("Average deletion execution time: " + 
        (double)executionTime[DELETE]/numOfOps[DELETE] + "ms");
    out.println("Average create execution time: " + 
        (double)executionTime[CREATE]/numOfOps[CREATE] + "ms");
    out.println("Average write_close execution time: " + 
        (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms");
  }
  if (totalTime != 0) { 
    out.println("Average operations per second: " + 
        (double)totalOps/totalTime +"ops/s");
  }
  out.println();
}
 
Example 4
Source Project: twill   Source File: FileContextLocationFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new instance.
 *
 * @param configuration the hadoop configuration
 * @param pathBase base path for all non-absolute location created through this {@link LocationFactory}.
 */
public FileContextLocationFactory(final Configuration configuration, String pathBase) {
  this.configuration = configuration;
  this.pathBase = new Path(pathBase.startsWith("/") ? pathBase : "/" + pathBase);

  int maxCacheSize = configuration.getInt(Configs.Keys.FILE_CONTEXT_CACHE_MAX_SIZE,
                                          Configs.Defaults.FILE_CONTEXT_CACHE_MAX_SIZE);
  this.fileContextCache = CacheBuilder
    .newBuilder()
    .weakKeys()
    .weakValues()
    .maximumSize(maxCacheSize)
    .build(new CacheLoader<UserGroupInformation, FileContext>() {
      @Override
      public FileContext load(UserGroupInformation ugi) throws Exception {
        return ugi.doAs(new PrivilegedExceptionAction<FileContext>() {
          @Override
          public FileContext run() throws UnsupportedFileSystemException {
            return FileContext.getFileContext(configuration);
          }
        });
      }
    });
}
 
Example 5
Source Project: big-c   Source File: AppLogAggregatorImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void doAppLogAggregationPostCleanUp() {
  // Remove the local app-log-dirs
  List<Path> localAppLogDirs = new ArrayList<Path>();
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logPath = new Path(rootLogDir, applicationId);
    try {
      // check if log dir exists
      lfs.getFileStatus(logPath);
      localAppLogDirs.add(logPath);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue);
      continue;
    } catch (IOException fe) {
      continue;
    }
  }

  if (localAppLogDirs.size() > 0) {
    this.delService.delete(this.userUgi.getShortUserName(), null,
      localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
}
 
Example 6
Source Project: big-c   Source File: ViewFs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 
Example 7
Source Project: big-c   Source File: LoadGenerator.java    License: Apache License 2.0 6 votes vote down vote up
protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
  out.println("Result of running LoadGenerator against fileSystem: " + 
  FileContext.getFileContext().getDefaultFileSystem().getUri());
  if (numOfOps[OPEN] != 0) {
    out.println("Average open execution time: " + 
        (double)executionTime[OPEN]/numOfOps[OPEN] + "ms");
  }
  if (numOfOps[LIST] != 0) {
    out.println("Average list execution time: " + 
        (double)executionTime[LIST]/numOfOps[LIST] + "ms");
  }
  if (numOfOps[DELETE] != 0) {
    out.println("Average deletion execution time: " + 
        (double)executionTime[DELETE]/numOfOps[DELETE] + "ms");
    out.println("Average create execution time: " + 
        (double)executionTime[CREATE]/numOfOps[CREATE] + "ms");
    out.println("Average write_close execution time: " + 
        (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms");
  }
  if (totalTime != 0) { 
    out.println("Average operations per second: " + 
        (double)totalOps/totalTime +"ops/s");
  }
  out.println();
}
 
Example 8
Source Project: hadoop   Source File: TestDiskFailures.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws AccessControlException,
    FileNotFoundException, UnsupportedFileSystemException, IOException {
  localFS = FileContext.getLocalFSFileContext();
  localFS.delete(new Path(localFSDirBase.getAbsolutePath()), true);
  localFSDirBase.mkdirs();
  // Do not start cluster here
}
 
Example 9
Source Project: hadoop   Source File: ResourceLocalizationService.java    License: Apache License 2.0 5 votes vote down vote up
private void submitDirForDeletion(String userName, Path dir) {
  try {
    lfs.getFileStatus(dir);
    delService.delete(userName, dir, new Path[] {});
  } catch (UnsupportedFileSystemException ue) {
    LOG.warn("Local dir " + dir + " is an unsupported filesystem", ue);
  } catch (IOException ie) {
    // ignore
    return;
  }
}
 
Example 10
Source Project: hadoop   Source File: ContainersLauncher.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  try {
    //TODO Is this required?
    FileContext.getLocalFSFileContext(conf);
  } catch (UnsupportedFileSystemException e) {
    throw new YarnRuntimeException("Failed to start ContainersLauncher", e);
  }
  super.serviceInit(conf);
}
 
Example 11
Source Project: hadoop   Source File: NonAggregatingLogHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void run() {
  List<Path> localAppLogDirs = new ArrayList<Path>();
  FileContext lfs = getLocalFileContext(getConfig());
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logDir = new Path(rootLogDir, applicationId.toString());
    try {
      lfs.getFileStatus(logDir);
      localAppLogDirs.add(logDir);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Unsupported file system used for log dir " + logDir, ue);
      continue;
    } catch (IOException ie) {
      continue;
    }
  }

  // Inform the application before the actual delete itself, so that links
  // to logs will no longer be there on NM web-UI.
  NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle(
    new ApplicationEvent(this.applicationId,
      ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
  if (localAppLogDirs.size() > 0) {
    NonAggregatingLogHandler.this.delService.delete(user, null,
      (Path[]) localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
  try {
    NonAggregatingLogHandler.this.stateStore.removeLogDeleter(
        this.applicationId);
  } catch (IOException e) {
    LOG.error("Error removing log deletion state", e);
  }
}
 
Example 12
Source Project: hadoop   Source File: DockerContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
public DockerContainerExecutor() {
  try {
    this.lfs = FileContext.getLocalFSFileContext();
    this.dockerImagePattern = Pattern.compile(DOCKER_IMAGE_PATTERN);
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example 13
Source Project: hadoop   Source File: DeletionService.java    License: Apache License 2.0 5 votes vote down vote up
static final FileContext getLfs() {
  try {
    return FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example 14
Source Project: hadoop   Source File: DefaultContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
public DefaultContainerExecutor() {
  try {
    this.lfs = FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example 15
Source Project: hadoop   Source File: BaseContainerManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
public BaseContainerManagerTest() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  localDir =
      new File("target", this.getClass().getSimpleName() + "-localDir")
          .getAbsoluteFile();
  localLogDir =
      new File("target", this.getClass().getSimpleName() + "-localLogDir")
          .getAbsoluteFile();
  remoteLogDir =
    new File("target", this.getClass().getSimpleName() + "-remoteLogDir")
        .getAbsoluteFile();
  tmpDir = new File("target", this.getClass().getSimpleName() + "-tmpDir");
}
 
Example 16
Source Project: hadoop   Source File: TestDeletionService.java    License: Apache License 2.0 5 votes vote down vote up
private static final FileContext getLfs() {
  try {
    return FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 
Example 17
Source Project: hadoop   Source File: TestNodeManagerShutdown.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  tmpDir.mkdirs();
  logsDir.mkdirs();
  remoteLogsDir.mkdirs();
  nmLocalDir.mkdirs();

  // Construct the Container-id
  cId = createContainerId();
}
 
Example 18
Source Project: hadoop   Source File: TestNodeManagerResync.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  tmpDir.mkdirs();
  logsDir.mkdirs();
  remoteLogsDir.mkdirs();
  nmLocalDir.mkdirs();
  syncBarrier = new CyclicBarrier(2);
}
 
Example 19
Source Project: hadoop   Source File: LocalContainerLauncher.java    License: Apache License 2.0 5 votes vote down vote up
public LocalContainerLauncher(AppContext context,
                              TaskUmbilicalProtocol umbilical) {
  super(LocalContainerLauncher.class.getName());
  this.context = context;
  this.umbilical = umbilical;
      // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us
      // (TODO/FIXME:  pointless to use RPC to talk to self; should create
      // LocalTaskAttemptListener or similar:  implement umbilical protocol
      // but skip RPC stuff)

  try {
    curFC = FileContext.getFileContext(curDir.toURI());
  } catch (UnsupportedFileSystemException ufse) {
    LOG.error("Local filesystem " + curDir.toURI().toString()
              + " is unsupported?? (should never happen)");
  }

  // Save list of files/dirs that are supposed to be present so can delete
  // any extras created by one task before starting subsequent task.  Note
  // that there's no protection against deleted or renamed localization;
  // users who do that get what they deserve (and will have to disable
  // uberization in order to run correctly).
  File[] curLocalFiles = curDir.listFiles();
  localizedFiles = new HashSet<File>(curLocalFiles.length);
  for (int j = 0; j < curLocalFiles.length; ++j) {
    localizedFiles.add(curLocalFiles[j]);
  }

  // Relocalization note/future FIXME (per chrisdo, 20110315):  At moment,
  // full localization info is in AppSubmissionContext passed from client to
  // RM and then to NM for AM-container launch:  no difference between AM-
  // localization and MapTask- or ReduceTask-localization, so can assume all
  // OK.  Longer-term, will need to override uber-AM container-localization
  // request ("needed resources") with union of regular-AM-resources + task-
  // resources (and, if maps and reduces ever differ, then union of all three
  // types), OR will need localizer service/API that uber-AM can request
  // after running (e.g., "localizeForTask()" or "localizeForMapTask()").
}
 
Example 20
Source Project: hadoop   Source File: YARNRunner.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)}
 * but allowing injecting {@link ClientCache}. Enable mocking and testing.
 * @param conf the configuration object
 * @param resMgrDelegate the resource manager delegate
 * @param clientCache the client cache object.
 */
public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate,
    ClientCache clientCache) {
  this.conf = conf;
  try {
    this.resMgrDelegate = resMgrDelegate;
    this.clientCache = clientCache;
    this.defaultFileContext = FileContext.getFileContext(this.conf);
  } catch (UnsupportedFileSystemException ufe) {
    throw new RuntimeException("Error in instantiating YarnClient", ufe);
  }
}
 
Example 21
Source Project: hadoop   Source File: JobHistoryUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get default file system URI for the cluster (used to ensure consistency
 * of history done/staging locations) over different context
 *
 * @return Default file context
 */
private static FileContext getDefaultFileContext() {
  // If FS_DEFAULT_NAME_KEY was set solely by core-default.xml then we ignore
  // ignore it. This prevents defaulting history paths to file system specified
  // by core-default.xml which would not make sense in any case. For a test
  // case to exploit this functionality it should create core-site.xml
  FileContext fc = null;
  Configuration defaultConf = new Configuration();
  String[] sources;
  sources = defaultConf.getPropertySources(
      CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
  if (sources != null &&
      (!Arrays.asList(sources).contains("core-default.xml") ||
      sources.length > 1)) {
    try {
      fc = FileContext.getFileContext(defaultConf);
      LOG.info("Default file system [" +
                fc.getDefaultFileSystem().getUri() + "]");
    } catch (UnsupportedFileSystemException e) {
      LOG.error("Unable to create default file context [" +
          defaultConf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) +
          "]",
          e);
    }
  }
  else {
    LOG.info("Default file system is set solely " +
        "by core-default.xml therefore -  ignoring");
  }

  return fc;
}
 
Example 22
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This constructor has the signature needed by
 * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
 * 
 * @param theUri which must be that of ViewFs
 * @param conf
 * @throws IOException
 * @throws URISyntaxException 
 */
ViewFs(final URI theUri, final Configuration conf) throws IOException,
    URISyntaxException {
  super(theUri, FsConstants.VIEWFS_SCHEME, false, -1);
  creationTime = Time.now();
  ugi = UserGroupInformation.getCurrentUser();
  config = conf;
  // Now build  client side view (i.e. client side mount table) from config.
  String authority = theUri.getAuthority();
  fsState = new InodeTree<AbstractFileSystem>(conf, authority) {

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        String pathString = uri.getPath();
        if (pathString.isEmpty()) {
          pathString = "/";
        }
        return new ChRootedFs(
            AbstractFileSystem.createFileSystem(uri, config),
            new Path(pathString));
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(
        final INodeDir<AbstractFileSystem> dir) throws URISyntaxException {
      return new InternalDirOfViewFs(dir, creationTime, ugi, getUri());
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      throw new UnsupportedFileSystemException("mergefs not implemented yet");
      // return MergeFs.createMergeFs(mergeFsURIList, config);
    }
  };
}
 
Example 23
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FileStatus getFileLinkStatus(final Path f)
   throws AccessControlException, FileNotFoundException,
   UnsupportedFileSystemException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), false); // do not follow mount link
  return res.targetFileSystem.getFileLinkStatus(res.remainingPath);
}
 
Example 24
Source Project: hadoop   Source File: ViewFs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  throw readOnlyMountTable("create", f);
}
 
Example 25
Source Project: hadoop   Source File: TestViewFsConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected=FileAlreadyExistsException.class)
public void testInvalidConfig() throws IOException, URISyntaxException {
  Configuration conf = new Configuration();
  ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
      new Path("file:///dir2").toUri());
  ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3",
      new Path("file:///dir3").toUri());
  
  class Foo { };
  
   new InodeTree<Foo>(conf, null) {

    @Override
    protected
    Foo getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        return null;
    }

    @Override
    protected
    Foo getTargetFileSystem(
        org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo>
                                        dir)
      throws URISyntaxException {
      return null;
    }

    @Override
    protected
    Foo getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      return null;
    }
  };
}
 
Example 26
Source Project: dremio-oss   Source File: HadoopFileSystemWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void createSymlink(Path target, Path link, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    underlyingFs.createSymlink(target, link, createParent);
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}
 
Example 27
Source Project: dremio-oss   Source File: HadoopFileSystemWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FileStatus getFileLinkStatus(Path f) throws AccessControlException, FileNotFoundException,
    UnsupportedFileSystemException, IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    return underlyingFs.getFileLinkStatus(f);
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}
 
Example 28
Source Project: big-c   Source File: TestDiskFailures.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws AccessControlException,
    FileNotFoundException, UnsupportedFileSystemException, IOException {
  localFS = FileContext.getLocalFSFileContext();
  localFS.delete(new Path(localFSDirBase.getAbsolutePath()), true);
  localFSDirBase.mkdirs();
  // Do not start cluster here
}
 
Example 29
Source Project: big-c   Source File: ResourceLocalizationService.java    License: Apache License 2.0 5 votes vote down vote up
private void submitDirForDeletion(String userName, Path dir) {
  try {
    lfs.getFileStatus(dir);
    delService.delete(userName, dir, new Path[] {});
  } catch (UnsupportedFileSystemException ue) {
    LOG.warn("Local dir " + dir + " is an unsupported filesystem", ue);
  } catch (IOException ie) {
    // ignore
    return;
  }
}
 
Example 30
Source Project: big-c   Source File: ContainersLauncher.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  try {
    //TODO Is this required?
    FileContext.getLocalFSFileContext(conf);
  } catch (UnsupportedFileSystemException e) {
    throw new YarnRuntimeException("Failed to start ContainersLauncher", e);
  }
  super.serviceInit(conf);
}