org.apache.hadoop.fs.UnsupportedFileSystemException Java Examples
The following examples show how to use
org.apache.hadoop.fs.UnsupportedFileSystemException.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LoadGenerator.java From hadoop with Apache License 2.0 | 6 votes |
protected static void printResults(PrintStream out) throws UnsupportedFileSystemException { out.println("Result of running LoadGenerator against fileSystem: " + FileContext.getFileContext().getDefaultFileSystem().getUri()); if (numOfOps[OPEN] != 0) { out.println("Average open execution time: " + (double)executionTime[OPEN]/numOfOps[OPEN] + "ms"); } if (numOfOps[LIST] != 0) { out.println("Average list execution time: " + (double)executionTime[LIST]/numOfOps[LIST] + "ms"); } if (numOfOps[DELETE] != 0) { out.println("Average deletion execution time: " + (double)executionTime[DELETE]/numOfOps[DELETE] + "ms"); out.println("Average create execution time: " + (double)executionTime[CREATE]/numOfOps[CREATE] + "ms"); out.println("Average write_close execution time: " + (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms"); } if (totalTime != 0) { out.println("Average operations per second: " + (double)totalOps/totalTime +"ops/s"); } out.println(); }
Example #2
Source File: AppLogAggregatorImpl.java From hadoop with Apache License 2.0 | 6 votes |
private void doAppLogAggregationPostCleanUp() { // Remove the local app-log-dirs List<Path> localAppLogDirs = new ArrayList<Path>(); for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) { Path logPath = new Path(rootLogDir, applicationId); try { // check if log dir exists lfs.getFileStatus(logPath); localAppLogDirs.add(logPath); } catch (UnsupportedFileSystemException ue) { LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue); continue; } catch (IOException fe) { continue; } } if (localAppLogDirs.size() > 0) { this.delService.delete(this.userUgi.getShortUserName(), null, localAppLogDirs.toArray(new Path[localAppLogDirs.size()])); } }
Example #3
Source File: LoadGenerator.java From big-c with Apache License 2.0 | 6 votes |
protected static void printResults(PrintStream out) throws UnsupportedFileSystemException { out.println("Result of running LoadGenerator against fileSystem: " + FileContext.getFileContext().getDefaultFileSystem().getUri()); if (numOfOps[OPEN] != 0) { out.println("Average open execution time: " + (double)executionTime[OPEN]/numOfOps[OPEN] + "ms"); } if (numOfOps[LIST] != 0) { out.println("Average list execution time: " + (double)executionTime[LIST]/numOfOps[LIST] + "ms"); } if (numOfOps[DELETE] != 0) { out.println("Average deletion execution time: " + (double)executionTime[DELETE]/numOfOps[DELETE] + "ms"); out.println("Average create execution time: " + (double)executionTime[CREATE]/numOfOps[CREATE] + "ms"); out.println("Average write_close execution time: " + (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms"); } if (totalTime != 0) { out.println("Average operations per second: " + (double)totalOps/totalTime +"ops/s"); } out.println(); }
Example #4
Source File: FileContextLocationFactory.java From twill with Apache License 2.0 | 6 votes |
/** * Creates a new instance. * * @param configuration the hadoop configuration * @param pathBase base path for all non-absolute location created through this {@link LocationFactory}. */ public FileContextLocationFactory(final Configuration configuration, String pathBase) { this.configuration = configuration; this.pathBase = new Path(pathBase.startsWith("/") ? pathBase : "/" + pathBase); int maxCacheSize = configuration.getInt(Configs.Keys.FILE_CONTEXT_CACHE_MAX_SIZE, Configs.Defaults.FILE_CONTEXT_CACHE_MAX_SIZE); this.fileContextCache = CacheBuilder .newBuilder() .weakKeys() .weakValues() .maximumSize(maxCacheSize) .build(new CacheLoader<UserGroupInformation, FileContext>() { @Override public FileContext load(UserGroupInformation ugi) throws Exception { return ugi.doAs(new PrivilegedExceptionAction<FileContext>() { @Override public FileContext run() throws UnsupportedFileSystemException { return FileContext.getFileContext(configuration); } }); } }); }
Example #5
Source File: ViewFs.java From hadoop with Apache License 2.0 | 6 votes |
@Override public FSDataOutputStream createInternal(final Path f, final EnumSet<CreateFlag> flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException { InodeTree.ResolveResult<AbstractFileSystem> res; try { res = fsState.resolve(getUriPath(f), false); } catch (FileNotFoundException e) { if (createParent) { throw readOnlyMountTable("create", f); } else { throw e; } } assert(res.remainingPath != null); return res.targetFileSystem.createInternal(res.remainingPath, flag, absolutePermission, bufferSize, replication, blockSize, progress, checksumOpt, createParent); }
Example #6
Source File: ViewFs.java From big-c with Apache License 2.0 | 6 votes |
@Override public FSDataOutputStream createInternal(final Path f, final EnumSet<CreateFlag> flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException { InodeTree.ResolveResult<AbstractFileSystem> res; try { res = fsState.resolve(getUriPath(f), false); } catch (FileNotFoundException e) { if (createParent) { throw readOnlyMountTable("create", f); } else { throw e; } } assert(res.remainingPath != null); return res.targetFileSystem.createInternal(res.remainingPath, flag, absolutePermission, bufferSize, replication, blockSize, progress, checksumOpt, createParent); }
Example #7
Source File: AppLogAggregatorImpl.java From big-c with Apache License 2.0 | 6 votes |
private void doAppLogAggregationPostCleanUp() { // Remove the local app-log-dirs List<Path> localAppLogDirs = new ArrayList<Path>(); for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) { Path logPath = new Path(rootLogDir, applicationId); try { // check if log dir exists lfs.getFileStatus(logPath); localAppLogDirs.add(logPath); } catch (UnsupportedFileSystemException ue) { LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue); continue; } catch (IOException fe) { continue; } } if (localAppLogDirs.size() > 0) { this.delService.delete(this.userUgi.getShortUserName(), null, localAppLogDirs.toArray(new Path[localAppLogDirs.size()])); } }
Example #8
Source File: ContainersLauncher.java From big-c with Apache License 2.0 | 5 votes |
@Override protected void serviceInit(Configuration conf) throws Exception { try { //TODO Is this required? FileContext.getLocalFSFileContext(conf); } catch (UnsupportedFileSystemException e) { throw new YarnRuntimeException("Failed to start ContainersLauncher", e); } super.serviceInit(conf); }
Example #9
Source File: YARNRunner.java From incubator-tez with Apache License 2.0 | 5 votes |
/** * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)} * but allowing injecting {@link ClientCache}. Enable mocking and testing. * @param conf the configuration object * @param resMgrDelegate the resource manager delegate * @param clientCache the client cache object. */ public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate, ClientCache clientCache) { this.conf = conf; this.tezConf = new TezConfiguration(conf); try { this.resMgrDelegate = resMgrDelegate; this.clientCache = clientCache; this.defaultFileContext = FileContext.getFileContext(this.conf); } catch (UnsupportedFileSystemException ufe) { throw new RuntimeException("Error in instantiating YarnClient", ufe); } }
Example #10
Source File: DeletionService.java From incubator-tajo with Apache License 2.0 | 5 votes |
static final FileContext getLfs() { try { return FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
Example #11
Source File: DeletionService.java From tajo with Apache License 2.0 | 5 votes |
static final FileContext getLfs() { try { return FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
Example #12
Source File: FileContextUtils.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
public static FileContext getFileContext(@NotNull Path path, @Nullable Configuration conf) throws UnsupportedFileSystemException { Preconditions.checkNotNull(path, "path"); URI pathUri = path.toUri(); if (pathUri.getScheme() != null) { return FileContext.getFileContext(pathUri, conf == null ? new Configuration() : conf); } else { return FileContext.getFileContext(conf == null ? new Configuration() : conf); } }
Example #13
Source File: TestViewFsConfig.java From big-c with Apache License 2.0 | 5 votes |
@Test(expected=FileAlreadyExistsException.class) public void testInvalidConfig() throws IOException, URISyntaxException { Configuration conf = new Configuration(); ConfigUtil.addLink(conf, "/internalDir/linkToDir2", new Path("file:///dir2").toUri()); ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3", new Path("file:///dir3").toUri()); class Foo { }; new InodeTree<Foo>(conf, null) { @Override protected Foo getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { return null; } @Override protected Foo getTargetFileSystem( org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo> dir) throws URISyntaxException { return null; } @Override protected Foo getTargetFileSystem(URI[] mergeFsURIList) throws URISyntaxException, UnsupportedFileSystemException { return null; } }; }
Example #14
Source File: ViewFs.java From big-c with Apache License 2.0 | 5 votes |
@Override public FSDataOutputStream createInternal(final Path f, final EnumSet<CreateFlag> flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException { throw readOnlyMountTable("create", f); }
Example #15
Source File: ViewFs.java From big-c with Apache License 2.0 | 5 votes |
@Override public FileStatus getFileLinkStatus(final Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(getUriPath(f), false); // do not follow mount link return res.targetFileSystem.getFileLinkStatus(res.remainingPath); }
Example #16
Source File: DeletionService.java From big-c with Apache License 2.0 | 5 votes |
static final FileContext getLfs() { try { return FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
Example #17
Source File: YARNRunner.java From tez with Apache License 2.0 | 5 votes |
/** * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)} * but allowing injecting {@link ClientCache}. Enable mocking and testing. * @param conf the configuration object * @param resMgrDelegate the resource manager delegate * @param clientCache the client cache object. */ public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate, ClientCache clientCache) { this.conf = conf; this.tezConf = new TezConfiguration(conf); try { this.resMgrDelegate = resMgrDelegate; this.clientCache = clientCache; this.defaultFileContext = FileContext.getFileContext(this.conf); } catch (UnsupportedFileSystemException ufe) { throw new RuntimeException("Error in instantiating YarnClient", ufe); } }
Example #18
Source File: LocalContainerLauncher.java From big-c with Apache License 2.0 | 5 votes |
public LocalContainerLauncher(AppContext context, TaskUmbilicalProtocol umbilical) { super(LocalContainerLauncher.class.getName()); this.context = context; this.umbilical = umbilical; // umbilical: MRAppMaster creates (taskAttemptListener), passes to us // (TODO/FIXME: pointless to use RPC to talk to self; should create // LocalTaskAttemptListener or similar: implement umbilical protocol // but skip RPC stuff) try { curFC = FileContext.getFileContext(curDir.toURI()); } catch (UnsupportedFileSystemException ufse) { LOG.error("Local filesystem " + curDir.toURI().toString() + " is unsupported?? (should never happen)"); } // Save list of files/dirs that are supposed to be present so can delete // any extras created by one task before starting subsequent task. Note // that there's no protection against deleted or renamed localization; // users who do that get what they deserve (and will have to disable // uberization in order to run correctly). File[] curLocalFiles = curDir.listFiles(); localizedFiles = new HashSet<File>(curLocalFiles.length); for (int j = 0; j < curLocalFiles.length; ++j) { localizedFiles.add(curLocalFiles[j]); } // Relocalization note/future FIXME (per chrisdo, 20110315): At moment, // full localization info is in AppSubmissionContext passed from client to // RM and then to NM for AM-container launch: no difference between AM- // localization and MapTask- or ReduceTask-localization, so can assume all // OK. Longer-term, will need to override uber-AM container-localization // request ("needed resources") with union of regular-AM-resources + task- // resources (and, if maps and reduces ever differ, then union of all three // types), OR will need localizer service/API that uber-AM can request // after running (e.g., "localizeForTask()" or "localizeForMapTask()"). }
Example #19
Source File: TestNodeManagerShutdown.java From big-c with Apache License 2.0 | 5 votes |
@Before public void setup() throws UnsupportedFileSystemException { localFS = FileContext.getLocalFSFileContext(); tmpDir.mkdirs(); logsDir.mkdirs(); remoteLogsDir.mkdirs(); nmLocalDir.mkdirs(); // Construct the Container-id cId = createContainerId(); }
Example #20
Source File: ViewFs.java From big-c with Apache License 2.0 | 5 votes |
/** * This constructor has the signature needed by * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}. * * @param theUri which must be that of ViewFs * @param conf * @throws IOException * @throws URISyntaxException */ ViewFs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException { super(theUri, FsConstants.VIEWFS_SCHEME, false, -1); creationTime = Time.now(); ugi = UserGroupInformation.getCurrentUser(); config = conf; // Now build client side view (i.e. client side mount table) from config. String authority = theUri.getAuthority(); fsState = new InodeTree<AbstractFileSystem>(conf, authority) { @Override protected AbstractFileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { String pathString = uri.getPath(); if (pathString.isEmpty()) { pathString = "/"; } return new ChRootedFs( AbstractFileSystem.createFileSystem(uri, config), new Path(pathString)); } @Override protected AbstractFileSystem getTargetFileSystem( final INodeDir<AbstractFileSystem> dir) throws URISyntaxException { return new InternalDirOfViewFs(dir, creationTime, ugi, getUri()); } @Override protected AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList) throws URISyntaxException, UnsupportedFileSystemException { throw new UnsupportedFileSystemException("mergefs not implemented yet"); // return MergeFs.createMergeFs(mergeFsURIList, config); } }; }
Example #21
Source File: BaseContainerManagerTest.java From big-c with Apache License 2.0 | 5 votes |
public BaseContainerManagerTest() throws UnsupportedFileSystemException { localFS = FileContext.getLocalFSFileContext(); localDir = new File("target", this.getClass().getSimpleName() + "-localDir") .getAbsoluteFile(); localLogDir = new File("target", this.getClass().getSimpleName() + "-localLogDir") .getAbsoluteFile(); remoteLogDir = new File("target", this.getClass().getSimpleName() + "-remoteLogDir") .getAbsoluteFile(); tmpDir = new File("target", this.getClass().getSimpleName() + "-tmpDir"); }
Example #22
Source File: DefaultContainerExecutor.java From big-c with Apache License 2.0 | 5 votes |
public DefaultContainerExecutor() { try { this.lfs = FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
Example #23
Source File: TestNodeManagerResync.java From big-c with Apache License 2.0 | 5 votes |
@Before public void setup() throws UnsupportedFileSystemException { localFS = FileContext.getLocalFSFileContext(); tmpDir.mkdirs(); logsDir.mkdirs(); remoteLogsDir.mkdirs(); nmLocalDir.mkdirs(); syncBarrier = new CyclicBarrier(2); }
Example #24
Source File: DockerContainerExecutor.java From big-c with Apache License 2.0 | 5 votes |
public DockerContainerExecutor() { try { this.lfs = FileContext.getLocalFSFileContext(); this.dockerImagePattern = Pattern.compile(DOCKER_IMAGE_PATTERN); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
Example #25
Source File: NonAggregatingLogHandler.java From big-c with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public void run() { List<Path> localAppLogDirs = new ArrayList<Path>(); FileContext lfs = getLocalFileContext(getConfig()); for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) { Path logDir = new Path(rootLogDir, applicationId.toString()); try { lfs.getFileStatus(logDir); localAppLogDirs.add(logDir); } catch (UnsupportedFileSystemException ue) { LOG.warn("Unsupported file system used for log dir " + logDir, ue); continue; } catch (IOException ie) { continue; } } // Inform the application before the actual delete itself, so that links // to logs will no longer be there on NM web-UI. NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle( new ApplicationEvent(this.applicationId, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED)); if (localAppLogDirs.size() > 0) { NonAggregatingLogHandler.this.delService.delete(user, null, (Path[]) localAppLogDirs.toArray(new Path[localAppLogDirs.size()])); } try { NonAggregatingLogHandler.this.stateStore.removeLogDeleter( this.applicationId); } catch (IOException e) { LOG.error("Error removing log deletion state", e); } }
Example #26
Source File: ResourceLocalizationService.java From big-c with Apache License 2.0 | 5 votes |
private void submitDirForDeletion(String userName, Path dir) { try { lfs.getFileStatus(dir); delService.delete(userName, dir, new Path[] {}); } catch (UnsupportedFileSystemException ue) { LOG.warn("Local dir " + dir + " is an unsupported filesystem", ue); } catch (IOException ie) { // ignore return; } }
Example #27
Source File: TestDiskFailures.java From big-c with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { localFS = FileContext.getLocalFSFileContext(); localFS.delete(new Path(localFSDirBase.getAbsolutePath()), true); localFSDirBase.mkdirs(); // Do not start cluster here }
Example #28
Source File: HadoopFileSystemWrapper.java From dremio-oss with Apache License 2.0 | 5 votes |
@Override public FileStatus getFileLinkStatus(Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) { return underlyingFs.getFileLinkStatus(f); } catch(FSError e) { throw propagateFSError(e); } }
Example #29
Source File: HadoopFileSystemWrapper.java From dremio-oss with Apache License 2.0 | 5 votes |
@Override public void createSymlink(Path target, Path link, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException { try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) { underlyingFs.createSymlink(target, link, createParent); } catch(FSError e) { throw propagateFSError(e); } }
Example #30
Source File: TestViewFsConfig.java From hadoop with Apache License 2.0 | 5 votes |
@Test(expected=FileAlreadyExistsException.class) public void testInvalidConfig() throws IOException, URISyntaxException { Configuration conf = new Configuration(); ConfigUtil.addLink(conf, "/internalDir/linkToDir2", new Path("file:///dir2").toUri()); ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3", new Path("file:///dir3").toUri()); class Foo { }; new InodeTree<Foo>(conf, null) { @Override protected Foo getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { return null; } @Override protected Foo getTargetFileSystem( org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo> dir) throws URISyntaxException { return null; } @Override protected Foo getTargetFileSystem(URI[] mergeFsURIList) throws URISyntaxException, UnsupportedFileSystemException { return null; } }; }