Java Code Examples for org.apache.hadoop.fs.FilterFileSystem

The following examples show how to use org.apache.hadoop.fs.FilterFileSystem. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestChRootedFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDeleteOnExitPathHandling() throws IOException {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
      
  URI chrootUri = URI.create("mockfs://foo/a/b");
  ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
  FileSystem mockFs = ((FilterFileSystem)chrootFs.getRawFileSystem())
      .getRawFileSystem();
  
  // ensure delete propagates the correct path
  Path chrootPath = new Path("/c");
  Path rawPath = new Path("/a/b/c");
  chrootFs.delete(chrootPath, false);
  verify(mockFs).delete(eq(rawPath), eq(false));
  reset(mockFs);
 
  // fake that the path exists for deleteOnExit
  FileStatus stat = mock(FileStatus.class);
  when(mockFs.getFileStatus(eq(rawPath))).thenReturn(stat);
  // ensure deleteOnExit propagates the correct path
  chrootFs.deleteOnExit(chrootPath);
  chrootFs.close();
  verify(mockFs).delete(eq(rawPath), eq(true));
}
 
Example 2
Source Project: big-c   Source File: TestChRootedFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDeleteOnExitPathHandling() throws IOException {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
      
  URI chrootUri = URI.create("mockfs://foo/a/b");
  ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
  FileSystem mockFs = ((FilterFileSystem)chrootFs.getRawFileSystem())
      .getRawFileSystem();
  
  // ensure delete propagates the correct path
  Path chrootPath = new Path("/c");
  Path rawPath = new Path("/a/b/c");
  chrootFs.delete(chrootPath, false);
  verify(mockFs).delete(eq(rawPath), eq(false));
  reset(mockFs);
 
  // fake that the path exists for deleteOnExit
  FileStatus stat = mock(FileStatus.class);
  when(mockFs.getFileStatus(eq(rawPath))).thenReturn(stat);
  // ensure deleteOnExit propagates the correct path
  chrootFs.deleteOnExit(chrootPath);
  chrootFs.close();
  verify(mockFs).delete(eq(rawPath), eq(true));
}
 
Example 3
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static FileSystem getRawFileSystem(FileSystem fileSystem)
{
    if (fileSystem instanceof FilterFileSystem) {
        return getRawFileSystem(((FilterFileSystem) fileSystem).getRawFileSystem());
    }
    return fileSystem;
}
 
Example 4
Source Project: presto   Source File: TestRubixCaching.java    License: Apache License 2.0 5 votes vote down vote up
private static CachingFileSystem<?> unwrapCachingFileSystem(FileSystem fileSystem)
{
    if (fileSystem instanceof CachingFileSystem) {
        return (CachingFileSystem<?>) fileSystem;
    }
    if (fileSystem instanceof FilterFileSystem) {
        return unwrapCachingFileSystem(((FilterFileSystem) fileSystem).getRawFileSystem());
    }
    throw new IllegalStateException();
}
 
Example 5
Source Project: hadoop   Source File: TestMRApps.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public void testSetupDistributedCacheConflicts() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI archive = new URI("mockfs://mock/tmp/something.zip#something");
  Path archivePath = new Path(archive);
  URI file = new URI("mockfs://mock/tmp/something.txt#something");
  Path filePath = new Path(file);
  
  when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  
  DistributedCache.addCacheArchive(archive, conf);
  conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
  DistributedCache.addCacheFile(file, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
  
  assertEquals(1, localResources.size());
  LocalResource lr = localResources.get("something");
  //Archive wins
  assertNotNull(lr);
  assertEquals(10l, lr.getSize());
  assertEquals(10l, lr.getTimestamp());
  assertEquals(LocalResourceType.ARCHIVE, lr.getType());
}
 
Example 6
Source Project: hadoop   Source File: TestMRApps.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public void testSetupDistributedCacheConflictsFiles() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI file = new URI("mockfs://mock/tmp/something.zip#something");
  Path filePath = new Path(file);
  URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
  Path file2Path = new Path(file2);
  
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  when(mockFs.resolvePath(file2Path)).thenReturn(file2Path);
  
  DistributedCache.addCacheFile(file, conf);
  DistributedCache.addCacheFile(file2, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
  
  assertEquals(1, localResources.size());
  LocalResource lr = localResources.get("something");
  //First one wins
  assertNotNull(lr);
  assertEquals(10l, lr.getSize());
  assertEquals(10l, lr.getTimestamp());
  assertEquals(LocalResourceType.FILE, lr.getType());
}
 
Example 7
Source Project: hadoop   Source File: TestChRootedFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that ChRootedFileSystem delegates calls for every ACL method to the
 * underlying FileSystem with all Path arguments translated as required to
 * enforce chroot.
 */
@Test
public void testAclMethodsPathTranslation() throws IOException {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);

  URI chrootUri = URI.create("mockfs://foo/a/b");
  ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
  FileSystem mockFs = ((FilterFileSystem)chrootFs.getRawFileSystem())
      .getRawFileSystem();

  Path chrootPath = new Path("/c");
  Path rawPath = new Path("/a/b/c");
  List<AclEntry> entries = Collections.emptyList();

  chrootFs.modifyAclEntries(chrootPath, entries);
  verify(mockFs).modifyAclEntries(rawPath, entries);

  chrootFs.removeAclEntries(chrootPath, entries);
  verify(mockFs).removeAclEntries(rawPath, entries);

  chrootFs.removeDefaultAcl(chrootPath);
  verify(mockFs).removeDefaultAcl(rawPath);

  chrootFs.removeAcl(chrootPath);
  verify(mockFs).removeAcl(rawPath);

  chrootFs.setAcl(chrootPath, entries);
  verify(mockFs).setAcl(rawPath, entries);

  chrootFs.getAclStatus(chrootPath);
  verify(mockFs).getAclStatus(rawPath);
}
 
Example 8
Source Project: big-c   Source File: TestMRApps.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public void testSetupDistributedCacheConflicts() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI archive = new URI("mockfs://mock/tmp/something.zip#something");
  Path archivePath = new Path(archive);
  URI file = new URI("mockfs://mock/tmp/something.txt#something");
  Path filePath = new Path(file);
  
  when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  
  DistributedCache.addCacheArchive(archive, conf);
  conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
  DistributedCache.addCacheFile(file, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
  
  assertEquals(1, localResources.size());
  LocalResource lr = localResources.get("something");
  //Archive wins
  assertNotNull(lr);
  assertEquals(10l, lr.getSize());
  assertEquals(10l, lr.getTimestamp());
  assertEquals(LocalResourceType.ARCHIVE, lr.getType());
}
 
Example 9
Source Project: big-c   Source File: TestMRApps.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public void testSetupDistributedCacheConflictsFiles() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI file = new URI("mockfs://mock/tmp/something.zip#something");
  Path filePath = new Path(file);
  URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
  Path file2Path = new Path(file2);
  
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  when(mockFs.resolvePath(file2Path)).thenReturn(file2Path);
  
  DistributedCache.addCacheFile(file, conf);
  DistributedCache.addCacheFile(file2, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
  
  assertEquals(1, localResources.size());
  LocalResource lr = localResources.get("something");
  //First one wins
  assertNotNull(lr);
  assertEquals(10l, lr.getSize());
  assertEquals(10l, lr.getTimestamp());
  assertEquals(LocalResourceType.FILE, lr.getType());
}
 
Example 10
Source Project: big-c   Source File: TestChRootedFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that ChRootedFileSystem delegates calls for every ACL method to the
 * underlying FileSystem with all Path arguments translated as required to
 * enforce chroot.
 */
@Test
public void testAclMethodsPathTranslation() throws IOException {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);

  URI chrootUri = URI.create("mockfs://foo/a/b");
  ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
  FileSystem mockFs = ((FilterFileSystem)chrootFs.getRawFileSystem())
      .getRawFileSystem();

  Path chrootPath = new Path("/c");
  Path rawPath = new Path("/a/b/c");
  List<AclEntry> entries = Collections.emptyList();

  chrootFs.modifyAclEntries(chrootPath, entries);
  verify(mockFs).modifyAclEntries(rawPath, entries);

  chrootFs.removeAclEntries(chrootPath, entries);
  verify(mockFs).removeAclEntries(rawPath, entries);

  chrootFs.removeDefaultAcl(chrootPath);
  verify(mockFs).removeDefaultAcl(rawPath);

  chrootFs.removeAcl(chrootPath);
  verify(mockFs).removeAcl(rawPath);

  chrootFs.setAcl(chrootPath, entries);
  verify(mockFs).setAcl(rawPath, entries);

  chrootFs.getAclStatus(chrootPath);
  verify(mockFs).getAclStatus(rawPath);
}
 
Example 11
Source Project: hbase   Source File: RecoverLeaseFSUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Recover the lease from HDFS, retrying multiple times.
 */
public static void recoverFileLease(FileSystem fs, Path p, Configuration conf,
  CancelableProgressable reporter) throws IOException {
  if (fs instanceof FilterFileSystem) {
    fs = ((FilterFileSystem) fs).getRawFileSystem();
  }
  // lease recovery not needed for local file system case.
  if (!(fs instanceof DistributedFileSystem)) {
    return;
  }
  recoverDFSFileLease((DistributedFileSystem) fs, p, conf, reporter);
}
 
Example 12
Source Project: RDFS   Source File: DFSUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static DistributedFileSystem convertToDFS(FileSystem fs) {
  // for RaidDFS
  if (fs instanceof FilterFileSystem) {
    fs = ((FilterFileSystem) fs).getRawFileSystem();
  }
  if (fs instanceof DistributedFileSystem)
    return (DistributedFileSystem) fs;
  else
    return null;
}
 
Example 13
Source Project: presto   Source File: TestRubixCaching.java    License: Apache License 2.0 4 votes vote down vote up
private void assertRawFileSystemInstanceOf(FileSystem actual, Class<? extends FileSystem> expectedType)
{
    assertInstanceOf(actual, FilterFileSystem.class);
    FileSystem rawFileSystem = ((FilterFileSystem) actual).getRawFileSystem();
    assertInstanceOf(rawFileSystem, expectedType);
}
 
Example 14
Source Project: hadoop   Source File: TestMRApps.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("deprecation")
@Test (timeout = 30000)
public void testSetupDistributedCache() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI archive = new URI("mockfs://mock/tmp/something.zip");
  Path archivePath = new Path(archive);
  URI file = new URI("mockfs://mock/tmp/something.txt#something");
  Path filePath = new Path(file);
  
  when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  
  DistributedCache.addCacheArchive(archive, conf);
  conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
  DistributedCache.addCacheFile(file, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
  assertEquals(2, localResources.size());
  LocalResource lr = localResources.get("something.zip");
  assertNotNull(lr);
  assertEquals(10l, lr.getSize());
  assertEquals(10l, lr.getTimestamp());
  assertEquals(LocalResourceType.ARCHIVE, lr.getType());
  lr = localResources.get("something");
  assertNotNull(lr);
  assertEquals(11l, lr.getSize());
  assertEquals(11l, lr.getTimestamp());
  assertEquals(LocalResourceType.FILE, lr.getType());
}
 
Example 15
Source Project: big-c   Source File: TestMRApps.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("deprecation")
@Test (timeout = 30000)
public void testSetupDistributedCache() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI archive = new URI("mockfs://mock/tmp/something.zip");
  Path archivePath = new Path(archive);
  URI file = new URI("mockfs://mock/tmp/something.txt#something");
  Path filePath = new Path(file);
  
  when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  
  DistributedCache.addCacheArchive(archive, conf);
  conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
  DistributedCache.addCacheFile(file, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
  assertEquals(2, localResources.size());
  LocalResource lr = localResources.get("something.zip");
  assertNotNull(lr);
  assertEquals(10l, lr.getSize());
  assertEquals(10l, lr.getTimestamp());
  assertEquals(LocalResourceType.ARCHIVE, lr.getType());
  lr = localResources.get("something");
  assertNotNull(lr);
  assertEquals(11l, lr.getSize());
  assertEquals(11l, lr.getTimestamp());
  assertEquals(LocalResourceType.FILE, lr.getType());
}
 
Example 16
Source Project: hbase   Source File: TestCleanerChore.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void retriesIOExceptionInStatus() throws Exception {
  Stoppable stop = new StoppableImplementation();
  Configuration conf = UTIL.getConfiguration();
  Path testDir = UTIL.getDataTestDir();
  FileSystem fs = UTIL.getTestFileSystem();
  String confKey = "hbase.test.cleaner.delegates";

  Path child = new Path(testDir, "child");
  Path file = new Path(child, "file");
  fs.mkdirs(child);
  fs.create(file).close();
  assertTrue("test file didn't get created.", fs.exists(file));
  final AtomicBoolean fails = new AtomicBoolean(true);

  FilterFileSystem filtered = new FilterFileSystem(fs) {
    public FileStatus[] listStatus(Path f) throws IOException {
      if (fails.get()) {
        throw new IOException("whomp whomp.");
      }
      return fs.listStatus(f);
    }
  };

  AllValidPaths chore =
    new AllValidPaths("test-retry-ioe", stop, conf, filtered, testDir, confKey, POOL);

  // trouble talking to the filesystem
  Boolean result = chore.runCleaner();

  // verify that it couldn't clean the files.
  assertTrue("test rig failed to inject failure.", fs.exists(file));
  assertTrue("test rig failed to inject failure.", fs.exists(child));
  // and verify that it accurately reported the failure.
  assertFalse("chore should report that it failed.", result);

  // filesystem is back
  fails.set(false);
  result = chore.runCleaner();

  // verify everything is gone.
  assertFalse("file should have been destroyed.", fs.exists(file));
  assertFalse("directory should have been destroyed.", fs.exists(child));
  // and verify that it accurately reported success.
  assertTrue("chore should claim it succeeded.", result);
}