Java Code Examples for org.apache.hadoop.fs.FileSystem#closeAll()

The following examples show how to use org.apache.hadoop.fs.FileSystem#closeAll() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: RDFS   File: DFSLocationsRoot.java    License: Apache License 2.0 6 votes vote down vote up
public void disconnect() {
  Thread closeThread = new Thread() {
    /* @inheritDoc */
    @Override
    public void run() {
      try {
        System.out.printf("Closing all opened File Systems...\n");
        FileSystem.closeAll();
        System.out.printf("File Systems closed\n");

      } catch (IOException ioe) {
        ioe.printStackTrace();
      }
    }
  };

  // Wait 5 seconds for the connections to be closed
  closeThread.start();
  try {
    closeThread.join(5000);

  } catch (InterruptedException ie) {
    // Ignore
  }
}
 
Example 2
Source Project: hadoop   File: TestFileInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testListLocatedStatus() throws Exception {
  Configuration conf = getConfiguration();
  conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
  conf.setBoolean("fs.test.impl.disable.cache", false);
  conf.set(FileInputFormat.INPUT_DIR, "test:///a1/a2");
  MockFileSystem mockFs =
      (MockFileSystem) new Path("test:///").getFileSystem(conf);
  Assert.assertEquals("listLocatedStatus already called",
      0, mockFs.numListLocatedStatusCalls);
  Job job = Job.getInstance(conf);
  FileInputFormat<?, ?> fileInputFormat = new TextInputFormat();
  List<InputSplit> splits = fileInputFormat.getSplits(job);
  Assert.assertEquals("Input splits are not correct", 2, splits.size());
  Assert.assertEquals("listLocatedStatuss calls",
      1, mockFs.numListLocatedStatusCalls);
  FileSystem.closeAll();
}
 
Example 3
Source Project: hadoop-gpu   File: DFSLocationsRoot.java    License: Apache License 2.0 6 votes vote down vote up
public void disconnect() {
  Thread closeThread = new Thread() {
    /* @inheritDoc */
    @Override
    public void run() {
      try {
        System.out.printf("Closing all opened File Systems...\n");
        FileSystem.closeAll();
        System.out.printf("File Systems closed\n");

      } catch (IOException ioe) {
        ioe.printStackTrace();
      }
    }
  };

  // Wait 5 seconds for the connections to be closed
  closeThread.start();
  try {
    closeThread.join(5000);

  } catch (InterruptedException ie) {
    // Ignore
  }
}
 
Example 4
Source Project: big-c   File: TestFileInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testListLocatedStatus() throws Exception {
  Configuration conf = getConfiguration();
  conf.setBoolean("fs.test.impl.disable.cache", false);
  conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
  conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR,
      "test:///a1/a2");
  MockFileSystem mockFs =
      (MockFileSystem) new Path("test:///").getFileSystem(conf);
  Assert.assertEquals("listLocatedStatus already called",
      0, mockFs.numListLocatedStatusCalls);
  JobConf job = new JobConf(conf);
  TextInputFormat fileInputFormat = new TextInputFormat();
  fileInputFormat.configure(job);
  InputSplit[] splits = fileInputFormat.getSplits(job, 1);
  Assert.assertEquals("Input splits are not correct", 2, splits.length);
  Assert.assertEquals("listLocatedStatuss calls",
      1, mockFs.numListLocatedStatusCalls);
  FileSystem.closeAll();
}
 
Example 5
Source Project: big-c   File: TestDistributedFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFileSystemCloseAll() throws Exception {
  Configuration conf = getTestConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
  URI address = FileSystem.getDefaultUri(conf);

  try {
    FileSystem.closeAll();

    conf = getTestConfiguration();
    FileSystem.setDefaultUri(conf, address);
    FileSystem.get(conf);
    FileSystem.get(conf);
    FileSystem.closeAll();
  }
  finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 
Example 6
Source Project: hadoop-gpu   File: TestTrash.java    License: Apache License 2.0 6 votes vote down vote up
public static void trashNonDefaultFS(Configuration conf) throws IOException {
  conf.set("fs.trash.interval", "10"); // 10 minute
  // attempt non-default FileSystem trash
  {
    final FileSystem lfs = FileSystem.getLocal(conf);
    Path p = TEST_DIR;
    Path f = new Path(p, "foo/bar");
    if (lfs.exists(p)) {
      lfs.delete(p, true);
    }
    try {
      f = writeFile(lfs, f);

      FileSystem.closeAll();
      FileSystem localFs = FileSystem.get(URI.create("file:///"), conf);
      Trash lTrash = new Trash(localFs, conf);
      lTrash.moveToTrash(f.getParent());
      checkTrash(localFs, lTrash.getCurrentTrashDir(), f);
    } finally {
      if (lfs.exists(p)) {
        lfs.delete(p, true);
      }
    }
  }
}
 
Example 7
@Override
@Before
public void setUp() throws IOException {
  super.setUp();
  // Without this closeAll() call, setting of FileSystem block size is
  // not effective and will be old block size set in earlier test.
  FileSystem.closeAll();
}
 
Example 8
@Override
public void stop() {
	try {
		FileSystem.closeAll();
		logger.info("Closing the Hadoop FileSystem");
	} catch (IOException e) {
		logger.error("Unable to close Hadoop FileSystem", e);
	}
	this.running = false;
}
 
Example 9
Source Project: RDFS   File: TestBlockFixer.java    License: Apache License 2.0 5 votes vote down vote up
public static DistributedFileSystem getDFS(
      Configuration conf, FileSystem dfs) throws IOException {
  Configuration clientConf = new Configuration(conf);
  clientConf.set("fs.hdfs.impl",
                 "org.apache.hadoop.hdfs.DistributedFileSystem");
  clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
  URI dfsUri = dfs.getUri();
  FileSystem.closeAll();
  return (DistributedFileSystem) FileSystem.get(dfsUri, clientConf);
}
 
Example 10
Source Project: stratosphere   File: CancellingTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@After
public void stopCluster() throws Exception {
	if (this.executor != null) {
		this.executor.stop();
		this.executor = null;
		FileSystem.closeAll();
		System.gc();
	}
}
 
Example 11
@Override
@Before
public void setUp() throws IOException {
  super.setUp();
  // Without this closeAll() call, setting of FileSystem block size is
  // not effective and will be old block size set in earlier test.
  FileSystem.closeAll();
}
 
Example 12
Source Project: stratosphere   File: AbstractTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@After
public void stopCluster() throws Exception {
	try {
		if (this.executor != null) {
			this.executor.stop();
			this.executor = null;
			FileSystem.closeAll();
			System.gc();
		}
	} finally {
		deleteAllTempFiles();
	}
}
 
Example 13
private void testConcurrentCommitTaskWithSubDir(int version) throws Exception {
    final Job job = Job.getInstance();
    FileOutputFormat.setOutputPath(job, outDir);
    final Configuration conf = job.getConfiguration();
    conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
    conf.setInt(FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
    
    conf.setClass("fs.file.impl", RLFS.class, FileSystem.class);
    FileSystem.closeAll();
    
    final JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
    final FileOutputCommitter amCommitter = new SafeFileOutputCommitter(outDir, jContext);
    amCommitter.setupJob(jContext);
    
    final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
    taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
    taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);
    
    final TextOutputFormat[] tof = new TextOutputFormat[2];
    for (int i = 0; i < tof.length; i++) {
        tof[i] = new TextOutputFormat() {
            @Override
            public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException {
                final FileOutputCommitter foc = (FileOutputCommitter) getOutputCommitter(context);
                return new Path(new Path(foc.getWorkPath(), SUB_DIR), getUniqueFile(context, getOutputName(context), extension));
            }
        };
    }
    
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    try {
        for (int i = 0; i < taCtx.length; i++) {
            final int taskIdx = i;
            executor.submit((Callable<Void>) () -> {
                final OutputCommitter outputCommitter = tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
                outputCommitter.setupTask(taCtx[taskIdx]);
                final RecordWriter rw = tof[taskIdx].getRecordWriter(taCtx[taskIdx]);
                writeOutput(rw, taCtx[taskIdx]);
                outputCommitter.commitTask(taCtx[taskIdx]);
                return null;
            });
        }
    } finally {
        executor.shutdown();
        while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
            LOG.info("Awaiting thread termination!");
        }
    }
    
    amCommitter.commitJob(jContext);
    final RawLocalFileSystem lfs = new RawLocalFileSystem();
    lfs.setConf(conf);
    assertFalse("Must not end up with sub_dir/sub_dir", lfs.exists(new Path(OUT_SUB_DIR, SUB_DIR)));
    
    // validate output
    validateContent(OUT_SUB_DIR);
    FileUtil.fullyDelete(new File(outDir.toString()));
}
 
Example 14
@Before
public void closeFilesystems() throws IOException {
  // prevent the same mockfs instance from being reused due to FS cache
  FileSystem.closeAll();
}
 
Example 15
@After
public void after() throws IOException {
  FileSystem.closeAll();
}
 
Example 16
Source Project: mr4c   File: MR4CGenericOptionsParser.java    License: Apache License 2.0 4 votes vote down vote up
/**
  * This should be called after tests to get rid of FileSystem side effects
*/
public static void cleanup() throws IOException {
	// GenericOptionsParser will cache FileSystem objects based on the config it is updating
	FileSystem.closeAll();
}
 
Example 17
@Before
public void closeFilesystems() throws IOException {
  // prevent the same mockfs instance from being reused due to FS cache
  FileSystem.closeAll();
}
 
Example 18
Source Project: big-c   File: TestHftpFileSystem.java    License: Apache License 2.0 4 votes vote down vote up
@After
public void resetFileSystems() throws IOException {
  FileSystem.closeAll();
}
 
Example 19
@Before
public void clearFileSystemCache() throws IOException {
  FileSystem.closeAll();
}
 
Example 20
Source Project: ignite   File: HadoopFileSystemsUtils.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Clears Hadoop {@link FileSystem} cache.
 *
 * @throws IOException On error.
 */
public static void clearFileSystemCache() throws IOException {
    FileSystem.closeAll();
}