Java Code Examples for org.apache.hadoop.fs.FileSystem.closeAll()

The following are Jave code examples for showing how to use closeAll() of the org.apache.hadoop.fs.FileSystem class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
+ Save this method
Example 1
Project: hadoop   File: TestFileInputFormat.java   View Source Code Vote up 6 votes
@Test
public void testListLocatedStatus() throws Exception {
  Configuration conf = getConfiguration();
  conf.setBoolean("fs.test.impl.disable.cache", false);
  conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
  conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR,
      "test:///a1/a2");
  MockFileSystem mockFs =
      (MockFileSystem) new Path("test:///").getFileSystem(conf);
  Assert.assertEquals("listLocatedStatus already called",
      0, mockFs.numListLocatedStatusCalls);
  JobConf job = new JobConf(conf);
  TextInputFormat fileInputFormat = new TextInputFormat();
  fileInputFormat.configure(job);
  InputSplit[] splits = fileInputFormat.getSplits(job, 1);
  Assert.assertEquals("Input splits are not correct", 2, splits.length);
  Assert.assertEquals("listLocatedStatuss calls",
      1, mockFs.numListLocatedStatusCalls);
  FileSystem.closeAll();
}
 
Example 2
Project: hadoop   File: TestFileInputFormat.java   View Source Code Vote up 6 votes
@Test
public void testListLocatedStatus() throws Exception {
  Configuration conf = getConfiguration();
  conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
  conf.setBoolean("fs.test.impl.disable.cache", false);
  conf.set(FileInputFormat.INPUT_DIR, "test:///a1/a2");
  MockFileSystem mockFs =
      (MockFileSystem) new Path("test:///").getFileSystem(conf);
  Assert.assertEquals("listLocatedStatus already called",
      0, mockFs.numListLocatedStatusCalls);
  Job job = Job.getInstance(conf);
  FileInputFormat<?, ?> fileInputFormat = new TextInputFormat();
  List<InputSplit> splits = fileInputFormat.getSplits(job);
  Assert.assertEquals("Input splits are not correct", 2, splits.size());
  Assert.assertEquals("listLocatedStatuss calls",
      1, mockFs.numListLocatedStatusCalls);
  FileSystem.closeAll();
}
 
Example 3
Project: hadoop   File: TestDistributedFileSystem.java   View Source Code Vote up 6 votes
@Test
public void testFileSystemCloseAll() throws Exception {
  Configuration conf = getTestConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
  URI address = FileSystem.getDefaultUri(conf);

  try {
    FileSystem.closeAll();

    conf = getTestConfiguration();
    FileSystem.setDefaultUri(conf, address);
    FileSystem.get(conf);
    FileSystem.get(conf);
    FileSystem.closeAll();
  }
  finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 
Example 4
Project: hadoop   File: TestStreamXmlMultipleRecords.java   View Source Code Vote up 5 votes
@Override
@Before
public void setUp() throws IOException {
  super.setUp();
  // Without this closeAll() call, setting of FileSystem block size is
  // not effective and will be old block size set in earlier test.
  FileSystem.closeAll();
}
 
Example 5
Project: hadoop   File: TestAggregatedLogDeletionService.java   View Source Code Vote up 4 votes
@Before
public void closeFilesystems() throws IOException {
  // prevent the same mockfs instance from being reused due to FS cache
  FileSystem.closeAll();
}
 
Example 6
Project: hadoop   File: TestFileOutputCommitter.java   View Source Code Vote up 4 votes
private void testConcurrentCommitTaskWithSubDir(int version)
    throws Exception {
  final Job job = Job.getInstance();
  FileOutputFormat.setOutputPath(job, outDir);
  final Configuration conf = job.getConfiguration();
  conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
  conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
      version);

  conf.setClass("fs.file.impl", RLFS.class, FileSystem.class);
  FileSystem.closeAll();

  final JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
  final FileOutputCommitter amCommitter =
      new FileOutputCommitter(outDir, jContext);
  amCommitter.setupJob(jContext);

  final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
  taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
  taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);

  final TextOutputFormat[] tof = new TextOutputFormat[2];
  for (int i = 0; i < tof.length; i++) {
    tof[i] = new TextOutputFormat() {
      @Override
      public Path getDefaultWorkFile(TaskAttemptContext context,
          String extension) throws IOException {
        final FileOutputCommitter foc = (FileOutputCommitter)
            getOutputCommitter(context);
        return new Path(new Path(foc.getWorkPath(), SUB_DIR),
            getUniqueFile(context, getOutputName(context), extension));
      }
    };
  }

  final ExecutorService executor = Executors.newFixedThreadPool(2);
  try {
    for (int i = 0; i < taCtx.length; i++) {
      final int taskIdx = i;
      executor.submit(new Callable<Void>() {
        @Override
        public Void call() throws IOException, InterruptedException {
          final OutputCommitter outputCommitter =
              tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
          outputCommitter.setupTask(taCtx[taskIdx]);
          final RecordWriter rw =
              tof[taskIdx].getRecordWriter(taCtx[taskIdx]);
          writeOutput(rw, taCtx[taskIdx]);
          outputCommitter.commitTask(taCtx[taskIdx]);
          return null;
        }
      });
    }
  } finally {
    executor.shutdown();
    while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
      LOG.info("Awaiting thread termination!");
    }
  }

  amCommitter.commitJob(jContext);
  final RawLocalFileSystem lfs = new RawLocalFileSystem();
  lfs.setConf(conf);
  assertFalse("Must not end up with sub_dir/sub_dir",
      lfs.exists(new Path(OUT_SUB_DIR, SUB_DIR)));

  // validate output
  validateContent(OUT_SUB_DIR);
  FileUtil.fullyDelete(new File(outDir.toString()));
}
 
Example 7
Project: hadoop   File: TestHftpFileSystem.java   View Source Code Vote up 4 votes
@After
public void resetFileSystems() throws IOException {
  FileSystem.closeAll();
}