Java Code Examples for org.apache.hadoop.fs.FileSystem.exists()

The following are Jave code examples for showing how to use exists() of the org.apache.hadoop.fs.FileSystem class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: hadoop   File: MapReduceTestUtil.java   Source Code and License Vote up 7 votes
public static Job createJob(Configuration conf, Path inDir, Path outDir, 
    int numInputFiles, int numReds, String input) throws IOException {
  Job job = Job.getInstance(conf);
  FileSystem fs = FileSystem.get(conf);
  if (fs.exists(outDir)) {
    fs.delete(outDir, true);
  }
  if (fs.exists(inDir)) {
    fs.delete(inDir, true);
  }
  fs.mkdirs(inDir);
  for (int i = 0; i < numInputFiles; ++i) {
    DataOutputStream file = fs.create(new Path(inDir, "part-" + i));
    file.writeBytes(input);
    file.close();
  }    

  FileInputFormat.setInputPaths(job, inDir);
  FileOutputFormat.setOutputPath(job, outDir);
  job.setNumReduceTasks(numReds);
  return job;
}
 
Example 2
Project: hadoop   File: TestCopyCommitter.java   Source Code and License Vote up 6 votes
private boolean checkDirectoryPermissions(FileSystem fs, String targetBase,
                                          FsPermission sourcePerm) throws IOException {
  Path base = new Path(targetBase);

  Stack<Path> stack = new Stack<Path>();
  stack.push(base);
  while (!stack.isEmpty()) {
    Path file = stack.pop();
    if (!fs.exists(file)) continue;
    FileStatus[] fStatus = fs.listStatus(file);
    if (fStatus == null || fStatus.length == 0) continue;

    for (FileStatus status : fStatus) {
      if (status.isDirectory()) {
        stack.push(status.getPath());
        Assert.assertEquals(status.getPermission(), sourcePerm);
      }
    }
  }
  return true;
}
 
Example 3
Project: ditb   File: IntegrationTestBigLinkedList.java   Source Code and License Vote up 6 votes
static SortedSet<byte []> readKeysToSearch(final Configuration conf)
throws IOException, InterruptedException {
  Path keysInputDir = new Path(conf.get(SEARCHER_INPUTDIR_KEY));
  FileSystem fs = FileSystem.get(conf);
  SortedSet<byte []> result = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
  if (!fs.exists(keysInputDir)) {
    throw new FileNotFoundException(keysInputDir.toString());
  }
  if (!fs.isDirectory(keysInputDir)) {
    throw new UnsupportedOperationException("TODO");
  } else {
    RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(keysInputDir, false);
    while(iterator.hasNext()) {
      LocatedFileStatus keyFileStatus = iterator.next();
      // Skip "_SUCCESS" file.
      if (keyFileStatus.getPath().getName().startsWith("_")) continue;
      result.addAll(readFileToSearch(conf, fs, keyFileStatus));
    }
  }
  return result;
}
 
Example 4
Project: ditb   File: HRegionFileSystem.java   Source Code and License Vote up 6 votes
/**
 * Remove the region from the table directory, archiving the region's hfiles.
 *
 * @param conf       the {@link Configuration} to use
 * @param fs         {@link FileSystem} from which to remove the region
 * @param tableDir   {@link Path} to where the table is being stored
 * @param regionInfo {@link HRegionInfo} for region to be deleted
 * @throws IOException if the request cannot be completed
 */
public static void deleteRegionFromFileSystem(final Configuration conf, final FileSystem fs,
    final Path tableDir, final HRegionInfo regionInfo) throws IOException {
  HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
  Path regionDir = regionFs.getRegionDir();

  if (!fs.exists(regionDir)) {
    LOG.warn("Trying to delete a region that do not exists on disk: " + regionDir);
    return;
  }

  if (LOG.isDebugEnabled()) {
    LOG.debug("DELETING region " + regionDir);
  }

  // Archive region
  Path rootDir = FSUtils.getRootDir(conf);
  HFileArchiver.archiveRegion(fs, rootDir, tableDir, regionDir);

  // Delete empty region dir
  if (!fs.delete(regionDir, true)) {
    LOG.warn("Failed delete of " + regionDir);
  }
}
 
Example 5
Project: circus-train   File: S3MapReduceCpTestUtils.java   Source Code and License Vote up 6 votes
public static boolean checkIfFoldersAreInSync(FileSystem fs, String targetBase, String sourceBase)
  throws IOException {
  Path base = new Path(targetBase);

  Stack<Path> stack = new Stack<>();
  stack.push(base);
  while (!stack.isEmpty()) {
    Path file = stack.pop();
    if (!fs.exists(file)) {
      continue;
    }
    FileStatus[] fStatus = fs.listStatus(file);
    if (fStatus == null || fStatus.length == 0) {
      continue;
    }

    for (FileStatus status : fStatus) {
      if (status.isDirectory()) {
        stack.push(status.getPath());
      }
      assertTrue(
          fs.exists(new Path(sourceBase + "/" + PathUtil.getRelativePath(new Path(targetBase), status.getPath()))));
    }
  }
  return true;
}
 
Example 6
Project: hadoop   File: TestLogsCLI.java   Source Code and License Vote up 5 votes
private static void createContainerLogInLocalDir(Path appLogsDir,
    ContainerId containerId, FileSystem fs) throws Exception {
  Path containerLogsDir = new Path(appLogsDir, containerId.toString());
  if (fs.exists(containerLogsDir)) {
    fs.delete(containerLogsDir, true);
  }
  assertTrue(fs.mkdirs(containerLogsDir));
  Writer writer =
      new FileWriter(new File(containerLogsDir.toString(), "sysout"));
  writer.write("Hello " + containerId + "!");
  writer.close();
}
 
Example 7
Project: hadoop   File: TestResourceUsageEmulators.java   Source Code and License Vote up 5 votes
static long testInitialization(String id, Configuration conf) 
throws IOException {
  Path testPath = getInitFilePath(id);
  FileSystem fs = FileSystem.getLocal(conf);
  return fs.exists(testPath) 
         ? fs.getFileStatus(testPath).getModificationTime() 
         : 0;
}
 
Example 8
Project: ditb   File: TestRecoveredEdits.java   Source Code and License Vote up 5 votes
/**
 * HBASE-12782 ITBLL fails for me if generator does anything but 5M per maptask.
 * Create a region. Close it. Then copy into place a file to replay, one that is bigger than
 * configured flush size so we bring on lots of flushes.  Then reopen and confirm all edits
 * made it in.
 * @throws IOException
 */
@Test (timeout=60000)
public void testReplayWorksThoughLotsOfFlushing() throws IOException {
  Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
  // Set it so we flush every 1M or so.  Thats a lot.
  conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
  // The file of recovered edits has a column family of 'meta'. Also has an encoded regionname
  // of 4823016d8fca70b25503ee07f4c6d79f which needs to match on replay.
  final String encodedRegionName = "4823016d8fca70b25503ee07f4c6d79f";
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(testName.getMethodName()));
  final String columnFamily = "meta";
  byte [][] columnFamilyAsByteArray = new byte [][] {Bytes.toBytes(columnFamily)};
  htd.addFamily(new HColumnDescriptor(columnFamily));
  HRegionInfo hri = new HRegionInfo(htd.getTableName()) {
    @Override
    public synchronized String getEncodedName() {
      return encodedRegionName;
    }

    // Cache the name because lots of lookups.
    private byte [] encodedRegionNameAsBytes = null;
    @Override
    public synchronized byte[] getEncodedNameAsBytes() {
      if (encodedRegionNameAsBytes == null) {
        this.encodedRegionNameAsBytes = Bytes.toBytes(getEncodedName());
      }
      return this.encodedRegionNameAsBytes;
    }
  };
  Path hbaseRootDir = TEST_UTIL.getDataTestDir();
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  Path tableDir = FSUtils.getTableDir(hbaseRootDir, htd.getTableName());
  HRegionFileSystem hrfs =
      new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tableDir, hri);
  if (fs.exists(hrfs.getRegionDir())) {
    LOG.info("Region directory already exists. Deleting.");
    fs.delete(hrfs.getRegionDir(), true);
  }
  HRegion region = HRegion.createHRegion(hri, hbaseRootDir, conf, htd, null);
  assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
  List<String> storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
  // There should be no store files.
  assertTrue(storeFiles.isEmpty());
  region.close();
  Path regionDir = region.getRegionDir(hbaseRootDir, hri);
  Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);
  // This is a little fragile getting this path to a file of 10M of edits.
  Path recoveredEditsFile = new Path(
    System.getProperty("test.build.classes", "target/test-classes"),
      "0000000000000016310");
  // Copy this file under the region's recovered.edits dir so it is replayed on reopen.
  Path destination = new Path(recoveredEditsDir, recoveredEditsFile.getName());
  fs.copyToLocalFile(recoveredEditsFile, destination);
  assertTrue(fs.exists(destination));
  // Now the file 0000000000000016310 is under recovered.edits, reopen the region to replay.
  region = HRegion.openHRegion(region, null);
  assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
  storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
  // Our 0000000000000016310 is 10MB. Most of the edits are for one region. Lets assume that if
  // we flush at 1MB, that there are at least 3 flushed files that are there because of the
  // replay of edits.
  assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10);
  // Now verify all edits made it into the region.
  int count = verifyAllEditsMadeItIn(fs, conf, recoveredEditsFile, region);
  LOG.info("Checked " + count + " edits made it in");
}
 
Example 9
Project: hadoop   File: TestMapperReducerCleanup.java   Source Code and License Vote up 5 votes
private Path createInput() throws IOException {
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.getLocal(conf);
  Path inputPath = getInputPath();

  // Clear the input directory if it exists, first.
  if (fs.exists(inputPath)) {
    fs.delete(inputPath, true);
  }

  // Create an input file
  createInputFile(inputPath, 0, 10);

  return inputPath;
}
 
Example 10
Project: ditb   File: TableMapReduceUtil.java   Source Code and License Vote up 5 votes
/**
 * Add the jars containing the given classes to the job's configuration
 * such that JobClient will ship them to the cluster and add them to
 * the DistributedCache.
 */
public static void addDependencyJars(Configuration conf,
    Class<?>... classes) throws IOException {

  FileSystem localFs = FileSystem.getLocal(conf);
  Set<String> jars = new HashSet<String>();
  // Add jars that are already in the tmpjars variable
  jars.addAll(conf.getStringCollection("tmpjars"));

  // add jars as we find them to a map of contents jar name so that we can avoid
  // creating new jars for classes that have already been packaged.
  Map<String, String> packagedClasses = new HashMap<String, String>();

  // Add jars containing the specified classes
  for (Class<?> clazz : classes) {
    if (clazz == null) continue;

    Path path = findOrCreateJar(clazz, localFs, packagedClasses);
    if (path == null) {
      LOG.warn("Could not find jar for class " + clazz +
               " in order to ship it to the cluster.");
      continue;
    }
    if (!localFs.exists(path)) {
      LOG.warn("Could not validate jar file " + path + " for class "
               + clazz);
      continue;
    }
    jars.add(path.toString());
  }
  if (jars.isEmpty()) return;

  conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
}
 
Example 11
Project: hadoop   File: CleanerService.java   Source Code and License Vote up 5 votes
/**
 * To ensure there are not multiple instances of the SCM running on a given
 * cluster, a global pid file is used. This file contains the hostname of the
 * machine that owns the pid file.
 *
 * @return true if the pid file was written, false otherwise
 * @throws YarnException
 */
private boolean writeGlobalCleanerPidFile() throws YarnException {
  String root =
      conf.get(YarnConfiguration.SHARED_CACHE_ROOT,
          YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT);
  Path pidPath = new Path(root, GLOBAL_CLEANER_PID);
  try {
    FileSystem fs = FileSystem.get(this.conf);

    if (fs.exists(pidPath)) {
      return false;
    }

    FSDataOutputStream os = fs.create(pidPath, false);
    // write the hostname and the process id in the global cleaner pid file
    final String ID = ManagementFactory.getRuntimeMXBean().getName();
    os.writeUTF(ID);
    os.close();
    // add it to the delete-on-exit to ensure it gets deleted when the JVM
    // exits
    fs.deleteOnExit(pidPath);
  } catch (IOException e) {
    throw new YarnException(e);
  }
  LOG.info("Created the global cleaner pid file at " + pidPath.toString());
  return true;
}
 
Example 12
Project: alluxio   File: HdfsAndAlluxioUtils_update.java   Source Code and License Vote up 5 votes
/**
 * 此方法用于删除文件
 *
 * @param fileSystemInfo
 *            文件系统信息
 * @param path
 *            文件路径
 * @return 删除文件是否成功
 */
public static boolean delete(FileSystemInfo fileSystemInfo, String path) {
	FileSystem fs = getFileSystem(fileSystemInfo);
	Path uri = new Path(path);
	try {
		if (fs.exists(uri)) {
			return fs.delete(uri, true);
		}
	} catch (IOException e) {
		e.printStackTrace();
	} finally {
		closeFileSystem(fs);
	}
	return false;
}
 
Example 13
Project: hadoop   File: TestEncryptedTransfer.java   Source Code and License Vote up 5 votes
private static void writeTestDataToFile(FileSystem fs) throws IOException {
  OutputStream out = null;
  if (!fs.exists(TEST_PATH)) {
    out = fs.create(TEST_PATH);
  } else {
    out = fs.append(TEST_PATH);
  }
  out.write(PLAIN_TEXT.getBytes());
  out.close();
}
 
Example 14
Project: ditb   File: TestHFileCleaner.java   Source Code and License Vote up 5 votes
@Test
public void testRemovesEmptyDirectories() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  // no cleaner policies = delete all files
  conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
  Server server = new DummyServer();
  Path archivedHfileDir = new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);

  // setup the cleaner
  FileSystem fs = UTIL.getDFSCluster().getFileSystem();
  HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir);

  // make all the directories for archiving files
  Path table = new Path(archivedHfileDir, "table");
  Path region = new Path(table, "regionsomthing");
  Path family = new Path(region, "fam");
  Path file = new Path(family, "file12345");
  fs.mkdirs(family);
  if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family);
  fs.create(file).close();
  if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file);

  // run the chore to cleanup the files (and the directories above it)
  cleaner.chore();

  // make sure all the parent directories get removed
  assertFalse("family directory not removed for empty directory", fs.exists(family));
  assertFalse("region directory not removed for empty directory", fs.exists(region));
  assertFalse("table directory not removed for empty directory", fs.exists(table));
  assertTrue("archive directory", fs.exists(archivedHfileDir));
}
 
Example 15
Project: hadoop   File: LoadTypedBytes.java   Source Code and License Vote up 5 votes
/**
 * The main driver for <code>LoadTypedBytes</code>.
 */
public int run(String[] args) throws Exception {
  if (args.length == 0) {
    System.err.println("Too few arguments!");
    printUsage();
    return 1;
  }
  Path path = new Path(args[0]);
  FileSystem fs = path.getFileSystem(getConf());
  if (fs.exists(path)) {
    System.err.println("given path exists already!");
    return -1;
  }
  TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
  SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path,
    TypedBytesWritable.class, TypedBytesWritable.class);
  try {
    TypedBytesWritable key = new TypedBytesWritable();
    TypedBytesWritable value = new TypedBytesWritable();
    byte[] rawKey = tbinput.readRaw();
    while (rawKey != null) {
      byte[] rawValue = tbinput.readRaw();
      key.set(rawKey, 0, rawKey.length);
      value.set(rawValue, 0, rawValue.length);
      writer.append(key, value);
      rawKey = tbinput.readRaw();
    }
  } finally {
    writer.close();
  }
  return 0;
}
 
Example 16
Project: aliyun-maxcompute-data-collectors   File: TestHiveImport.java   Source Code and License Vote up 5 votes
/** Test that we can generate a file containing the DDL and not import. */
@Test
public void testGenerateOnly() throws IOException {
  final String TABLE_NAME = "GenerateOnly";
  setCurTableName(TABLE_NAME);
  setNumCols(1);

  // Figure out where our target generated .q file is going to be.
  SqoopOptions options = getSqoopOptions(getArgv(false, null),
      new ImportTool());
  Path ddlFile = new Path(new Path(options.getCodeOutputDir()),
      TABLE_NAME + ".q");
  FileSystem fs = FileSystem.getLocal(new Configuration());

  // If it's already there, remove it before running the test to ensure
  // that it's the current test that generated the file.
  if (fs.exists(ddlFile)) {
    if (!fs.delete(ddlFile, false)) {
      LOG.warn("Could not delete previous ddl file: " + ddlFile);
    }
  }

  // Run a basic import, but specify that we're just generating definitions.
  String [] types = { "INTEGER" };
  String [] vals = { "42" };
  runImportTest(TABLE_NAME, types, vals, null, getCodeGenArgs(),
      new CodeGenTool());

  // Test that the generated definition file exists.
  assertTrue("Couldn't find expected ddl file", fs.exists(ddlFile));

  Path hiveImportPath = new Path(new Path(options.getWarehouseDir()),
      TABLE_NAME);
  assertFalse("Import actually happened!", fs.exists(hiveImportPath));
}
 
Example 17
Project: ditb   File: TestHQuorumPeer.java   Source Code and License Vote up 5 votes
@Before public void setup() throws IOException {
  // Set it to a non-standard port.
  TEST_UTIL.getConfiguration().setInt(HConstants.ZOOKEEPER_CLIENT_PORT,
      PORT_NO);
  this.dataDir = TEST_UTIL.getDataTestDir(this.getClass().getName());
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  if (fs.exists(this.dataDir)) {
    if (!fs.delete(this.dataDir, true)) {
      throw new IOException("Failed cleanup of " + this.dataDir);
    }
  }
  if (!fs.mkdirs(this.dataDir)) {
    throw new IOException("Failed create of " + this.dataDir);
  }
}
 
Example 18
Project: hadoop   File: MapFile.java   Source Code and License Vote up 4 votes
/**
 * This method attempts to fix a corrupt MapFile by re-creating its index.
 * @param fs filesystem
 * @param dir directory containing the MapFile data and index
 * @param keyClass key class (has to be a subclass of Writable)
 * @param valueClass value class (has to be a subclass of Writable)
 * @param dryrun do not perform any changes, just report what needs to be done
 * @return number of valid entries in this MapFile, or -1 if no fixing was needed
 * @throws Exception
 */
public static long fix(FileSystem fs, Path dir,
                       Class<? extends Writable> keyClass,
                       Class<? extends Writable> valueClass, boolean dryrun,
                       Configuration conf) throws Exception {
  String dr = (dryrun ? "[DRY RUN ] " : "");
  Path data = new Path(dir, DATA_FILE_NAME);
  Path index = new Path(dir, INDEX_FILE_NAME);
  int indexInterval = conf.getInt(Writer.INDEX_INTERVAL, 128);
  if (!fs.exists(data)) {
    // there's nothing we can do to fix this!
    throw new Exception(dr + "Missing data file in " + dir + ", impossible to fix this.");
  }
  if (fs.exists(index)) {
    // no fixing needed
    return -1;
  }
  SequenceFile.Reader dataReader = 
    new SequenceFile.Reader(conf, SequenceFile.Reader.file(data));
  if (!dataReader.getKeyClass().equals(keyClass)) {
    throw new Exception(dr + "Wrong key class in " + dir + ", expected" + keyClass.getName() +
                        ", got " + dataReader.getKeyClass().getName());
  }
  if (!dataReader.getValueClass().equals(valueClass)) {
    throw new Exception(dr + "Wrong value class in " + dir + ", expected" + valueClass.getName() +
                        ", got " + dataReader.getValueClass().getName());
  }
  long cnt = 0L;
  Writable key = ReflectionUtils.newInstance(keyClass, conf);
  Writable value = ReflectionUtils.newInstance(valueClass, conf);
  SequenceFile.Writer indexWriter = null;
  if (!dryrun) {
    indexWriter = 
      SequenceFile.createWriter(conf, 
                                SequenceFile.Writer.file(index), 
                                SequenceFile.Writer.keyClass(keyClass), 
                                SequenceFile.Writer.valueClass
                                  (LongWritable.class));
  }
  try {
    long pos = 0L;
    LongWritable position = new LongWritable();
    while(dataReader.next(key, value)) {
      cnt++;
      if (cnt % indexInterval == 0) {
        position.set(pos);
        if (!dryrun) indexWriter.append(key, position);
      }
      pos = dataReader.getPosition();
    }
  } catch(Throwable t) {
    // truncated data file. swallow it.
  }
  dataReader.close();
  if (!dryrun) indexWriter.close();
  return cnt;
}
 
Example 19
Project: ditb   File: WALPerformanceEvaluation.java   Source Code and License Vote up 4 votes
private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException {
  if (fs.exists(dir)) {
    fs.delete(dir, true);
  }
}
 
Example 20
Project: alluxio   File: HdfsAndAlluxioUtils_update.java   Source Code and License Vote up 3 votes
/**
 * 此方法用于检测分布式系统中文件是否存在
 *
 * @param uri
 *            uri
 * @param fs
 *            FileSystem
 * @param path
 *            path
 * @throws IOException
 */
private static void pathNotExistCheck(String uri, FileSystem fs, Path path) throws IOException {
	if (!fs.exists(path)) {
		throw new RuntimeException(NOT_EXEXTS_EXECEPTION_MSG + uri);
	}
}