Java Code Examples for org.apache.hadoop.fs.Path#SEPARATOR

The following examples show how to use org.apache.hadoop.fs.Path#SEPARATOR . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: tez   File: TestTezCommonUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testTezRecoveryStagingPath() throws Exception {
  String strAppId = "testAppId";
  Path stageDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
  Path confStageDir = TezCommonUtils.getRecoveryPath(stageDir, conf);
  String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
      + TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId + Path.SEPARATOR
      + TezConstants.DAG_RECOVERY_DATA_DIR_NAME;
  Assert.assertEquals(confStageDir.toString(), expectedDir);
}
 
Example 2
Source Project: attic-apex-core   File: FSStorageAgent.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object load(int operatorId, long windowId) throws IOException
{
  Path lPath = new Path(path + Path.SEPARATOR + String.valueOf(operatorId) + Path.SEPARATOR + Long.toHexString(windowId));
  logger.debug("Loading: {}", lPath);

  FSDataInputStream stream = fileContext.open(lPath);
  try {
    return retrieve(stream);
  } finally {
    stream.close();
  }
}
 
Example 3
Source Project: RDFS   File: NamenodeFsck.java    License: Apache License 2.0 5 votes vote down vote up
private void listCorruptOpenFiles() throws IOException {
  int matchedCorruptFilesCount = 0;
  // directory representation of path
  String pathdir = path.endsWith(Path.SEPARATOR) ? path : path + Path.SEPARATOR;
  FileStatus pathFileStatus = nn.getNamesystem().getFileInfo(pathdir);
  List<FileStatus> corruptFileStatusList = new ArrayList<FileStatus>();
  checkForCorruptOpenFiles(pathFileStatus, corruptFileStatusList);

  for (FileStatus fileStatus : corruptFileStatusList) {
    String currentPath = fileStatus.getPath().toString();
    if (currentPath.startsWith(pathdir) || currentPath.equals(path)) {
      matchedCorruptFilesCount++;
      
      // print the header before listing first item
      if (matchedCorruptFilesCount == 1 ) {
        out.println("Here are a few files that may be corrupted:");
        out.println("===========================================");
      }
      
      out.println(currentPath);
    }
  }

  out.println();
  out.println(buildSummaryResultForListCorruptFiles(matchedCorruptFilesCount,
      path));

}
 
Example 4
Source Project: big-c   File: FSDirRenameOp.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @deprecated Use {@link #renameToInt(FSDirectory, String, String,
 * boolean, Options.Rename...)}
 */
@Deprecated
@SuppressWarnings("deprecation")
private static boolean renameTo(FSDirectory fsd, FSPermissionChecker pc,
    String src, String dst, boolean logRetryCache) throws IOException {
  // Rename does not operate on link targets
  // Do not resolveLink when checking permissions of src and dst
  // Check write access to parent of src
  final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
  // Note: We should not be doing this.  This is move() not renameTo().
  final String actualDst = fsd.isDir(dst) ?
      dst + Path.SEPARATOR + new Path(src).getName() : dst;
  final INodesInPath dstIIP = fsd.getINodesInPath4Write(actualDst, false);
  if (fsd.isPermissionEnabled()) {
    fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
        false);
    // Check write access to ancestor of dst
    fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null,
        null, false);
  }

  if (NameNode.stateChangeLog.isDebugEnabled()) {
    NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src + " to "
        + dst);
  }
  final long mtime = Time.now();
  boolean stat = false;
  fsd.writeLock();
  try {
    stat = unprotectedRenameTo(fsd, src, actualDst, srcIIP, dstIIP, mtime);
  } finally {
    fsd.writeUnlock();
  }
  if (stat) {
    fsd.getEditLog().logRename(src, actualDst, mtime, logRetryCache);
    return true;
  }
  return false;
}
 
Example 5
Source Project: phoenix   File: LocalIndexIT.java    License: Apache License 2.0 5 votes vote down vote up
private void copyLocalIndexHFiles(Configuration conf, RegionInfo fromRegion, RegionInfo toRegion, boolean move)
        throws IOException {
    Path root = FSUtils.getRootDir(conf);

    Path seondRegion = new Path(FSUtils.getTableDir(root, fromRegion.getTable()) + Path.SEPARATOR
            + fromRegion.getEncodedName() + Path.SEPARATOR + "L#0/");
    Path hfilePath = FSUtils.getCurrentFileSystem(conf).listFiles(seondRegion, true).next().getPath();
    Path firstRegionPath = new Path(FSUtils.getTableDir(root, toRegion.getTable()) + Path.SEPARATOR
            + toRegion.getEncodedName() + Path.SEPARATOR + "L#0/");
    FileSystem currentFileSystem = FSUtils.getCurrentFileSystem(conf);
    assertTrue(FileUtil.copy(currentFileSystem, hfilePath, currentFileSystem, firstRegionPath, move, conf));
}
 
Example 6
Source Project: Bats   File: FSRecoveryHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void save(Object state) throws IOException
{

  if (fs.exists(snapshotBackupPath)) {
    throw new IllegalStateException("Found previous backup " + snapshotBackupPath);
  }

  if (fs.exists(snapshotPath)) {
    LOG.debug("Backup {} to {}", snapshotPath, snapshotBackupPath);
    fs.rename(snapshotPath, snapshotBackupPath);
  }

  LOG.debug("Writing checkpoint to {}", snapshotPath);
  try (FSDataOutputStream fsOutputStream = fs.create(snapshotPath);
      ObjectOutputStream oos = new ObjectOutputStream(fsOutputStream)) {
    oos.writeObject(state);
  }
  // remove snapshot backup
  if (fs.exists(snapshotBackupPath) && !fs.delete(snapshotBackupPath, false)) {
    throw new IOException("Failed to remove " + snapshotBackupPath);
  }

  // remove log backup
  Path logBackup = new Path(basedir + Path.SEPARATOR + FILE_LOG_BACKUP);
  if (fs.exists(logBackup) && !fs.delete(logBackup, false)) {
    throw new IOException("Failed to remove " + logBackup);
  }

}
 
Example 7
Source Project: hadoop   File: FSDirRenameOp.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @deprecated Use {@link #renameToInt(FSDirectory, String, String,
 * boolean, Options.Rename...)}
 */
@Deprecated
@SuppressWarnings("deprecation")
private static boolean renameTo(FSDirectory fsd, FSPermissionChecker pc,
    String src, String dst, boolean logRetryCache) throws IOException {
  // Rename does not operate on link targets
  // Do not resolveLink when checking permissions of src and dst
  // Check write access to parent of src
  final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
  // Note: We should not be doing this.  This is move() not renameTo().
  final String actualDst = fsd.isDir(dst) ?
      dst + Path.SEPARATOR + new Path(src).getName() : dst;
  final INodesInPath dstIIP = fsd.getINodesInPath4Write(actualDst, false);
  if (fsd.isPermissionEnabled()) {
    fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
        false);
    // Check write access to ancestor of dst
    fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null,
        null, false);
  }

  if (NameNode.stateChangeLog.isDebugEnabled()) {
    NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src + " to "
        + dst);
  }
  final long mtime = Time.now();
  boolean stat = false;
  fsd.writeLock();
  try {
    stat = unprotectedRenameTo(fsd, src, actualDst, srcIIP, dstIIP, mtime);
  } finally {
    fsd.writeUnlock();
  }
  if (stat) {
    fsd.getEditLog().logRename(src, actualDst, mtime, logRetryCache);
    return true;
  }
  return false;
}
 
Example 8
Source Project: big-c   File: TestHadoopArchives.java    License: Apache License 2.0 5 votes vote down vote up
@Test
/*
 * Tests copying from archive file system to a local file system
 */
public void testCopyToLocal() throws Exception {
  final String fullHarPathStr = makeArchive();

  // make path to copy the file to:
  final String tmpDir
    = System.getProperty("test.build.data","build/test/data") + "/work-dir/har-fs-tmp";
  final Path tmpPath = new Path(tmpDir);
  final LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
  localFs.delete(tmpPath, true);
  localFs.mkdirs(tmpPath);
  assertTrue(localFs.exists(tmpPath));
  
  // Create fresh HarFs:
  final HarFileSystem harFileSystem = new HarFileSystem(fs);
  try {
    final URI harUri = new URI(fullHarPathStr);
    harFileSystem.initialize(harUri, fs.getConf());
    
    final Path sourcePath = new Path(fullHarPathStr + Path.SEPARATOR + "a");
    final Path targetPath = new Path(tmpPath, "straus");
    // copy the Har file to a local file system:
    harFileSystem.copyToLocalFile(false, sourcePath, targetPath);
    FileStatus straus = localFs.getFileStatus(targetPath);
    // the file should contain just 1 character:
    assertEquals(1, straus.getLen());
  } finally {
    harFileSystem.close();
    localFs.delete(tmpPath, true);      
  }
}
 
Example 9
@Override
public void save(Object state) throws IOException
{

  if (fs.exists(snapshotBackupPath)) {
    throw new IllegalStateException("Found previous backup " + snapshotBackupPath);
  }

  if (fs.exists(snapshotPath)) {
    LOG.debug("Backup {} to {}", snapshotPath, snapshotBackupPath);
    fs.rename(snapshotPath, snapshotBackupPath);
  }

  LOG.debug("Writing checkpoint to {}", snapshotPath);
  try (FSDataOutputStream fsOutputStream = fs.create(snapshotPath);
      ObjectOutputStream oos = new ObjectOutputStream(fsOutputStream)) {
    oos.writeObject(state);
  }
  // remove snapshot backup
  if (fs.exists(snapshotBackupPath) && !fs.delete(snapshotBackupPath, false)) {
    throw new IOException("Failed to remove " + snapshotBackupPath);
  }

  // remove log backup
  Path logBackup = new Path(basedir + Path.SEPARATOR + FILE_LOG_BACKUP);
  if (fs.exists(logBackup) && !fs.delete(logBackup, false)) {
    throw new IOException("Failed to remove " + logBackup);
  }

}
 
Example 10
@Test
public void testScannerFilterForDuplicates() throws InterruptedException
{
  String filePath = testMeta.dataDirectory + Path.SEPARATOR + "file0.txt";
  testMeta.scanner = new MockScanner();
  testMeta.fileSplitterInput.setScanner(testMeta.scanner);
  testMeta.fileSplitterInput.getScanner().setScanIntervalMillis(500);
  testMeta.fileSplitterInput.getScanner().setFilePatternRegularExp(".*[.]txt");
  testMeta.fileSplitterInput.getScanner().setFiles(filePath);

  testMeta.fileSplitterInput.setup(testMeta.context);
  testMeta.fileSplitterInput.beginWindow(1);
  testMeta.scanner.semaphore.acquire();

  testMeta.fileSplitterInput.emitTuples();
  testMeta.fileSplitterInput.endWindow();

  testMeta.fileSplitterInput.beginWindow(2);
  testMeta.fileSplitterInput.emitTuples();
  testMeta.fileSplitterInput.endWindow();

  Assert.assertEquals("File metadata", 1, testMeta.fileMetadataSink.collectedTuples.size());
  for (Object fileMetadata : testMeta.fileMetadataSink.collectedTuples) {
    FileSplitterInput.FileMetadata metadata = (FileSplitterInput.FileMetadata)fileMetadata;
    Assert.assertTrue("path: " + metadata.getFilePath(), testMeta.filePaths.contains(metadata.getFilePath()));
    Assert.assertNotNull("name: ", metadata.getFileName());
  }

  testMeta.fileMetadataSink.collectedTuples.clear();
  testMeta.fileSplitterInput.teardown();
}
 
Example 11
Source Project: big-c   File: PathData.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Given a child of this directory, use the directory's path and the child's
 * basename to construct the string to the child.  This preserves relative
 * paths since Path will fully qualify.
 * @param childPath a path contained within this directory
 * @return String of the path relative to this directory
 */
private String getStringForChildPath(Path childPath) {
  String basename = childPath.getName();
  if (Path.CUR_DIR.equals(toString())) {
    return basename;
  }
  // check getPath() so scheme slashes aren't considered part of the path
  String separator = uri.getPath().endsWith(Path.SEPARATOR)
      ? "" : Path.SEPARATOR;
  return uriToString(uri, inferredSchemeFromPath) + separator + basename;
}
 
Example 12
Source Project: hbase   File: TestHMobStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCommitFile() throws Exception {
  final Configuration conf = HBaseConfiguration.create();
  init(name.getMethodName(), conf, true);
  String targetPathName = MobUtils.formatDate(new Date());
  Path targetPath = new Path(store.getPath(), (targetPathName
      + Path.SEPARATOR + mobFilePath.getName()));
  fs.delete(targetPath, true);
  Assert.assertFalse(fs.exists(targetPath));
  //commit file
  store.commitFile(mobFilePath, targetPath);
  Assert.assertTrue(fs.exists(targetPath));
}
 
Example 13
Source Project: tez   File: Fetcher.java    License: Apache License 2.0 5 votes vote down vote up
private final String getMapOutputFile(String pathComponent) {
  String outputPath = Constants.TEZ_RUNTIME_TASK_OUTPUT_DIR + Path.SEPARATOR +
      pathComponent + Path.SEPARATOR +
      Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING;

  if(ShuffleUtils.isTezShuffleHandler(conf)) {
    return Constants.DAG_PREFIX + this.dagIdentifier + Path.SEPARATOR +
        outputPath;
  }
  return outputPath;
}
 
Example 14
Source Project: tez   File: TestHistoryParser.java    License: Apache License 2.0 5 votes vote down vote up
private DagInfo getDagInfo(String dagId) throws TezException {
  //Parse downloaded contents
  File downloadedFile = new File(DOWNLOAD_DIR
      + Path.SEPARATOR + dagId + ".zip");
  ATSFileParser parser = new ATSFileParser(Arrays.asList(downloadedFile));
  DagInfo dagInfo = parser.getDAGData(dagId);
  assertTrue(dagInfo.getDagId().equals(dagId));
  return dagInfo;
}
 
Example 15
Source Project: big-c   File: TestDistCpSync.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test the sync returns false in the following scenarios:
 * 1. the source/target dir are not snapshottable dir
 * 2. the source/target does not have the given snapshots
 * 3. changes have been made in target
 */
@Test
public void testFallback() throws Exception {
  // the source/target dir are not snapshottable dir
  Assert.assertFalse(DistCpSync.sync(options, conf));
  // make sure the source path has been updated to the snapshot path
  final Path spath = new Path(source,
      HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2");
  Assert.assertEquals(spath, options.getSourcePaths().get(0));

  // reset source path in options
  options.setSourcePaths(Arrays.asList(source));
  // the source/target does not have the given snapshots
  dfs.allowSnapshot(source);
  dfs.allowSnapshot(target);
  Assert.assertFalse(DistCpSync.sync(options, conf));
  Assert.assertEquals(spath, options.getSourcePaths().get(0));

  // reset source path in options
  options.setSourcePaths(Arrays.asList(source));
  dfs.createSnapshot(source, "s1");
  dfs.createSnapshot(source, "s2");
  dfs.createSnapshot(target, "s1");
  Assert.assertTrue(DistCpSync.sync(options, conf));

  // reset source paths in options
  options.setSourcePaths(Arrays.asList(source));
  // changes have been made in target
  final Path subTarget = new Path(target, "sub");
  dfs.mkdirs(subTarget);
  Assert.assertFalse(DistCpSync.sync(options, conf));
  // make sure the source path has been updated to the snapshot path
  Assert.assertEquals(spath, options.getSourcePaths().get(0));

  // reset source paths in options
  options.setSourcePaths(Arrays.asList(source));
  dfs.delete(subTarget, true);
  Assert.assertTrue(DistCpSync.sync(options, conf));
}
 
Example 16
Source Project: hadoop   File: MRApps.java    License: Apache License 2.0 4 votes vote down vote up
public static Path getEndJobCommitSuccessFile(Configuration conf, String user,
    JobId jobId) {
  Path endCommitFile = new Path(MRApps.getStagingAreaDir(conf, user),
      jobId.toString() + Path.SEPARATOR + "COMMIT_SUCCESS");
  return endCommitFile;
}
 
Example 17
Source Project: RDFS   File: TaskTracker.java    License: Apache License 2.0 4 votes vote down vote up
static String getCacheSubdir() {
  return TaskTracker.SUBDIR + Path.SEPARATOR + TaskTracker.CACHEDIR;
}
 
Example 18
Source Project: big-c   File: TestHadoopArchives.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testReadFileContent() throws Exception {
  fileList.add(createFile(inputPath, fs, "c c"));
  final Path sub1 = new Path(inputPath, "sub 1");
  fs.mkdirs(sub1);
  fileList.add(createFile(inputPath, fs, sub1.getName(), "file x y z"));
  fileList.add(createFile(inputPath, fs, sub1.getName(), "file"));
  fileList.add(createFile(inputPath, fs, sub1.getName(), "x"));
  fileList.add(createFile(inputPath, fs, sub1.getName(), "y"));
  fileList.add(createFile(inputPath, fs, sub1.getName(), "z"));
  final Path sub2 = new Path(inputPath, "sub 1 with suffix");
  fs.mkdirs(sub2);
  fileList.add(createFile(inputPath, fs, sub2.getName(), "z"));
  // Generate a big binary file content:
  final byte[] binContent = prepareBin();
  fileList.add(createFile(inputPath, fs, binContent, sub2.getName(), "bin"));
  fileList.add(createFile(inputPath, fs, new byte[0], sub2.getName(), "zero-length"));

  final String fullHarPathStr = makeArchive();

  // Create fresh HarFs:
  final HarFileSystem harFileSystem = new HarFileSystem(fs);
  try {
    final URI harUri = new URI(fullHarPathStr);
    harFileSystem.initialize(harUri, fs.getConf());
    // now read the file content and compare it against the expected:
    int readFileCount = 0;
    for (final String pathStr0 : fileList) {
      final Path path = new Path(fullHarPathStr + Path.SEPARATOR + pathStr0);
      final String baseName = path.getName();
      final FileStatus status = harFileSystem.getFileStatus(path);
      if (status.isFile()) {
        // read the file:
        final byte[] actualContentSimple = readAllSimple(
            harFileSystem.open(path), true);
        
        final byte[] actualContentBuffer = readAllWithBuffer(
            harFileSystem.open(path), true);
        assertArrayEquals(actualContentSimple, actualContentBuffer);
        
        final byte[] actualContentFully = readAllWithReadFully(
            actualContentSimple.length,
            harFileSystem.open(path), true);
        assertArrayEquals(actualContentSimple, actualContentFully);
        
        final byte[] actualContentSeek = readAllWithSeek(
            actualContentSimple.length,
            harFileSystem.open(path), true);
        assertArrayEquals(actualContentSimple, actualContentSeek);
        
        final byte[] actualContentRead4
        = readAllWithRead4(harFileSystem.open(path), true);
        assertArrayEquals(actualContentSimple, actualContentRead4);
        
        final byte[] actualContentSkip = readAllWithSkip(
            actualContentSimple.length, 
            harFileSystem.open(path), 
            harFileSystem.open(path), 
            true);
        assertArrayEquals(actualContentSimple, actualContentSkip);
        
        if ("bin".equals(baseName)) {
          assertArrayEquals(binContent, actualContentSimple);
        } else if ("zero-length".equals(baseName)) {
          assertEquals(0, actualContentSimple.length);
        } else {
          String actual = new String(actualContentSimple, "UTF-8");
          assertEquals(baseName, actual);
        }
        readFileCount++;
      }
    }
    assertEquals(fileList.size(), readFileCount);
  } finally {
    harFileSystem.close();
  }
}
 
Example 19
private static String getTaskLogFile(LogName filter) {
  return ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR +
      filter.toString();
}
 
Example 20
Source Project: hadoop-gpu   File: TaskTracker.java    License: Apache License 2.0 4 votes vote down vote up
static String getPidFile(String jobid, 
                         String taskid, 
                         boolean isCleanup) {
  return  getLocalTaskDir(jobid, taskid, isCleanup)
          + Path.SEPARATOR + PID;
}