Java Code Examples for org.apache.hadoop.fs.Path#makeQualified()

The following examples show how to use org.apache.hadoop.fs.Path#makeQualified() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KosmosFileSystem.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public FileStatus getFileStatus(Path path) throws IOException {
Path absolute = makeAbsolute(path);
       String srep = absolute.toUri().getPath();
       if (!kfsImpl.exists(srep)) {
         throw new FileNotFoundException("File " + path + " does not exist.");
       }
       if (kfsImpl.isDirectory(srep)) {
           // System.out.println("Status of path: " + path + " is dir");
           return new FileStatus(0, true, 1, 0, kfsImpl.getModificationTime(srep), 
                                 path.makeQualified(this));
       } else {
           // System.out.println("Status of path: " + path + " is file");
           return new FileStatus(kfsImpl.filesize(srep), false, 
                                 kfsImpl.getReplication(srep),
                                 getDefaultBlockSize(),
                                 kfsImpl.getModificationTime(srep),
                                 path.makeQualified(this));
       }
   }
 
Example 2
Source File: FileOutputCommitter.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
Path getTempTaskOutputPath(TaskAttemptContext taskContext) {
  JobConf conf = taskContext.getJobConf();
  Path outputPath = FileOutputFormat.getOutputPath(conf);
  if (outputPath != null) {
    Path p = new Path(outputPath,
                   (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR +
                    "_" + taskContext.getTaskAttemptID().toString()));
    try {
      FileSystem fs = p.getFileSystem(conf);
      return p.makeQualified(fs);
    } catch (IOException ie) {
      LOG.warn(StringUtils .stringifyException(ie));
      return p;
    }
  }
  return null;
}
 
Example 3
Source File: TestBulkLoadHFiles.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadTooMayHFiles() throws Exception {
  Path dir = util.getDataTestDirOnTestFS("testLoadTooMayHFiles");
  FileSystem fs = util.getTestFileSystem();
  dir = dir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
  Path familyDir = new Path(dir, Bytes.toString(FAMILY));

  byte[] from = Bytes.toBytes("begin");
  byte[] to = Bytes.toBytes("end");
  for (int i = 0; i <= MAX_FILES_PER_REGION_PER_FAMILY; i++) {
    HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_" + i),
      FAMILY, QUALIFIER, from, to, 1000);
  }

  try {
    BulkLoadHFiles.create(util.getConfiguration())
      .bulkLoad(TableName.valueOf("mytable_testLoadTooMayHFiles"), dir);
    fail("Bulk loading too many files should fail");
  } catch (IOException ie) {
    assertTrue(ie.getMessage()
      .contains("Trying to load more than " + MAX_FILES_PER_REGION_PER_FAMILY + " hfiles"));
  }
}
 
Example 4
Source File: S3FileSystem.java    From RDFS with Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus[] listStatus(Path f) throws IOException {
  Path absolutePath = makeAbsolute(f);
  INode inode = store.retrieveINode(absolutePath);
  if (inode == null) {
    return null;
  }
  if (inode.isFile()) {
    return new FileStatus[] {
      new S3FileStatus(f.makeQualified(this), inode)
    };
  }
  ArrayList<FileStatus> ret = new ArrayList<FileStatus>();
  for (Path p : store.listSubPaths(absolutePath)) {
    ret.add(getFileStatus(p.makeQualified(this)));
  }
  return ret.toArray(new FileStatus[0]);
}
 
Example 5
Source File: S3FileSystem.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus[] listStatus(Path f) throws IOException {
  Path absolutePath = makeAbsolute(f);
  INode inode = store.retrieveINode(absolutePath);
  if (inode == null) {
    throw new FileNotFoundException("File " + f + " does not exist.");
  }
  if (inode.isFile()) {
    return new FileStatus[] {
      new S3FileStatus(f.makeQualified(this), inode)
    };
  }
  ArrayList<FileStatus> ret = new ArrayList<FileStatus>();
  for (Path p : store.listSubPaths(absolutePath)) {
    ret.add(getFileStatus(p.makeQualified(this)));
  }
  return ret.toArray(new FileStatus[0]);
}
 
Example 6
Source File: CrailHadoopFileSystem.java    From crail with Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus getFileStatus(Path path) throws IOException {
	CrailNode directFile = null;
	try {
		directFile = dfs.lookup(path.toUri().getRawPath()).get();
	} catch (Exception e) {
		throw new IOException(e);
	}
	if (directFile == null) {
		throw new FileNotFoundException("File does not exist: " + path);
	}
	FsPermission permission = FsPermission.getFileDefault();
	if (directFile.getType().isDirectory()) {
		permission = FsPermission.getDirDefault();
	}
	FileStatus status = new FileStatus(directFile.getCapacity(), directFile.getType().isContainer(), CrailConstants.SHADOW_REPLICATION, CrailConstants.BLOCK_SIZE, directFile.getModificationTime(), directFile.getModificationTime(), permission, CrailConstants.USER, CrailConstants.USER, path.makeQualified(this.getUri(), this.workingDir));
	return status;
}
 
Example 7
Source File: TeraSort.java    From hadoop-book with Apache License 2.0 6 votes vote down vote up
public int run(String[] args) throws Exception {
  LOG.info("starting");
  JobConf job = (JobConf) getConf();
  Path inputDir = new Path(args[0]);
  inputDir = inputDir.makeQualified(inputDir.getFileSystem(job));
  Path partitionFile = new Path(inputDir, TeraInputFormat.PARTITION_FILENAME);
  URI partitionUri = new URI(partitionFile.toString() +
                             "#" + TeraInputFormat.PARTITION_FILENAME);
  TeraInputFormat.setInputPaths(job, new Path(args[0]));
  FileOutputFormat.setOutputPath(job, new Path(args[1]));
  job.setJobName("TeraSort");
  job.setJarByClass(TeraSort.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setInputFormat(TeraInputFormat.class);
  job.setOutputFormat(TeraOutputFormat.class);
  job.setPartitionerClass(TotalOrderPartitioner.class);
  TeraInputFormat.writePartitionFile(job, partitionFile);
  DistributedCache.addCacheFile(partitionUri, job);
  DistributedCache.createSymlink(job);
  job.setInt("dfs.replication", 1);
  TeraOutputFormat.setFinalSync(job, true);
  JobClient.runJob(job);
  LOG.info("done");
  return 0;
}
 
Example 8
Source File: KosmosFileSystem.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
public FileStatus getFileStatus(Path path) throws IOException {
Path absolute = makeAbsolute(path);
       String srep = absolute.toUri().getPath();
       if (!kfsImpl.exists(srep)) {
         throw new FileNotFoundException("File " + path + " does not exist.");
       }
       if (kfsImpl.isDirectory(srep)) {
           // System.out.println("Status of path: " + path + " is dir");
           return new FileStatus(0, true, 1, 0, kfsImpl.getModificationTime(srep), 
                                 path.makeQualified(this));
       } else {
           // System.out.println("Status of path: " + path + " is file");
           return new FileStatus(kfsImpl.filesize(srep), false, 
                                 kfsImpl.getReplication(srep),
                                 getDefaultBlockSize(),
                                 kfsImpl.getModificationTime(srep),
                                 path.makeQualified(this));
       }
   }
 
Example 9
Source File: SplittingOutputStream.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
/** Initialize the OutputStream to the next file to write to.
 */
private void openNextFile() throws IOException {
  StringBuffer sb = new StringBuffer();
  Formatter fmt = new Formatter(sb);
  fmt.format("%05d", this.fileNum++);
  String filename = filePrefix + fmt.toString();
  if (codec != null) {
    filename = filename + codec.getDefaultExtension();
  }
  Path destFile = new Path(destDir, filename);
  FileSystem fs = destFile.getFileSystem(conf);
  LOG.debug("Opening next output file: " + destFile);
  if (fs.exists(destFile)) {
    Path canonicalDest = destFile.makeQualified(fs);
    throw new IOException("Destination file " + canonicalDest
        + " already exists");
  }

  OutputStream fsOut = fs.create(destFile);

  // Count how many actual bytes hit HDFS.
  this.countingFilterStream = new CountingOutputStream(fsOut);

  if (codec != null) {
    // Wrap that in a compressing stream.
    this.writeStream = codec.createOutputStream(this.countingFilterStream);
  } else {
    // Write to the counting stream directly.
    this.writeStream = this.countingFilterStream;
  }
}
 
Example 10
Source File: SmallBlockS3FileSystem.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up
/**
 * FileStatus for S3 file systems.
 */
@Override
public FileStatus getFileStatus(Path f)  throws IOException {
  INode inode = store.retrieveINode(makeAbsolute(f));
  if (inode == null) {
    throw new FileNotFoundException(f + ": No such file or directory.");
  }
  return new S3FileStatus(f.makeQualified(this), inode);
}
 
Example 11
Source File: TestBulkLoadHFiles.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test that tags survive through a bulk load that needs to split hfiles. This test depends on the
 * "hbase.client.rpc.codec" = KeyValueCodecWithTags so that the client can get tags in the
 * responses.
 */
@Test
public void testTagsSurviveBulkLoadSplit() throws Exception {
  Path dir = util.getDataTestDirOnTestFS(tn.getMethodName());
  FileSystem fs = util.getTestFileSystem();
  dir = dir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
  Path familyDir = new Path(dir, Bytes.toString(FAMILY));
  // table has these split points
  byte[][] tableSplitKeys = new byte[][] { Bytes.toBytes("aaa"), Bytes.toBytes("fff"),
    Bytes.toBytes("jjj"), Bytes.toBytes("ppp"), Bytes.toBytes("uuu"), Bytes.toBytes("zzz"), };

  // creating an hfile that has values that span the split points.
  byte[] from = Bytes.toBytes("ddd");
  byte[] to = Bytes.toBytes("ooo");
  HFileTestUtil.createHFileWithTags(util.getConfiguration(), fs,
    new Path(familyDir, tn.getMethodName() + "_hfile"), FAMILY, QUALIFIER, from, to, 1000);
  int expectedRows = 1000;

  TableName tableName = TableName.valueOf(tn.getMethodName());
  TableDescriptor htd = buildHTD(tableName, BloomType.NONE);
  util.getAdmin().createTable(htd, tableSplitKeys);

  BulkLoadHFiles.create(util.getConfiguration()).bulkLoad(tableName, dir);

  Table table = util.getConnection().getTable(tableName);
  try {
    assertEquals(expectedRows, countRows(table));
    HFileTestUtil.verifyTags(table);
  } finally {
    table.close();
  }

  util.deleteTable(tableName);
}
 
Example 12
Source File: TestDFSShell.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public void testCount() throws Exception {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
  DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
  FsShell shell = new FsShell();
  shell.setConf(conf);

  try {
    String root = createTree(dfs, "count");

    // Verify the counts
    runCount(root, 2, 4, conf);
    runCount(root + "2", 2, 1, conf);
    runCount(root + "2/f1", 0, 1, conf);
    runCount(root + "2/sub", 1, 0, conf);

    final FileSystem localfs = FileSystem.getLocal(conf);
    Path localpath = new Path(TEST_ROOT_DIR, "testcount");
    localpath = localpath.makeQualified(localfs);
    localfs.mkdirs(localpath);

    final String localstr = localpath.toString();
    System.out.println("localstr=" + localstr);
    runCount(localstr, 1, 0, conf);
    assertEquals(0, new Count(new String[]{root, localstr}, 0, conf).runAll());
  } finally {
    try {
      dfs.close();
    } catch (Exception e) {
    }
    cluster.shutdown();
  }
}
 
Example 13
Source File: NativeAzureFileSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
private FileStatus newDirectory(FileMetadata meta, Path path) {
  return new FileStatus (
      0,
      true,
      1,
      blockSize,
      meta == null ? 0 : meta.getLastModified(),
      0,
      meta == null ? FsPermission.getDefault() : meta.getPermissionStatus().getPermission(),
      meta == null ? "" : meta.getPermissionStatus().getUserName(),
      meta == null ? "" : meta.getPermissionStatus().getGroupName(),
      path.makeQualified(getUri(), getWorkingDirectory()));
}
 
Example 14
Source File: TestExportSnapshotV1NoCluster.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Setup for test. Returns path to test data dir. Sets configuration into the passed
 * hctu.getConfiguration.
 */
static Path setup(FileSystem fs, HBaseCommonTestingUtility hctu) throws IOException {
  Path testDir = hctu.getDataTestDir().makeQualified(fs.getUri(), fs.getWorkingDirectory());
  hctu.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  hctu.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
  hctu.getConfiguration().setInt("hbase.client.pause", 250);
  hctu.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  hctu.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
  hctu.getConfiguration().setInt("mapreduce.map.maxattempts", 10);
  hctu.getConfiguration().set(HConstants.HBASE_DIR, testDir.toString());
  return testDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
}
 
Example 15
Source File: HftpFileSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public FSDataInputStream open(Path f, int buffersize) throws IOException {
  f = f.makeQualified(getUri(), getWorkingDirectory());
  String path = "/data" + ServletUtil.encodePath(f.toUri().getPath());
  String query = addDelegationTokenParam("ugi=" + getEncodedUgiParameter());
  URL u = getNamenodeURL(path, query);
  return new FSDataInputStream(new RangeHeaderInputStream(connectionFactory, u));
}
 
Example 16
Source File: S3FileSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * FileStatus for S3 file systems. 
 */
@Override
public FileStatus getFileStatus(Path f)  throws IOException {
  INode inode = store.retrieveINode(makeAbsolute(f));
  if (inode == null) {
    throw new FileNotFoundException(f + ": No such file or directory.");
  }
  return new S3FileStatus(f.makeQualified(this), inode);
}
 
Example 17
Source File: CommonFSUtils.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static boolean isValidWALRootDir(Path walDir, final Configuration c) throws IOException {
  Path rootDir = getRootDir(c);
  FileSystem fs = walDir.getFileSystem(c);
  Path qualifiedWalDir = walDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
  if (!qualifiedWalDir.equals(rootDir)) {
    if (qualifiedWalDir.toString().startsWith(rootDir.toString() + "/")) {
      throw new IllegalStateException("Illegal WAL directory specified. " +
          "WAL directories are not permitted to be under the root directory if set.");
    }
  }
  return true;
}
 
Example 18
Source File: CsvBlurMapper.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
@Override
protected void setup(Context context) throws IOException, InterruptedException {
  super.setup(context);
  Configuration configuration = context.getConfiguration();
  _autoGenerateRecordIdAsHashOfData = isAutoGenerateRecordIdAsHashOfData(configuration);
  _autoGenerateRowIdAsHashOfData = isAutoGenerateRowIdAsHashOfData(configuration);
  if (_autoGenerateRecordIdAsHashOfData || _autoGenerateRowIdAsHashOfData) {
    try {
      _digest = MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
      throw new IOException(e);
    }
  }
  _columnNameMap = getFamilyAndColumnNameMap(configuration);
  _separator = new String(Base64.decodeBase64(configuration.get(BLUR_CSV_SEPARATOR_BASE64, _separator)), UTF_8);
  _splitter = Splitter.on(_separator);
  Path fileCurrentlyProcessing = getCurrentFile(context);
  Collection<String> families = configuration.getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILIES);
  OUTER: for (String family : families) {
    Collection<String> pathStrCollection = configuration
        .getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILY_PREFIX + family);
    for (String pathStr : pathStrCollection) {
      Path path = new Path(pathStr);
      FileSystem fileSystem = path.getFileSystem(configuration);
      path = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory());
      if (isParent(path, fileCurrentlyProcessing)) {
        _familyFromPath = family;
        _familyNotInFile = true;
        break OUTER;
      }
    }
  }
}
 
Example 19
Source File: NativeS3FileSystem.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
private FileStatus newDirectory(Path path) {
  return new FileStatus(0, true, 1, MAX_S3_FILE_SIZE, 0,
      path.makeQualified(this));
}
 
Example 20
Source File: FileSystemUtil.java    From emodb with Apache License 2.0 4 votes vote down vote up
/**
 * Qualifies a path so it includes the schema and authority from the root path.
 */
private static Path qualified(Path rootPath, Path path) {
    URI rootUri = rootPath.toUri();
    return path.makeQualified(rootUri, new Path(rootUri.getPath()));
}