Java Code Examples for org.apache.hadoop.fs.Path#makeQualified()

The following examples show how to use org.apache.hadoop.fs.Path#makeQualified() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: RDFS   File: KosmosFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
public FileStatus getFileStatus(Path path) throws IOException {
Path absolute = makeAbsolute(path);
       String srep = absolute.toUri().getPath();
       if (!kfsImpl.exists(srep)) {
         throw new FileNotFoundException("File " + path + " does not exist.");
       }
       if (kfsImpl.isDirectory(srep)) {
           // System.out.println("Status of path: " + path + " is dir");
           return new FileStatus(0, true, 1, 0, kfsImpl.getModificationTime(srep), 
                                 path.makeQualified(this));
       } else {
           // System.out.println("Status of path: " + path + " is file");
           return new FileStatus(kfsImpl.filesize(srep), false, 
                                 kfsImpl.getReplication(srep),
                                 getDefaultBlockSize(),
                                 kfsImpl.getModificationTime(srep),
                                 path.makeQualified(this));
       }
   }
 
Example 2
Source Project: hbase   File: TestBulkLoadHFiles.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadTooMayHFiles() throws Exception {
  Path dir = util.getDataTestDirOnTestFS("testLoadTooMayHFiles");
  FileSystem fs = util.getTestFileSystem();
  dir = dir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
  Path familyDir = new Path(dir, Bytes.toString(FAMILY));

  byte[] from = Bytes.toBytes("begin");
  byte[] to = Bytes.toBytes("end");
  for (int i = 0; i <= MAX_FILES_PER_REGION_PER_FAMILY; i++) {
    HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_" + i),
      FAMILY, QUALIFIER, from, to, 1000);
  }

  try {
    BulkLoadHFiles.create(util.getConfiguration())
      .bulkLoad(TableName.valueOf("mytable_testLoadTooMayHFiles"), dir);
    fail("Bulk loading too many files should fail");
  } catch (IOException ie) {
    assertTrue(ie.getMessage()
      .contains("Trying to load more than " + MAX_FILES_PER_REGION_PER_FAMILY + " hfiles"));
  }
}
 
Example 3
Source Project: RDFS   File: S3FileSystem.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus[] listStatus(Path f) throws IOException {
  Path absolutePath = makeAbsolute(f);
  INode inode = store.retrieveINode(absolutePath);
  if (inode == null) {
    return null;
  }
  if (inode.isFile()) {
    return new FileStatus[] {
      new S3FileStatus(f.makeQualified(this), inode)
    };
  }
  ArrayList<FileStatus> ret = new ArrayList<FileStatus>();
  for (Path p : store.listSubPaths(absolutePath)) {
    ret.add(getFileStatus(p.makeQualified(this)));
  }
  return ret.toArray(new FileStatus[0]);
}
 
Example 4
Source Project: hadoop   File: S3FileSystem.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus[] listStatus(Path f) throws IOException {
  Path absolutePath = makeAbsolute(f);
  INode inode = store.retrieveINode(absolutePath);
  if (inode == null) {
    throw new FileNotFoundException("File " + f + " does not exist.");
  }
  if (inode.isFile()) {
    return new FileStatus[] {
      new S3FileStatus(f.makeQualified(this), inode)
    };
  }
  ArrayList<FileStatus> ret = new ArrayList<FileStatus>();
  for (Path p : store.listSubPaths(absolutePath)) {
    ret.add(getFileStatus(p.makeQualified(this)));
  }
  return ret.toArray(new FileStatus[0]);
}
 
Example 5
Source Project: crail   File: CrailHadoopFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus getFileStatus(Path path) throws IOException {
	CrailNode directFile = null;
	try {
		directFile = dfs.lookup(path.toUri().getRawPath()).get();
	} catch (Exception e) {
		throw new IOException(e);
	}
	if (directFile == null) {
		throw new FileNotFoundException("File does not exist: " + path);
	}
	FsPermission permission = FsPermission.getFileDefault();
	if (directFile.getType().isDirectory()) {
		permission = FsPermission.getDirDefault();
	}
	FileStatus status = new FileStatus(directFile.getCapacity(), directFile.getType().isContainer(), CrailConstants.SHADOW_REPLICATION, CrailConstants.BLOCK_SIZE, directFile.getModificationTime(), directFile.getModificationTime(), permission, CrailConstants.USER, CrailConstants.USER, path.makeQualified(this.getUri(), this.workingDir));
	return status;
}
 
Example 6
Source Project: hadoop-gpu   File: KosmosFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
public FileStatus getFileStatus(Path path) throws IOException {
Path absolute = makeAbsolute(path);
       String srep = absolute.toUri().getPath();
       if (!kfsImpl.exists(srep)) {
         throw new FileNotFoundException("File " + path + " does not exist.");
       }
       if (kfsImpl.isDirectory(srep)) {
           // System.out.println("Status of path: " + path + " is dir");
           return new FileStatus(0, true, 1, 0, kfsImpl.getModificationTime(srep), 
                                 path.makeQualified(this));
       } else {
           // System.out.println("Status of path: " + path + " is file");
           return new FileStatus(kfsImpl.filesize(srep), false, 
                                 kfsImpl.getReplication(srep),
                                 getDefaultBlockSize(),
                                 kfsImpl.getModificationTime(srep),
                                 path.makeQualified(this));
       }
   }
 
Example 7
Source Project: hadoop-book   File: TeraSort.java    License: Apache License 2.0 6 votes vote down vote up
public int run(String[] args) throws Exception {
  LOG.info("starting");
  JobConf job = (JobConf) getConf();
  Path inputDir = new Path(args[0]);
  inputDir = inputDir.makeQualified(inputDir.getFileSystem(job));
  Path partitionFile = new Path(inputDir, TeraInputFormat.PARTITION_FILENAME);
  URI partitionUri = new URI(partitionFile.toString() +
                             "#" + TeraInputFormat.PARTITION_FILENAME);
  TeraInputFormat.setInputPaths(job, new Path(args[0]));
  FileOutputFormat.setOutputPath(job, new Path(args[1]));
  job.setJobName("TeraSort");
  job.setJarByClass(TeraSort.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setInputFormat(TeraInputFormat.class);
  job.setOutputFormat(TeraOutputFormat.class);
  job.setPartitionerClass(TotalOrderPartitioner.class);
  TeraInputFormat.writePartitionFile(job, partitionFile);
  DistributedCache.addCacheFile(partitionUri, job);
  DistributedCache.createSymlink(job);
  job.setInt("dfs.replication", 1);
  TeraOutputFormat.setFinalSync(job, true);
  JobClient.runJob(job);
  LOG.info("done");
  return 0;
}
 
Example 8
Source Project: hadoop-gpu   File: FileOutputCommitter.java    License: Apache License 2.0 6 votes vote down vote up
Path getTempTaskOutputPath(TaskAttemptContext taskContext) {
  JobConf conf = taskContext.getJobConf();
  Path outputPath = FileOutputFormat.getOutputPath(conf);
  if (outputPath != null) {
    Path p = new Path(outputPath,
                   (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR +
                    "_" + taskContext.getTaskAttemptID().toString()));
    try {
      FileSystem fs = p.getFileSystem(conf);
      return p.makeQualified(fs);
    } catch (IOException ie) {
      LOG.warn(StringUtils .stringifyException(ie));
      return p;
    }
  }
  return null;
}
 
Example 9
Source Project: big-c   File: S3FileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * FileStatus for S3 file systems. 
 */
@Override
public FileStatus getFileStatus(Path f)  throws IOException {
  INode inode = store.retrieveINode(makeAbsolute(f));
  if (inode == null) {
    throw new FileNotFoundException(f + ": No such file or directory.");
  }
  return new S3FileStatus(f.makeQualified(this), inode);
}
 
Example 10
@Override
protected void setup(Context context) throws IOException, InterruptedException {
  super.setup(context);
  Configuration configuration = context.getConfiguration();
  _autoGenerateRecordIdAsHashOfData = isAutoGenerateRecordIdAsHashOfData(configuration);
  _autoGenerateRowIdAsHashOfData = isAutoGenerateRowIdAsHashOfData(configuration);
  if (_autoGenerateRecordIdAsHashOfData || _autoGenerateRowIdAsHashOfData) {
    try {
      _digest = MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
      throw new IOException(e);
    }
  }
  _columnNameMap = getFamilyAndColumnNameMap(configuration);
  _separator = new String(Base64.decodeBase64(configuration.get(BLUR_CSV_SEPARATOR_BASE64, _separator)), UTF_8);
  _splitter = Splitter.on(_separator);
  Path fileCurrentlyProcessing = getCurrentFile(context);
  Collection<String> families = configuration.getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILIES);
  OUTER: for (String family : families) {
    Collection<String> pathStrCollection = configuration
        .getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILY_PREFIX + family);
    for (String pathStr : pathStrCollection) {
      Path path = new Path(pathStr);
      FileSystem fileSystem = path.getFileSystem(configuration);
      path = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory());
      if (isParent(path, fileCurrentlyProcessing)) {
        _familyFromPath = family;
        _familyNotInFile = true;
        break OUTER;
      }
    }
  }
}
 
Example 11
Source Project: hbase   File: CommonFSUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static boolean isValidWALRootDir(Path walDir, final Configuration c) throws IOException {
  Path rootDir = getRootDir(c);
  FileSystem fs = walDir.getFileSystem(c);
  Path qualifiedWalDir = walDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
  if (!qualifiedWalDir.equals(rootDir)) {
    if (qualifiedWalDir.toString().startsWith(rootDir.toString() + "/")) {
      throw new IllegalStateException("Illegal WAL directory specified. " +
          "WAL directories are not permitted to be under the root directory if set.");
    }
  }
  return true;
}
 
Example 12
/** Initialize the OutputStream to the next file to write to.
 */
private void openNextFile() throws IOException {
  StringBuffer sb = new StringBuffer();
  Formatter fmt = new Formatter(sb);
  fmt.format("%05d", this.fileNum++);
  String filename = filePrefix + fmt.toString();
  if (codec != null) {
    filename = filename + codec.getDefaultExtension();
  }
  Path destFile = new Path(destDir, filename);
  FileSystem fs = destFile.getFileSystem(conf);
  LOG.debug("Opening next output file: " + destFile);
  if (fs.exists(destFile)) {
    Path canonicalDest = destFile.makeQualified(fs);
    throw new IOException("Destination file " + canonicalDest
        + " already exists");
  }

  OutputStream fsOut = fs.create(destFile);

  // Count how many actual bytes hit HDFS.
  this.countingFilterStream = new CountingOutputStream(fsOut);

  if (codec != null) {
    // Wrap that in a compressing stream.
    this.writeStream = codec.createOutputStream(this.countingFilterStream);
  } else {
    // Write to the counting stream directly.
    this.writeStream = this.countingFilterStream;
  }
}
 
Example 13
Source Project: hadoop   File: HftpFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FSDataInputStream open(Path f, int buffersize) throws IOException {
  f = f.makeQualified(getUri(), getWorkingDirectory());
  String path = "/data" + ServletUtil.encodePath(f.toUri().getPath());
  String query = addDelegationTokenParam("ugi=" + getEncodedUgiParameter());
  URL u = getNamenodeURL(path, query);
  return new FSDataInputStream(new RangeHeaderInputStream(connectionFactory, u));
}
 
Example 14
/**
 * Setup for test. Returns path to test data dir. Sets configuration into the passed
 * hctu.getConfiguration.
 */
static Path setup(FileSystem fs, HBaseCommonTestingUtility hctu) throws IOException {
  Path testDir = hctu.getDataTestDir().makeQualified(fs.getUri(), fs.getWorkingDirectory());
  hctu.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  hctu.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
  hctu.getConfiguration().setInt("hbase.client.pause", 250);
  hctu.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  hctu.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
  hctu.getConfiguration().setInt("mapreduce.map.maxattempts", 10);
  hctu.getConfiguration().set(HConstants.HBASE_DIR, testDir.toString());
  return testDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
}
 
Example 15
Source Project: big-c   File: NativeAzureFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
private FileStatus newDirectory(FileMetadata meta, Path path) {
  return new FileStatus (
      0,
      true,
      1,
      blockSize,
      meta == null ? 0 : meta.getLastModified(),
      0,
      meta == null ? FsPermission.getDefault() : meta.getPermissionStatus().getPermission(),
      meta == null ? "" : meta.getPermissionStatus().getUserName(),
      meta == null ? "" : meta.getPermissionStatus().getGroupName(),
      path.makeQualified(getUri(), getWorkingDirectory()));
}
 
Example 16
Source Project: RDFS   File: TestDFSShell.java    License: Apache License 2.0 5 votes vote down vote up
public void testCount() throws Exception {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
  DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
  FsShell shell = new FsShell();
  shell.setConf(conf);

  try {
    String root = createTree(dfs, "count");

    // Verify the counts
    runCount(root, 2, 4, conf);
    runCount(root + "2", 2, 1, conf);
    runCount(root + "2/f1", 0, 1, conf);
    runCount(root + "2/sub", 1, 0, conf);

    final FileSystem localfs = FileSystem.getLocal(conf);
    Path localpath = new Path(TEST_ROOT_DIR, "testcount");
    localpath = localpath.makeQualified(localfs);
    localfs.mkdirs(localpath);

    final String localstr = localpath.toString();
    System.out.println("localstr=" + localstr);
    runCount(localstr, 1, 0, conf);
    assertEquals(0, new Count(new String[]{root, localstr}, 0, conf).runAll());
  } finally {
    try {
      dfs.close();
    } catch (Exception e) {
    }
    cluster.shutdown();
  }
}
 
Example 17
Source Project: hbase   File: TestBulkLoadHFiles.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test that tags survive through a bulk load that needs to split hfiles. This test depends on the
 * "hbase.client.rpc.codec" = KeyValueCodecWithTags so that the client can get tags in the
 * responses.
 */
@Test
public void testTagsSurviveBulkLoadSplit() throws Exception {
  Path dir = util.getDataTestDirOnTestFS(tn.getMethodName());
  FileSystem fs = util.getTestFileSystem();
  dir = dir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
  Path familyDir = new Path(dir, Bytes.toString(FAMILY));
  // table has these split points
  byte[][] tableSplitKeys = new byte[][] { Bytes.toBytes("aaa"), Bytes.toBytes("fff"),
    Bytes.toBytes("jjj"), Bytes.toBytes("ppp"), Bytes.toBytes("uuu"), Bytes.toBytes("zzz"), };

  // creating an hfile that has values that span the split points.
  byte[] from = Bytes.toBytes("ddd");
  byte[] to = Bytes.toBytes("ooo");
  HFileTestUtil.createHFileWithTags(util.getConfiguration(), fs,
    new Path(familyDir, tn.getMethodName() + "_hfile"), FAMILY, QUALIFIER, from, to, 1000);
  int expectedRows = 1000;

  TableName tableName = TableName.valueOf(tn.getMethodName());
  TableDescriptor htd = buildHTD(tableName, BloomType.NONE);
  util.getAdmin().createTable(htd, tableSplitKeys);

  BulkLoadHFiles.create(util.getConfiguration()).bulkLoad(tableName, dir);

  Table table = util.getConnection().getTable(tableName);
  try {
    assertEquals(expectedRows, countRows(table));
    HFileTestUtil.verifyTags(table);
  } finally {
    table.close();
  }

  util.deleteTable(tableName);
}
 
Example 18
/**
 * FileStatus for S3 file systems.
 */
@Override
public FileStatus getFileStatus(Path f)  throws IOException {
  INode inode = store.retrieveINode(makeAbsolute(f));
  if (inode == null) {
    throw new FileNotFoundException(f + ": No such file or directory.");
  }
  return new S3FileStatus(f.makeQualified(this), inode);
}
 
Example 19
Source Project: emodb   File: FileSystemUtil.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Qualifies a path so it includes the schema and authority from the root path.
 */
private static Path qualified(Path rootPath, Path path) {
    URI rootUri = rootPath.toUri();
    return path.makeQualified(rootUri, new Path(rootUri.getPath()));
}
 
Example 20
Source Project: hadoop-gpu   File: NativeS3FileSystem.java    License: Apache License 2.0 4 votes vote down vote up
private FileStatus newDirectory(Path path) {
  return new FileStatus(0, true, 1, MAX_S3_FILE_SIZE, 0,
      path.makeQualified(this));
}