Java Code Examples for org.apache.hadoop.hdfs.DistributedFileSystem#concat()
The following examples show how to use
org.apache.hadoop.hdfs.DistributedFileSystem#concat() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HDFSConcat.java From hadoop with Apache License 2.0 | 6 votes |
|
public static void main(String... args) throws IOException {
if(args.length < 2) {
System.err.println("Usage HDFSConcat target srcs..");
System.exit(0);
}
Configuration conf = new Configuration();
String uri = conf.get("fs.default.name", def_uri);
Path path = new Path(uri);
DistributedFileSystem dfs =
(DistributedFileSystem)FileSystem.get(path.toUri(), conf);
Path [] srcs = new Path[args.length-1];
for(int i=1; i<args.length; i++) {
srcs[i-1] = new Path(args[i]);
}
dfs.concat(new Path(args[0]), srcs);
}
Example 2
| Source File: HDFSConcat.java From big-c with Apache License 2.0 | 6 votes |
|
public static void main(String... args) throws IOException {
if(args.length < 2) {
System.err.println("Usage HDFSConcat target srcs..");
System.exit(0);
}
Configuration conf = new Configuration();
String uri = conf.get("fs.default.name", def_uri);
Path path = new Path(uri);
DistributedFileSystem dfs =
(DistributedFileSystem)FileSystem.get(path.toUri(), conf);
Path [] srcs = new Path[args.length-1];
for(int i=1; i<args.length; i++) {
srcs[i-1] = new Path(args[i]);
}
dfs.concat(new Path(args[0]), srcs);
}
Example 3
| Source File: HDFSConcat.java From RDFS with Apache License 2.0 | 6 votes |
|
/**
* @param args
*/
public static void main(String... args) throws IOException {
if(args.length < 3) {
System.err.println("Usage HDFSConcat (restricted/unrestricted) target srcs..");
System.exit(0);
}
Configuration conf = new Configuration();
String uri = conf.get("fs.default.name", def_uri);
Path path = new Path(uri);
DistributedFileSystem dfs =
(DistributedFileSystem)FileSystem.get(path.toUri(), conf);
Path [] srcs = new Path[args.length-2];
for(int i=2; i<args.length; i++) {
srcs[i-2] = new Path(args[i]);
}
dfs.concat(new Path(args[1]), srcs, !args[0].equals("unrestricted"));
}
Example 4
| Source File: HDFSTool.java From WIFIProbe with Apache License 2.0 | 5 votes |
|
public static void concat(String dir) throws IOException {
String directory = NodeConfig.HDFS_PATH + dir;
Configuration conf = new Configuration();
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.get(URI.create(directory), conf);
FileStatus fileList[] = fs.listStatus(new Path(directory));
if (fileList.length>=2) {
ArrayList<Path> srcs = new ArrayList<Path>(fileList.length);
for (FileStatus fileStatus : fileList) {
if ( fileStatus.isFile() &&
(fileStatus.getLen()&~fileStatus.getBlockSize())<fileStatus.getBlockSize()/2 ) {
srcs.add(fileStatus.getPath());
}
}
if (srcs.size()>=2) {
Logger.println("come to here");
Path appended = srcs.get(0);
Path[] sources = new Path[srcs.size()-1];
for (int i=0; i<srcs.size()-1; i++) {
sources[i] = srcs.get(i+1);
}
Logger.println(fs==null);
Logger.println(appended==null);
Logger.println(sources==null);
fs.concat(appended, sources);
Logger.println("concat to : " + appended.getName());
Logger.println(Arrays.toString(sources));
}
fs.close();
}
}
Example 5
| Source File: IndexScrutinyToolIT.java From phoenix with Apache License 2.0 | 4 votes |
|
/**
* Tests that with the output to file option set, the scrutiny tool outputs invalid rows to file
*/
@Test public void testOutputInvalidRowsToFile() throws Exception {
insertOneValid_OneBadVal_OneMissingTarget();
String[]
argValues =
getArgValues(schemaName, dataTableName, indexTableName, 10L, SourceTable.DATA_TABLE_SOURCE, true, OutputFormat.FILE, null);
runScrutiny(argValues);
// check the output files
Path outputPath = CsvBulkImportUtil.getOutputPath(new Path(outputDir), dataTableFullName);
DistributedFileSystem fs = getUtility().getDFSCluster().getFileSystem();
List<Path> paths = Lists.newArrayList();
Path firstPart = null;
for (FileStatus outputFile : fs.listStatus(outputPath)) {
if (outputFile.getPath().getName().startsWith("part")) {
if (firstPart == null) {
firstPart = outputFile.getPath();
} else {
paths.add(outputFile.getPath());
}
}
}
if (dataTableDdl.contains("SALT_BUCKETS")) {
fs.concat(firstPart, paths.toArray(new Path[0]));
}
Path outputFilePath = firstPart;
assertTrue(fs.exists(outputFilePath));
FSDataInputStream fsDataInputStream = fs.open(outputFilePath);
BufferedReader reader = new BufferedReader(new InputStreamReader(fsDataInputStream));
TreeSet<String> lines = Sets.newTreeSet();
try {
String line = null;
while ((line = reader.readLine()) != null) {
lines.add(line);
}
} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(fsDataInputStream);
}
Iterator<String> lineIterator = lines.iterator();
assertEquals("[2, name-2, " + new Timestamp(testTime).toString() + ", 95123]\t[2, name-2, "
+ new Timestamp(testTime).toString() + ", 9999]", lineIterator.next());
assertEquals("[3, name-3, " + new Timestamp(testTime).toString() + ", 95123]\tTarget row not found",
lineIterator.next());
}
Example 6
| Source File: OfflineEditsViewerHelper.java From RDFS with Apache License 2.0 | 4 votes |
|
/**
* Run file operations to create edits for all op codes
* to be tested.
*/
private void runOperations() throws IOException {
LOG.info("Creating edits by performing fs operations");
// no check, if it's not it throws an exception which is what we want
DistributedFileSystem dfs =
(DistributedFileSystem)cluster.getFileSystem();
// OP_ADD 0, OP_SET_GENSTAMP 10
Path pathFileCreate = new Path("/file_create");
FSDataOutputStream s = dfs.create(pathFileCreate);
// OP_CLOSE 9
s.close();
// OP_RENAME 1
Path pathFileMoved = new Path("/file_moved");
dfs.rename(pathFileCreate, pathFileMoved);
// OP_DELETE 2
dfs.delete(pathFileMoved, false);
// OP_MKDIR 3
Path pathDirectoryMkdir = new Path("/directory_mkdir");
dfs.mkdirs(pathDirectoryMkdir);
// OP_SET_REPLICATION 4
s = dfs.create(pathFileCreate);
s.close();
dfs.setReplication(pathFileCreate, (short)1);
// OP_SET_PERMISSIONS 7
Short permission = 0777;
dfs.setPermission(pathFileCreate, new FsPermission(permission));
// OP_SET_OWNER 8
dfs.setOwner(pathFileCreate, new String("newOwner"), null);
// OP_CLOSE 9 see above
// OP_SET_GENSTAMP 10 see above
// OP_SET_NS_QUOTA 11 obsolete
// OP_CLEAR_NS_QUOTA 12 obsolete
// OP_TIMES 13
long mtime = 1285195527000L; // Wed, 22 Sep 2010 22:45:27 GMT
long atime = mtime;
dfs.setTimes(pathFileCreate, mtime, atime);
// OP_SET_QUOTA 14
dfs.setQuota(pathDirectoryMkdir, 1000L, FSConstants.QUOTA_DONT_SET);
// OP_CONCAT_DELETE 16
Path pathConcatTarget = new Path("/file_concat_target");
Path[] pathConcatFiles = new Path[2];
pathConcatFiles[0] = new Path("/file_concat_0");
pathConcatFiles[1] = new Path("/file_concat_1");
long length = blockSize * 3; // multiple of blocksize for concat
short replication = 1;
long seed = 1;
DFSTestUtil.createFile(dfs, pathConcatTarget, length, replication, seed);
DFSTestUtil.createFile(dfs, pathConcatFiles[0], length, replication, seed);
DFSTestUtil.createFile(dfs, pathConcatFiles[1], length, replication, seed);
dfs.concat(pathConcatTarget, pathConcatFiles, false);
// sync to disk, otherwise we parse partial edits
cluster.getNameNode().getFSImage().getEditLog().logSync();
dfs.close();
}
