Java Code Examples for org.apache.hadoop.mapreduce.MapReduceTestUtil#readOutput()
The following examples show how to use
org.apache.hadoop.mapreduce.MapReduceTestUtil#readOutput() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TestMRFieldSelection.java From hadoop with Apache License 2.0 | 5 votes |
|
public static void launch() throws Exception {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 10;
Path outDir = new Path(testDir, "output_for_field_selection_test");
Path inDir = new Path(testDir, "input_for_field_selection_test");
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
constructInputOutputData(inputData, expectedOutput, numOfInputLines);
conf.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
conf.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
conf.set(
FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");
Job job = MapReduceTestUtil.createJob(conf, inDir, outDir,
1, 1, inputData.toString());
job.setMapperClass(FieldSelectionMapper.class);
job.setReducerClass(FieldSelectionReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(1);
job.waitForCompletion(true);
assertTrue("Job Failed!", job.isSuccessful());
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
String outdata = MapReduceTestUtil.readOutput(outDir, conf);
assertEquals("Outputs doesnt match.",expectedOutput.toString(), outdata);
fs.delete(outDir, true);
}
Example 2
| Source File: TestMRFieldSelection.java From big-c with Apache License 2.0 | 5 votes |
|
public static void launch() throws Exception {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 10;
Path outDir = new Path(testDir, "output_for_field_selection_test");
Path inDir = new Path(testDir, "input_for_field_selection_test");
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
constructInputOutputData(inputData, expectedOutput, numOfInputLines);
conf.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
conf.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
conf.set(
FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");
Job job = MapReduceTestUtil.createJob(conf, inDir, outDir,
1, 1, inputData.toString());
job.setMapperClass(FieldSelectionMapper.class);
job.setReducerClass(FieldSelectionReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(1);
job.waitForCompletion(true);
assertTrue("Job Failed!", job.isSuccessful());
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
String outdata = MapReduceTestUtil.readOutput(outDir, conf);
assertEquals("Outputs doesnt match.",expectedOutput.toString(), outdata);
fs.delete(outDir, true);
}
Example 3
| Source File: TestFieldSelection.java From hadoop with Apache License 2.0 | 4 votes |
|
public static void launch() throws Exception {
JobConf conf = new JobConf(TestFieldSelection.class);
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 10;
Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
fs.delete(OUTPUT_DIR, true);
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
TestMRFieldSelection.constructInputOutputData(inputData,
expectedOutput, numOfInputLines);
FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
fileOut.write(inputData.toString().getBytes("utf-8"));
fileOut.close();
System.out.println("inputData:");
System.out.println(inputData.toString());
JobConf job = new JobConf(conf, TestFieldSelection.class);
FileInputFormat.setInputPaths(job, INPUT_DIR);
job.setInputFormat(TextInputFormat.class);
job.setMapperClass(FieldSelectionMapReduce.class);
job.setReducerClass(FieldSelectionMapReduce.class);
FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormat(TextOutputFormat.class);
job.setNumReduceTasks(1);
job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");
JobClient.runJob(job);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path outPath = new Path(OUTPUT_DIR, "part-00000");
String outdata = MapReduceTestUtil.readOutput(outPath,job);
assertEquals(expectedOutput.toString(),outdata);
fs.delete(OUTPUT_DIR, true);
fs.delete(INPUT_DIR, true);
}
Example 4
| Source File: TestAggregates.java From hadoop with Apache License 2.0 | 4 votes |
|
public static void launch() throws Exception {
JobConf conf = new JobConf(TestAggregates.class);
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 20;
Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test");
Path INPUT_DIR = new Path("build/test/input_for_aggregates_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
fs.delete(OUTPUT_DIR, true);
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
expectedOutput.append("max\t19\n");
expectedOutput.append("min\t1\n");
FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
for (int i = 1; i < numOfInputLines; i++) {
expectedOutput.append("count_").append(idFormat.format(i));
expectedOutput.append("\t").append(i).append("\n");
inputData.append(idFormat.format(i));
for (int j = 1; j < i; j++) {
inputData.append(" ").append(idFormat.format(i));
}
inputData.append("\n");
}
expectedOutput.append("value_as_string_max\t9\n");
expectedOutput.append("value_as_string_min\t1\n");
expectedOutput.append("uniq_count\t15\n");
fileOut.write(inputData.toString().getBytes("utf-8"));
fileOut.close();
System.out.println("inputData:");
System.out.println(inputData.toString());
JobConf job = new JobConf(conf, TestAggregates.class);
FileInputFormat.setInputPaths(job, INPUT_DIR);
job.setInputFormat(TextInputFormat.class);
FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
job.setOutputFormat(TextOutputFormat.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(1);
job.setMapperClass(ValueAggregatorMapper.class);
job.setReducerClass(ValueAggregatorReducer.class);
job.setCombinerClass(ValueAggregatorCombiner.class);
job.setInt("aggregator.descriptor.num", 1);
job.set("aggregator.descriptor.0",
"UserDefined,org.apache.hadoop.mapred.lib.aggregate.AggregatorTests");
job.setLong("aggregate.max.num.unique.values", 14);
JobClient.runJob(job);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path outPath = new Path(OUTPUT_DIR, "part-00000");
String outdata = MapReduceTestUtil.readOutput(outPath,job);
System.out.println("full out data:");
System.out.println(outdata.toString());
outdata = outdata.substring(0, expectedOutput.toString().length());
assertEquals(expectedOutput.toString(),outdata);
//fs.delete(OUTPUT_DIR);
fs.delete(INPUT_DIR, true);
}
Example 5
| Source File: TestJobSysDirWithDFS.java From hadoop with Apache License 2.0 | 4 votes |
|
public static TestResult launchWordCount(JobConf conf,
Path inDir,
Path outDir,
String input,
int numMaps,
int numReduces,
String sysDir) throws IOException {
FileSystem inFs = inDir.getFileSystem(conf);
FileSystem outFs = outDir.getFileSystem(conf);
outFs.delete(outDir, true);
if (!inFs.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
{
DataOutputStream file = inFs.create(new Path(inDir, "part-0"));
file.writeBytes(input);
file.close();
}
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
// the keys are words (strings)
conf.setOutputKeyClass(Text.class);
// the values are counts (ints)
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(WordCount.MapClass.class);
conf.setCombinerClass(WordCount.Reduce.class);
conf.setReducerClass(WordCount.Reduce.class);
FileInputFormat.setInputPaths(conf, inDir);
FileOutputFormat.setOutputPath(conf, outDir);
conf.setNumMapTasks(numMaps);
conf.setNumReduceTasks(numReduces);
conf.set(JTConfig.JT_SYSTEM_DIR, "/tmp/subru/mapred/system");
JobClient jobClient = new JobClient(conf);
RunningJob job = jobClient.runJob(conf);
// Checking that the Job Client system dir is not used
assertFalse(FileSystem.get(conf).exists(
new Path(conf.get(JTConfig.JT_SYSTEM_DIR))));
// Check if the Job Tracker system dir is propogated to client
assertFalse(sysDir.contains("/tmp/subru/mapred/system"));
assertTrue(sysDir.contains("custom"));
return new TestResult(job, MapReduceTestUtil.readOutput(outDir, conf));
}
Example 6
| Source File: TestMapReduceAggregates.java From hadoop with Apache License 2.0 | 4 votes |
|
public static void launch() throws Exception {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 20;
Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test");
Path INPUT_DIR = new Path("build/test/input_for_aggregates_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
fs.delete(OUTPUT_DIR, true);
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
expectedOutput.append("max\t19\n");
expectedOutput.append("min\t1\n");
FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
for (int i = 1; i < numOfInputLines; i++) {
expectedOutput.append("count_").append(idFormat.format(i));
expectedOutput.append("\t").append(i).append("\n");
inputData.append(idFormat.format(i));
for (int j = 1; j < i; j++) {
inputData.append(" ").append(idFormat.format(i));
}
inputData.append("\n");
}
expectedOutput.append("value_as_string_max\t9\n");
expectedOutput.append("value_as_string_min\t1\n");
expectedOutput.append("uniq_count\t15\n");
fileOut.write(inputData.toString().getBytes("utf-8"));
fileOut.close();
System.out.println("inputData:");
System.out.println(inputData.toString());
conf.setInt(ValueAggregatorJobBase.DESCRIPTOR_NUM, 1);
conf.set(ValueAggregatorJobBase.DESCRIPTOR + ".0",
"UserDefined,org.apache.hadoop.mapreduce.lib.aggregate.AggregatorTests");
conf.setLong(UniqValueCount.MAX_NUM_UNIQUE_VALUES, 14);
Job job = Job.getInstance(conf);
FileInputFormat.setInputPaths(job, INPUT_DIR);
job.setInputFormatClass(TextInputFormat.class);
FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
job.setOutputFormatClass(TextOutputFormat.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(1);
job.setMapperClass(ValueAggregatorMapper.class);
job.setReducerClass(ValueAggregatorReducer.class);
job.setCombinerClass(ValueAggregatorCombiner.class);
job.waitForCompletion(true);
assertTrue(job.isSuccessful());
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
String outdata = MapReduceTestUtil.readOutput(OUTPUT_DIR, conf);
System.out.println("full out data:");
System.out.println(outdata.toString());
outdata = outdata.substring(0, expectedOutput.toString().length());
assertEquals(expectedOutput.toString(),outdata);
fs.delete(OUTPUT_DIR, true);
fs.delete(INPUT_DIR, true);
}
Example 7
| Source File: TestFieldSelection.java From big-c with Apache License 2.0 | 4 votes |
|
public static void launch() throws Exception {
JobConf conf = new JobConf(TestFieldSelection.class);
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 10;
Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
fs.delete(OUTPUT_DIR, true);
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
TestMRFieldSelection.constructInputOutputData(inputData,
expectedOutput, numOfInputLines);
FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
fileOut.write(inputData.toString().getBytes("utf-8"));
fileOut.close();
System.out.println("inputData:");
System.out.println(inputData.toString());
JobConf job = new JobConf(conf, TestFieldSelection.class);
FileInputFormat.setInputPaths(job, INPUT_DIR);
job.setInputFormat(TextInputFormat.class);
job.setMapperClass(FieldSelectionMapReduce.class);
job.setReducerClass(FieldSelectionMapReduce.class);
FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormat(TextOutputFormat.class);
job.setNumReduceTasks(1);
job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");
JobClient.runJob(job);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path outPath = new Path(OUTPUT_DIR, "part-00000");
String outdata = MapReduceTestUtil.readOutput(outPath,job);
assertEquals(expectedOutput.toString(),outdata);
fs.delete(OUTPUT_DIR, true);
fs.delete(INPUT_DIR, true);
}
Example 8
| Source File: TestAggregates.java From big-c with Apache License 2.0 | 4 votes |
|
public static void launch() throws Exception {
JobConf conf = new JobConf(TestAggregates.class);
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 20;
Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test");
Path INPUT_DIR = new Path("build/test/input_for_aggregates_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
fs.delete(OUTPUT_DIR, true);
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
expectedOutput.append("max\t19\n");
expectedOutput.append("min\t1\n");
FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
for (int i = 1; i < numOfInputLines; i++) {
expectedOutput.append("count_").append(idFormat.format(i));
expectedOutput.append("\t").append(i).append("\n");
inputData.append(idFormat.format(i));
for (int j = 1; j < i; j++) {
inputData.append(" ").append(idFormat.format(i));
}
inputData.append("\n");
}
expectedOutput.append("value_as_string_max\t9\n");
expectedOutput.append("value_as_string_min\t1\n");
expectedOutput.append("uniq_count\t15\n");
fileOut.write(inputData.toString().getBytes("utf-8"));
fileOut.close();
System.out.println("inputData:");
System.out.println(inputData.toString());
JobConf job = new JobConf(conf, TestAggregates.class);
FileInputFormat.setInputPaths(job, INPUT_DIR);
job.setInputFormat(TextInputFormat.class);
FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
job.setOutputFormat(TextOutputFormat.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(1);
job.setMapperClass(ValueAggregatorMapper.class);
job.setReducerClass(ValueAggregatorReducer.class);
job.setCombinerClass(ValueAggregatorCombiner.class);
job.setInt("aggregator.descriptor.num", 1);
job.set("aggregator.descriptor.0",
"UserDefined,org.apache.hadoop.mapred.lib.aggregate.AggregatorTests");
job.setLong("aggregate.max.num.unique.values", 14);
JobClient.runJob(job);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path outPath = new Path(OUTPUT_DIR, "part-00000");
String outdata = MapReduceTestUtil.readOutput(outPath,job);
System.out.println("full out data:");
System.out.println(outdata.toString());
outdata = outdata.substring(0, expectedOutput.toString().length());
assertEquals(expectedOutput.toString(),outdata);
//fs.delete(OUTPUT_DIR);
fs.delete(INPUT_DIR, true);
}
Example 9
| Source File: TestJobSysDirWithDFS.java From big-c with Apache License 2.0 | 4 votes |
|
public static TestResult launchWordCount(JobConf conf,
Path inDir,
Path outDir,
String input,
int numMaps,
int numReduces,
String sysDir) throws IOException {
FileSystem inFs = inDir.getFileSystem(conf);
FileSystem outFs = outDir.getFileSystem(conf);
outFs.delete(outDir, true);
if (!inFs.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
{
DataOutputStream file = inFs.create(new Path(inDir, "part-0"));
file.writeBytes(input);
file.close();
}
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
// the keys are words (strings)
conf.setOutputKeyClass(Text.class);
// the values are counts (ints)
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(WordCount.MapClass.class);
conf.setCombinerClass(WordCount.Reduce.class);
conf.setReducerClass(WordCount.Reduce.class);
FileInputFormat.setInputPaths(conf, inDir);
FileOutputFormat.setOutputPath(conf, outDir);
conf.setNumMapTasks(numMaps);
conf.setNumReduceTasks(numReduces);
conf.set(JTConfig.JT_SYSTEM_DIR, "/tmp/subru/mapred/system");
JobClient jobClient = new JobClient(conf);
RunningJob job = jobClient.runJob(conf);
// Checking that the Job Client system dir is not used
assertFalse(FileSystem.get(conf).exists(
new Path(conf.get(JTConfig.JT_SYSTEM_DIR))));
// Check if the Job Tracker system dir is propogated to client
assertFalse(sysDir.contains("/tmp/subru/mapred/system"));
assertTrue(sysDir.contains("custom"));
return new TestResult(job, MapReduceTestUtil.readOutput(outDir, conf));
}
Example 10
| Source File: TestMapReduceAggregates.java From big-c with Apache License 2.0 | 4 votes |
|
public static void launch() throws Exception {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 20;
Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test");
Path INPUT_DIR = new Path("build/test/input_for_aggregates_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
fs.delete(OUTPUT_DIR, true);
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
expectedOutput.append("max\t19\n");
expectedOutput.append("min\t1\n");
FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
for (int i = 1; i < numOfInputLines; i++) {
expectedOutput.append("count_").append(idFormat.format(i));
expectedOutput.append("\t").append(i).append("\n");
inputData.append(idFormat.format(i));
for (int j = 1; j < i; j++) {
inputData.append(" ").append(idFormat.format(i));
}
inputData.append("\n");
}
expectedOutput.append("value_as_string_max\t9\n");
expectedOutput.append("value_as_string_min\t1\n");
expectedOutput.append("uniq_count\t15\n");
fileOut.write(inputData.toString().getBytes("utf-8"));
fileOut.close();
System.out.println("inputData:");
System.out.println(inputData.toString());
conf.setInt(ValueAggregatorJobBase.DESCRIPTOR_NUM, 1);
conf.set(ValueAggregatorJobBase.DESCRIPTOR + ".0",
"UserDefined,org.apache.hadoop.mapreduce.lib.aggregate.AggregatorTests");
conf.setLong(UniqValueCount.MAX_NUM_UNIQUE_VALUES, 14);
Job job = Job.getInstance(conf);
FileInputFormat.setInputPaths(job, INPUT_DIR);
job.setInputFormatClass(TextInputFormat.class);
FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
job.setOutputFormatClass(TextOutputFormat.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(1);
job.setMapperClass(ValueAggregatorMapper.class);
job.setReducerClass(ValueAggregatorReducer.class);
job.setCombinerClass(ValueAggregatorCombiner.class);
job.waitForCompletion(true);
assertTrue(job.isSuccessful());
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
String outdata = MapReduceTestUtil.readOutput(OUTPUT_DIR, conf);
System.out.println("full out data:");
System.out.println(outdata.toString());
outdata = outdata.substring(0, expectedOutput.toString().length());
assertEquals(expectedOutput.toString(),outdata);
fs.delete(OUTPUT_DIR, true);
fs.delete(INPUT_DIR, true);
}
