Java Code Examples for org.apache.hadoop.mrunit.mapreduce.MapReduceDriver

The following examples show how to use org.apache.hadoop.mrunit.mapreduce.MapReduceDriver. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: rya   Source File: FullTableSizeTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFullTableSize() throws IOException {

    Value value = new Value(new byte[0]);

    Mutation m = new Mutation(new Text("subjectpredicateobject" + DELIM + "FullTableCardinality"));
    m.put(new Text("FullTableCardinality"), new Text("15"), new Value(new byte[0]));

    new MapReduceDriver<Key, Value, Text, IntWritable, Text, Mutation>()
            .withMapper(new FullTableSize.FullTableMapper()).withInput(new Key(new Text("entry1")), value)
            .withInput(new Key(new Text("entry2")), value).withInput(new Key(new Text("entry3")), value)
            .withInput(new Key(new Text("entry4")), value).withInput(new Key(new Text("entry5")), value)
            .withInput(new Key(new Text("entry6")), value).withInput(new Key(new Text("entry7")), value)
            .withInput(new Key(new Text("entry8")), value).withInput(new Key(new Text("entry9")), value)
            .withInput(new Key(new Text("entry10")), value).withInput(new Key(new Text("entry11")), value)
            .withInput(new Key(new Text("entry12")), value).withInput(new Key(new Text("entry13")), value)
            .withInput(new Key(new Text("entry14")), value).withInput(new Key(new Text("entry15")), value)
            .withCombiner(new FullTableSize.FullTableCombiner()).withReducer(new FullTableSize.FullTableReducer())
            .withOutput(new Text(""), m).runTest();

}
 
Example 2
Source Project: ApprovalTests.Java   Source File: HadoopApprovals.java    License: Apache License 2.0 6 votes vote down vote up
public static void verifyMapReduce(SmartMapper mapper, SmartReducer reducer, Object key, Object input)
    throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  MapReduceDriver mapReduceDriver = new MapReduceDriver();
  mapReduceDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  mapReduceDriver = new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
  writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapReduceDriver.withInput(writableKey, writableValue);
  mapReduceDriver.setMapper(mapper);
  mapReduceDriver.setReducer(reducer);
  List finalResults = mapReduceDriver.run();
  String text = String.format("[%s]\n\n -> maps via %s to -> \n\n%s\n\n -> reduces via %s to -> \n\n%s", input,
      mapper.getClass().getSimpleName(), ArrayUtils.toString(results, Echo.INSTANCE),
      reducer.getClass().getSimpleName(), ArrayUtils.toString(finalResults, Echo.INSTANCE));
  Approvals.verify(text);
}
 
Example 3
Source Project: kylin-on-parquet-v2   Source File: NDCuboidMapperTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    NDCuboidMapper mapper = new NDCuboidMapper();
    CuboidReducer reducer = new CuboidReducer();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example 4
Source Project: dkpro-c4corpus   Source File: MRUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp()
{
    SMSCDRMapper mapper = new SMSCDRMapper();
    SMSCDRReducer reducer = new SMSCDRReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example 5
Source Project: kylin   Source File: NDCuboidMapperTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    NDCuboidMapper mapper = new NDCuboidMapper();
    CuboidReducer reducer = new CuboidReducer();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example 6
Source Project: Kylin   Source File: NDCuboidMapperTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));

    NDCuboidMapper mapper = new NDCuboidMapper();
    CuboidReducer reducer = new CuboidReducer();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example 7
Source Project: hiped2   Source File: IdentityMapReduceTest.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
  mapper = new Mapper<Text, Text, Text, Text>();
  reducer = new Reducer<Text, Text, Text, Text>();
  driver = new MapReduceDriver<Text, Text, Text, Text, Text, Text>(mapper, reducer);
}