org.apache.hadoop.mrunit.mapreduce.MapReduceDriver Java Examples

The following examples show how to use org.apache.hadoop.mrunit.mapreduce.MapReduceDriver. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FullTableSizeTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testFullTableSize() throws IOException {

    Value value = new Value(new byte[0]);

    Mutation m = new Mutation(new Text("subjectpredicateobject" + DELIM + "FullTableCardinality"));
    m.put(new Text("FullTableCardinality"), new Text("15"), new Value(new byte[0]));

    new MapReduceDriver<Key, Value, Text, IntWritable, Text, Mutation>()
            .withMapper(new FullTableSize.FullTableMapper()).withInput(new Key(new Text("entry1")), value)
            .withInput(new Key(new Text("entry2")), value).withInput(new Key(new Text("entry3")), value)
            .withInput(new Key(new Text("entry4")), value).withInput(new Key(new Text("entry5")), value)
            .withInput(new Key(new Text("entry6")), value).withInput(new Key(new Text("entry7")), value)
            .withInput(new Key(new Text("entry8")), value).withInput(new Key(new Text("entry9")), value)
            .withInput(new Key(new Text("entry10")), value).withInput(new Key(new Text("entry11")), value)
            .withInput(new Key(new Text("entry12")), value).withInput(new Key(new Text("entry13")), value)
            .withInput(new Key(new Text("entry14")), value).withInput(new Key(new Text("entry15")), value)
            .withCombiner(new FullTableSize.FullTableCombiner()).withReducer(new FullTableSize.FullTableReducer())
            .withOutput(new Text(""), m).runTest();

}
 
Example #2
Source File: HadoopApprovals.java    From ApprovalTests.Java with Apache License 2.0 6 votes vote down vote up
public static void verifyMapReduce(SmartMapper mapper, SmartReducer reducer, Object key, Object input)
    throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  MapReduceDriver mapReduceDriver = new MapReduceDriver();
  mapReduceDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  mapReduceDriver = new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
  writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapReduceDriver.withInput(writableKey, writableValue);
  mapReduceDriver.setMapper(mapper);
  mapReduceDriver.setReducer(reducer);
  List finalResults = mapReduceDriver.run();
  String text = String.format("[%s]\n\n -> maps via %s to -> \n\n%s\n\n -> reduces via %s to -> \n\n%s", input,
      mapper.getClass().getSimpleName(), ArrayUtils.toString(results, Echo.INSTANCE),
      reducer.getClass().getSimpleName(), ArrayUtils.toString(finalResults, Echo.INSTANCE));
  Approvals.verify(text);
}
 
Example #3
Source File: NDCuboidMapperTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    NDCuboidMapper mapper = new NDCuboidMapper();
    CuboidReducer reducer = new CuboidReducer();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example #4
Source File: MRUnitTest.java    From dkpro-c4corpus with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp()
{
    SMSCDRMapper mapper = new SMSCDRMapper();
    SMSCDRReducer reducer = new SMSCDRReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example #5
Source File: NDCuboidMapperTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    NDCuboidMapper mapper = new NDCuboidMapper();
    CuboidReducer reducer = new CuboidReducer();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example #6
Source File: NDCuboidMapperTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));

    NDCuboidMapper mapper = new NDCuboidMapper();
    CuboidReducer reducer = new CuboidReducer();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example #7
Source File: IdentityMapReduceTest.java    From hiped2 with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
  mapper = new Mapper<Text, Text, Text, Text>();
  reducer = new Reducer<Text, Text, Text, Text>();
  driver = new MapReduceDriver<Text, Text, Text, Text, Text, Text>(mapper, reducer);
}