org.apache.hadoop.mrunit.mapreduce.ReduceDriver Java Examples

The following examples show how to use org.apache.hadoop.mrunit.mapreduce.ReduceDriver. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HadoopApprovals.java    From ApprovalTests.Java with Apache License 2.0 6 votes vote down vote up
public static void verifyReducer(SmartReducer reducer, Object key, Object... values) throws Exception
{
  List list = new ArrayList();
  for (Object value : values)
  {
    list.add(WritableUtils.createWritable(value, reducer.getValueInType()));
  }
  ReduceDriver reduceDriver = new ReduceDriver<Text, LongWritable, Text, LongWritable>();
  reduceDriver.withInput(WritableUtils.createWritable(key, reducer.getKeyInType()), list);
  reduceDriver.setReducer(reducer);
  List results = reduceDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  String header = String.format("(%s, %s)\n\n -> reduces via %s to -> \n", key, list,
      reducer.getClass().getSimpleName());
  Approvals.verifyAll(header, results, Echo.INSTANCE);
}
 
Example #2
Source File: CardinalityIdentityReducerTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testCIReducerOneConstant() throws InterruptedException, IOException {

  TripleEntry te = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("object"));
  CardList cL1 = new CardList(1, 2, 3, 0, 0, 0);
  CardList cL2 = new CardList(4, 5, 6, 0, 0, 0);
  CardList cl = new CardList(5, 7, 9, 0, 0, 0);
  List<CardList> list = new ArrayList<CardList>();
  list.add(cL1);
  list.add(cL2);

  Text row = new Text(te.getFirstPos().toString() + DELIM + te.getFirst().toString());
  Mutation m1 = new Mutation(row);
  m1.put(new Text(te.getKeyPos().toString() + "subject"), new Text(cl.getcardS().toString()), new Value(new byte[0]));
  Mutation m2 = new Mutation(row);
  m2.put(new Text(te.getKeyPos().toString() + "predicate"), new Text(cl.getcardP().toString()), new Value(new byte[0]));
  Mutation m3 = new Mutation(row);
  m3.put(new Text(te.getKeyPos().toString() + "object"), new Text(cl.getcardO().toString()), new Value(new byte[0]));
  Text table = new Text("");

  new ReduceDriver<TripleEntry,CardList,Text,Mutation>().withReducer(new JoinSelectStatisticsSum.CardinalityIdentityReducer()).withInput(te, list)
      .withOutput(table, m1).withOutput(table, m2).withOutput(table, m3).runTest();

}
 
Example #3
Source File: CardinalityIdentityReducerTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testCIReducerTwoConstant() throws InterruptedException, IOException {

  TripleEntry te = new TripleEntry(new Text("urn:gem:etype#1234"), new Text("urn:gem#pred"), new Text("subject"), new Text("predicate"), new Text("object"));
  CardList cL1 = new CardList(1, 2, 3, 0, 0, 0);
  CardList cL2 = new CardList(4, 5, 6, 0, 0, 0);
  CardList cl = new CardList(5, 7, 9, 0, 0, 0);
  List<CardList> list = new ArrayList<CardList>();
  list.add(cL1);
  list.add(cL2);

  Text row = new Text(te.getFirstPos().toString() + te.getSecondPos().toString() + DELIM + te.getFirst().toString() + DELIM + te.getSecond());
  Mutation m1 = new Mutation(row);
  m1.put(new Text(te.getKeyPos().toString() + "subject"), new Text(cl.getcardS().toString()), new Value(new byte[0]));
  Mutation m2 = new Mutation(row);
  m2.put(new Text(te.getKeyPos().toString() + "predicate"), new Text(cl.getcardP().toString()), new Value(new byte[0]));
  Mutation m3 = new Mutation(row);
  m3.put(new Text(te.getKeyPos().toString() + "object"), new Text(cl.getcardO().toString()), new Value(new byte[0]));
  Text table = new Text("");

  new ReduceDriver<TripleEntry,CardList,Text,Mutation>().withReducer(new JoinSelectStatisticsSum.CardinalityIdentityReducer()).withInput(te, list)
      .withOutput(table, m1).withOutput(table, m2).withOutput(table, m3).runTest();

}
 
Example #4
Source File: CardinalityIdentityReducerTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testJoinTwoVars() throws InterruptedException, IOException {

  TripleEntry te = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("predicateobject"));
  CardList cL1 = new CardList(0, 0, 0, 1, 2, 3);
  CardList cL2 = new CardList(0, 0, 0, 4, 5, 6);
  CardList cl = new CardList(0, 0, 0, 5, 7, 9);
  List<CardList> list = new ArrayList<CardList>();
  list.add(cL1);
  list.add(cL2);

  Text row = new Text(te.getFirstPos().toString() + DELIM + te.getFirst().toString());
  Mutation m1 = new Mutation(row);
  m1.put(new Text(te.getKeyPos().toString() + "subjectpredicate"), new Text(cl.getcardSP().toString()), new Value(new byte[0]));
  Mutation m2 = new Mutation(row);
  m2.put(new Text(te.getKeyPos().toString() + "predicateobject"), new Text(cl.getcardPO().toString()), new Value(new byte[0]));
  Mutation m3 = new Mutation(row);
  m3.put(new Text(te.getKeyPos().toString() + "objectsubject"), new Text(cl.getcardSO().toString()), new Value(new byte[0]));
  Text table = new Text("");

  new ReduceDriver<TripleEntry,CardList,Text,Mutation>().withReducer(new JoinSelectStatisticsSum.CardinalityIdentityReducer()).withInput(te, list)
      .withOutput(table, m1).withOutput(table, m2).withOutput(table, m3).runTest();

}
 
Example #5
Source File: CardinalityIdentityReducerTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testJoinTwoVarsReverseOrder() throws InterruptedException, IOException {

  TripleEntry te = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("objectpredicate"));
  CardList cL1 = new CardList(0, 0, 0, 1, 2, 3);
  CardList cL2 = new CardList(0, 0, 0, 4, 5, 6);
  CardList cl = new CardList(0, 0, 0, 5, 7, 9);
  List<CardList> list = new ArrayList<CardList>();
  list.add(cL1);
  list.add(cL2);

  Text row = new Text(te.getFirstPos().toString() + DELIM + te.getFirst().toString());
  Mutation m1 = new Mutation(row);
  m1.put(new Text("predicateobject" + "predicatesubject"), new Text(cl.getcardSP().toString()), new Value(new byte[0]));
  Mutation m2 = new Mutation(row);
  m2.put(new Text("predicateobject" + "objectpredicate"), new Text(cl.getcardPO().toString()), new Value(new byte[0]));
  Mutation m3 = new Mutation(row);
  m3.put(new Text("predicateobject" + "subjectobject"), new Text(cl.getcardSO().toString()), new Value(new byte[0]));
  Text table = new Text("");

  new ReduceDriver<TripleEntry,CardList,Text,Mutation>().withReducer(new JoinSelectStatisticsSum.CardinalityIdentityReducer()).withInput(te, list)
      .withOutput(table, m1).withOutput(table, m2).withOutput(table, m3).runTest();

}
 
Example #6
Source File: JoinReducerTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleConstCard() throws InterruptedException, IOException {

  CompositeType ct = new CompositeType("urn:gem:etype#1234", 1);
  TripleEntry te = new TripleEntry("urn:gem#pred", "urn:gem:etype#4567", "predicate", "object", "subject");
  CardinalityType c5 = new CardinalityType(45, "object", 0);
  CardinalityType c1 = new CardinalityType(25, "subject", 2);
  CardinalityType c2 = new CardinalityType(27, "predicate", 2);
  CardinalityType c3 = new CardinalityType(29, "object", 2);
  CardinalityType c4 = new CardinalityType(31, "predicate", 1);
  List<TripleCard> list = new ArrayList<TripleCard>();
  list.add(new TripleCard(c1));
  list.add(new TripleCard(c2));
  list.add(new TripleCard(c3));
  list.add(new TripleCard(c4));
  list.add(new TripleCard(c5));
  list.add(new TripleCard(te));
  System.out.println("List is " + list);

  new ReduceDriver<CompositeType,TripleCard,TripleEntry,CardList>().withReducer(new JoinSelectAggregate.JoinReducer()).withInput(ct, list)
      .withOutput(te, new CardList(25, 31, 45, 0, 0, 0)).runTest();

}
 
Example #7
Source File: JoinReducerTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testTwoTripleEntry() throws InterruptedException, IOException {

  CompositeType ct = new CompositeType("urn:gem:etype#1234", 1);
  TripleEntry te1 = new TripleEntry("urn:gem#pred", "urn:gem:etype#4567", "predicate", "object", "subject");
  TripleEntry te2 = new TripleEntry("urn:gem#8910", "urn:gem:etype#4567", "subject", "predicate", "object");
  CardinalityType c5 = new CardinalityType(45, "object", 0);
  CardinalityType c1 = new CardinalityType(25, "subject", 2);
  CardinalityType c2 = new CardinalityType(27, "predicate", 2);
  CardinalityType c3 = new CardinalityType(29, "object", 2);
  CardinalityType c4 = new CardinalityType(31, "predicate", 1);
  List<TripleCard> list = new ArrayList<TripleCard>();
  list.add(new TripleCard(c1));
  list.add(new TripleCard(c2));
  list.add(new TripleCard(c3));
  list.add(new TripleCard(c4));
  list.add(new TripleCard(c5));
  list.add(new TripleCard(te1));
  list.add(new TripleCard(te2));
  System.out.println("List is " + list);

  new ReduceDriver<CompositeType,TripleCard,TripleEntry,CardList>().withReducer(new JoinSelectAggregate.JoinReducer()).withInput(ct, list)
      .withOutput(te1, new CardList(25, 31, 45, 0, 0, 0)).withOutput(te2, new CardList(25, 31, 45, 0, 0, 0)).runTest();

}
 
Example #8
Source File: ForwardChainTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testReducerJoin() throws Exception {
    ResourceWritable rw = new ResourceWritable();
    rw.set(TestUtils.uri("y"));
    List<Fact> facts = new LinkedList<>();
    facts.add(X_SUB_Y);
    facts.add(Y_SUB_Z);
    ReduceDriver<ResourceWritable, Fact, Fact,
        NullWritable> driver = new ReduceDriver<>();
    driver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, 1);
    driver.withReducer(new ForwardChain.ReasoningReducer(schema))
        .withInput(rw, facts)
        .withMultiOutput(MRReasoningUtils.INTERMEDIATE_OUT,
            X_SUB_Z, NullWritable.get())
        .runTest();
}
 
Example #9
Source File: DuplicateEliminationTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testEliminateOld() throws Exception {
    List<Derivation> facts = new LinkedList<>();
    facts.add(X_SUB_Y_INV.getDerivation());
    facts.add(X_SUB_Y.getDerivation());
    X_SUB_Y.unsetDerivation();
    ReduceDriver<Fact, Derivation, Fact, NullWritable> driver = new ReduceDriver<>();
    driver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, 1);
    driver.withReducer(new DuplicateElimination.DuplicateEliminationReducer())
        .withInput(X_SUB_Y, facts)
        .runTest();
}
 
Example #10
Source File: CubeReducerTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));

    CuboidReducer reducer = new CuboidReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #11
Source File: CommunityCompressionTest.java    From distributed-graph-analytics with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
    CommunityCompression.Map mapper = new CommunityCompression.Map();
    mapDriver = MapDriver.newMapDriver(mapper);
    CommunityCompression.Reduce reducer = new CommunityCompression.Reduce();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #12
Source File: LouvainTableSynthesizerTest.java    From distributed-graph-analytics with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
    LouvainTableSynthesizerMapper mapper = new LouvainTableSynthesizerMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
    LouvainTableSynthesizerReducer reducer = new LouvainTableSynthesizerReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #13
Source File: DuplicateEliminationTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testInconsistencyOld() throws Exception {
    List<Derivation> facts = new LinkedList<>();
    facts.add(X_DISJOINT.clone());
    ReduceDriver<Fact, Derivation, Fact, NullWritable> driver = new ReduceDriver<>();
    driver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, 2);
    driver.withReducer(new DuplicateElimination.DuplicateEliminationReducer())
        .withInput(Fact.NONE, facts)
        .runTest();
}
 
Example #14
Source File: DuplicateEliminationTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testInconsistencyReduce() throws Exception {
    List<Derivation> facts = new LinkedList<>();
    facts.add(X_DISJOINT.clone());
    facts.add(X_DISJOINT.clone());
    ReduceDriver<Fact, Derivation, Fact, NullWritable> driver = new ReduceDriver<>();
    driver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, 1);
    driver.withReducer(new DuplicateElimination.DuplicateEliminationReducer())
        .withInput(Fact.NONE, facts)
        .withMultiOutput(MRReasoningUtils.INCONSISTENT_OUT,
            X_DISJOINT, NullWritable.get())
        .runTest();
}
 
Example #15
Source File: DuplicateEliminationTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testRetainSimplest() throws Exception {
    List<Derivation> facts = new LinkedList<>();
    facts.add(Y_SUPER_X_INV.getDerivation());
    facts.add(Y_SUPER_X.getDerivation());
    Fact unset = Y_SUPER_X.clone();
    unset.unsetDerivation();
    ReduceDriver<Fact, Derivation, Fact, NullWritable> driver = new ReduceDriver<>();
    driver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, 1);
    driver.withReducer(new DuplicateElimination.DuplicateEliminationReducer())
        .withInput(unset, facts)
        .withMultiOutput(MRReasoningUtils.INTERMEDIATE_OUT,
            Y_SUPER_X, NullWritable.get())
        .runTest();
}
 
Example #16
Source File: ForwardChainTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testReducerInference() throws Exception {
    schema.processTriple(INV.getTriple());
    schema.closure();
    ResourceWritable rw = new ResourceWritable();
    rw.set(TestUtils.uri("y"));
    List<Fact> facts = new LinkedList<>();
    facts.add(X_SUB_Y);
    new ReduceDriver<ResourceWritable, Fact, Fact, NullWritable>()
        .withReducer(new ForwardChain.ReasoningReducer(schema))
        .withInput(rw, facts)
        .withMultiOutput(MRReasoningUtils.INTERMEDIATE_OUT,
            Y_SUPER_X, NullWritable.get())
        .runTest();
}
 
Example #17
Source File: JoinReducerTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoConstCard() throws InterruptedException, IOException {

  CompositeType ct1 = new CompositeType("urn:gem#pred" + DELIM + "urn:gem:etype#1234", 1);
  TripleEntry te1 = new TripleEntry("uri:testSubject", "", "subject", "", "predicateobject");
  TripleEntry te2 = new TripleEntry("uri:testSubject", "", "subject", "", "objectpredicate");

  CardinalityType c5 = new CardinalityType(45, "subjectobject", 0);
  CardinalityType c1 = new CardinalityType(25, "subjectobject", 2);
  CardinalityType c2 = new CardinalityType(27, "predicateobject", 5);
  CardinalityType c3 = new CardinalityType(29, "predicateobject", 2);
  CardinalityType c4 = new CardinalityType(31, "subjectpredicate", 1);
  CardinalityType c6 = new CardinalityType(56, "subjectpredicate", 2);

  List<TripleCard> list1 = new ArrayList<TripleCard>();

  list1.add(new TripleCard(c1));
  list1.add(new TripleCard(c2));
  list1.add(new TripleCard(c3));
  list1.add(new TripleCard(c4));
  list1.add(new TripleCard(c5));
  list1.add(new TripleCard(c6));
  list1.add(new TripleCard(te1));
  list1.add(new TripleCard(te2));

  // System.out.println("List is " + list);

  new ReduceDriver<CompositeType,TripleCard,TripleEntry,CardList>().withReducer(new JoinSelectAggregate.JoinReducer()).withInput(ct1, list1)
      .withOutput(te1, new CardList(0, 0, 0, 31, 29, 45)).withOutput(te2, new CardList(0, 0, 0, 31, 29, 45)).runTest();

}
 
Example #18
Source File: CubeReducerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta"));

    CuboidReducer reducer = new CuboidReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #19
Source File: FactDistinctColumnsReducerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    FactDistinctColumnsReducer factDistinctColumnsReducer = new FactDistinctColumnsReducer();
    reduceDriver = ReduceDriver.newReduceDriver(factDistinctColumnsReducer);
}
 
Example #20
Source File: MRUnitTest.java    From dkpro-c4corpus with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp()
{
    SMSCDRMapper mapper = new SMSCDRMapper();
    SMSCDRReducer reducer = new SMSCDRReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example #21
Source File: CubeReducerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta"));

    CuboidReducer reducer = new CuboidReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #22
Source File: ColumnCardinalityReducerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
    ColumnCardinalityReducer reducer = new ColumnCardinalityReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #23
Source File: ColumnCardinalityReducerTest.java    From kylin with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
    ColumnCardinalityReducer reducer = new ColumnCardinalityReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #24
Source File: RangeKeyDistributionReducerTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
    RangeKeyDistributionReducer reducer = new RangeKeyDistributionReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #25
Source File: RandomKeyDistributionReducerTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up
@Before
@SuppressWarnings({ "rawtypes", "unchecked" })
public void setUp() {
    RandomKeyDistributionReducer reducer = new RandomKeyDistributionReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #26
Source File: ColumnCardinalityReducerTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
    ColumnCardinalityReducer reducer = new ColumnCardinalityReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #27
Source File: IdentityReduceTest.java    From hiped2 with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
  reducer = new Reducer<Text, Text, Text, Text>();
  driver = new ReduceDriver<Text, Text, Text, Text>(reducer);
}
 
Example #28
Source File: AggregationPhaseTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.HOURS.toString());
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.MILLISECONDS.toString());
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  aggPhaseConfig = AggregationPhaseConfig.fromThirdEyeConfig(thirdeyeConfig);

  // Mapper config
  AggregationMapper mapper = new AggregationMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  AggregationReducer reducer = new AggregationReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  Schema reducerSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  configuration.set(AggregationPhaseConstants.AGG_PHASE_AVRO_SCHEMA.toString(), reducerSchema.toString());

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(AggregationPhaseConstants.AGG_PHASE_OUTPUT_PATH.toString(), outputPath);
  setUpAvroSerialization(reduceDriver.getConfiguration(), reducerSchema);

}
 
Example #29
Source File: TopkPhaseTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION_NAMES.toString(), "d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION.toString() + ".d3", "xyz2");
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);

  // Mapper config
  TopKPhaseMapper mapper = new TopKPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  TopKPhaseReducer reducer = new TopKPhaseReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(TopKPhaseConstants.TOPK_PHASE_OUTPUT_PATH.toString(), outputPath);

}
 
Example #30
Source File: DBScanMapReduceTest.java    From geowave with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws IOException {

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "10,10");

  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      10);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          DBScanMapReduce.class,
          HullParameters.Hull.PROJECTION_CLASS),
      SimpleFeatureProjection.class,
      Projection.class);

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);
  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);

  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  serializations();
}