org.apache.hadoop.mrunit.mapreduce.MapDriver Java Examples

The following examples show how to use org.apache.hadoop.mrunit.mapreduce.MapDriver. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HadoopApprovals.java    From ApprovalTests.Java with Apache License 2.0 6 votes vote down vote up
public static void verifyMapReduce(SmartMapper mapper, SmartReducer reducer, Object key, Object input)
    throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  MapReduceDriver mapReduceDriver = new MapReduceDriver();
  mapReduceDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  mapReduceDriver = new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
  writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapReduceDriver.withInput(writableKey, writableValue);
  mapReduceDriver.setMapper(mapper);
  mapReduceDriver.setReducer(reducer);
  List finalResults = mapReduceDriver.run();
  String text = String.format("[%s]\n\n -> maps via %s to -> \n\n%s\n\n -> reduces via %s to -> \n\n%s", input,
      mapper.getClass().getSimpleName(), ArrayUtils.toString(results, Echo.INSTANCE),
      reducer.getClass().getSimpleName(), ArrayUtils.toString(finalResults, Echo.INSTANCE));
  Approvals.verify(text);
}
 
Example #2
Source File: InMemCuboidMapperTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    inMemCuboidMapper = new InMemCuboidMapper<>();
    mapDriver = MapDriver.newMapDriver(inMemCuboidMapper);

    PowerMockito.stub(PowerMockito.method(CuboidSchedulerUtil.class, "getCuboidSchedulerByMode", CubeSegment.class,
            String.class)).toReturn(cube.getCuboidScheduler());
    IMRBatchCubingInputSide mockInputSide = createMockInputSide();
    PowerMockito.stub(PowerMockito.method(MRUtil.class, "getBatchCubingInputSide")).toReturn(mockInputSide);

}
 
Example #3
Source File: DuplicateEliminationTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testTableMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    TripleRowResolver trr = new WholeRowTripleResolver();
    Map<TABLE_LAYOUT,TripleRow> map = trr.serialize(rya);
    TripleRow tr = map.get(TABLE_LAYOUT.SPO);
    byte[] b = new byte[0];
    Key key = new Key(tr.getRow(), tr.getColumnFamily(),
        tr.getColumnQualifier(), b, 1);
    Value val = new Value(b);
    new MapDriver<Key, Value, Fact, Derivation>()
        .withMapper(new DuplicateElimination.DuplicateTableMapper())
        .withInput(key, val)
        .withOutput(X_SUB_Y, X_SUB_Y.getDerivation())
        .runTest();
}
 
Example #4
Source File: ForwardChainTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testTableMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    TripleRowResolver trr = new WholeRowTripleResolver();
    Map<TABLE_LAYOUT,TripleRow> map = trr.serialize(rya);
    TripleRow tr = map.get(TABLE_LAYOUT.SPO);
    byte[] b = new byte[0];
    Key key = new Key(tr.getRow(), tr.getColumnFamily(),
        tr.getColumnQualifier(), b, 1);
    Value val = new Value(b);
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<Key, Value, ResourceWritable, Fact>()
        .withMapper(new ForwardChain.TableMapper(schema))
        .withInput(key, val)
        .withOutput(rw1, X_SUB_Y)
        .withOutput(rw2, X_SUB_Y)
        .runTest();
}
 
Example #5
Source File: ForwardChainTest.java    From rya with Apache License 2.0 6 votes vote down vote up
@Test
public void testRdfMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    RyaStatementWritable rsw = new RyaStatementWritable();
    rsw.setRyaStatement(rya);
    LongWritable l = new LongWritable();
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<LongWritable, RyaStatementWritable, ResourceWritable,
            Fact>()
        .withMapper(new ForwardChain.RdfMapper(schema))
        .withInput(l, rsw)
        .withOutput(rw1, X_SUB_Y)
        .withOutput(rw2, X_SUB_Y)
        .runTest();
}
 
Example #6
Source File: LouvainTableSynthesizerTest.java    From distributed-graph-analytics with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
    LouvainTableSynthesizerMapper mapper = new LouvainTableSynthesizerMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
    LouvainTableSynthesizerReducer reducer = new LouvainTableSynthesizerReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #7
Source File: HadoopApprovals.java    From ApprovalTests.Java with Apache License 2.0 5 votes vote down vote up
public static void verifyMapping(SmartMapper mapper, Object key, Object input) throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  String header = String.format("[%s]\n\n -> maps via %s to -> \n", input, mapper.getClass().getSimpleName());
  Approvals.verifyAll(header, results, Echo.INSTANCE);
}
 
Example #8
Source File: DerivedColumnTransformationTest.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  DerivedColumnTransformationPhaseMapper mapper = new DerivedColumnTransformationPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");

  ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  Schema inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  Schema outputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(TRANSFORMATION_SCHEMA));
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_SCHEMA.toString(),
      outputSchema.toString());

  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_TOPK_PATH.toString(),
      ClassLoader.getSystemResource(TOPK_PATH).toString());

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_PATH.toString(), outputPath);

}
 
Example #9
Source File: DerivedColumnNoTransformationTest.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  DerivedColumnNoTransformationPhaseMapper mapper = new DerivedColumnNoTransformationPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");


  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");

  ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  Schema inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  Schema outputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(NO_TRANSFORMATION_SCHEMA));
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_SCHEMA.toString(),
      outputSchema.toString());

  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_TOPK_PATH.toString(),
      TOPK_PATH);

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_PATH.toString(), outputPath);
}
 
Example #10
Source File: BaseCuboidMapperTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));

    BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>();
    mapDriver = MapDriver.newMapDriver(mapper);
}
 
Example #11
Source File: CommunityCompressionTest.java    From distributed-graph-analytics with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
    CommunityCompression.Map mapper = new CommunityCompression.Map();
    mapDriver = MapDriver.newMapDriver(mapper);
    CommunityCompression.Reduce reducer = new CommunityCompression.Reduce();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 
Example #12
Source File: DuplicateEliminationTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testInconsistencyMapperOutput() throws Exception {
    Fact empty = new Fact();
    empty.setDerivation(X_DISJOINT);
    new MapDriver<Derivation, NullWritable, Fact, Derivation>()
        .withMapper(new DuplicateElimination.InconsistencyMapper())
        .withInput(X_DISJOINT, NullWritable.get())
        .withOutput(empty, X_DISJOINT)
        .runTest();
}
 
Example #13
Source File: DuplicateEliminationTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testRdfMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    RyaStatementWritable rsw = new RyaStatementWritable();
    rsw.setRyaStatement(rya);
    LongWritable l = new LongWritable();
    new MapDriver<LongWritable, RyaStatementWritable, Fact,
        Derivation>()
        .withMapper(new DuplicateElimination.DuplicateRdfMapper())
        .withInput(l, rsw)
        .withOutput(X_SUB_Y, X_SUB_Y.getDerivation())
        .runTest();
}
 
Example #14
Source File: DuplicateEliminationTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testFileMapperOutput() throws Exception {
    new MapDriver<Fact, NullWritable, Fact, Derivation>()
        .withMapper(new DuplicateElimination.DuplicateFileMapper())
        .withInput(X_SUB_Y, NullWritable.get())
        .withOutput(X_SUB_Y, X_SUB_Y.getDerivation())
        .runTest();
}
 
Example #15
Source File: LindenMapredTest.java    From linden with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void init() throws IOException {
  LindenMapper mapper = new LindenMapper();
  mDriver = MapDriver.newMapDriver(mapper);
  int numShards = 1;
  Shard[] shards = LindenJob.createShards(indexPath, numShards);
  Shard.setIndexShards(mDriver.getConfiguration(), shards);
}
 
Example #16
Source File: JoinSelectStatisticsSumTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testFullTripleEntry() throws InterruptedException, IOException {

  TripleEntry te1 = new TripleEntry(new Text("urn:gem:etype#1234"), new Text("urn:gem#pred"), new Text("subject"), new Text("predicate"), new Text("object"));
  CardList cl = new CardList(34, 52, 63, 0, 0, 0);
  TripleEntry te2 = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("object"));
  TripleEntry te3 = new TripleEntry(new Text("urn:gem#pred"), new Text(""), new Text("predicate"), new Text(""), new Text("object"));

  new MapDriver<TripleEntry,CardList,TripleEntry,CardList>().withMapper(new JoinSelectStatisticsSum.CardinalityIdentityMapper()).withInput(te1, cl)
      .withOutput(te2, cl).withOutput(te3, cl).withOutput(te1, cl).runTest();

}
 
Example #17
Source File: MRUnitTest.java    From dkpro-c4corpus with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp()
{
    SMSCDRMapper mapper = new SMSCDRMapper();
    SMSCDRReducer reducer = new SMSCDRReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 
Example #18
Source File: JoinSelectProspectOutputTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testOutput() throws InterruptedException, IOException {

    String s = "urn:gem:etype#1234";
    String p = "urn:gem#pred";

    String ts = "798497748386999999";
    
    Text t1 = new Text(TripleValueType.subject.name() + DELIM + s + DELIM + 1);
    Text t2 = new Text(TripleValueType.predicate.name() + DELIM + p + DELIM + 2);
    Text t3 = new Text(TripleValueType.subjectpredicate.name() + DELIM + s + DELIM + p + DELIM + ts);

    byte[] b = new byte[0];
    byte[] c = "25".getBytes();
    byte[] d = "47".getBytes();
    byte[] e = "15".getBytes();

    Key key1 = new Key(t1.getBytes(), b, b, b, 1);
    Key key2 = new Key(t2.getBytes(), b, b, b, 1);
    Key key3 = new Key(t3.getBytes(), b, b, b, 1);
    Value val1 = new Value(c);
    Value val2 = new Value(d);
    Value val3 = new Value(e);
    
   

    // System.out.println("Keys are " + key1 + " and " + key2);

    new MapDriver<Key, Value, CompositeType, TripleCard>()
            .withMapper(new JoinSelectProspectOutput.CardinalityMapper())
            .withInput(key1, val1)
            .withInput(key2, val2)
            .withInput(key3, val3)
            .withOutput(new CompositeType(s, 1), new TripleCard(new CardinalityType(25, "subject", 1)))
            .withOutput(new CompositeType(p, 1), new TripleCard(new CardinalityType(47, "predicate", 2)))
            .withOutput(new CompositeType(s + DELIM + p, 1),
                    new TripleCard(new CardinalityType(15, "subjectpredicate", Long.parseLong(ts)))).runTest();

}
 
Example #19
Source File: CardinalityMapperTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testOutput() throws InterruptedException, IOException {

  String s = "urn:gem:etype#1234";
  String p = "urn:gem#pred";

  Text t1 = new Text(TripleValueType.subject.name() + DELIM + s + DELIM + 1);
  Text t2 = new Text(TripleValueType.predicate.name() + DELIM + p + DELIM + 2);
  Text t3 = new Text(TripleValueType.subjectpredicate.name() + DELIM + s + DELIM + p + DELIM + 3);

  byte[] b = new byte[0];
  byte[] c = "25".getBytes();
  byte[] d = "47".getBytes();
  byte[] e = "15".getBytes();

  Key key1 = new Key(t1.getBytes(), b, b, b, 1);
  Key key2 = new Key(t2.getBytes(), b, b, b, 1);
  Key key3 = new Key(t3.getBytes(), b, b, b, 1);
  Value val1 = new Value(c);
  Value val2 = new Value(d);
  Value val3 = new Value(e);

  // System.out.println("Keys are " + key1 + " and " + key2);

  new MapDriver<Key,Value,CompositeType,TripleCard>().withMapper(new JoinSelectProspectOutput.CardinalityMapper()).withInput(key1, val1)
      .withInput(key2, val2).withInput(key3, val3).withOutput(new CompositeType(s, 1), new TripleCard(new CardinalityType(25, "subject", 1)))
      .withOutput(new CompositeType(p, 1), new TripleCard(new CardinalityType(47, "predicate", 2)))
      .withOutput(new CompositeType(s + DELIM + p, 1), new TripleCard(new CardinalityType(15, "subjectpredicate", 3))).runTest();

}
 
Example #20
Source File: FactDistinctColumnsMapperTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    FactDistinctColumnsMapper<LongWritable> factDistinctColumnsMapper = new FactDistinctColumnsMapper<>();
    mapDriver = MapDriver.newMapDriver(factDistinctColumnsMapper);
}
 
Example #21
Source File: CalculateStatsFromBaseCuboidMapperTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    CalculateStatsFromBaseCuboidMapper calStatsFromBasicCuboidMapper = new CalculateStatsFromBaseCuboidMapper();
    mapDriver = MapDriver.newMapDriver(calStatsFromBasicCuboidMapper);
}
 
Example #22
Source File: ForwardChainTest.java    From rya with Apache License 2.0 5 votes vote down vote up
@Test
public void testFileMapperOutput() throws Exception {
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<Fact, NullWritable, ResourceWritable, Fact>()
        .withMapper(new ForwardChain.FileMapper(schema))
        .withInput(X_SUB_Y, NullWritable.get())
        .withOutput(rw1, X_SUB_Y)
        .withOutput(rw2, X_SUB_Y)
        .runTest();
}
 
Example #23
Source File: ColumnCardinalityMapperTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Before
public void setUp() {
    ColumnCardinalityMapper mapper = new ColumnCardinalityMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}
 
Example #24
Source File: NNMapReduceTest.java    From geowave with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws IOException {
  GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put(
      "memory",
      new MemoryStoreFactoryFamily());
  final NNMapReduce.NNMapper<SimpleFeature> nnMapper = new NNMapReduce.NNMapper<>();
  final NNMapReduce.NNReducer<SimpleFeature, Text, Text, Boolean> nnReducer =
      new NNMapReduce.NNSimpleFeatureIDOutputReducer();

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "0.0002,0.0002");

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          CommonParameters.Common.DISTANCE_FUNCTION_CLASS),
      FeatureCentroidOrthodromicDistanceFn.class,
      DistanceFn.class);
  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      0.001);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);

  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
  internalAdapterId = InternalAdapterStoreImpl.getLazyInitialAdapterId(adapter.getTypeName());
  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      internalAdapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      internalAdapterId);

  serializations();
}
 
Example #25
Source File: DBScanMapReduceTest.java    From geowave with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws IOException {

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "10,10");

  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      10);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          DBScanMapReduce.class,
          HullParameters.Hull.PROJECTION_CLASS),
      SimpleFeatureProjection.class,
      Projection.class);

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);
  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);

  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  serializations();
}
 
Example #26
Source File: TopkPhaseTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION_NAMES.toString(), "d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION.toString() + ".d3", "xyz2");
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);

  // Mapper config
  TopKPhaseMapper mapper = new TopKPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  TopKPhaseReducer reducer = new TopKPhaseReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(TopKPhaseConstants.TOPK_PHASE_OUTPUT_PATH.toString(), outputPath);

}
 
Example #27
Source File: AggregationPhaseTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.HOURS.toString());
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.MILLISECONDS.toString());
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  aggPhaseConfig = AggregationPhaseConfig.fromThirdEyeConfig(thirdeyeConfig);

  // Mapper config
  AggregationMapper mapper = new AggregationMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  AggregationReducer reducer = new AggregationReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  Schema reducerSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  configuration.set(AggregationPhaseConstants.AGG_PHASE_AVRO_SCHEMA.toString(), reducerSchema.toString());

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(AggregationPhaseConstants.AGG_PHASE_OUTPUT_PATH.toString(), outputPath);
  setUpAvroSerialization(reduceDriver.getConfiguration(), reducerSchema);

}
 
Example #28
Source File: CsvBlurMapperTest.java    From incubator-retired-blur with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws IOException {
  _mapper = new CsvBlurMapper();
  _mapDriver = MapDriver.newMapDriver(_mapper);
}
 
Example #29
Source File: IdentityMapJUnitAssertsTest.java    From hiped2 with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
  mapper = new Mapper<Text, Text, Text, Text>();
  driver = new MapDriver<Text, Text, Text, Text>(mapper);
}
 
Example #30
Source File: IdentityMapJUnitTest.java    From hiped2 with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() {
  mapper = new Mapper<Text, Text, Text, Text>();
  driver = new MapDriver<Text, Text, Text, Text>(mapper);
}