Java Code Examples for org.apache.hadoop.mrunit.mapreduce.MapDriver#newMapDriver()

The following examples show how to use org.apache.hadoop.mrunit.mapreduce.MapDriver#newMapDriver() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: InMemCuboidMapperTest.java From kylin with Apache License 2.0

6 votes

@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    inMemCuboidMapper = new InMemCuboidMapper<>();
    mapDriver = MapDriver.newMapDriver(inMemCuboidMapper);

    PowerMockito.stub(PowerMockito.method(CuboidSchedulerUtil.class, "getCuboidSchedulerByMode", CubeSegment.class,
            String.class)).toReturn(cube.getCuboidScheduler());
    IMRBatchCubingInputSide mockInputSide = createMockInputSide();
    PowerMockito.stub(PowerMockito.method(MRUtil.class, "getBatchCubingInputSide")).toReturn(mockInputSide);

}

Example 2

Source File: LindenMapredTest.java From linden with Apache License 2.0

5 votes

@BeforeClass
public static void init() throws IOException {
  LindenMapper mapper = new LindenMapper();
  mDriver = MapDriver.newMapDriver(mapper);
  int numShards = 1;
  Shard[] shards = LindenJob.createShards(indexPath, numShards);
  Shard.setIndexShards(mDriver.getConfiguration(), shards);
}

Example 3

Source File: DerivedColumnTransformationTest.java From incubator-pinot with Apache License 2.0

5 votes

@Before
public void setUp() throws Exception {
  DerivedColumnTransformationPhaseMapper mapper = new DerivedColumnTransformationPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");

  ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  Schema inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  Schema outputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(TRANSFORMATION_SCHEMA));
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_SCHEMA.toString(),
      outputSchema.toString());

  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_TOPK_PATH.toString(),
      ClassLoader.getSystemResource(TOPK_PATH).toString());

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_PATH.toString(), outputPath);

}

Example 4

Source File: DerivedColumnNoTransformationTest.java From incubator-pinot with Apache License 2.0

5 votes

@Before
public void setUp() throws Exception {
  DerivedColumnNoTransformationPhaseMapper mapper = new DerivedColumnNoTransformationPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");


  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");

  ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  Schema inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  Schema outputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(NO_TRANSFORMATION_SCHEMA));
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_SCHEMA.toString(),
      outputSchema.toString());

  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_TOPK_PATH.toString(),
      TOPK_PATH);

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_PATH.toString(), outputPath);
}

Example 5

Source File: BaseCuboidMapperTest.java From Kylin with Apache License 2.0

5 votes

@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));

    BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>();
    mapDriver = MapDriver.newMapDriver(mapper);
}

Example 6

Source File: CommunityCompressionTest.java From distributed-graph-analytics with Apache License 2.0

5 votes

@Before
public void setUp() {
    CommunityCompression.Map mapper = new CommunityCompression.Map();
    mapDriver = MapDriver.newMapDriver(mapper);
    CommunityCompression.Reduce reducer = new CommunityCompression.Reduce();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}

Example 7

Source File: LouvainTableSynthesizerTest.java From distributed-graph-analytics with Apache License 2.0

5 votes

@Before
public void setUp() {
    LouvainTableSynthesizerMapper mapper = new LouvainTableSynthesizerMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
    LouvainTableSynthesizerReducer reducer = new LouvainTableSynthesizerReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}

Example 8

Source File: CalculateStatsFromBaseCuboidMapperTest.java From kylin with Apache License 2.0

5 votes

@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    CalculateStatsFromBaseCuboidMapper calStatsFromBasicCuboidMapper = new CalculateStatsFromBaseCuboidMapper();
    mapDriver = MapDriver.newMapDriver(calStatsFromBasicCuboidMapper);
}

Example 9

Source File: FactDistinctColumnsMapperTest.java From kylin with Apache License 2.0

5 votes

@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    FactDistinctColumnsMapper<LongWritable> factDistinctColumnsMapper = new FactDistinctColumnsMapper<>();
    mapDriver = MapDriver.newMapDriver(factDistinctColumnsMapper);
}

Example 10

Source File: MRUnitTest.java From dkpro-c4corpus with Apache License 2.0

5 votes

@Before
public void setUp()
{
    SMSCDRMapper mapper = new SMSCDRMapper();
    SMSCDRReducer reducer = new SMSCDRReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}

Example 11

Source File: CubeHFileMapperTest.java From kylin with Apache License 2.0

4 votes

@Before
public void setUp() {
    CubeHFileMapper mapper = new CubeHFileMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}

Example 12

Source File: CubeHFileMapperTest.java From Kylin with Apache License 2.0

4 votes

@Before
public void setUp() {
    CubeHFileMapper mapper = new CubeHFileMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}

Example 13

Source File: NNMapReduceTest.java From geowave with Apache License 2.0

4 votes

@Before
public void setUp() throws IOException {
  GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put(
      "memory",
      new MemoryStoreFactoryFamily());
  final NNMapReduce.NNMapper<SimpleFeature> nnMapper = new NNMapReduce.NNMapper<>();
  final NNMapReduce.NNReducer<SimpleFeature, Text, Text, Boolean> nnReducer =
      new NNMapReduce.NNSimpleFeatureIDOutputReducer();

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "0.0002,0.0002");

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          CommonParameters.Common.DISTANCE_FUNCTION_CLASS),
      FeatureCentroidOrthodromicDistanceFn.class,
      DistanceFn.class);
  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      0.001);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);

  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
  internalAdapterId = InternalAdapterStoreImpl.getLazyInitialAdapterId(adapter.getTypeName());
  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      internalAdapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      internalAdapterId);

  serializations();
}

Example 14

Source File: RandomKeyDistributionMapperTest.java From Kylin with Apache License 2.0

4 votes

@Before
@SuppressWarnings({ "rawtypes", "unchecked" })
public void setUp() {
    RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}

Example 15

Source File: DBScanMapReduceTest.java From geowave with Apache License 2.0

4 votes

@Before
public void setUp() throws IOException {

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "10,10");

  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      10);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          DBScanMapReduce.class,
          HullParameters.Hull.PROJECTION_CLASS),
      SimpleFeatureProjection.class,
      Projection.class);

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);
  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);

  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  serializations();
}

Example 16

Source File: ColumnCardinalityMapperTest.java From Kylin with Apache License 2.0

4 votes

@SuppressWarnings({ "rawtypes", "unchecked" })
@Before
public void setUp() {
    ColumnCardinalityMapper mapper = new ColumnCardinalityMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}

Example 17

Source File: CsvBlurMapperTest.java From incubator-retired-blur with Apache License 2.0

4 votes

@Before
public void setUp() throws IOException {
  _mapper = new CsvBlurMapper();
  _mapDriver = MapDriver.newMapDriver(_mapper);
}

Example 18

Source File: AggregationPhaseTest.java From incubator-pinot with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.HOURS.toString());
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.MILLISECONDS.toString());
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  aggPhaseConfig = AggregationPhaseConfig.fromThirdEyeConfig(thirdeyeConfig);

  // Mapper config
  AggregationMapper mapper = new AggregationMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  AggregationReducer reducer = new AggregationReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  Schema reducerSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  configuration.set(AggregationPhaseConstants.AGG_PHASE_AVRO_SCHEMA.toString(), reducerSchema.toString());

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(AggregationPhaseConstants.AGG_PHASE_OUTPUT_PATH.toString(), outputPath);
  setUpAvroSerialization(reduceDriver.getConfiguration(), reducerSchema);

}

Example 19

Source File: TopkPhaseTest.java From incubator-pinot with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION_NAMES.toString(), "d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION.toString() + ".d3", "xyz2");
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);

  // Mapper config
  TopKPhaseMapper mapper = new TopKPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  TopKPhaseReducer reducer = new TopKPhaseReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(TopKPhaseConstants.TOPK_PHASE_OUTPUT_PATH.toString(), outputPath);

}

Example 20

Source File: CubeHFileMapperTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Before
public void setUp() {
    CubeHFileMapper mapper = new CubeHFileMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}