org.apache.hadoop.mrunit.types.Pair Java Examples

The following examples show how to use org.apache.hadoop.mrunit.types.Pair. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KMeansDistortionMapReduceTest.java    From geowave with Apache License 2.0 6 votes vote down vote up
@Test
public void testReducer() throws IOException {

  reduceDriver.addInput(
      new Text("g1"),
      Arrays.asList(new CountofDoubleWritable(0.34, 1), new CountofDoubleWritable(0.75, 1)));
  reduceDriver.addInput(
      new Text("g2"),
      Arrays.asList(new CountofDoubleWritable(0.34, 1), new CountofDoubleWritable(0.25, 1)));

  final List<Pair<GeoWaveOutputKey, DistortionEntry>> results = reduceDriver.run();
  assertEquals(1, results.size());

  assertTrue(results.get(0).getSecond().getGroupId().equals("g1"));
  assertTrue(results.get(0).getSecond().getClusterCount().equals(1));
  // TODO: floating point error?
  assertTrue(results.get(0).getSecond().getDistortionValue().equals(3.6697247706422016));
}
 
Example #2
Source File: MRUnitJUnitAsserts.java    From hiped2 with Apache License 2.0 6 votes vote down vote up
public static <K, V, T extends TestDriver<K, V, T>> void assertOutputs(
    TestDriver<K, V, T> driver, List<Pair<K, V>> actuals) {

  List<Pair<K, V>> expected = driver.getExpectedOutputs();

  assertEquals("Number of expected records don't match actual number",
      expected.size(), actuals.size());

  // make sure all actual outputs are in the expected set,
  // and at the proper position.
  for (int i = 0; i < expected.size(); i++) {
    Pair<K, V> actual = actuals.get(i);
    Pair<K, V> expect = expected.get(i);
    assertEquals("Records don't match at position " + i,
        expect, actual);
  }
}
 
Example #3
Source File: RandomKeyDistributionMapperTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void test() throws IOException {
    List<Text> data = new ArrayList<Text>();
    for (int i = 0; i < 1001; i++) {
        data.add(new Text(String.valueOf(i)));
    }

    for (Text t : data) {
        mapDriver.addInput(t, new Text("abc"));
    }

    mapDriver.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, "100");
    List<Pair<Text, LongWritable>> result = mapDriver.run();
    assertEquals(100, result.size());

    for (Pair<Text, LongWritable> p : result) {
        System.out.println(p.getFirst());
    }
}
 
Example #4
Source File: RandomKeyDistributionReducerTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void test() throws IOException {
    List<Text> data = new ArrayList<Text>();
    for (int i = 0; i < 1001; i++) {
        data.add(new Text(String.valueOf(i)));
    }
    for (Text t : data) {
        reduceDriver.addInput(t, new ArrayList<LongWritable>());
    }

    reduceDriver.getConfiguration().set(BatchConstants.REGION_NUMBER, "2");
    List<Pair<Text, LongWritable>> result = reduceDriver.run();

    assertEquals(2, result.size());

    for (Pair<Text, LongWritable> p : result) {
        System.out.println(p.getFirst());
    }
}
 
Example #5
Source File: FactDistinctColumnsReducerTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
private void testNormalDim() throws IOException {
    setConfigurations();
    setMultipleOutputs(BatchConstants.CFG_OUTPUT_COLUMN, reduceDriver.getConfiguration(),
            SequenceFileOutputFormat.class, NullWritable.class, Text.class);
    setMultipleOutputs(BatchConstants.CFG_OUTPUT_DICT, reduceDriver.getConfiguration(),
            SequenceFileOutputFormat.class, NullWritable.class, ArrayPrimitiveWritable.class);
    setMultipleOutputs(BatchConstants.CFG_OUTPUT_PARTITION, reduceDriver.getConfiguration(), TextOutputFormat.class,
            NullWritable.class, LongWritable.class);

    int nDimReducers = cubeDesc.getRowkey().getRowKeyColumns().length;
    setContextTaskId(nDimReducers - 1);

    ByteBuffer tmpBuf = ByteBuffer.allocate(4096);
    String val = "100";
    tmpBuf.put(Bytes.toBytes(val));
    Text outputKey1 = new Text();
    outputKey1.set(tmpBuf.array(), 0, tmpBuf.position());
    SelfDefineSortableKey key1 = new SelfDefineSortableKey();
    key1.init(outputKey1, (byte) 0);

    reduceDriver.setInput(key1, ImmutableList.of(new Text()));
    List<Pair<NullWritable, Text>> result = reduceDriver.run();
    assertEquals(0, result.size());
}
 
Example #6
Source File: LouvainTableSynthesizerTest.java    From distributed-graph-analytics with Apache License 2.0 6 votes vote down vote up
@Test
public void testReducer() throws Exception {
    List<Text> list = new LinkedList<Text>();
    list.add(new Text("1\t2:0"));
    list.add(new Text("2\t2:0"));
    list.add(new Text("3\t2:0"));
    list.add(new Text("6:1"));
    list.add(new Text("5\t2:0"));
    reduceDriver.addInput(new Text("2"), list);
    reduceDriver.addOutput(new Text("1\t2\t6"), NullWritable.get());
    reduceDriver.addOutput(new Text("2\t2\t6"), NullWritable.get());
    reduceDriver.addOutput(new Text("3\t2\t6"), NullWritable.get());
    reduceDriver.addOutput(new Text("5\t2\t6"), NullWritable.get());
    List<Pair<Text, NullWritable>> results = reduceDriver.run();
    assertTrue(results.size() == 4);
}
 
Example #7
Source File: DBScanMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
private List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> getReducerDataFromMapperInput(
    final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults) {
  final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> reducerInputSet =
      new ArrayList<>();
  for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
    getListFor(pair.getFirst(), reducerInputSet).add(pair.getSecond());
  }
  return reducerInputSet;
}
 
Example #8
Source File: ColumnCardinalityMapperTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testMapperOnComma() throws IOException {
    mapDriver.clearInput();
    LongWritable inputKey1 = new LongWritable(1);
    LongWritable inputKey2 = new LongWritable(2);
    LongWritable inputKey3 = new LongWritable(3);
    LongWritable inputKey4 = new LongWritable(4);
    LongWritable inputKey5 = new LongWritable(5);
    LongWritable inputKey6 = new LongWritable(6);
    LongWritable inputKey7 = new LongWritable(7);

    mapDriver.addInput(inputKey1, new Text());
    mapDriver.addInput(inputKey2, new Text(strArr));
    mapDriver.addInput(inputKey3, new Text(strArr));
    mapDriver.addInput(inputKey4, new Text(strArr));
    mapDriver.addInput(inputKey5, new Text(strArr));
    mapDriver.addInput(inputKey6, new Text(strArr));
    mapDriver.addInput(inputKey7, new Text(strArr));

    List<Pair<IntWritable, BytesWritable>> result = mapDriver.run();

    assertEquals(9, result.size());

    int key1 = result.get(0).getFirst().get();
    BytesWritable value1 = result.get(0).getSecond();
    byte[] bytes = value1.getBytes();
    HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
    hllc.readRegisters(ByteBuffer.wrap(bytes));
    System.out.println("ab\177ab".length());
    assertTrue(key1 > 0);
    assertEquals(1, hllc.getCountEstimate());
}
 
Example #9
Source File: IdentityMapReduceTest.java    From hiped2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testIdentityMapper() throws IOException {
  List<Pair<Text, Text>> results = driver
      .withInput(new Text("foo"), new Text("bar"))
      .withInput(new Text("foo2"), new Text("bar2"))
      .withOutput(new Text("foo"), new Text("bar"))
      .withOutput(new Text("foo2"), new Text("bar2"))
      .run();

  MRUnitJUnitAsserts.assertOutputs(driver, results);
}
 
Example #10
Source File: IdentityReduceTest.java    From hiped2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testIdentityMapper() throws IOException {
  List<Pair<Text, Text>> results = driver
      .withInput(new Text("foo"), Arrays.asList(new Text("bar1"), new Text("bar2")))
      .withOutput(new Text("foo"), new Text("bar1"))
      .withOutput(new Text("foo"), new Text("bar2"))
      .run();

  MRUnitJUnitAsserts.assertOutputs(driver, results);
}
 
Example #11
Source File: IdentityMapJUnitTest.java    From hiped2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testIdentityMapper() throws IOException {
  List<Pair<Text, Text>> results = driver
      .withInput(new Text("foo"), new Text("bar"))
      .run();

  assertEquals(1, results.size());
  assertEquals(new Text("foo"), results.get(0).getFirst());
  assertEquals(new Text("bar"), results.get(0).getSecond());
}
 
Example #12
Source File: PipelineTest.java    From hiped2 with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  mapper1 = new IdentityMapper<Text, Text>();
  reducer1 = new IdentityReducer<Text, Text>();
  mapper2 = new IdentityMapper<Text, Text>();
  reducer2 = new IdentityReducer<Text, Text>();
  driver = new PipelineMapReduceDriver<Text, Text, Text, Text>();
  driver.addMapReduce(new Pair<Mapper, Reducer>(mapper1, reducer1));
  driver.addMapReduce(new Pair<Mapper, Reducer>(mapper2, reducer2));
}
 
Example #13
Source File: PipelineTest.java    From hiped2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testIdentityMapper() throws IOException {
  List<Pair<Text, Text>> results = driver
      .withInput(new Text("foo"), new Text("bar"))
      .withInput(new Text("foo2"), new Text("bar2"))
      .withOutput(new Text("foo"), new Text("bar"))
      .withOutput(new Text("foo2"), new Text("bar2"))
      .run();

  MRUnitJUnitAsserts.assertOutputs(driver, results);
}
 
Example #14
Source File: IdentityMapJUnitAssertsTest.java    From hiped2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testIdentityMapper() throws IOException {
  List<Pair<Text, Text>> results = driver
      .withInput(new Text("foo"), new Text("bar"))
      .withOutput(new Text("foo"), new Text("bar"))
      .run();

  MRUnitJUnitAsserts.assertOutputs(driver, results);
}
 
Example #15
Source File: TopkPhaseTest.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
private List<Pair<BytesWritable,List<BytesWritable>>> generateTestReduceData(List<Pair<BytesWritable, BytesWritable>> result) throws Exception {
  List<Pair<BytesWritable, List<BytesWritable>>> inputRecords = new ArrayList<>();
  Map<BytesWritable, List<BytesWritable>> inputMap = new TreeMap<>();

  for (Pair<BytesWritable, BytesWritable> pair : result) {
    inputMap.put(pair.getFirst(), Lists.newArrayList(pair.getSecond()));
  }
  for (Entry<BytesWritable, List<BytesWritable>> listPair : inputMap.entrySet()) {
    inputRecords.add(new Pair<BytesWritable, List<BytesWritable>>(listPair.getKey(), listPair.getValue()));
  }
  return inputRecords;
}
 
Example #16
Source File: CubeReducerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testReducer() throws Exception {

    reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");

    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());

    Text key1 = new Text("72010ustech");
    List<Text> values1 = new ArrayList<Text>();
    values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
    values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
    values1.add(newValueText(codec, "10", "10", "10", 1, 300));

    Text key2 = new Text("1tech");
    List<Text> values2 = new ArrayList<Text>();
    values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
    values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));

    Text key3 = new Text("0");
    List<Text> values3 = new ArrayList<Text>();
    values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

    reduceDriver.withInput(key1, values1);
    reduceDriver.withInput(key2, values2);
    reduceDriver.withInput(key3, values3);

    List<Pair<Text, Text>> result = reduceDriver.run();

    Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3, 600));
    Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2, 1500));
    Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

    assertEquals(3, result.size());

    assertTrue(result.contains(p1));
    assertTrue(result.contains(p2));
    assertTrue(result.contains(p3));
}
 
Example #17
Source File: DBScanMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
private List<AdapterWithObjectWritable> getListFor(
    final PartitionDataWritable pd,
    final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> reducerInputSet) {
  for (final Pair<PartitionDataWritable, List<AdapterWithObjectWritable>> pair : reducerInputSet) {
    if (pair.getFirst().compareTo(pd) == 0) {
      return pair.getSecond();
    }
  }
  final List<AdapterWithObjectWritable> newPairList = new ArrayList<>();
  reducerInputSet.add(new Pair(pd, newPairList));
  return newPairList;
}
 
Example #18
Source File: DBScanMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
private PartitionData getPartitionDataFor(
    final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults,
    final String id,
    final boolean primary) {
  for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
    if (((FeatureWritable) pair.getSecond().getObjectWritable().get()).getFeature().getID().equals(
        id) && (pair.getFirst().getPartitionData().isPrimary() == primary)) {
      return pair.getFirst().getPartitionData();
    }
  }
  return null;
}
 
Example #19
Source File: DBScanMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
@Test
public void test8With4() throws IOException {

  final Random r = new Random(3434);
  for (int i = 0; i < 8; i++) {
    final SimpleFeature feature =
        createTestFeature(
            "f" + i,
            new Coordinate(
                round(30.0 + (r.nextGaussian() * 0.00001)),
                round(30.0 + (r.nextGaussian() * 0.00001))));
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature.getID())), feature);
  }

  final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults =
      mapDriver.run();

  final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions =
      getReducerDataFromMapperInput(mapperResults);

  reduceDriver.addAll(partitions);

  reduceDriver.getConfiguration().setInt(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          ClusteringParameters.Clustering.MINIMUM_SIZE),
      4);

  final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
  assertEquals(1, reduceResults.size());
}
 
Example #20
Source File: DBScanMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
@Test
public void testScale() throws IOException {

  final Random r = new Random(3434);
  for (int i = 0; i < 10000; i++) {
    final SimpleFeature feature =
        createTestFeature(
            "f" + i,
            new Coordinate(
                round(30.0 + (r.nextGaussian() * 0.0001)),
                round(30.0 + (r.nextGaussian() * 0.0001))));
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature.getID())), feature);
  }

  final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults =
      mapDriver.run();

  final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions =
      getReducerDataFromMapperInput(mapperResults);

  reduceDriver.addAll(partitions);

  reduceDriver.getConfiguration().setInt(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          ClusteringParameters.Clustering.MINIMUM_SIZE),
      10);

  final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
  assertTrue(reduceResults.size() > 0);
}
 
Example #21
Source File: NNMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
private Text find(final List<Pair<Text, Text>> outputSet, final String key) {
  for (final Pair<Text, Text> item : outputSet) {
    if (key.equals(item.getFirst().toString())) {
      return item.getSecond();
    }
  }
  return null;
}
 
Example #22
Source File: NNMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
private List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> getReducerDataFromMapperInput(
    final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults) {
  final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> reducerInputSet =
      new ArrayList<>();
  for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
    getListFor(pair.getFirst(), reducerInputSet).add(pair.getSecond());
  }
  return reducerInputSet;
}
 
Example #23
Source File: NNMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
private List<AdapterWithObjectWritable> getListFor(
    final PartitionDataWritable pd,
    final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> reducerInputSet) {
  for (final Pair<PartitionDataWritable, List<AdapterWithObjectWritable>> pair : reducerInputSet) {
    if (pair.getFirst().compareTo(pd) == 0) {
      return pair.getSecond();
    }
  }
  final List<AdapterWithObjectWritable> newPairList = new ArrayList<>();
  reducerInputSet.add(new Pair(pd, newPairList));
  return newPairList;
}
 
Example #24
Source File: NNMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
private List<PartitionData> getPartitionDataFor(
    final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults,
    final String id,
    final boolean primary) {
  final ArrayList<PartitionData> results = new ArrayList<>();
  for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
    if (((FeatureWritable) pair.getSecond().getObjectWritable().get()).getFeature().getID().equals(
        id) && (pair.getFirst().partitionData.isPrimary() == primary)) {
      results.add(pair.getFirst().partitionData);
    }
  }
  return results;
}
 
Example #25
Source File: KMeansDistortionMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
@Test
public void testMapper() throws IOException {

  final GeoWaveInputKey inputKey = new GeoWaveInputKey();
  inputKey.setInternalAdapterId(adapterId);
  inputKey.setDataId(new ByteArray("abc".getBytes()));

  final ObjectWritable ow = new ObjectWritable();
  ow.set(
      new FeatureWritable(
          ftype,
          AnalyticFeature.createGeometryFeature(
              ftype,
              batchId,
              "123",
              "fred",
              grp1,
              20.30203,
              factory.createPoint(new Coordinate(02.33, 0.23)),
              new String[] {"extra1"},
              new double[] {0.022},
              1,
              1,
              0)));

  mapDriver.withInput(inputKey, ow);

  final List<Pair<Text, CountofDoubleWritable>> results = mapDriver.run();
  // output key has the dataID adjusted to contain the rank
  assertEquals(results.get(0).getFirst().toString(), grp1);
  // output value is the same as input value
  assertEquals(results.get(0).getSecond().getValue(), 0.0, 0.0001);
}
 
Example #26
Source File: KSamplerMapReduceTest.java    From geowave with Apache License 2.0 5 votes vote down vote up
@Test
public void testMapperWithZeroRank() throws IOException {
  capturedObjects.clear();
  mapDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          KSamplerMapReduce.class,
          SampleParameters.Sample.SAMPLE_RANK_FUNCTION),
      TestSamplingNoRankFunction.class,
      SamplingRankFunction.class);

  final GeoWaveInputKey inputKey = new GeoWaveInputKey();
  inputKey.setInternalAdapterId(internalAdapterId);
  inputKey.setDataId(new ByteArray("abc".getBytes()));

  final ObjectWritable ow = new ObjectWritable();
  ow.set(new TestObjectWritable(new TestObject(new Coordinate(25.4, 25.6), "abc")));

  final GeoWaveInputKey outputKey = new GeoWaveInputKey();
  outputKey.setInternalAdapterId(internalAdapterId);

  final ByteBuffer keyBuf = ByteBuffer.allocate(64);
  keyBuf.putDouble(0.0);
  keyBuf.putInt(3);
  keyBuf.put(inputKey.getDataId().getBytes());
  outputKey.setDataId(new ByteArray(keyBuf.array()));

  mapDriver.withInput(inputKey, ow);

  final List<Pair<GeoWaveInputKey, ObjectWritable>> results = mapDriver.run();

  assertEquals(0, results.size());

  // results from sample rank function to make sure it was provided the
  // correct object
  assertEquals(1, capturedObjects.size());
  assertEquals("abc", ((TestObject) capturedObjects.get(0)).id);
}
 
Example #27
Source File: ColumnCardinalityReducerTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testReducer() throws IOException {
    IntWritable key1 = new IntWritable(1);
    List<BytesWritable> values1 = new ArrayList<BytesWritable>();
    values1.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr)));

    IntWritable key2 = new IntWritable(2);
    List<BytesWritable> values2 = new ArrayList<BytesWritable>();
    values2.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " x")));

    IntWritable key3 = new IntWritable(3);
    List<BytesWritable> values3 = new ArrayList<BytesWritable>();
    values3.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xx")));

    IntWritable key4 = new IntWritable(4);
    List<BytesWritable> values4 = new ArrayList<BytesWritable>();
    values4.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xxx")));

    IntWritable key5 = new IntWritable(5);
    List<BytesWritable> values5 = new ArrayList<BytesWritable>();
    values5.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xxxx")));

    reduceDriver.withInput(key1, values1);
    reduceDriver.withInput(key2, values2);
    reduceDriver.withInput(key3, values3);
    reduceDriver.withInput(key4, values4);
    reduceDriver.withInput(key5, values5);

    List<Pair<IntWritable, LongWritable>> result = reduceDriver.run();

    assertEquals(5, result.size());

    int outputKey1 = result.get(0).getFirst().get();
    LongWritable value1 = result.get(0).getSecond();
    assertTrue(outputKey1 == 1);
    assertTrue((10 == value1.get()) || (9 == value1.get()));

}
 
Example #28
Source File: CubeHFileMapperTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Test
@Ignore("not maintaining")
public void testMapper2() throws IOException {
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name);

    mapDriver.addInput(new Text("52010tech"), new Text("35.432"));

    List<Pair<RowKeyWritable, KeyValue>> result = mapDriver.run();

    assertEquals(2, result.size());

    byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 };
    ImmutableBytesWritable key = new ImmutableBytesWritable(bytes);

    Pair<RowKeyWritable, KeyValue> p1 = result.get(0);
    Pair<RowKeyWritable, KeyValue> p2 = result.get(1);

    assertEquals(key, p1.getFirst());
    assertEquals("cf1", new String(p1.getSecond().getFamily(), StandardCharsets.UTF_8));
    assertEquals("usd_amt", new String(p1.getSecond().getQualifier(), StandardCharsets.UTF_8));
    assertEquals("35.43", new String(p1.getSecond().getValue(), StandardCharsets.UTF_8));

    assertEquals(key, p2.getFirst());
    assertEquals("cf1", new String(p2.getSecond().getFamily(), StandardCharsets.UTF_8));
    assertEquals("item_count", new String(p2.getSecond().getQualifier(), StandardCharsets.UTF_8));
    assertEquals("2", new String(p2.getSecond().getValue(), StandardCharsets.UTF_8));
}
 
Example #29
Source File: ColumnCardinalityReducerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testReducer() throws IOException {
    IntWritable key1 = new IntWritable(1);
    List<BytesWritable> values1 = new ArrayList<BytesWritable>();
    values1.add(new BytesWritable(getBytes(strArr)));

    IntWritable key2 = new IntWritable(2);
    List<BytesWritable> values2 = new ArrayList<BytesWritable>();
    values2.add(new BytesWritable(getBytes(strArr + " x")));

    IntWritable key3 = new IntWritable(3);
    List<BytesWritable> values3 = new ArrayList<BytesWritable>();
    values3.add(new BytesWritable(getBytes(strArr + " xx")));

    IntWritable key4 = new IntWritable(4);
    List<BytesWritable> values4 = new ArrayList<BytesWritable>();
    values4.add(new BytesWritable(getBytes(strArr + " xxx")));

    IntWritable key5 = new IntWritable(5);
    List<BytesWritable> values5 = new ArrayList<BytesWritable>();
    values5.add(new BytesWritable(getBytes(strArr + " xxxx")));

    reduceDriver.withInput(key1, values1);
    reduceDriver.withInput(key2, values2);
    reduceDriver.withInput(key3, values3);
    reduceDriver.withInput(key4, values4);
    reduceDriver.withInput(key5, values5);

    List<Pair<IntWritable, LongWritable>> result = reduceDriver.run();

    assertEquals(5, result.size());

    int outputKey1 = result.get(0).getFirst().get();
    LongWritable value1 = result.get(0).getSecond();
    assertTrue(outputKey1 == 1);
    assertTrue((10 == value1.get()) || (9 == value1.get()));

}
 
Example #30
Source File: FactDistinctColumnsMapperTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testMapper() throws IOException {
    Configuration configuration = mapDriver.getConfiguration();
    configuration.set(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, "100");
    configuration.set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_1_new_segment");
    configuration.set(BatchConstants.CFG_CUBE_SEGMENT_ID, "198va32a-a33e-4b69-83dd-0bb8b1f8c53b");
    HCatRecord value1 = new DefaultHCatRecord(11);
    value1.set(0, "2012-08-16");
    value1.set(1, "48027");
    value1.set(2, "0");
    value1.set(3, "Home & Garden");
    value1.set(4, "Cheese & Crackers");
    value1.set(5, "Cheese & Crackers");
    value1.set(6, "48027");
    value1.set(7, "16");
    value1.set(8, "10000010");
    value1.set(9, "204.28");
    value1.set(10, "5");
    mapDriver.addInput(new LongWritable(0), value1);

    List<Pair<SelfDefineSortableKey, Text>> result = mapDriver.run();
    int colsNeedDictSize = cubeDesc.getAllColumnsNeedDictionaryBuilt().size();
    int cuboidsCnt = cubeDesc.getAllCuboids().size();

    assertEquals(
            colsNeedDictSize + (cubeDesc.getRowkey().getRowKeyColumns().length - colsNeedDictSize) * 2 + cuboidsCnt,
            result.size());
}