Java Code Examples for org.apache.kylin.metadata.filter.TupleFilterSerializer#deserialize()

The following examples show how to use org.apache.kylin.metadata.filter.TupleFilterSerializer#deserialize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FragmentFileSearcher.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private TupleFilter convertFilter(FragmentMetaInfo fragmentMetaInfo, TupleFilter rootFilter,
        ColumnarRecordCodec recordCodec, final TblColRef[] dimensions, final IDimensionEncodingMap dimEncodingMap, //
        final Set<TblColRef> unEvaluableColumnCollector, long timezoneOffset) {
    Map<TblColRef, Integer> colMapping = Maps.newHashMap();
    for (int i = 0; i < dimensions.length; i++) {
        colMapping.put(dimensions[i], i);
    }
    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, null, StringCodeSystem.INSTANCE);
    TupleFilter filter = TupleFilterSerializer.deserialize(bytes, StringCodeSystem.INSTANCE);

    BuiltInFunctionTransformer builtInFunctionTransformer = new BuiltInFunctionTransformer(dimEncodingMap);
    filter = builtInFunctionTransformer.transform(filter);
    FragmentFilterConverter fragmentFilterConverter = new FragmentFilterConverter(fragmentMetaInfo, unEvaluableColumnCollector,
            colMapping, recordCodec);
    fragmentFilterConverter.setTimezoneOffset(timezoneOffset);
    filter = fragmentFilterConverter.transform(filter);

    filter = new FilterOptimizeTransformer().transform(filter);
    return filter;
}

Example 2

Source File: CaseTupleExpression.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Override
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
    int nWhenEntries = BytesUtil.readVInt(buffer);
    List<Pair<TupleFilter, TupleExpression>> whenList = Lists.newArrayListWithExpectedSize(nWhenEntries);
    for (int i = 0; i < nWhenEntries; i++) {
        TupleFilter tupleFilter = TupleFilterSerializer.deserialize(BytesUtil.readByteArray(buffer), cs);
        TupleExpression tupleExpression = TupleExpressionSerializer.deserialize(BytesUtil.readByteArray(buffer),
                cs);
        whenList.add(new Pair<>(tupleFilter, tupleExpression));
    }
    this.whenList = whenList;
    int flag = BytesUtil.readVInt(buffer);
    if (flag == 1) {
        this.elseExpr = TupleExpressionSerializer.deserialize(BytesUtil.readByteArray(buffer), cs);
    }
}

Example 3

Source File: FilterSerializeTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testSerialize12() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCaseFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}

Example 4

Source File: CoprocessorFilter.java From kylin with Apache License 2.0

5 votes

public static CoprocessorFilter fromFilter(final IDimensionEncodingMap dimEncMap, TupleFilter rootFilter, FilterDecorator.FilterConstantsTreatment filterConstantsTreatment) {
    // translate constants into dictionary IDs via a serialize copy
    FilterDecorator filterDecorator = new FilterDecorator(dimEncMap, filterConstantsTreatment);
    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator, DictCodeSystem.INSTANCE);
    TupleFilter copy = TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);

    return new CoprocessorFilter(copy, filterDecorator.getInevaluableColumns());
}

Example 5

Source File: FilterEvaluateTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testEvaluate08() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "4");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, number - matcheCounts[0] - matcheCounts[1] + matcheCounts[2]);
}

Example 6

Source File: FilterSerializeTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testSerialize10() {
    List<TblColRef> groups = buildGroups();
    TupleFilter orFilter = buildOrFilter(groups);
    TupleFilter andFilter = buildAndFilter(groups);

    LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
    logicFilter.addChild(orFilter);
    logicFilter.addChild(andFilter);

    byte[] bytes = TupleFilterSerializer.serialize(logicFilter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(logicFilter, newFilter);
}

Example 7

Source File: FilterEvaluateTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testEvaluate07() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "3");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, 0);
}

Example 8

Source File: FilterEvaluateTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testEvaluate06() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "2");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 1;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[1] - matcheCounts[2]);
}

Example 9

Source File: FilterEvaluateTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testEvaluate03() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildAndFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[2]);
}

Example 10

Source File: FilterEvaluateTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testEvaluate07() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "3");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, 0);
}

Example 11

Source File: FilterSerializeTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testSerialize01() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}

Example 12

Source File: FilterEvaluateTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testEvaluate04() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "0");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[2]);
}

Example 13

Source File: FilterEvaluateTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testEvaluate03() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildAndFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[2]);
}

Example 14

Source File: FilterSerializeTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testSerialize03() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildAndFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}

Example 15

Source File: GTUtil.java From kylin with Apache License 2.0

5 votes

private static TupleFilter convertFilter(TupleFilter rootFilter, final GTInfo info, //
        final Map<TblColRef, Integer> colMapping, final boolean encodeConstants, //
        final Set<TblColRef> unevaluatableColumnCollector) {

    IFilterCodeSystem<ByteArray> filterCodeSystem = wrap(info.codeSystem.getComparator());

    GTConvertDecorator decorator = new GTConvertDecorator(unevaluatableColumnCollector, colMapping, info,
            encodeConstants);

    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, decorator, filterCodeSystem);
    return TupleFilterSerializer.deserialize(bytes, filterCodeSystem);
}

Example 16

Source File: FilterEvaluateTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testEvaluate02() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildOrFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[0] + matcheCounts[1] - matcheCounts[2]);
}

Example 17

Source File: FilterEvaluateTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testEvaluate03() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildAndFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[2]);
}

Example 18

Source File: FilterEvaluateTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testEvaluate05() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "1");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[0] - matcheCounts[2]);
}

Example 19

Source File: FilterSerializeTest.java From kylin with Apache License 2.0

4 votes

private void assertFilterSerDe(TupleFilter filter) {
    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    compareFilter(filter, newFilter);
}

Example 20

Source File: CoprocessorFilter.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public static CoprocessorFilter deserialize(byte[] filterBytes) {
    TupleFilter filter = (filterBytes == null || filterBytes.length == 0) ? null //
            : TupleFilterSerializer.deserialize(filterBytes, DictCodeSystem.INSTANCE);
    return new CoprocessorFilter(filter, null);
}