Java Code Examples for org.apache.kylin.metadata.filter.TupleFilterSerializer#serialize()

The following examples show how to use org.apache.kylin.metadata.filter.TupleFilterSerializer#serialize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CaseTupleExpression.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
    BytesUtil.writeVInt(whenList.size(), buffer);
    for (Pair<TupleFilter, TupleExpression> whenEntry : whenList) {
        byte[] whenBytes = TupleFilterSerializer.serialize(whenEntry.getFirst(), cs);
        BytesUtil.writeByteArray(whenBytes, buffer);

        byte[] thenBytes = TupleExpressionSerializer.serialize(whenEntry.getSecond(), cs);
        BytesUtil.writeByteArray(thenBytes, buffer);
    }
    if (elseExpr != null) {
        BytesUtil.writeVInt(1, buffer);
        byte[] elseBytes = TupleExpressionSerializer.serialize(elseExpr, cs);
        BytesUtil.writeByteArray(elseBytes, buffer);
    } else {
        BytesUtil.writeVInt(-1, buffer);
    }
}
 
Example 2
Source File: FragmentFileSearcher.java    From kylin with Apache License 2.0 6 votes vote down vote up
private TupleFilter convertFilter(FragmentMetaInfo fragmentMetaInfo, TupleFilter rootFilter,
        ColumnarRecordCodec recordCodec, final TblColRef[] dimensions, final IDimensionEncodingMap dimEncodingMap, //
        final Set<TblColRef> unEvaluableColumnCollector, long timezoneOffset) {
    Map<TblColRef, Integer> colMapping = Maps.newHashMap();
    for (int i = 0; i < dimensions.length; i++) {
        colMapping.put(dimensions[i], i);
    }
    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, null, StringCodeSystem.INSTANCE);
    TupleFilter filter = TupleFilterSerializer.deserialize(bytes, StringCodeSystem.INSTANCE);

    BuiltInFunctionTransformer builtInFunctionTransformer = new BuiltInFunctionTransformer(dimEncodingMap);
    filter = builtInFunctionTransformer.transform(filter);
    FragmentFilterConverter fragmentFilterConverter = new FragmentFilterConverter(fragmentMetaInfo, unEvaluableColumnCollector,
            colMapping, recordCodec);
    fragmentFilterConverter.setTimezoneOffset(timezoneOffset);
    filter = fragmentFilterConverter.transform(filter);

    filter = new FilterOptimizeTransformer().transform(filter);
    return filter;
}
 
Example 3
Source File: FragmentFileSearcher.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private TupleFilter convertFilter(FragmentMetaInfo fragmentMetaInfo, TupleFilter rootFilter,
        ColumnarRecordCodec recordCodec, final TblColRef[] dimensions, final IDimensionEncodingMap dimEncodingMap, //
        final Set<TblColRef> unEvaluableColumnCollector, long timezoneOffset) {
    Map<TblColRef, Integer> colMapping = Maps.newHashMap();
    for (int i = 0; i < dimensions.length; i++) {
        colMapping.put(dimensions[i], i);
    }
    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, null, StringCodeSystem.INSTANCE);
    TupleFilter filter = TupleFilterSerializer.deserialize(bytes, StringCodeSystem.INSTANCE);

    BuiltInFunctionTransformer builtInFunctionTransformer = new BuiltInFunctionTransformer(dimEncodingMap);
    filter = builtInFunctionTransformer.transform(filter);
    FragmentFilterConverter fragmentFilterConverter = new FragmentFilterConverter(fragmentMetaInfo, unEvaluableColumnCollector,
            colMapping, recordCodec);
    fragmentFilterConverter.setTimezoneOffset(timezoneOffset);
    filter = fragmentFilterConverter.transform(filter);

    filter = new FilterOptimizeTransformer().transform(filter);
    return filter;
}
 
Example 4
Source File: CoprocessorFilter.java    From kylin with Apache License 2.0 5 votes vote down vote up
public static CoprocessorFilter fromFilter(final IDimensionEncodingMap dimEncMap, TupleFilter rootFilter, FilterDecorator.FilterConstantsTreatment filterConstantsTreatment) {
    // translate constants into dictionary IDs via a serialize copy
    FilterDecorator filterDecorator = new FilterDecorator(dimEncMap, filterConstantsTreatment);
    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator, DictCodeSystem.INSTANCE);
    TupleFilter copy = TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);

    return new CoprocessorFilter(copy, filterDecorator.getInevaluableColumns());
}
 
Example 5
Source File: FilterEvaluateTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate08() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "4");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, number - matcheCounts[0] - matcheCounts[1] + matcheCounts[2]);
}
 
Example 6
Source File: FilterSerializeTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialize10() {
    List<TblColRef> groups = buildGroups();
    TupleFilter orFilter = buildOrFilter(groups);
    TupleFilter andFilter = buildAndFilter(groups);

    LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
    logicFilter.addChild(orFilter);
    logicFilter.addChild(andFilter);

    byte[] bytes = TupleFilterSerializer.serialize(logicFilter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(logicFilter, newFilter);
}
 
Example 7
Source File: GTUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private static TupleFilter convertFilter(TupleFilter rootFilter, final GTInfo info, //
        final Map<TblColRef, Integer> colMapping, final boolean encodeConstants, //
        final Set<TblColRef> unevaluatableColumnCollector) {

    IFilterCodeSystem<ByteArray> filterCodeSystem = wrap(info.codeSystem.getComparator());

    GTConvertDecorator decorator = new GTConvertDecorator(unevaluatableColumnCollector, colMapping, info,
            encodeConstants);

    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, decorator, filterCodeSystem);
    return TupleFilterSerializer.deserialize(bytes, filterCodeSystem);
}
 
Example 8
Source File: FilterEvaluateTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate07() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "3");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, 0);
}
 
Example 9
Source File: FilterEvaluateTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate08() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "4");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, number - matcheCounts[0] - matcheCounts[1] + matcheCounts[2]);
}
 
Example 10
Source File: FilterEvaluateTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate01() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildEQCompareFilter(groups, 1);

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[1]);
}
 
Example 11
Source File: FilterEvaluateTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate06() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "2");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 1;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[1] - matcheCounts[2]);
}
 
Example 12
Source File: FilterEvaluateTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate01() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareFilter(groups, 1);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[1]);
}
 
Example 13
Source File: FilterEvaluateTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate04() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "0");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[2]);
}
 
Example 14
Source File: FilterEvaluateTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate03() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildAndFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[2]);
}
 
Example 15
Source File: FilterEvaluateTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate06() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "2");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 1;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[1] - matcheCounts[2]);
}
 
Example 16
Source File: FilterEvaluateTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate05() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "1");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[0] - matcheCounts[2]);
}
 
Example 17
Source File: FilterEvaluateTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate00() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildEQCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[0]);
}
 
Example 18
Source File: FilterEvaluateTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate06() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "2");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 1;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[1] - matcheCounts[2]);
}
 
Example 19
Source File: FilterEvaluateTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate04() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "0");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[2]);
}
 
Example 20
Source File: FilterSerializeTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialize13() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "0");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}