org.apache.kylin.metadata.filter.LogicalTupleFilter Java Examples

The following examples show how to use org.apache.kylin.metadata.filter.LogicalTupleFilter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: DerivedFilterTranslator.java From kylin with Apache License 2.0

6 votes

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}

Example #2

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

6 votes

private TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}

Example #3

Source File: DictGridTableTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@SuppressWarnings("unused")
private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter)
        throws IOException {
    long start = System.currentTimeMillis();
    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setFilterPushDown(filter).createGTScanRequest();
    IGTScanner scanner = table.scan(req);
    int i = 0;
    for (GTRecord r : scanner) {
        i++;
    }
    scanner.close();
    long end = System.currentTimeMillis();
    System.out.println(
            (end - start) + "ms with filter cache enabled=" + FilterResultCache.DEFAULT_OPTION + ", " + i + " rows");
}

Example #4

Source File: DictGridTableTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void verifyConvertFilterConstants1() {
    GTInfo info = table.getInfo();

    TableDesc extTable = TableDesc.mockup("ext");
    TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
    TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");

    CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
    CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10");
    LogicalTupleFilter filter = and(fComp1, fComp2);

    List<TblColRef> colMapping = Lists.newArrayList();
    colMapping.add(extColA);
    colMapping.add(extColB);

    TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
    assertEquals(
            "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]",
            newFilter.toString());
}

Example #5

Source File: DerivedFilterTranslator.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}

Example #6

Source File: DerivedFilterTranslator.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}

Example #7

Source File: GTCubeStorageQueryBase.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

protected Set<CompareTupleFilter> findSingleValuesCompFilters(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return Collections.emptySet();
    }

    Set<CompareTupleFilter> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1
                    && compFilter.getColumn() != null) {
                result.add(compFilter);
            }
        }
    }
    return result;
}

Example #8

Source File: DictGridTableTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void verifyConvertFilterConstants4() {
    GTInfo info = table.getInfo();

    TableDesc extTable = TableDesc.mockup("ext");
    TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
    TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");

    CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
    CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15");
    LogicalTupleFilter filter = and(fComp1, fComp2);

    List<TblColRef> colMapping = Lists.newArrayList();
    colMapping.add(extColA);
    colMapping.add(extColB);

    // $1 in ("9", "10", "15") has only "10" left
    TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
    assertEquals(
            "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]",
            newFilter.toString());
}

Example #9

Source File: DictGridTableTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void verifyScanWithUnevaluatableFilter() throws IOException {
    GTInfo info = table.getInfo();

    CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
    ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1));
    LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1)));
    LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable);

    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" })
            .setFilterPushDown(filter).createGTScanRequest();

    // note the unEvaluatable column 1 in filter is added to group by
    assertEquals(
            "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [], []], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]",
            req.toString());

    doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]",
            "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]",
            "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]");
}

Example #10

Source File: DerivedFilterTranslator.java From Kylin with Apache License 2.0

6 votes

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}

Example #11

Source File: DictGridTableTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void verifyScanWithEvaluatableFilter() throws IOException {
    GTInfo info = table.getInfo();

    CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
    CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
    LogicalTupleFilter filter = and(fComp1, fComp2);

    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" })
            .setFilterPushDown(filter).createGTScanRequest();
    // note the evaluatable column 1 in filter is added to returned columns but not in group by
    assertEquals(
            "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]",
            req.toString());

    doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]",
            "[1421366400000, 20, null, 40, null]");
}

Example #12

Source File: DerivedFilterTranslator.java From Kylin with Apache License 2.0

6 votes

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}

Example #13

Source File: InvertIndexSearcher.java From kylin with Apache License 2.0

6 votes

public EvalResult doEval(TupleFilter filter) {
    if (filter == null)
        return EvalResult.ALL_MATCH;

    if (filter instanceof LogicalTupleFilter)
        return doEvalLogical((LogicalTupleFilter) filter);

    if (filter instanceof CompareTupleFilter)
        return doEvalCompare((CompareTupleFilter) filter);

    if (filter instanceof ConstantTupleFilter) {
        if (filter.getValues().size() == 0) {
            return new EvalResult();
        } else if (filter.getValues().size() > 0) {
            return EvalResult.ALL_MATCH;
        }
    }

    return EvalResult.ALL_MATCH; // unable to evaluate
}

Example #14

Source File: ScanRangePlannerBase.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != TupleFilter.FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}

Example #15

Source File: StreamingDataQueryPlanner.java From kylin with Apache License 2.0

6 votes

protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}

Example #16

Source File: DerivedFilterTranslator.java From kylin with Apache License 2.0

6 votes

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}

Example #17

Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0

6 votes

protected Set<CompareTupleFilter> findSingleValuesCompFilters(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return Collections.emptySet();
    }

    Set<CompareTupleFilter> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1
                    && compFilter.getColumn() != null) {
                result.add(compFilter);
            }
        }
    }
    return result;
}

Example #18

Source File: DictGridTableTest.java From kylin with Apache License 2.0

6 votes

@Test
public void verifyScanWithUnevaluatableFilter() throws IOException {
    GTInfo info = table.getInfo();

    CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
    ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1));
    LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1)));
    LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable);

    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" })
            .setFilterPushDown(filter).createGTScanRequest();

    // note the unEvaluatable column 1 in filter is added to group by
    assertEquals(
            "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [], []], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]",
            req.toString());

    doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]",
            "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]",
            "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]");
}

Example #19

Source File: DictGridTableTest.java From kylin with Apache License 2.0

6 votes

@Test
public void verifyScanWithEvaluatableFilter() throws IOException {
    GTInfo info = table.getInfo();

    CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
    CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
    LogicalTupleFilter filter = and(fComp1, fComp2);

    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" })
            .setFilterPushDown(filter).createGTScanRequest();
    // note the evaluatable column 1 in filter is added to returned columns but not in group by
    assertEquals(
            "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]",
            req.toString());

    doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]",
            "[1421366400000, 20, null, 40, null]");
}

Example #20

Source File: InvertIndexSearcher.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public EvalResult doEval(TupleFilter filter) {
    if (filter == null)
        return EvalResult.ALL_MATCH;

    if (filter instanceof LogicalTupleFilter)
        return doEvalLogical((LogicalTupleFilter) filter);

    if (filter instanceof CompareTupleFilter)
        return doEvalCompare((CompareTupleFilter) filter);

    if (filter instanceof ConstantTupleFilter) {
        if (filter.getValues().size() == 0) {
            return new EvalResult();
        } else if (filter.getValues().size() > 0) {
            return EvalResult.ALL_MATCH;
        }
    }

    return EvalResult.ALL_MATCH; // unable to evaluate
}

Example #21

Source File: DictGridTableTest.java From kylin with Apache License 2.0

6 votes

@SuppressWarnings("unused")
private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter)
        throws IOException {
    long start = System.currentTimeMillis();
    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setFilterPushDown(filter).createGTScanRequest();
    IGTScanner scanner = table.scan(req);
    int i = 0;
    for (GTRecord r : scanner) {
        i++;
    }
    scanner.close();
    long end = System.currentTimeMillis();
    System.out.println(
            (end - start) + "ms with filter cache enabled=" + FilterResultCache.DEFAULT_OPTION + ", " + i + " rows");
}

Example #22

Source File: DictGridTableTest.java From kylin with Apache License 2.0

6 votes

@Test
public void verifyConvertFilterConstants1() {
    GTInfo info = table.getInfo();

    TableDesc extTable = TableDesc.mockup("ext");
    TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
    TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");

    CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
    CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10");
    LogicalTupleFilter filter = and(fComp1, fComp2);

    List<TblColRef> colMapping = Lists.newArrayList();
    colMapping.add(extColA);
    colMapping.add(extColB);

    TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
    assertEquals(
            "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]",
            newFilter.toString());
}

Example #23

Source File: StreamingDataQueryPlanner.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}

Example #24

Source File: DictGridTableTest.java From kylin with Apache License 2.0

6 votes

@Test
public void verifyConvertFilterConstants4() {
    GTInfo info = table.getInfo();

    TableDesc extTable = TableDesc.mockup("ext");
    TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
    TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");

    CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
    CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15");
    LogicalTupleFilter filter = and(fComp1, fComp2);

    List<TblColRef> colMapping = Lists.newArrayList();
    colMapping.add(extColA);
    colMapping.add(extColB);

    // $1 in ("9", "10", "15") has only "10" left
    TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
    assertEquals(
            "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]",
            newFilter.toString());
}

Example #25

Source File: ScanRangePlannerBase.java From kylin with Apache License 2.0

6 votes

protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != TupleFilter.FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}

Example #26

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

5 votes

private TupleFilter replaceChildren(TupleFilter filter, List<TupleFilter> newChildren) {
    if (filter instanceof LogicalTupleFilter) {
        LogicalTupleFilter r = new LogicalTupleFilter(filter.getOperator());
        r.addChildren(newChildren);
        return r;
    } else
        throw new IllegalStateException("Cannot replaceChildren on " + filter);
}

Example #27

Source File: BitMapFilterEvaluator.java From Kylin with Apache License 2.0

5 votes

private ConciseSet evalLogical(LogicalTupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();

    switch (filter.getOperator()) {
    case AND:
        return evalLogicalAnd(children);
    case OR:
        return evalLogicalOr(children);
    case NOT:
        return evalLogicalNot(children);
    default:
        throw new IllegalStateException("Unsupported operator " + filter.getOperator());
    }
}

Example #28

Source File: BitMapFilterEvaluator.java From Kylin with Apache License 2.0

5 votes

/**
 * @param filter
 * @return a set of records that match the filter; or null if filter is null or unable to evaluate
 */
public ConciseSet evaluate(TupleFilter filter) {
    if (filter == null)
        return null;

    if (filter instanceof LogicalTupleFilter)
        return evalLogical((LogicalTupleFilter) filter);

    if (filter instanceof CompareTupleFilter)
        return evalCompare((CompareTupleFilter) filter);

    return null; // unable to evaluate
}

Example #29

Source File: StorageMockUtils.java From kylin with Apache License 2.0

5 votes

public TupleFilter buildAndFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
    CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
    LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
    andFilter.addChild(compareFilter1);
    andFilter.addChild(compareFilter2);
    return andFilter;
}

Example #30

Source File: StorageMockUtils.java From kylin with Apache License 2.0

5 votes

public TupleFilter buildOrFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
    CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
    LogicalTupleFilter logicFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
    logicFilter.addChild(compareFilter1);
    logicFilter.addChild(compareFilter2);
    return logicFilter;
}