org.apache.kylin.metadata.filter.CompareTupleFilter Java Examples

The following examples show how to use org.apache.kylin.metadata.filter.CompareTupleFilter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FilterBaseTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
protected CompareTupleFilter buildINCompareFilter(TblColRef dateColumn) throws ParseException {
    CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    ColumnTupleFilter columnFilter = new ColumnTupleFilter(dateColumn);
    compareFilter.addChild(columnFilter);

    List<String> inValues = Lists.newArrayList();
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
    Date startDate = simpleDateFormat.parse("1970-01-01");
    Date endDate = simpleDateFormat.parse("2100-01-01");
    Calendar start = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    start.setTime(startDate);
    Calendar end = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    end.setTime(endDate);
    for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
        inValues.add(simpleDateFormat.format(date));
    }

    ConstantTupleFilter constantFilter = new ConstantTupleFilter(inValues);
    compareFilter.addChild(constantFilter);
    return compareFilter;
}
 
Example #2
Source File: TupleExpressionSerializerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testSerialization() {
    TblColRef colD = TblColRef.mockup(t, 1, "C1", "decimal");
    TblColRef colM = TblColRef.mockup(t, 2, "C2", "string");
    BigDecimal value = BigDecimal.valueOf(10L);

    ColumnTupleFilter colFilter = new ColumnTupleFilter(colD);
    ConstantTupleFilter constFilter = new ConstantTupleFilter("col");
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
    compareFilter.addChild(colFilter);
    compareFilter.addChild(constFilter);

    ColumnTupleExpression colTuple = new ColumnTupleExpression(colM);
    NumberTupleExpression constTuple = new NumberTupleExpression(value);

    Pair<TupleFilter, TupleExpression> whenEntry = new Pair<TupleFilter, TupleExpression>(compareFilter, colTuple);
    CaseTupleExpression caseTuple = new CaseTupleExpression(Lists.newArrayList(whenEntry), constTuple);

    byte[] result = TupleExpressionSerializer.serialize(caseTuple, StringCodeSystem.INSTANCE);

    TupleExpression desTuple = TupleExpressionSerializer.deserialize(result, StringCodeSystem.INSTANCE);
    assertEquals(caseTuple, desTuple);
}
 
Example #3
Source File: DictGridTableTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void verifyConvertFilterConstants4() {
    GTInfo info = table.getInfo();

    TableDesc extTable = TableDesc.mockup("ext");
    TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
    TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");

    CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
    CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15");
    LogicalTupleFilter filter = and(fComp1, fComp2);

    List<TblColRef> colMapping = Lists.newArrayList();
    colMapping.add(extColA);
    colMapping.add(extColB);

    // $1 in ("9", "10", "15") has only "10" left
    TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
    assertEquals(
            "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]",
            newFilter.toString());
}
 
Example #4
Source File: InvertIndexSearcher.java    From kylin with Apache License 2.0 6 votes vote down vote up
public EvalResult doEval(TupleFilter filter) {
    if (filter == null)
        return EvalResult.ALL_MATCH;

    if (filter instanceof LogicalTupleFilter)
        return doEvalLogical((LogicalTupleFilter) filter);

    if (filter instanceof CompareTupleFilter)
        return doEvalCompare((CompareTupleFilter) filter);

    if (filter instanceof ConstantTupleFilter) {
        if (filter.getValues().size() == 0) {
            return new EvalResult();
        } else if (filter.getValues().size() > 0) {
            return EvalResult.ALL_MATCH;
        }
    }

    return EvalResult.ALL_MATCH; // unable to evaluate
}
 
Example #5
Source File: OLAPContext.java    From kylin with Apache License 2.0 6 votes vote down vote up
private void bindVariable(TupleFilter filter, DataContext dataContext) {
    if (filter == null) {
        return;
    }

    for (TupleFilter childFilter : filter.getChildren()) {
        bindVariable(childFilter, dataContext);
    }

    if (filter instanceof CompareTupleFilter && dataContext != null) {
        CompareTupleFilter compFilter = (CompareTupleFilter) filter;
        for (Map.Entry<String, Object> entry : compFilter.getVariables().entrySet()) {
            String variable = entry.getKey();
            Object value = dataContext.get(variable);
            if (value != null) {
                String str = value.toString();
                str = transferDateTimeColumnToMillis(compFilter, str);
                compFilter.clearPreviousVariableValues(variable);
                compFilter.bindVariable(variable, str);
            }

        }
    }
}
 
Example #6
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
protected TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
    if (filter == null)
        return filter;

    if (filter instanceof CompareTupleFilter) {
        return translateDerivedInCompare((CompareTupleFilter) filter, collector);
    }

    List<TupleFilter> children = (List<TupleFilter>) filter.getChildren();
    List<TupleFilter> newChildren = Lists.newArrayListWithCapacity(children.size());
    boolean modified = false;
    for (TupleFilter child : children) {
        TupleFilter translated = translateDerived(child, collector);
        newChildren.add(translated);
        if (child != translated)
            modified = true;
    }
    if (modified) {
        filter = replaceChildren(filter, newChildren);
    }
    return filter;
}
 
Example #7
Source File: InvertIndexSearcher.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private EvalResult doEvalCompareEqual(CompareTupleFilter filter) {
    EvalResult result = new EvalResult();
    String column = filter.getColumn().getName();
    byte[] value = null;
    if (filter.getFirstValue() instanceof ByteArray) {
        value = ((ByteArray)filter.getFirstValue()).array();
    } else if (filter.getFirstValue() instanceof byte[]) {
        value = (byte[])filter.getFirstValue();
    } else if (filter.getFirstValue() instanceof String) {
        value = Bytes.toBytes((String) filter.getFirstValue());
    }
    ColInvertIndexSearcher colSearcher = colIndexSearchers.get(column);
    if (colSearcher == null) {
        return EvalResult.ALL_MATCH;
    }
    ImmutableRoaringBitmap bitmap = colSearcher.searchValue(value);
    if (bitmap != null) {
        result.bitmap = bitmap;
    }
    return result;
}
 
Example #8
Source File: DerivedFilterTranslator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
 
Example #9
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private Collection<ColumnValueRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters, CubeSegment cubeSegment) {
    Map<TblColRef, ColumnValueRange> rangeMap = new HashMap<TblColRef, ColumnValueRange>();
    for (TupleFilter filter : andFilters) {
        if ((filter instanceof CompareTupleFilter) == false) {
            continue;
        }

        CompareTupleFilter comp = (CompareTupleFilter) filter;
        if (comp.getColumn() == null) {
            continue;
        }

        ColumnValueRange range = new ColumnValueRange(comp.getColumn(), comp.getValues(), comp.getOperator());
        andMerge(range, rangeMap);

    }
    return rangeMap.values();
}
 
Example #10
Source File: StreamingDataQueryPlanner.java    From kylin with Apache License 2.0 6 votes vote down vote up
private boolean canSkipForAndFilter(TupleFilter andFilter, CompareFilterTimeRangeChecker timeRangeChecker) {
    for (TupleFilter filter : andFilter.getChildren()) {
        if (!(filter instanceof CompareTupleFilter)) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return true;
            } else {
                continue;
            }
        }
        CompareTupleFilter comp = (CompareTupleFilter) filter;
        TblColRef column = comp.getColumn();
        if (column == null || !TimeDerivedColumnType.isTimeDerivedColumn(column.getName())) {
            continue;
        }
        TimeDerivedColumnType timeDerivedColumnType = TimeDerivedColumnType.getTimeDerivedColumnType(column
                .getName());

        CheckResult checkResult = timeRangeChecker.check(comp, timeDerivedColumnType, timezoneOffset);
        if (checkResult == CheckResult.EXCLUDED) {
            return true;
        } else {
            continue;
        }
    }
    return false;
}
 
Example #11
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
protected Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Set<CompareTupleFilter> compareTupleFilterSet = findSingleValuesCompFilters(filter);

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (CompareTupleFilter compFilter : compareTupleFilterSet) {
        TblColRef tblColRef = compFilter.getColumn();
        if (cubeDesc.isExtendedColumn(tblColRef)) {
            throw new CubeDesc.CannotFilterExtendedColumnException(tblColRef);
        }
        if (cubeDesc.isDerived(compFilter.getColumn())) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(tblColRef);
            if (hostInfo.isOneToOne) {
                resultD.addAll(Arrays.asList(hostInfo.columns));
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(compFilter.getColumn());
        }
    }
    return resultD;
}
 
Example #12
Source File: DictGridTableTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void verifyScanWithUnevaluatableFilter() throws IOException {
    GTInfo info = table.getInfo();

    CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
    ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1));
    LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1)));
    LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable);

    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" })
            .setFilterPushDown(filter).createGTScanRequest();

    // note the unEvaluatable column 1 in filter is added to group by
    assertEquals(
            "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [], []], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]",
            req.toString());

    doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]",
            "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]",
            "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]");
}
 
Example #13
Source File: StreamingDataQueryPlanner.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private boolean canSkipForAndFilter(TupleFilter andFilter, CompareFilterTimeRangeChecker timeRangeChecker) {
    for (TupleFilter filter : andFilter.getChildren()) {
        if (!(filter instanceof CompareTupleFilter)) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return true;
            } else {
                continue;
            }
        }
        CompareTupleFilter comp = (CompareTupleFilter) filter;
        TblColRef column = comp.getColumn();
        if (column == null || !TimeDerivedColumnType.isTimeDerivedColumn(column.getName())) {
            continue;
        }
        TimeDerivedColumnType timeDerivedColumnType = TimeDerivedColumnType.getTimeDerivedColumnType(column
                .getName());

        CheckResult checkResult = timeRangeChecker.check(comp, timeDerivedColumnType, timezoneOffset);
        if (checkResult == CheckResult.EXCLUDED) {
            return true;
        } else {
            continue;
        }
    }
    return false;
}
 
Example #14
Source File: DictGridTableTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void verifyScanWithEvaluatableFilter() throws IOException {
    GTInfo info = table.getInfo();

    CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
    CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
    LogicalTupleFilter filter = and(fComp1, fComp2);

    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" })
            .setFilterPushDown(filter).createGTScanRequest();
    // note the evaluatable column 1 in filter is added to returned columns but not in group by
    assertEquals(
            "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]",
            req.toString());

    doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]",
            "[1421366400000, 20, null, 40, null]");
}
 
Example #15
Source File: InvertIndexSearcher.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public EvalResult doEval(TupleFilter filter) {
    if (filter == null)
        return EvalResult.ALL_MATCH;

    if (filter instanceof LogicalTupleFilter)
        return doEvalLogical((LogicalTupleFilter) filter);

    if (filter instanceof CompareTupleFilter)
        return doEvalCompare((CompareTupleFilter) filter);

    if (filter instanceof ConstantTupleFilter) {
        if (filter.getValues().size() == 0) {
            return new EvalResult();
        } else if (filter.getValues().size() > 0) {
            return EvalResult.ALL_MATCH;
        }
    }

    return EvalResult.ALL_MATCH; // unable to evaluate
}
 
Example #16
Source File: DerivedFilterTranslator.java    From kylin with Apache License 2.0 6 votes vote down vote up
private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
 
Example #17
Source File: OLAPContext.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private void bindVariable(TupleFilter filter, DataContext dataContext) {
    if (filter == null) {
        return;
    }

    for (TupleFilter childFilter : filter.getChildren()) {
        bindVariable(childFilter, dataContext);
    }

    if (filter instanceof CompareTupleFilter && dataContext != null) {
        CompareTupleFilter compFilter = (CompareTupleFilter) filter;
        for (Map.Entry<String, Object> entry : compFilter.getVariables().entrySet()) {
            String variable = entry.getKey();
            Object value = dataContext.get(variable);
            if (value != null) {
                String str = value.toString();
                str = transferDateTimeColumnToMillis(compFilter, str);
                compFilter.clearPreviousVariableValues(variable);
                compFilter.bindVariable(variable, str);
            }

        }
    }
}
 
Example #18
Source File: DictGridTableTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void verifyScanWithEvaluatableFilter() throws IOException {
    GTInfo info = table.getInfo();

    CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
    CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
    LogicalTupleFilter filter = and(fComp1, fComp2);

    GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null)
            .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" })
            .setFilterPushDown(filter).createGTScanRequest();
    // note the evaluatable column 1 in filter is added to returned columns but not in group by
    assertEquals(
            "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]",
            req.toString());

    doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]",
            "[1421366400000, 20, null, 40, null]");
}
 
Example #19
Source File: StorageMockUtils.java    From kylin with Apache License 2.0 5 votes vote down vote up
public TupleFilter buildOrFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
    CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
    LogicalTupleFilter logicFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
    logicFilter.addChild(compareFilter1);
    logicFilter.addChild(compareFilter2);
    return logicFilter;
}
 
Example #20
Source File: FilterBaseTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
protected TupleFilter buildAndFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0);
    CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1);
    LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter.addChild(compareFilter1);
    andFilter.addChild(compareFilter2);
    return andFilter;
}
 
Example #21
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 5 votes vote down vote up
private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null)
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isExtendedColumn(derived)) {
        throw new CubeDesc.CannotFilterExtendedColumnException(derived);
    }
    if (!cubeDesc.isDerived(derived))
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    ILookupTable lookup = cubeDesc.getHostInfo(derived).type == CubeDesc.DeriveType.PK_FK ? null
            : getLookupStringTableForDerived(derived, hostInfo);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    try {
        if (lookup != null) {
            lookup.close();
        }
    } catch (IOException e) {
        logger.error("error when close lookup table.", e);
    }
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(translatedFilter, collector);
    }
    return translatedFilter;
}
 
Example #22
Source File: StorageMockUtils.java    From kylin with Apache License 2.0 5 votes vote down vote up
public TupleFilter buildAndFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
    CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
    LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
    andFilter.addChild(compareFilter1);
    andFilter.addChild(compareFilter2);
    return andFilter;
}
 
Example #23
Source File: SegmentPruner.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public static boolean satisfy(CompareTupleFilter comp, String minVal, String maxVal) {

        // When both min and max are null, it means all cells of the column are null.
        // In such case, return true to let query engine scan the segment, since the
        // result of null comparison is query engine specific.
        if (minVal == null && maxVal == null)
            return true;
        
        // pass on non-constant filter
        if (comp.getChildren().size() > 1 && !(comp.getChildren().get(1) instanceof ConstantTupleFilter))
            return true;

        TblColRef col = comp.getColumn();
        DataTypeOrder order = col.getType().getOrder();
        String filterVal = toString(comp.getFirstValue());
        
        switch (comp.getOperator()) {
        case EQ:
        case IN:
            for (String filterValue : (Set<String>) comp.getValues()) {
                if (order.compare(filterValue, maxVal) <= 0 && order.compare(minVal, filterValue) <= 0)
                    return true;
            }
            return false;
        case LT:
            return order.compare(minVal, filterVal) < 0;
        case LTE:
            return order.compare(minVal, filterVal) <= 0;
        case GT:
            return order.compare(maxVal, filterVal) > 0;
        case GTE:
            return order.compare(maxVal, filterVal) >= 0;
        case NEQ:
        case NOTIN:
        case ISNULL:
        case ISNOTNULL:
        default:
            return true;
        }
    }
 
Example #24
Source File: ScanRangePlannerBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) {
    Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>();
    for (TupleFilter filter : andFilters) {
        if ((filter instanceof CompareTupleFilter) == false) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return null;
            } else {
                continue;
            }
        }

        CompareTupleFilter comp = (CompareTupleFilter) filter;
        if (comp.getColumn() == null) {
            continue;
        }

        @SuppressWarnings("unchecked")
        ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(), comp.getOperator());
        ColumnRange existing = rangeMap.get(newRange.column);
        if (existing == null) {
            rangeMap.put(newRange.column, newRange);
        } else {
            existing.andMerge(newRange);
        }
    }
    return rangeMap.values();
}
 
Example #25
Source File: StorageMockUtils.java    From kylin with Apache License 2.0 5 votes vote down vote up
public CompareTupleFilter buildFilter3(TblColRef column) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
    ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
    compareFilter.addChild(columnFilter1);
    ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
    compareFilter.addChild(constantFilter1);
    return compareFilter;
}
 
Example #26
Source File: BitMapFilterEvaluator.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private void dropNull(ConciseSet set, CompareTupleFilter filter) {
    if (set == null)
        return;

    ConciseSet nullSet = evalCompareIsNull(filter);
    set.removeAll(nullSet);
}
 
Example #27
Source File: OLAPAggregateRel.java    From kylin with Apache License 2.0 5 votes vote down vote up
private TupleFilter getIsNotNullFilterFromExpression(TupleExpression tupleExpr) {
    TupleFilter filter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.ISNOTNULL);
    if (tupleExpr instanceof ColumnTupleExpression) {
        filter.addChild(new ColumnTupleFilter(((ColumnTupleExpression) tupleExpr).getColumn()));
    } else if (tupleExpr instanceof ConstantTupleExpression) {
        filter.addChild(new ConstantTupleFilter(((ConstantTupleExpression) tupleExpr).getValue()));
    } else {
        logger.warn("Cannot get IsNullFilter from Expression {}", tupleExpr);
        filter = null;
    }
    return filter;
}
 
Example #28
Source File: GTScannerBenchmark.java    From kylin with Apache License 2.0 5 votes vote down vote up
private CompareTupleFilter gt(ColumnTupleFilter col, int v) {
    CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT);
    r.addChild(col);

    int c = col.getColumn().getColumnDesc().getZeroBasedIndex();
    int len = info.getCodeSystem().maxCodeLength(c);
    ByteArray bytes = new ByteArray(len);
    BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len);
    r.addChild(new ConstantTupleFilter(bytes));

    return r;
}
 
Example #29
Source File: TupleExpressionVisitor.java    From kylin with Apache License 2.0 5 votes vote down vote up
private CaseTupleExpression getCaseTupleExpression(RexCall call) {
    List<Pair<TupleFilter, TupleExpression>> whenList = Lists
            .newArrayListWithExpectedSize(call.operands.size() / 2);
    TupleExpression elseExpr = null;

    TupleFilterVisitor filterVistor = new TupleFilterVisitor(inputRowType);
    for (int i = 0; i < call.operands.size() - 1; i += 2) {
        if (call.operands.get(i) instanceof RexCall) {
            RexCall whenCall = (RexCall) call.operands.get(i);
            CompareTupleFilter.CompareResultType compareResultType = RexUtil.getCompareResultType(whenCall);
            if (compareResultType == CompareTupleFilter.CompareResultType.AlwaysTrue) {
                elseExpr = call.operands.get(i + 1).accept(this);
                break;
            } else if (compareResultType == CompareTupleFilter.CompareResultType.AlwaysFalse) {
                continue;
            }
            TupleFilter whenFilter = whenCall.accept(filterVistor);
            whenFilter = new FilterOptimizeTransformer().transform(whenFilter);

            TupleExpression thenExpr = call.operands.get(i + 1).accept(this);
            whenList.add(new Pair<>(whenFilter, thenExpr));
        }
    }
    if (elseExpr == null && call.operands.size() % 2 == 1) {
        RexNode elseNode = call.operands.get(call.operands.size() - 1);
        if (!(elseNode instanceof RexLiteral && ((RexLiteral) elseNode).getValue() == null)) {
            elseExpr = elseNode.accept(this);
        }
    }

    DataType dataType = DataType.getType(OLAPTable.DATATYPE_MAPPING.get(call.type.getSqlTypeName()));
    CaseTupleExpression tuple = new CaseTupleExpression(dataType, whenList, elseExpr);
    tuple.setDigest(call.toString());
    return tuple;
}
 
Example #30
Source File: GTScannerBenchmark2.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private CompareTupleFilter gt(ColumnTupleFilter col, int v) {
    CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT);
    r.addChild(col);

    int c = col.getColumn().getColumnDesc().getZeroBasedIndex();
    int len = info.getCodeSystem().maxCodeLength(c);
    ByteArray bytes = new ByteArray(len);
    BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len);
    r.addChild(new ConstantTupleFilter(bytes));

    return r;
}