Java Code Examples for org.apache.kylin.metadata.filter.CompareTupleFilter#getColumn()

The following examples show how to use org.apache.kylin.metadata.filter.CompareTupleFilter#getColumn() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StreamingDataQueryPlanner.java From kylin with Apache License 2.0

6 votes

private boolean canSkipForAndFilter(TupleFilter andFilter, CompareFilterTimeRangeChecker timeRangeChecker) {
    for (TupleFilter filter : andFilter.getChildren()) {
        if (!(filter instanceof CompareTupleFilter)) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return true;
            } else {
                continue;
            }
        }
        CompareTupleFilter comp = (CompareTupleFilter) filter;
        TblColRef column = comp.getColumn();
        if (column == null || !TimeDerivedColumnType.isTimeDerivedColumn(column.getName())) {
            continue;
        }
        TimeDerivedColumnType timeDerivedColumnType = TimeDerivedColumnType.getTimeDerivedColumnType(column
                .getName());

        CheckResult checkResult = timeRangeChecker.check(comp, timeDerivedColumnType, timezoneOffset);
        if (checkResult == CheckResult.EXCLUDED) {
            return true;
        } else {
            continue;
        }
    }
    return false;
}

Example 2

Source File: GTCubeStorageQueryBase.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
protected Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Set<CompareTupleFilter> compareTupleFilterSet = findSingleValuesCompFilters(filter);

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (CompareTupleFilter compFilter : compareTupleFilterSet) {
        TblColRef tblColRef = compFilter.getColumn();
        if (cubeDesc.isExtendedColumn(tblColRef)) {
            throw new CubeDesc.CannotFilterExtendedColumnException(tblColRef);
        }
        if (cubeDesc.isDerived(compFilter.getColumn())) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(tblColRef);
            if (hostInfo.isOneToOne) {
                resultD.addAll(Arrays.asList(hostInfo.columns));
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(compFilter.getColumn());
        }
    }
    return resultD;
}

Example 3

Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0

6 votes

protected Set<CompareTupleFilter> findSingleValuesCompFilters(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return Collections.emptySet();
    }

    Set<CompareTupleFilter> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1
                    && compFilter.getColumn() != null) {
                result.add(compFilter);
            }
        }
    }
    return result;
}

Example 4

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

6 votes

private Collection<ColumnValueRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters, CubeSegment cubeSegment) {
    Map<TblColRef, ColumnValueRange> rangeMap = new HashMap<TblColRef, ColumnValueRange>();
    for (TupleFilter filter : andFilters) {
        if ((filter instanceof CompareTupleFilter) == false) {
            continue;
        }

        CompareTupleFilter comp = (CompareTupleFilter) filter;
        if (comp.getColumn() == null) {
            continue;
        }

        ColumnValueRange range = new ColumnValueRange(comp.getColumn(), comp.getValues(), comp.getOperator());
        andMerge(range, rangeMap);

    }
    return rangeMap.values();
}

Example 5

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

6 votes

private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null || compf.getValues().isEmpty())
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isDerived(derived) == false)
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(compf, collector);
    }
    return translatedFilter;
}

Example 6

Source File: StreamingDataQueryPlanner.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private boolean canSkipForAndFilter(TupleFilter andFilter, CompareFilterTimeRangeChecker timeRangeChecker) {
    for (TupleFilter filter : andFilter.getChildren()) {
        if (!(filter instanceof CompareTupleFilter)) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return true;
            } else {
                continue;
            }
        }
        CompareTupleFilter comp = (CompareTupleFilter) filter;
        TblColRef column = comp.getColumn();
        if (column == null || !TimeDerivedColumnType.isTimeDerivedColumn(column.getName())) {
            continue;
        }
        TimeDerivedColumnType timeDerivedColumnType = TimeDerivedColumnType.getTimeDerivedColumnType(column
                .getName());

        CheckResult checkResult = timeRangeChecker.check(comp, timeDerivedColumnType, timezoneOffset);
        if (checkResult == CheckResult.EXCLUDED) {
            return true;
        } else {
            continue;
        }
    }
    return false;
}

Example 7

Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
protected Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Set<CompareTupleFilter> compareTupleFilterSet = findSingleValuesCompFilters(filter);

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (CompareTupleFilter compFilter : compareTupleFilterSet) {
        TblColRef tblColRef = compFilter.getColumn();
        if (cubeDesc.isExtendedColumn(tblColRef)) {
            throw new CubeDesc.CannotFilterExtendedColumnException(tblColRef);
        }
        if (cubeDesc.isDerived(compFilter.getColumn())) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(tblColRef);
            if (hostInfo.isOneToOne) {
                resultD.addAll(Arrays.asList(hostInfo.columns));
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(compFilter.getColumn());
        }
    }
    return resultD;
}

Example 8

Source File: OLAPContext.java From kylin with Apache License 2.0

5 votes

private String transferDateTimeColumnToMillis(CompareTupleFilter compFilter, String value) {
    TblColRef column = compFilter.getColumn();
    // To fix KYLIN-4157, when using PrepareStatement query, functions within WHERE will cause InternalErrorException
    if (Objects.isNull(column)){
        return value;
    }

    if (column.getType().isDateTimeFamily()){
        value = String.valueOf(DateFormat.stringToMillis(value));
    }
    return value;
}

Example 9

Source File: SegmentPruner.java From kylin with Apache License 2.0

5 votes

public static boolean satisfy(CompareTupleFilter comp, String minVal, String maxVal) {

        // When both min and max are null, it means all cells of the column are null.
        // In such case, return true to let query engine scan the segment, since the
        // result of null comparison is query engine specific.
        if (minVal == null && maxVal == null)
            return true;
        
        // pass on non-constant filter
        if (comp.getChildren().size() <= 1 || !isConstantValue(comp.getChildren().get(1)))
            return true;

        TblColRef col = comp.getColumn();
        DataTypeOrder order = col.getType().getOrder();
        String filterVal = toString(comp.getFirstValue());
        
        switch (comp.getOperator()) {
        case EQ:
        case IN:
            for (String filterValue : (Set<String>) comp.getValues()) {
                if (order.compare(filterValue, maxVal) <= 0 && order.compare(minVal, filterValue) <= 0)
                    return true;
            }
            return false;
        case LT:
            return order.compare(minVal, filterVal) < 0;
        case LTE:
            return order.compare(minVal, filterVal) <= 0;
        case GT:
            return order.compare(maxVal, filterVal) > 0;
        case GTE:
            return order.compare(maxVal, filterVal) >= 0;
        case NEQ:
        case NOTIN:
        case ISNULL:
        case ISNOTNULL:
        default:
            return true;
        }
    }

Example 10

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
private Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return (Set<TblColRef>) Collections.EMPTY_SET;
    }

    Set<TblColRef> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 && compFilter.getColumn() != null) {
                result.add(compFilter.getColumn());
            }
        }
    }

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (TblColRef col : result) {
        if (cubeDesc.isDerived(col)) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
            if (hostInfo.isOneToOne) {
                for (TblColRef hostCol : hostInfo.columns) {
                    resultD.add(hostCol);
                }
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(col);
        }
    }
    return resultD;
}

Example 11

Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0

5 votes

private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null)
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isExtendedColumn(derived)) {
        throw new CubeDesc.CannotFilterExtendedColumnException(derived);
    }
    if (!cubeDesc.isDerived(derived))
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    ILookupTable lookup = cubeDesc.getHostInfo(derived).type == CubeDesc.DeriveType.PK_FK ? null
            : getLookupStringTableForDerived(derived, hostInfo);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    try {
        if (lookup != null) {
            lookup.close();
        }
    } catch (IOException e) {
        logger.error("error when close lookup table.", e);
    }
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(translatedFilter, collector);
    }
    return translatedFilter;
}

Example 12

Source File: OLAPContext.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private String transferDateTimeColumnToMillis(CompareTupleFilter compFilter, String value) {
    TblColRef column = compFilter.getColumn();
    // To fix KYLIN-4157, when using PrepareStatement query, functions within WHERE will cause InternalErrorException
    if (Objects.isNull(column)){
        return value;
    }

    if (column.getType().isDateTimeFamily()){
        value = String.valueOf(DateFormat.stringToMillis(value));
    }
    return value;
}

Example 13

Source File: GTCubeStorageQueryBase.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null)
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isExtendedColumn(derived)) {
        throw new CubeDesc.CannotFilterExtendedColumnException(derived);
    }
    if (!cubeDesc.isDerived(derived))
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    ILookupTable lookup = cubeDesc.getHostInfo(derived).type == CubeDesc.DeriveType.PK_FK ? null
            : getLookupStringTableForDerived(derived, hostInfo);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    try {
        if (lookup != null) {
            lookup.close();
        }
    } catch (IOException e) {
        logger.error("error when close lookup table.", e);
    }
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(translatedFilter, collector);
    }
    return translatedFilter;
}

Example 14

Source File: OLAPFilterRel.java From Kylin with Apache License 2.0

5 votes

private CompareTupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    TblColRef inColumn = null;
    List<String> inValues = new LinkedList<String>();
    for (TupleFilter child : children) {
        if (child.getOperator() == FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (inColumn == null) {
                inColumn = column;
            }

            if (column == null || !column.equals(inColumn)) {
                return null;
            }
            inValues.addAll(compFilter.getValues());
        } else {
            return null;
        }
    }

    children.clear();

    CompareTupleFilter inFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    inFilter.addChild(new ColumnTupleFilter(inColumn));
    inFilter.addChild(new ConstantTupleFilter(inValues));
    return inFilter;
}

Example 15

Source File: TupleFilterVisitor.java From kylin with Apache License 2.0

4 votes

@VisibleForTesting
static TupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    if (children.isEmpty()) {
        return filter;
    }
    // key: inColumn
    // Value: first: inValues
    // Value: second: dynamicVariables
    Map<TblColRef, Pair<Set<Object>, Map<String, Object>>> inColumnMap = Maps.newHashMap();
    List<TupleFilter> extraFilters = Lists.newLinkedList();
    for (TupleFilter child : children) {
        if (child.getOperator() == TupleFilter.FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (column != null) {
                Pair<Set<Object>, Map<String, Object>> tmpValue = inColumnMap.get(column);
                if (tmpValue == null) {
                    Set<Object> inValues = Sets.newHashSet();
                    Map<String, Object> dynamicVariables = Maps.newHashMap();
                    tmpValue = new Pair<>(inValues, dynamicVariables);
                    inColumnMap.put(column, tmpValue);
                }

                tmpValue.getFirst().addAll(compFilter.getValues());
                tmpValue.getSecond().putAll(compFilter.getVariables());
                continue;
            }
        }
        extraFilters.add(child);
    }

    children.clear();

    TupleFilter ret = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
    ret.addChildren(extraFilters);
    for (Map.Entry<TblColRef, Pair<Set<Object>, Map<String, Object>>> entry : inColumnMap.entrySet()) {
        CompareTupleFilter inFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.IN);
        inFilter.addChild(new ColumnTupleFilter(entry.getKey()));
        inFilter.addChild(new ConstantTupleFilter(entry.getValue().getFirst()));
        inFilter.getVariables().putAll(entry.getValue().getSecond());
        ret.addChild(inFilter);
    }

    return ret.getChildren().size() == 1 ? ret.getChildren().get(0) : ret;
}

Example 16

Source File: TsConditionExtractor.java From kylin with Apache License 2.0

4 votes

private static Range<Long> extractTsConditionInternal(TupleFilter filter, TblColRef colRef) {
    if (filter == null) {
        return Range.all();
    }

    if (filter instanceof LogicalTupleFilter) {
        if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
            Range<Long> ret = Range.all();
            for (TupleFilter child : filter.getChildren()) {
                Range childRange = extractTsConditionInternal(child, colRef);
                if (childRange != null) {
                    if (ret.isConnected(childRange) && !ret.intersection(childRange).isEmpty()) {
                        ret = ret.intersection(childRange);
                    } else {
                        return null;
                    }
                } else {
                    return null;
                }
            }
            return ret.isEmpty() ? null : ret;
        } else {
            //for conditions like date > DATE'2000-11-11' OR date < DATE '1999-01-01'
            //we will use Ranges.all() rather than two ranges to represent them
            return Range.all();
        }
    }

    if (filter instanceof CompareTupleFilter) {
        CompareTupleFilter compareTupleFilter = (CompareTupleFilter) filter;
        if (compareTupleFilter.getColumn() == null)// column will be null at filters like " 1<>1"
            return Range.all();

        if (compareTupleFilter.getColumn().equals(colRef)) {
            Object firstValue = compareTupleFilter.getFirstValue();
            long t;
            switch (compareTupleFilter.getOperator()) {
            case EQ:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.closed(t, t);
            case LT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.lessThan(t);
            case LTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atMost(t);
            case GT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.greaterThan(t);
            case GTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atLeast(t);
            case NEQ:
            case IN://not handled for now
                break;
            default:
            }
        }
    }
    return Range.all();
}

Example 17

Source File: FilterDecorator.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
    filter = translator.transform(filter);

    // un-evaluatable filter is replaced with TRUE
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (!(filter instanceof CompareTupleFilter))
        return filter;

    // double check all internal of CompareTupleFilter is evaluatable
    if (!TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
        return filter;
    } else {

        // extract ColumnFilter & ConstantFilter
        CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
        TblColRef col = compareFilter.getColumn();

        if (col == null) {
            return filter;
        }

        Collection<String> constValues = (Collection<String>) compareFilter.getValues();
        if (constValues == null || constValues.isEmpty()) {
            return filter;
        }

        CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
        newCompareFilter.addChild(new ColumnTupleFilter(col));

        if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
            return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
        } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
            return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
        } else {
            throw new RuntimeException("should not reach here");
        }
    }
}

Example 18

Source File: FilterDecorator.java From kylin with Apache License 2.0

4 votes

@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
    filter = translator.transform(filter);

    // un-evaluatable filter is replaced with TRUE
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (!(filter instanceof CompareTupleFilter))
        return filter;

    // double check all internal of CompareTupleFilter is evaluatable
    if (!TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
        return filter;
    } else {

        // extract ColumnFilter & ConstantFilter
        CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
        TblColRef col = compareFilter.getColumn();

        if (col == null) {
            return filter;
        }

        Collection<String> constValues = (Collection<String>) compareFilter.getValues();
        if (constValues == null || constValues.isEmpty()) {
            return filter;
        }

        CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
        newCompareFilter.addChild(new ColumnTupleFilter(col));

        if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
            return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
        } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
            return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
        } else {
            throw new RuntimeException("should not reach here");
        }
    }
}

Example 19

Source File: TsConditionExtractor.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

private static Range<Long> extractTsConditionInternal(TupleFilter filter, TblColRef colRef) {
    if (filter == null) {
        return Range.all();
    }

    if (filter instanceof LogicalTupleFilter) {
        if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
            Range<Long> ret = Range.all();
            for (TupleFilter child : filter.getChildren()) {
                Range childRange = extractTsConditionInternal(child, colRef);
                if (childRange != null) {
                    if (ret.isConnected(childRange) && !ret.intersection(childRange).isEmpty()) {
                        ret = ret.intersection(childRange);
                    } else {
                        return null;
                    }
                } else {
                    return null;
                }
            }
            return ret.isEmpty() ? null : ret;
        } else {
            //for conditions like date > DATE'2000-11-11' OR date < DATE '1999-01-01'
            //we will use Ranges.all() rather than two ranges to represent them
            return Range.all();
        }
    }

    if (filter instanceof CompareTupleFilter) {
        CompareTupleFilter compareTupleFilter = (CompareTupleFilter) filter;
        if (compareTupleFilter.getColumn() == null)// column will be null at filters like " 1<>1"
            return Range.all();

        if (compareTupleFilter.getColumn().equals(colRef)) {
            Object firstValue = compareTupleFilter.getFirstValue();
            long t;
            switch (compareTupleFilter.getOperator()) {
            case EQ:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.closed(t, t);
            case LT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.lessThan(t);
            case LTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atMost(t);
            case GT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.greaterThan(t);
            case GTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atLeast(t);
            case NEQ:
            case IN://not handled for now
                break;
            default:
            }
        }
    }
    return Range.all();
}

Example 20

Source File: TupleFilterVisitor.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@VisibleForTesting
static TupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    if (children.isEmpty()) {
        return filter;
    }
    // key: inColumn
    // Value: first: inValues
    // Value: second: dynamicVariables
    Map<TblColRef, Pair<Set<Object>, Map<String, Object>>> inColumnMap = Maps.newHashMap();
    List<TupleFilter> extraFilters = Lists.newLinkedList();
    for (TupleFilter child : children) {
        if (child.getOperator() == TupleFilter.FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (column != null) {
                Pair<Set<Object>, Map<String, Object>> tmpValue = inColumnMap.get(column);
                if (tmpValue == null) {
                    Set<Object> inValues = Sets.newHashSet();
                    Map<String, Object> dynamicVariables = Maps.newHashMap();
                    tmpValue = new Pair<>(inValues, dynamicVariables);
                    inColumnMap.put(column, tmpValue);
                }

                tmpValue.getFirst().addAll(compFilter.getValues());
                tmpValue.getSecond().putAll(compFilter.getVariables());
                continue;
            }
        }
        extraFilters.add(child);
    }

    children.clear();

    TupleFilter ret = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
    ret.addChildren(extraFilters);
    for (Map.Entry<TblColRef, Pair<Set<Object>, Map<String, Object>>> entry : inColumnMap.entrySet()) {
        CompareTupleFilter inFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.IN);
        inFilter.addChild(new ColumnTupleFilter(entry.getKey()));
        inFilter.addChild(new ConstantTupleFilter(entry.getValue().getFirst()));
        inFilter.getVariables().putAll(entry.getValue().getSecond());
        ret.addChild(inFilter);
    }

    return ret.getChildren().size() == 1 ? ret.getChildren().get(0) : ret;
}