Java Code Examples for org.apache.kylin.metadata.filter.TupleFilter#getChildren()

The following examples show how to use org.apache.kylin.metadata.filter.TupleFilter#getChildren() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StreamingDataQueryPlanner.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private boolean canSkipForAndFilter(TupleFilter andFilter, CompareFilterTimeRangeChecker timeRangeChecker) {
    for (TupleFilter filter : andFilter.getChildren()) {
        if (!(filter instanceof CompareTupleFilter)) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return true;
            } else {
                continue;
            }
        }
        CompareTupleFilter comp = (CompareTupleFilter) filter;
        TblColRef column = comp.getColumn();
        if (column == null || !TimeDerivedColumnType.isTimeDerivedColumn(column.getName())) {
            continue;
        }
        TimeDerivedColumnType timeDerivedColumnType = TimeDerivedColumnType.getTimeDerivedColumnType(column
                .getName());

        CheckResult checkResult = timeRangeChecker.check(comp, timeDerivedColumnType, timezoneOffset);
        if (checkResult == CheckResult.EXCLUDED) {
            return true;
        } else {
            continue;
        }
    }
    return false;
}

Example 2

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

6 votes

private List<Collection<ColumnValueRange>> translateToOrAndDimRanges(TupleFilter flatFilter, CubeSegment cubeSegment) {
    List<Collection<ColumnValueRange>> result = Lists.newArrayList();

    if (flatFilter == null) {
        result.add(Collections.<ColumnValueRange> emptyList());
        return result;
    }

    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != FilterOperatorEnum.AND) {
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);
        }

        Collection<ColumnValueRange> andRanges = translateToAndDimRanges(andFilter.getChildren(), cubeSegment);

        result.add(andRanges);
    }

    return preprocessConstantConditions(result);
}

Example 3

Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
protected TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
    if (filter == null)
        return filter;

    if (filter instanceof CompareTupleFilter) {
        return translateDerivedInCompare((CompareTupleFilter) filter, collector);
    }

    List<TupleFilter> children = (List<TupleFilter>) filter.getChildren();
    List<TupleFilter> newChildren = Lists.newArrayListWithCapacity(children.size());
    boolean modified = false;
    for (TupleFilter child : children) {
        TupleFilter translated = translateDerived(child, collector);
        newChildren.add(translated);
        if (child != translated)
            modified = true;
    }
    if (modified) {
        filter = replaceChildren(filter, newChildren);
    }
    return filter;
}

Example 4

Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0

6 votes

protected Set<CompareTupleFilter> findSingleValuesCompFilters(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return Collections.emptySet();
    }

    Set<CompareTupleFilter> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1
                    && compFilter.getColumn() != null) {
                result.add(compFilter);
            }
        }
    }
    return result;
}

Example 5

Source File: StreamingDataQueryPlanner.java From kylin with Apache License 2.0

6 votes

private boolean canSkipForAndFilter(TupleFilter andFilter, CompareFilterTimeRangeChecker timeRangeChecker) {
    for (TupleFilter filter : andFilter.getChildren()) {
        if (!(filter instanceof CompareTupleFilter)) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return true;
            } else {
                continue;
            }
        }
        CompareTupleFilter comp = (CompareTupleFilter) filter;
        TblColRef column = comp.getColumn();
        if (column == null || !TimeDerivedColumnType.isTimeDerivedColumn(column.getName())) {
            continue;
        }
        TimeDerivedColumnType timeDerivedColumnType = TimeDerivedColumnType.getTimeDerivedColumnType(column
                .getName());

        CheckResult checkResult = timeRangeChecker.check(comp, timeDerivedColumnType, timezoneOffset);
        if (checkResult == CheckResult.EXCLUDED) {
            return true;
        } else {
            continue;
        }
    }
    return false;
}

Example 6

Source File: OLAPContext.java From kylin with Apache License 2.0

6 votes

private void bindVariable(TupleFilter filter, DataContext dataContext) {
    if (filter == null) {
        return;
    }

    for (TupleFilter childFilter : filter.getChildren()) {
        bindVariable(childFilter, dataContext);
    }

    if (filter instanceof CompareTupleFilter && dataContext != null) {
        CompareTupleFilter compFilter = (CompareTupleFilter) filter;
        for (Map.Entry<String, Object> entry : compFilter.getVariables().entrySet()) {
            String variable = entry.getKey();
            Object value = dataContext.get(variable);
            if (value != null) {
                String str = value.toString();
                str = transferDateTimeColumnToMillis(compFilter, str);
                compFilter.clearPreviousVariableValues(variable);
                compFilter.bindVariable(variable, str);
            }

        }
    }
}

Example 7

Source File: ScanRangePlannerBase.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) {
    List<Collection<ColumnRange>> result = Lists.newArrayList();

    if (flatFilter == null) {
        result.add(Collections.<ColumnRange> emptyList());
        return result;
    }

    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != TupleFilter.FilterOperatorEnum.AND)
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);

        Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren());
        if (andRanges != null) {
            result.add(andRanges);
        }
    }

    return preEvaluateConstantConditions(result);
}

Example 8

Source File: GTCubeStorageQueryBase.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
protected TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
    if (filter == null)
        return filter;

    if (filter instanceof CompareTupleFilter) {
        return translateDerivedInCompare((CompareTupleFilter) filter, collector);
    }

    List<TupleFilter> children = (List<TupleFilter>) filter.getChildren();
    List<TupleFilter> newChildren = Lists.newArrayListWithCapacity(children.size());
    boolean modified = false;
    for (TupleFilter child : children) {
        TupleFilter translated = translateDerived(child, collector);
        newChildren.add(translated);
        if (child != translated)
            modified = true;
    }
    if (modified) {
        filter = replaceChildren(filter, newChildren);
    }
    return filter;
}

Example 9

Source File: GTCubeStorageQueryBase.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

protected Set<CompareTupleFilter> findSingleValuesCompFilters(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return Collections.emptySet();
    }

    Set<CompareTupleFilter> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1
                    && compFilter.getColumn() != null) {
                result.add(compFilter);
            }
        }
    }
    return result;
}

Example 10

Source File: CubeEnumerator.java From Kylin with Apache License 2.0

6 votes

private void bindVariable(TupleFilter filter) {
    if (filter == null) {
        return;
    }

    for (TupleFilter childFilter : filter.getChildren()) {
        bindVariable(childFilter);
    }

    if (filter instanceof CompareTupleFilter && optiqContext != null) {
        CompareTupleFilter compFilter = (CompareTupleFilter) filter;
        for (Map.Entry<String, String> entry : compFilter.getVariables().entrySet()) {
            String variable = entry.getKey();
            Object value = optiqContext.get(variable);
            if (value != null) {
                compFilter.bindVariable(variable, value.toString());
            }

        }
    }
}

Example 11

Source File: GTCubeStorageQueryBase.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
    if (filter == null)
        return;

    if (filter instanceof ColumnTupleFilter) {
        collector.add(((ColumnTupleFilter) filter).getColumn());
    }
    for (TupleFilter child : filter.getChildren()) {
        collectColumnsRecursively(child, collector);
    }
}

Example 12

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
private Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return (Set<TblColRef>) Collections.EMPTY_SET;
    }

    Set<TblColRef> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 && compFilter.getColumn() != null) {
                result.add(compFilter.getColumn());
            }
        }
    }

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (TblColRef col : result) {
        if (cubeDesc.isDerived(col)) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
            if (hostInfo.isOneToOne) {
                for (TblColRef hostCol : hostInfo.columns) {
                    resultD.add(hostCol);
                }
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(col);
        }
    }
    return resultD;
}

Example 13

Source File: MassInTupleFilter.java From kylin with Apache License 2.0

5 votes

public static boolean containsMassInTupleFilter(TupleFilter filter) {
    if (filter == null)
        return false;

    if (filter instanceof MassInTupleFilter) {
        return true;
    }

    for (TupleFilter child : filter.getChildren()) {
        if (containsMassInTupleFilter(child))
            return true;
    }
    return false;
}

Example 14

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

5 votes

private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
    if (filter instanceof ColumnTupleFilter) {
        collectColumns(((ColumnTupleFilter) filter).getColumn(), collector);
    }
    for (TupleFilter child : filter.getChildren()) {
        collectColumnsRecursively(child, collector);
    }
}

Example 15

Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0

5 votes

private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
    if (filter == null)
        return;

    if (filter instanceof ColumnTupleFilter) {
        collector.add(((ColumnTupleFilter) filter).getColumn());
    }
    for (TupleFilter child : filter.getChildren()) {
        collectColumnsRecursively(child, collector);
    }
}

Example 16

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

5 votes

private void collectNonEvaluable(TupleFilter filter, Set<TblColRef> collector) {
    if (filter == null)
        return;

    if (filter.isEvaluable()) {
        for (TupleFilter child : filter.getChildren())
            collectNonEvaluable(child, collector);
    } else {
        collectColumnsRecursively(filter, collector);
    }
}

Example 17

Source File: TupleFilterVisitor.java From kylin with Apache License 2.0

4 votes

@VisibleForTesting
static TupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    if (children.isEmpty()) {
        return filter;
    }
    // key: inColumn
    // Value: first: inValues
    // Value: second: dynamicVariables
    Map<TblColRef, Pair<Set<Object>, Map<String, Object>>> inColumnMap = Maps.newHashMap();
    List<TupleFilter> extraFilters = Lists.newLinkedList();
    for (TupleFilter child : children) {
        if (child.getOperator() == TupleFilter.FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (column != null) {
                Pair<Set<Object>, Map<String, Object>> tmpValue = inColumnMap.get(column);
                if (tmpValue == null) {
                    Set<Object> inValues = Sets.newHashSet();
                    Map<String, Object> dynamicVariables = Maps.newHashMap();
                    tmpValue = new Pair<>(inValues, dynamicVariables);
                    inColumnMap.put(column, tmpValue);
                }

                tmpValue.getFirst().addAll(compFilter.getValues());
                tmpValue.getSecond().putAll(compFilter.getVariables());
                continue;
            }
        }
        extraFilters.add(child);
    }

    children.clear();

    TupleFilter ret = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
    ret.addChildren(extraFilters);
    for (Map.Entry<TblColRef, Pair<Set<Object>, Map<String, Object>>> entry : inColumnMap.entrySet()) {
        CompareTupleFilter inFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.IN);
        inFilter.addChild(new ColumnTupleFilter(entry.getKey()));
        inFilter.addChild(new ConstantTupleFilter(entry.getValue().getFirst()));
        inFilter.getVariables().putAll(entry.getValue().getSecond());
        ret.addChild(inFilter);
    }

    return ret.getChildren().size() == 1 ? ret.getChildren().get(0) : ret;
}

Example 18

Source File: TsConditionExtractor.java From kylin with Apache License 2.0

4 votes

private static Range<Long> extractTsConditionInternal(TupleFilter filter, TblColRef colRef) {
    if (filter == null) {
        return Range.all();
    }

    if (filter instanceof LogicalTupleFilter) {
        if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
            Range<Long> ret = Range.all();
            for (TupleFilter child : filter.getChildren()) {
                Range childRange = extractTsConditionInternal(child, colRef);
                if (childRange != null) {
                    if (ret.isConnected(childRange) && !ret.intersection(childRange).isEmpty()) {
                        ret = ret.intersection(childRange);
                    } else {
                        return null;
                    }
                } else {
                    return null;
                }
            }
            return ret.isEmpty() ? null : ret;
        } else {
            //for conditions like date > DATE'2000-11-11' OR date < DATE '1999-01-01'
            //we will use Ranges.all() rather than two ranges to represent them
            return Range.all();
        }
    }

    if (filter instanceof CompareTupleFilter) {
        CompareTupleFilter compareTupleFilter = (CompareTupleFilter) filter;
        if (compareTupleFilter.getColumn() == null)// column will be null at filters like " 1<>1"
            return Range.all();

        if (compareTupleFilter.getColumn().equals(colRef)) {
            Object firstValue = compareTupleFilter.getFirstValue();
            long t;
            switch (compareTupleFilter.getOperator()) {
            case EQ:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.closed(t, t);
            case LT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.lessThan(t);
            case LTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atMost(t);
            case GT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.greaterThan(t);
            case GTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atLeast(t);
            case NEQ:
            case IN://not handled for now
                break;
            default:
            }
        }
    }
    return Range.all();
}

Example 19

Source File: TsConditionExtractor.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

private static Range<Long> extractTsConditionInternal(TupleFilter filter, TblColRef colRef) {
    if (filter == null) {
        return Range.all();
    }

    if (filter instanceof LogicalTupleFilter) {
        if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
            Range<Long> ret = Range.all();
            for (TupleFilter child : filter.getChildren()) {
                Range childRange = extractTsConditionInternal(child, colRef);
                if (childRange != null) {
                    if (ret.isConnected(childRange) && !ret.intersection(childRange).isEmpty()) {
                        ret = ret.intersection(childRange);
                    } else {
                        return null;
                    }
                } else {
                    return null;
                }
            }
            return ret.isEmpty() ? null : ret;
        } else {
            //for conditions like date > DATE'2000-11-11' OR date < DATE '1999-01-01'
            //we will use Ranges.all() rather than two ranges to represent them
            return Range.all();
        }
    }

    if (filter instanceof CompareTupleFilter) {
        CompareTupleFilter compareTupleFilter = (CompareTupleFilter) filter;
        if (compareTupleFilter.getColumn() == null)// column will be null at filters like " 1<>1"
            return Range.all();

        if (compareTupleFilter.getColumn().equals(colRef)) {
            Object firstValue = compareTupleFilter.getFirstValue();
            long t;
            switch (compareTupleFilter.getOperator()) {
            case EQ:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.closed(t, t);
            case LT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.lessThan(t);
            case LTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atMost(t);
            case GT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.greaterThan(t);
            case GTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atLeast(t);
            case NEQ:
            case IN://not handled for now
                break;
            default:
            }
        }
    }
    return Range.all();
}

Example 20

Source File: TupleFilterVisitor.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@VisibleForTesting
static TupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    if (children.isEmpty()) {
        return filter;
    }
    // key: inColumn
    // Value: first: inValues
    // Value: second: dynamicVariables
    Map<TblColRef, Pair<Set<Object>, Map<String, Object>>> inColumnMap = Maps.newHashMap();
    List<TupleFilter> extraFilters = Lists.newLinkedList();
    for (TupleFilter child : children) {
        if (child.getOperator() == TupleFilter.FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (column != null) {
                Pair<Set<Object>, Map<String, Object>> tmpValue = inColumnMap.get(column);
                if (tmpValue == null) {
                    Set<Object> inValues = Sets.newHashSet();
                    Map<String, Object> dynamicVariables = Maps.newHashMap();
                    tmpValue = new Pair<>(inValues, dynamicVariables);
                    inColumnMap.put(column, tmpValue);
                }

                tmpValue.getFirst().addAll(compFilter.getValues());
                tmpValue.getSecond().putAll(compFilter.getVariables());
                continue;
            }
        }
        extraFilters.add(child);
    }

    children.clear();

    TupleFilter ret = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
    ret.addChildren(extraFilters);
    for (Map.Entry<TblColRef, Pair<Set<Object>, Map<String, Object>>> entry : inColumnMap.entrySet()) {
        CompareTupleFilter inFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.IN);
        inFilter.addChild(new ColumnTupleFilter(entry.getKey()));
        inFilter.addChild(new ConstantTupleFilter(entry.getValue().getFirst()));
        inFilter.getVariables().putAll(entry.getValue().getSecond());
        ret.addChild(inFilter);
    }

    return ret.getChildren().size() == 1 ? ret.getChildren().get(0) : ret;
}