org.apache.kylin.metadata.tuple.ITupleIterator Java Examples

private ITupleIterator queryStorage() {

        logger.debug("query storage...");
        // bind dynamic variables
        olapContext.bindVariable(optiqContext);

        // If olapContext is cached, then inherit it.
        if (!olapContext.isBorrowedContext) {
            olapContext.resetSQLDigest();
        }
        SQLDigest sqlDigest = olapContext.getSQLDigest();

        // query storage engine
        IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
        ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
                olapContext.returnTupleInfo);
        if (logger.isDebugEnabled()) {
            logger.debug("return TupleIterator...");
        }

        return iterator;
    }
 

private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", /*allCol*/ Collections.<TblColRef> emptySet(), /*join*/ null, //
                groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), //
                /*dynamicGroupByColumns*/ Collections.<TblColRef, TupleExpression> emptyMap(), //
                /*groupByExpression*/ false, //
                /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), //
                /*dynamicAggregations*/ Collections.<DynamicFunctionDesc> emptyList(), //
                /*runtimeDimensionColumns*/ Collections.<TblColRef> emptySet(), //
                /*runtimeMetricColumns*/ Collections.<TblColRef> emptySet(), //
                /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, //
                /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, false, false, new HashSet<MeasureDesc>());
        iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } finally {
        if (iterator != null)
            iterator.close();
    }
    return count;
}
 

private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
        iterator = storageEngine.search(context, sqlDigest);
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (iterator != null) {
            iterator.close();
        }
    }
    return count;
}
 

public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {

        this.context = context;
        int limit = context.getLimit();
        this.partialResultLimit = Math.max(limit, PARTIAL_DEFAULT_LIMIT);

        this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size());
        Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
        for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
            CubeSegmentTupleIterator segIter = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context);
            this.segmentIteratorList.add(segIter);
        }

        this.segmentIteratorIterator = this.segmentIteratorList.iterator();
        if (this.segmentIteratorIterator.hasNext()) {
            this.segmentIterator = this.segmentIteratorIterator.next();
        } else {
            this.segmentIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
        }
    }
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter);
    for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) {
        CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), //
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), //
                request.getMetrics(), request.getDynFuncs(), //
                request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    if (scanners.isEmpty())
        return ITupleIterator.EMPTY_TUPLE_ITERATOR;

    return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
            request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
            request.getContext(), sqlDigest);
}
 

private ITupleIterator queryStorage() {
    logger.debug("query storage...");

    // set connection properties
    setConnectionProperties();

    // bind dynamic variables
    bindVariable(olapContext.filter);



    // query storage engine
    IStorageEngine storageEngine = StorageEngineFactory.getStorageEngine(olapContext.realization);
    ITupleIterator iterator = storageEngine.search(olapContext.storageContext,olapContext.getSQLDigest());
    if (logger.isDebugEnabled()) {
        logger.debug("return TupleIterator...");
    }

    this.fieldIndexes = null;
    return iterator;
}
 

private ITupleIterator queryStorage() {

        logger.debug("query storage...");
        // bind dynamic variables
        olapContext.bindVariable(optiqContext);

        // If olapContext is cached, then inherit it.
        if (!olapContext.isBorrowedContext) {
            olapContext.resetSQLDigest();
        }
        SQLDigest sqlDigest = olapContext.getSQLDigest();

        // query storage engine
        IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
        ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
                olapContext.returnTupleInfo);
        if (logger.isDebugEnabled()) {
            logger.debug("return TupleIterator...");
        }

        return iterator;
    }
 

private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", /*allCol*/ Collections.<TblColRef> emptySet(), /*join*/ null, //
                groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), //
                /*dynamicGroupByColumns*/ Collections.<TblColRef, TupleExpression> emptyMap(), //
                /*groupByExpression*/ false, //
                /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), //
                /*dynamicAggregations*/ Collections.<DynamicFunctionDesc> emptyList(), //
                /*runtimeDimensionColumns*/ Collections.<TblColRef> emptySet(), //
                /*runtimeMetricColumns*/ Collections.<TblColRef> emptySet(), //
                /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, //
                /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, false, false, new HashSet<MeasureDesc>());
        iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } finally {
        if (iterator != null)
            iterator.close();
    }
    return count;
}
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter);
    for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) {
        CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), //
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), //
                request.getMetrics(), request.getDynFuncs(), //
                request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    if (scanners.isEmpty())
        return ITupleIterator.EMPTY_TUPLE_ITERATOR;

    return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
            request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
            request.getContext(), sqlDigest);
}
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    StreamingSegmentManager cubeDataStore = StreamingServer.getInstance().getStreamingSegmentManager(
            cubeInstance.getName());
    boolean enableStreamProfile = BackdoorToggles.isStreamingProfileEnable();
    StreamingQueryProfile queryProfile = new StreamingQueryProfile(QueryContextFacade.current().getQueryId(),
            System.currentTimeMillis());
    if (enableStreamProfile) {
        queryProfile.enableDetailProfile();
    }
    StreamingQueryProfile.set(queryProfile);
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);
    return cubeDataStore.getSearcher().search(returnTupleInfo, request.getFilter(), request.getHavingFilter(),
            request.getDimensions(), request.getGroups(), request.getMetrics(), context.isNeedStorageAggregation());
}
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) {
    String tableName = seg.getStorageLocationIdentifier();

    //HConnection is cached, so need not be closed
    HConnection conn = HBaseConnection.get(context.getConnUrl());
    try {
        return new EndpointTupleIterator(seg, sqlDigest.filter, sqlDigest.groupbyColumns, new ArrayList<>(sqlDigest.aggregations), context, conn);
    } catch (Throwable e) {
        e.printStackTrace();
        throw new IllegalStateException("Error when connecting to II htable " + tableName, e);
    }
}
 

private ITupleIterator transformToTupleIterator(final StreamingTupleConverter tupleConverter,
                                                final IStreamingSearchResult searchResult, final TupleInfo returnTupleInfo) {
    final Tuple tuple = new Tuple(returnTupleInfo);

    final Iterator<Record> recordIterator = searchResult.iterator();
    return new ITupleIterator() {
        @Override
        public void close() {
            try {
                searchResult.close();
            } catch (IOException e) {
                logger.warn("exception when close gtscanner", e);
            }
        }

        @Override
        public boolean hasNext() {
            return recordIterator.hasNext();
        }

        @Override
        public ITuple next() {
            tupleConverter.translateResult(recordIterator.next(), tuple);
            return tuple;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("not support");
        }
    };
}
 

public ITupleIterator search(TupleInfo returnTupleInfo, TupleFilter filter, TupleFilter havingFilter,
        Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics, boolean allowStorageAggregation) {
    StreamingSearchContext searchRequest = new StreamingSearchContext(cubeDesc, dimensions, groups, metrics,
            filter, havingFilter);
    IStreamingSearchResult searchResult = doSearch(searchRequest, -1, allowStorageAggregation);
    StreamingTupleConverter tupleConverter = new StreamingTupleConverter(searchRequest.getRespResultSchema(),
            returnTupleInfo);
    return transformToTupleIterator(tupleConverter, searchResult, returnTupleInfo);
}
 

@Override
public ITupleIterator search(final long minSegmentTime, final CubeInstance cube, final TupleInfo tupleInfo,
        final TupleFilter tupleFilter, final Set<TblColRef> dimensions, final Set<TblColRef> groups,
        final Set<FunctionDesc> metrics, final int storagePushDownLimit, final boolean allowStorageAggregation) {
    List<ReplicaSet> replicaSetsOfCube = assignmentsCache.getReplicaSetsByCube(cube.getName());
    int timeout = cube.getConfig().getStreamingRPCHttpReadTimeout() * 2;
    final QueuedStreamingTupleIterator result = new QueuedStreamingTupleIterator(replicaSetsOfCube.size(), timeout);
    final QueryContext query = QueryContextFacade.current();

    final CubeDesc cubeDesc = cube.getDescriptor();
    final ResponseResultSchema schema = new ResponseResultSchema(cubeDesc, dimensions, metrics);
    final StreamingTupleConverter tupleConverter = new StreamingTupleConverter(schema, tupleInfo);
    final RecordsSerializer recordsSerializer = new RecordsSerializer(schema);
    final DataRequest dataRequest = createDataRequest(query.getQueryId(), cube.getName(), minSegmentTime, tupleInfo,
            tupleFilter, dimensions, groups, metrics, storagePushDownLimit, allowStorageAggregation);

    logger.info("Query-{}:send request to stream receivers", query.getQueryId());
    for (final ReplicaSet rs : replicaSetsOfCube) {
        executorService.submit(new Runnable() {
            @Override
            public void run() {
                try {
                    Iterator<ITuple> tuplesBlock = search(dataRequest, cube, tupleConverter, recordsSerializer, rs,
                            tupleInfo);
                    result.addBlock(tuplesBlock);
                } catch (Exception e) {
                    result.setEndpointException(e);
                }
            }
        });
    }

    return result;
}
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    try {
        Thread.sleep(5 * 1000);
    } catch (InterruptedException e) {
        throw new KylinTimeoutException("Kylin query timeout");
    }
    return null;
}
 

@Override
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
    List<ITupleIterator> tupleIterators = Lists.newArrayList();
    for (int i = 0; i < realizations.length; i++) {
        if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest).capable) {
            ITupleIterator dataIterator = storageEngines[i].search(context, sqlDigest, returnTupleInfo);
            tupleIterators.add(dataIterator);
        }
    }
    // combine tuple iterator
    return new CompoundTupleIterator(tupleIterators);
}
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    StreamingSegmentManager cubeDataStore = StreamingServer.getInstance().getStreamingSegmentManager(
            cubeInstance.getName());
    boolean enableStreamProfile = BackdoorToggles.isStreamingProfileEnable();
    StreamingQueryProfile queryProfile = new StreamingQueryProfile(QueryContextFacade.current().getQueryId(),
            System.currentTimeMillis());
    if (enableStreamProfile) {
        queryProfile.enableDetailProfile();
    }
    StreamingQueryProfile.set(queryProfile);
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);
    return cubeDataStore.getSearcher().search(returnTupleInfo, request.getFilter(), request.getHavingFilter(),
            request.getDimensions(), request.getGroups(), request.getMetrics(), context.isNeedStorageAggregation());
}
 

private ITupleIterator transformToTupleIterator(final StreamingTupleConverter tupleConverter,
                                                final IStreamingSearchResult searchResult, final TupleInfo returnTupleInfo) {
    final Tuple tuple = new Tuple(returnTupleInfo);

    final Iterator<Record> recordIterator = searchResult.iterator();
    return new ITupleIterator() {
        @Override
        public void close() {
            try {
                searchResult.close();
            } catch (IOException e) {
                logger.warn("exception when close gtscanner", e);
            }
        }

        @Override
        public boolean hasNext() {
            return recordIterator.hasNext();
        }

        @Override
        public ITuple next() {
            tupleConverter.translateResult(recordIterator.next(), tuple);
            return tuple;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("not support");
        }
    };
}
 

public ITupleIterator search(TupleInfo returnTupleInfo, TupleFilter filter, TupleFilter havingFilter,
        Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics, boolean allowStorageAggregation) {
    StreamingSearchContext searchRequest = new StreamingSearchContext(cubeDesc, dimensions, groups, metrics,
            filter, havingFilter);
    IStreamingSearchResult searchResult = doSearch(searchRequest, -1, allowStorageAggregation);
    StreamingTupleConverter tupleConverter = new StreamingTupleConverter(searchRequest.getRespResultSchema(),
            returnTupleInfo);
    return transformToTupleIterator(tupleConverter, searchResult, returnTupleInfo);
}
 

@Override
public ITupleIterator search(final long minSegmentTime, final CubeInstance cube, final TupleInfo tupleInfo,
        final TupleFilter tupleFilter, final Set<TblColRef> dimensions, final Set<TblColRef> groups,
        final Set<FunctionDesc> metrics, final int storagePushDownLimit, final boolean allowStorageAggregation) {
    List<ReplicaSet> replicaSetsOfCube = assignmentsCache.getReplicaSetsByCube(cube.getName());
    int timeout = 120 * 1000; // timeout should be configurable
    final QueuedStreamingTupleIterator result = new QueuedStreamingTupleIterator(replicaSetsOfCube.size(), timeout);
    final QueryContext query = QueryContextFacade.current();

    final CubeDesc cubeDesc = cube.getDescriptor();
    final ResponseResultSchema schema = new ResponseResultSchema(cubeDesc, dimensions, metrics);
    final StreamingTupleConverter tupleConverter = new StreamingTupleConverter(schema, tupleInfo);
    final RecordsSerializer recordsSerializer = new RecordsSerializer(schema);
    final DataRequest dataRequest = createDataRequest(query.getQueryId(), cube.getName(), minSegmentTime, tupleInfo,
            tupleFilter, dimensions, groups, metrics, storagePushDownLimit, allowStorageAggregation);

    logger.info("Query-{}:send request to stream receivers", query.getQueryId());
    for (final ReplicaSet rs : replicaSetsOfCube) {
        executorService.submit(new Runnable() {
            @Override
            public void run() {
                try {
                    Iterator<ITuple> tuplesBlock = search(dataRequest, cube, tupleConverter, recordsSerializer, rs,
                            tupleInfo);
                    result.addBlock(tuplesBlock);
                } catch (Exception e) {
                    result.setEndpointException(e);
                }
            }
        });
    }

    return result;
}
 

@Override
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
    List<ITupleIterator> tupleIterators = Lists.newArrayList();
    for (int i = 0; i < realizations.length; i++) {
        if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest).capable) {
            ITupleIterator dataIterator = storageEngines[i].search(context, sqlDigest, returnTupleInfo);
            tupleIterators.add(dataIterator);
        }
    }
    // combine tuple iterator
    return new CompoundTupleIterator(tupleIterators);
}
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    long maxHistorySegmentTime = -1;
    StreamingDataQueryPlanner segmentsPlanner = new StreamingDataQueryPlanner(cubeInstance.getDescriptor(),
            request.getFilter());
    long current = System.currentTimeMillis();
    for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
        TSRange segmentRange = cubeSeg.getTSRange();
        if (segmentRange.end.v > maxHistorySegmentTime) {
            if (cubeSeg.getTSRange().end.v < current) {
                // In normal case, the segment for future time range is not reasonable in streaming case
                maxHistorySegmentTime = cubeSeg.getTSRange().end.v;
            }
        }
        CubeSegmentScanner scanner;

        if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
            logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
            continue;
        }

        if (segmentsPlanner.canSkip(segmentRange.start.v, segmentRange.end.v)) {
            logger.info("Skip cube segment {} because of not satisfy filter:{}", cubeSeg, request.getFilter());
            continue;
        }

        scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(),
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), request.getMetrics(),
                request.getDynFuncs(), request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    ITupleIterator historyResult;
    if (scanners.isEmpty()) {
        historyResult = ITupleIterator.EMPTY_TUPLE_ITERATOR;
    } else {
        historyResult = new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
                request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo, context, sqlDigest);
    }
    Set<TblColRef> dimensionsD = request.getDimensions();
    if (dimensionsD.isEmpty()) {
        dimensionsD = Sets.newHashSet(request.getCuboid().getColumns()); // temporary fix for query like: select count(1) from TABLE
    }

    ITupleIterator realTimeResult;
    if (segmentsPlanner.canSkip(maxHistorySegmentTime, Long.MAX_VALUE)) {
        logger.info("Skip scan realTime data, {}", maxHistorySegmentTime);
        realTimeResult = ITupleIterator.EMPTY_TUPLE_ITERATOR;
    } else {
        boolean isSelectAllQuery = isSelectAllQuery(request.getCuboid(), request.getGroups(), request.getFilter());
        int limitPushDown = isSelectAllQuery ? context.getFinalPushDownLimit() : Integer.MAX_VALUE;
        realTimeResult = realTimeSearchClient.search(maxHistorySegmentTime, cubeInstance, returnTupleInfo,
                request.getFilter(), dimensionsD, request.getGroups(), request.getMetrics(), limitPushDown,
                !isSelectAllQuery);
    }
    return new CompoundTupleIterator(Arrays.asList(historyResult, realTimeResult));
}
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    long maxHistorySegmentTime = -1;
    StreamingDataQueryPlanner segmentsPlanner = new StreamingDataQueryPlanner(cubeInstance.getDescriptor(),
            request.getFilter());
    for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
        TSRange segmentRange = cubeSeg.getTSRange();
        if (segmentRange.end.v > maxHistorySegmentTime) {
            maxHistorySegmentTime = cubeSeg.getTSRange().end.v;
        }
        CubeSegmentScanner scanner;

        if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
            logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
            continue;
        }

        if (segmentsPlanner.canSkip(segmentRange.start.v, segmentRange.end.v)) {
            logger.info("Skip cube segment {} because of not satisfy filter:{}", cubeSeg, request.getFilter());
            continue;
        }

        scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(),
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), request.getMetrics(),
                request.getDynFuncs(), request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    ITupleIterator historyResult;
    if (scanners.isEmpty()) {
        historyResult = ITupleIterator.EMPTY_TUPLE_ITERATOR;
    } else {
        historyResult = new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
                request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo, context, sqlDigest);
    }
    Set<TblColRef> dimensionsD = request.getDimensions();
    if (dimensionsD.isEmpty()) {
        dimensionsD = Sets.newHashSet(request.getCuboid().getColumns()); // temporary fix for query like: select count(1) from TABLE
    }

    ITupleIterator realTimeResult;
    if (segmentsPlanner.canSkip(maxHistorySegmentTime, Long.MAX_VALUE)) {
        logger.info("Skip scan realTime data, {}", maxHistorySegmentTime);
        realTimeResult = ITupleIterator.EMPTY_TUPLE_ITERATOR;
    } else {
        boolean isSelectAllQuery = isSelectAllQuery(request.getCuboid(), request.getGroups(), request.getFilter());
        int limitPushDown = isSelectAllQuery ? context.getFinalPushDownLimit() : Integer.MAX_VALUE;
        realTimeResult = realTimeSearchClient.search(maxHistorySegmentTime, cubeInstance, returnTupleInfo,
                request.getFilter(), dimensionsD, request.getGroups(), request.getMetrics(), limitPushDown,
                !isSelectAllQuery);
    }
    return new CompoundTupleIterator(Arrays.asList(historyResult, realTimeResult));
}
 

public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, HConnection conn) throws Throwable {

        String tableName = segment.getStorageLocationIdentifier();
        table = conn.getTable(tableName);
        factTableName = segment.getIIDesc().getFactTableName();

        if (rootFilter == null) {
            rootFilter = ConstantTupleFilter.TRUE;
        }

        if (groupBy == null) {
            groupBy = Sets.newHashSet();
        }

        if (measures == null) {
            measures = Lists.newArrayList();
        }

        //this method will change measures
        rewriteMeasureParameters(measures, segment.getColumns());

        this.seg = segment;
        this.context = context;
        this.measures = measures;

        this.columns = segment.getColumns();
        this.columnNames = getColumnNames(columns);

        this.tupleInfo = buildTupleInfo();
        this.tableRecordInfo = new TableRecordInfo(this.seg);

        this.pushedDownRowType = CoprocessorRowType.fromTableRecordInfo(tableRecordInfo, this.columns);
        this.pushedDownFilter = CoprocessorFilter.fromFilter(this.seg, rootFilter);

        for (TblColRef column : this.pushedDownFilter.getUnstrictlyFilteredColumns()) {
            groupBy.add(column);
        }

        this.pushedDownProjector = CoprocessorProjector.makeForEndpoint(tableRecordInfo, groupBy);
        this.pushedDownAggregators = EndpointAggregators.fromFunctions(tableRecordInfo, measures);

        IIProtos.IIRequest endpointRequest = prepareRequest();
        regionResponsesIterator = getResults(endpointRequest, table);

        if (this.regionResponsesIterator.hasNext()) {
            this.tupleIterator = new SingleRegionTupleIterator(this.regionResponsesIterator.next());
        } else {
            this.tupleIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
        }
    }
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) {

    Collection<TblColRef> groups = sqlDigest.groupbyColumns;
    TupleFilter filter = sqlDigest.filter;

    // build dimension & metrics
    Collection<TblColRef> dimensions = new HashSet<TblColRef>();
    Collection<FunctionDesc> metrics = new HashSet<FunctionDesc>();
    buildDimensionsAndMetrics(dimensions, metrics, sqlDigest);

    // all dimensions = groups + others
    Set<TblColRef> others = Sets.newHashSet(dimensions);
    others.removeAll(groups);

    // expand derived
    Set<TblColRef> derivedPostAggregation = Sets.newHashSet();
    Set<TblColRef> groupsD = expandDerived(groups, derivedPostAggregation);
    Set<TblColRef> othersD = expandDerived(others, derivedPostAggregation);
    othersD.removeAll(groupsD);
    derivedPostAggregation.removeAll(groups);

    // identify cuboid
    Set<TblColRef> dimensionsD = Sets.newHashSet();
    dimensionsD.addAll(groupsD);
    dimensionsD.addAll(othersD);
    Cuboid cuboid = identifyCuboid(dimensionsD);
    context.setCuboid(cuboid);

    // isExactAggregation? meaning: tuples returned from storage requires no further aggregation in query engine
    Set<TblColRef> singleValuesD = findSingleValueColumns(filter);
    boolean isExactAggregation = isExactAggregation(cuboid, groups, othersD, singleValuesD, derivedPostAggregation);
    context.setExactAggregation(isExactAggregation);

    // translate filter for scan range and compose returning groups for coprocessor, note:
    // - columns on non-evaluatable filter have to return
    // - columns on loosened filter (due to derived translation) have to return
    Set<TblColRef> groupsCopD = Sets.newHashSet(groupsD);
    collectNonEvaluable(filter, groupsCopD);
    TupleFilter filterD = translateDerived(filter, groupsCopD);

    // flatten to OR-AND filter, (A AND B AND ..) OR (C AND D AND ..) OR ..
    TupleFilter flatFilter = flattenToOrAndFilter(filterD);

    // translate filter into segment scan ranges
    List<HBaseKeyRange> scans = buildScanRanges(flatFilter, dimensionsD);

    // check involved measures, build value decoder for each each family:column
    List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHBaseMapping(), metrics, context);

    setThreshold(dimensionsD, valueDecoders, context); // set cautious threshold to prevent out of memory
    setCoprocessor(groupsCopD, valueDecoders, context); // enable coprocessor if beneficial
    setLimit(filter, context);

    HConnection conn = HBaseConnection.get(context.getConnUrl());
    return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context);
}
 

/**
 *
 * @param minSegmentTime minimum segment time to search for steaming data
 * @param cube
 * @param tupleInfo
 * @param tupleFilter
 * @param dimensions
 * @param groups
 * @param metrics
 * @return
 */
ITupleIterator search(long minSegmentTime, CubeInstance cube, TupleInfo tupleInfo, TupleFilter tupleFilter,
        Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics, int storagePushDownLimit,
        boolean allowStorageAggregation);
 

/**
 *
 * @param minSegmentTime minimum segment time to search for steaming data
 * @param cube
 * @param tupleInfo
 * @param tupleFilter
 * @param dimensions
 * @param groups
 * @param metrics
 * @return
 */
ITupleIterator search(long minSegmentTime, CubeInstance cube, TupleInfo tupleInfo, TupleFilter tupleFilter,
        Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics, int storagePushDownLimit,
        boolean allowStorageAggregation);
 

ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo); 

ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo); 

ITupleIterator search(StorageContext context, SQLDigest sqlDigest);