org.apache.kylin.metadata.tuple.ITuple Java Examples

private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
        iterator = storageEngine.search(context, sqlDigest);
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (iterator != null) {
            iterator.close();
        }
    }
    return count;
}
 

public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid,
        Set<TblColRef> selectedDimensions, List<TblColRef> rtGroups, Set<TblColRef> groups, //
        Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, SQLDigest sqlDigest) {
    this.context = context;
    this.scanners = scanners;

    Set<TblColRef> selectedDims = Sets.newHashSet(selectedDimensions);
    selectedDims.addAll(rtGroups);

    segmentCubeTupleIterators = Lists.newArrayList();
    for (CubeSegmentScanner scanner : scanners) {
        segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDims, selectedMetrics, returnTupleInfo, context));
    }

    if (context.mergeSortPartitionResults() && !sqlDigest.isRawQuery) {
        //query with limit
        logger.info("Using SortedIteratorMergerWithLimit to merge segment results");
        Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator();
        tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, groups, returnTupleInfo)).getIterator();
    } else {
        //normal case
        logger.info("Using Iterators.concat to merge segment results");
        tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
    }
}
 

private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", /*allCol*/ Collections.<TblColRef> emptySet(), /*join*/ null, //
                groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), //
                /*dynamicGroupByColumns*/ Collections.<TblColRef, TupleExpression> emptyMap(), //
                /*groupByExpression*/ false, //
                /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), //
                /*dynamicAggregations*/ Collections.<DynamicFunctionDesc> emptyList(), //
                /*runtimeDimensionColumns*/ Collections.<TblColRef> emptySet(), //
                /*runtimeMetricColumns*/ Collections.<TblColRef> emptySet(), //
                /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, //
                /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, false, false, new HashSet<MeasureDesc>());
        iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } finally {
        if (iterator != null)
            iterator.close();
    }
    return count;
}
 

@Override
public boolean evaluate(ITuple tuple) {
    // extract tuple value
    String extractType = null;
    String tupleValue = null;
    for (TupleFilter filter : this.children) {
        filter.evaluate(tuple);
        if (filter instanceof ConstantTupleFilter) {
            tupleValue = filter.getValues().iterator().next();
        } else if (filter instanceof CompareTupleFilter) {
            extractType = filter.getValues().iterator().next();
        }
    }

    // extract date
    this.date = extractDate(extractType, Integer.valueOf(tupleValue));
    return true;
}
 

@Override
public ITuple next() {
    if (!hasNext()) {
        throw new IllegalStateException("No more Tuple in the SingleRegionTupleIterator");
    }

    IIProtos.IIResponse.IIRow currentRow = rows.get(index);
    byte[] columnsBytes = currentRow.getColumns().toByteArray();
    this.tableRecord.setBytes(columnsBytes, 0, columnsBytes.length);
    if (currentRow.hasMeasures()) {
        byte[] measuresBytes = currentRow.getMeasures().toByteArray();

        this.measureValues = pushedDownAggregators.deserializeMetricValues(measuresBytes, 0);
    }

    index++;

    return makeTuple(this.tableRecord, this.measureValues);
}
 

@Override
public boolean moveNext() {
    if (cursor == null) {
        cursor = queryStorage();
    }

    if (!cursor.hasNext()) {
        return false;
    }

    ITuple tuple = cursor.next();
    if (tuple == null) {
        return false;
    }
    convertCurrentRow(tuple);
    return true;
}
 

public Iterator<ITuple> doSearch(DataRequest dataRequest, CubeInstance cube, StreamingTupleConverter tupleConverter,
        RecordsSerializer recordsSerializer, Node receiver, TupleInfo tupleInfo) throws Exception {
    String queryId = dataRequest.getQueryId();
    logger.info("send query to receiver " + receiver + " with query id:" + queryId);
    String url = "http://" + receiver.getHost() + ":" + receiver.getPort() + "/kylin/api/data/query";

    try {
        String content = JsonUtil.writeValueAsString(dataRequest);
        Stopwatch sw = new Stopwatch();
        sw.start();
        int connTimeout = cube.getConfig().getStreamingRPCHttpConnTimeout();
        int readTimeout = cube.getConfig().getStreamingRPCHttpReadTimeout();
        String msg = restService.postRequest(url, content, connTimeout, readTimeout);

        logger.info("query-{}: receive response from {} take time:{}", queryId, receiver, sw.elapsedMillis());
        if (failedReceivers.containsKey(receiver)) {
            failedReceivers.remove(receiver);
        }
        DataResponse response = JsonUtil.readValue(msg, DataResponse.class);
        logger.info("query-{}: receiver {} profile info:{}", queryId, receiver, response.getProfile());
        return deserializeResponse(tupleConverter, recordsSerializer, cube.getName(), tupleInfo, response);
    } catch (Exception e) {
        logger.error("error when search data from receiver:" + url, e);
        throw e;
    }
}
 

public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid,
        Set<TblColRef> selectedDimensions, List<TblColRef> rtGroups, Set<TblColRef> groups, //
        Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, SQLDigest sqlDigest) {
    this.context = context;
    this.scanners = scanners;

    Set<TblColRef> selectedDims = Sets.newHashSet(selectedDimensions);
    selectedDims.addAll(rtGroups);

    segmentCubeTupleIterators = Lists.newArrayList();
    for (CubeSegmentScanner scanner : scanners) {
        segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDims, selectedMetrics, returnTupleInfo, context));
    }

    if (context.mergeSortPartitionResults() && !sqlDigest.isRawQuery) {
        //query with limit
        logger.info("Using SortedIteratorMergerWithLimit to merge segment results");
        Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator();
        tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, groups, returnTupleInfo)).getIterator();
    } else {
        //normal case
        logger.info("Using Iterators.concat to merge segment results");
        tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
    }
}
 

private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", /*allCol*/ Collections.<TblColRef> emptySet(), /*join*/ null, //
                groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), //
                /*dynamicGroupByColumns*/ Collections.<TblColRef, TupleExpression> emptyMap(), //
                /*groupByExpression*/ false, //
                /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), //
                /*dynamicAggregations*/ Collections.<DynamicFunctionDesc> emptyList(), //
                /*runtimeDimensionColumns*/ Collections.<TblColRef> emptySet(), //
                /*runtimeMetricColumns*/ Collections.<TblColRef> emptySet(), //
                /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, //
                /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, false, false, new HashSet<MeasureDesc>());
        iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } finally {
        if (iterator != null)
            iterator.close();
    }
    return count;
}
 

private Object[] convertCurrentRow(ITuple tuple) {

        // build field index map
        if (this.fieldIndexes == null) {
            List<String> fields = tuple.getAllFields();
            int size = fields.size();
            this.fieldIndexes = new int[size];
            for (int i = 0; i < size; i++) {
                String field = fields.get(i);
                RelDataTypeField relField = olapContext.olapRowType.getField(field, true);
                if (relField != null) {
                    fieldIndexes[i] = relField.getIndex();
                } else {
                    fieldIndexes[i] = -1;
                }
            }
        }

        // set field value
        Object[] values = tuple.getAllValues();
        for (int i = 0, n = values.length; i < n; i++) {
            Object value = values[i];
            int index = fieldIndexes[i];
            if (index >= 0) {
                current[index] = value;
            }
        }

        return current;
    }
 

@Override
public boolean evaluate(ITuple tuple) {
    switch (this.operator) {
    case AND:
        return evalAnd(tuple);
    case OR:
        return evalOr(tuple);
    case NOT:
        return evalNot(tuple);
    default:
        return false;
    }
}
 

private boolean evalAnd(ITuple tuple) {
    for (TupleFilter filter : this.children) {
        if (!filter.evaluate(tuple)) {
            return false;
        }
    }
    return true;
}
 

@Override
public ITuple next() {
    // fetch next record
    if (next == null) {
        hasNext();
        if (next == null)
            throw new NoSuchElementException();
    }

    ITuple result = next;
    next = null;
    return result;
}
 

private ITupleIterator transformToTupleIterator(final StreamingTupleConverter tupleConverter,
                                                final IStreamingSearchResult searchResult, final TupleInfo returnTupleInfo) {
    final Tuple tuple = new Tuple(returnTupleInfo);

    final Iterator<Record> recordIterator = searchResult.iterator();
    return new ITupleIterator() {
        @Override
        public void close() {
            try {
                searchResult.close();
            } catch (IOException e) {
                logger.warn("exception when close gtscanner", e);
            }
        }

        @Override
        public boolean hasNext() {
            return recordIterator.hasNext();
        }

        @Override
        public ITuple next() {
            tupleConverter.translateResult(recordIterator.next(), tuple);
            return tuple;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("not support");
        }
    };
}
 

@Override
public boolean evaluate(ITuple tuple) {
    if (whenFilters.size() != thenFilters.size()) {
        elseFilter = whenFilters.remove(whenFilters.size() - 1);
    }
    boolean matched = false;
    for (int i = 0; i < whenFilters.size(); i++) {
        TupleFilter whenFilter = whenFilters.get(i);
        if (whenFilter.evaluate(tuple)) {
            TupleFilter thenFilter = thenFilters.get(i);
            thenFilter.evaluate(tuple);
            values = thenFilter.getValues();
            matched = true;
            break;
        }
    }
    if (!matched) {
        if (elseFilter != null) {
            elseFilter.evaluate(tuple);
            values = elseFilter.getValues();
        } else {
            values = Collections.emptyList();
        }
    }

    return true;
}
 

@Override
public boolean hasNext() {
    try {
        if (currentBlock.hasNext()) {
            return true;
        } else if (numConsumeBlocks < totalBlockNum) {
            while (numConsumeBlocks < totalBlockNum) {
                if (hasEndpointFail()) {
                    throw new RuntimeException("endpoint fail", endpointException);
                }
                Iterator<ITuple> ret = null;
                while (ret == null && endpointException == null && timeoutTS > System.currentTimeMillis()) {
                    ret = queue.poll(1000, MILLISECONDS);
                }
                currentBlock = ret;
                if (currentBlock == null) {
                    throw new RuntimeException("timeout when call stream rpc");
                }
                numConsumeBlocks++;
                if (currentBlock.hasNext()) {
                    return true;
                }
            }

        }
    } catch (InterruptedException e) {
        logger.error("interrupted", e);
        throw new RuntimeException("interrupted", e);
    }

    return false;
}
 

public void addBlock(Iterator<ITuple> tuples) {
    try {
        queue.put(tuples);
    } catch (InterruptedException e) {
        logger.error("interrupted", e);
        throw new RuntimeException("interrupted", e);
    }
}
 

@Override
public boolean moveNext() {
    try {
        if (cursor == null) {
            cursor = queryStorage();
        }

        if (!cursor.hasNext()) {
            return false;
        }

        ITuple tuple = cursor.next();
        if (tuple == null) {
            return false;
        }
        convertCurrentRow(tuple);
        return true;
    } catch (Exception e) {
        try {
            if (cursor != null) {
                cursor.close();
            }
        } catch (Exception ee) {
            logger.info("Error when closing cursor, ignore it", ee);
        }
        throw e;
    }
}
 

public Iterator<ITuple> doSearch(DataRequest dataRequest, CubeInstance cube, StreamingTupleConverter tupleConverter,
        RecordsSerializer recordsSerializer, Node receiver, TupleInfo tupleInfo) throws Exception {
    String queryId = dataRequest.getQueryId();
    String url = "http://" + receiver.getHost() + ":" + receiver.getPort() + "/kylin/api/data/query";

    try {
        int connTimeout = cube.getConfig().getStreamingRPCHttpConnTimeout();
        int readTimeout = cube.getConfig().getStreamingRPCHttpReadTimeout();
        dataRequest.setDeadline(System.currentTimeMillis() + (int)(readTimeout * 1.5));
        String content = JsonUtil.writeValueAsString(dataRequest);
        Stopwatch sw;
        sw = Stopwatch.createUnstarted();
        sw.start();
        String msg = restService.postRequest(url, content, connTimeout, readTimeout);

        logger.info("query-{}: receive response from {} take time:{}", queryId, receiver, sw.elapsed(MILLISECONDS));
        if (failedReceivers.containsKey(receiver)) {
            failedReceivers.remove(receiver);
        }
        DataResponse response = JsonUtil.readValue(msg, DataResponse.class);
        logger.info("query-{}: receiver {} profile info:{}", queryId, receiver, response.getProfile());
        return deserializeResponse(tupleConverter, recordsSerializer, cube.getName(), tupleInfo, response);
    } catch (Exception e) {
        logger.error("error when search data from receiver:" + url, e);
        throw e;
    }
}
 

private boolean evalOr(ITuple tuple) {
    for (TupleFilter filter : this.children) {
        if (filter.evaluate(tuple)) {
            return true;
        }
    }
    return false;
}
 

@Override
public ITupleIterator search(final long minSegmentTime, final CubeInstance cube, final TupleInfo tupleInfo,
        final TupleFilter tupleFilter, final Set<TblColRef> dimensions, final Set<TblColRef> groups,
        final Set<FunctionDesc> metrics, final int storagePushDownLimit, final boolean allowStorageAggregation) {
    List<ReplicaSet> replicaSetsOfCube = assignmentsCache.getReplicaSetsByCube(cube.getName());
    int timeout = cube.getConfig().getStreamingRPCHttpReadTimeout() * 2;
    final QueuedStreamingTupleIterator result = new QueuedStreamingTupleIterator(replicaSetsOfCube.size(), timeout);
    final QueryContext query = QueryContextFacade.current();

    final CubeDesc cubeDesc = cube.getDescriptor();
    final ResponseResultSchema schema = new ResponseResultSchema(cubeDesc, dimensions, metrics);
    final StreamingTupleConverter tupleConverter = new StreamingTupleConverter(schema, tupleInfo);
    final RecordsSerializer recordsSerializer = new RecordsSerializer(schema);
    final DataRequest dataRequest = createDataRequest(query.getQueryId(), cube.getName(), minSegmentTime, tupleInfo,
            tupleFilter, dimensions, groups, metrics, storagePushDownLimit, allowStorageAggregation);

    logger.info("Query-{}:send request to stream receivers", query.getQueryId());
    for (final ReplicaSet rs : replicaSetsOfCube) {
        executorService.submit(new Runnable() {
            @Override
            public void run() {
                try {
                    Iterator<ITuple> tuplesBlock = search(dataRequest, cube, tupleConverter, recordsSerializer, rs,
                            tupleInfo);
                    result.addBlock(tuplesBlock);
                } catch (Exception e) {
                    result.setEndpointException(e);
                }
            }
        });
    }

    return result;
}
 

@Override
public ITuple next() {
    // fetch next record
    if (next == null) {
        hasNext();
        if (next == null)
            throw new NoSuchElementException();
    }

    ITuple result = next;
    next = null;
    return result;
}
 

@Override
public ITuple next() {
    // fetch next record
    if (next == null) {
        hasNext();
        if (next == null)
            throw new NoSuchElementException();
    }

    ITuple result = next;
    next = null;
    return result;
}
 

@Override
public boolean moveNext() {
    try {
        if (cursor == null) {
            cursor = queryStorage();
        }

        if (!cursor.hasNext()) {
            return false;
        }

        ITuple tuple = cursor.next();
        if (tuple == null) {
            return false;
        }
        convertCurrentRow(tuple);
        return true;
    } catch (Exception e) {
        try {
            if (cursor != null) {
                cursor.close();
            }
        } catch (Exception ee) {
            logger.info("Error when closing cursor, ignore it", ee);
        }
        throw e;
    }
}
 

@Override
public ITuple next() {
    // fetch next record
    if (next == null) {
        hasNext();
        if (next == null)
            throw new NoSuchElementException();
    }

    ITuple result = next;
    next = null;
    return result;
}
 

private ITupleIterator transformToTupleIterator(final StreamingTupleConverter tupleConverter,
                                                final IStreamingSearchResult searchResult, final TupleInfo returnTupleInfo) {
    final Tuple tuple = new Tuple(returnTupleInfo);

    final Iterator<Record> recordIterator = searchResult.iterator();
    return new ITupleIterator() {
        @Override
        public void close() {
            try {
                searchResult.close();
            } catch (IOException e) {
                logger.warn("exception when close gtscanner", e);
            }
        }

        @Override
        public boolean hasNext() {
            return recordIterator.hasNext();
        }

        @Override
        public ITuple next() {
            tupleConverter.translateResult(recordIterator.next(), tuple);
            return tuple;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("not support");
        }
    };
}
 

@Override
public ITuple next() {
    if (scanCount++ % 100 == 1) {
        QueryContextFacade.current().checkMillisBeforeDeadline();
    }
    if (++scanCountDelta >= 1000) {
        flushScanCountDelta();
    }
    return tupleIterator.next();
}
 

@Override
public ITuple next() {
    rowsInAllMetric++;

    if (!hasNext()) {
        throw new IllegalStateException("No more ITuple in EndpointTupleIterator");
    }

    return this.tupleIterator.next();
}
 

@Override
public ITuple next() {
    ITuple t = null;
    while (hasNext()) {
        if (segmentIterator.hasNext()) {
            t = segmentIterator.next();
            scanCount++;
            break;
        } else {
            segmentIterator.close();
            segmentIterator = segmentIteratorIterator.next();
        }
    }
    return t;
}
 

@Override
public ITuple next() {
    if (scanCount++ % 100 == 1) {
        QueryContextFacade.current().checkMillisBeforeDeadline();
    }
    if (++scanCountDelta >= 1000) {
        flushScanCountDelta();
    }
    return tupleIterator.next();
}