com.facebook.presto.spi.Page Java Examples

@Override
public CompletableFuture<?> appendPage(Page page) {
    for (int position = 0; position < page.getPositionCount(); position++) {
        Upsert upsert = table.newUpsert();
        PartialRow row = upsert.getRow();
        int start = 0;
        if (generateUUID) {
            String id = String.format("%s-%08x", uuid, nextSubId++);
            row.addString(0, id);
            start = 1;
        }

        for (int channel = 0; channel < page.getChannelCount(); channel++) {
            appendColumn(row, page, position, channel, channel + start);
        }

        try {
            session.apply(upsert);
        } catch (KuduException e) {
            throw new RuntimeException(e);
        }
    }
    return NOT_BLOCKED;
}
 

@Override
public Page getNextPage()
{
    verify(pageBuilder.isEmpty());
    count = 0;
    for (int i = 0; i < ROWS_PER_REQUEST; i++) {
        if (!searchResult.hasNext()) {
            finished = true;
            break;
        }
        Map<String, Object> docMap = searchResult.next();
        count++;

        pageBuilder.declarePosition();
        ImmutableList.Builder<ArrayDocForeachFunc> funcs = ImmutableList.builder();
        for (int column = 0; column < columnTypes.size(); column++) {
            BlockBuilder output = pageBuilder.getBlockBuilder(column);
            Type type = columnTypes.get(column);
            Object value = docMap.get(columnNames.get(column));
            //如果是 array[] 字段 则下面会进行打平处理 es head展示时则只显示了array第一个元素
            if (isRowType(type) && value instanceof List && ((List) value).size() > 0 && ((List) value).get(0) instanceof Map) {
                final List<Map> listSourceMap = (List<Map>) value;
                appendTo(type, listSourceMap.get(0), output);
                //----下面是展平逻辑 目前存在如果有多个array字段 展平时笛卡尔积join问题---
                // 此处不做展平处理, 逻辑将和<es head `基本查询`>保持一致
                //funcs.add(new ArrayDocForeachFunc(column, docMap));
            }
            else {
                appendTo(type, value, output);
            }
        }
        funcs.build().forEach(func -> func.accept());
    }

    Page page = pageBuilder.build();
    pageBuilder.reset();
    return page;
}
 

@Override
public CompletableFuture<?> appendPage(Page page)
{
    List<Document> batch = new ArrayList<>(page.getPositionCount());

    for (int position = 0; position < page.getPositionCount(); position++) {
        Document.DocumentBuilder builder = Document.newDocument().setIndex(schemaTableName.getTableName());
        Map<String, Object> source = new HashMap<>();
        for (int channel = 0; channel < page.getChannelCount(); channel++) {
            ElasticsearchColumnHandle column = columns.get(channel);
            Object value = getObjectValue(column.getType(), page.getBlock(channel), position);

            if ("_id".equals(column.getName())) {
                builder.setId((String) value);
            }
            else if ("_type".equals(column.getName())) {
                builder.setType((String) value);
            }
            else {
                source.put(column.getName(), value);
            }
        }
        batch.add(builder.setSource(source).get());
    }

    // push
    client.insertMany(batch);
    return NOT_BLOCKED;
}
 

private void appendColumn(PartialRow row, Page page, int position, int channel, int destChannel) {
    Block block = page.getBlock(channel);
    Type type = columnTypes.get(destChannel);
    if (block.isNull(position)) {
        row.setNull(destChannel);
    } else if (TIMESTAMP.equals(type)) {
        row.addLong(destChannel, type.getLong(block, position) * 1000);
    } else if (REAL.equals(type)) {
        row.addFloat(destChannel, intBitsToFloat((int) type.getLong(block, position)));
    } else if (BIGINT.equals(type)) {
        row.addLong(destChannel, type.getLong(block, position));
    } else if (INTEGER.equals(type)) {
        row.addInt(destChannel, (int) type.getLong(block, position));
    } else if (SMALLINT.equals(type)) {
        row.addShort(destChannel, (short) type.getLong(block, position));
    } else if (TINYINT.equals(type)) {
        row.addByte(destChannel, (byte) type.getLong(block, position));
    } else if (BOOLEAN.equals(type)) {
        row.addBoolean(destChannel, type.getBoolean(block, position));
    } else if (DOUBLE.equals(type)) {
        row.addDouble(destChannel, type.getDouble(block, position));
    } else if (isVarcharType(type)) {
        Type originalType = originalColumnTypes.get(destChannel);
        if (DATE.equals(originalType)) {
            SqlDate date = (SqlDate) originalType.getObjectValue(connectorSession, block, position);
            LocalDateTime ldt = LocalDateTime.ofEpochSecond(TimeUnit.DAYS.toSeconds(date.getDays()), 0, ZoneOffset.UTC);
            byte[] bytes = ldt.format(DateTimeFormatter.ISO_LOCAL_DATE).getBytes(Charsets.UTF_8);
            row.addStringUtf8(destChannel, bytes);
        } else {
            row.addString(destChannel, type.getSlice(block, position).toStringUtf8());
        }
    } else if (VARBINARY.equals(type)) {
        row.addBinary(destChannel, type.getSlice(block, position).toByteBuffer());
    } else if (type instanceof DecimalType) {
        SqlDecimal sqlDecimal = (SqlDecimal) type.getObjectValue(connectorSession, block, position);
        row.addDecimal(destChannel, sqlDecimal.toBigDecimal());
    } else {
        throw new UnsupportedOperationException("Type is not supported: " + type);
    }
}
 

@Override
public CompletableFuture<?> appendPage(Page page)
{
    // For each position within the page, i.e. row
    for (int position = 0; position < page.getPositionCount(); ++position) {
        Type rowkeyType = columns.get(rowIdOrdinal).getType();
        Object rowKey = TypeUtils.readNativeValue(rowkeyType, page.getBlock(rowIdOrdinal), position);
        if (rowKey == null) {
            throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Column mapped as the Hbase row ID cannot be null");
        }
        Put put = new Put(toHbaseBytes(rowkeyType, rowKey));

        // For each channel within the page, i.e. column
        for (HbaseColumnHandle column : columns) {
            // Skip the row ID ordinal
            if (column.getOrdinal() == rowIdOrdinal) {
                continue;
            }
            // Get the type for this channel
            int channel = column.getOrdinal();

            // Read the value from the page and append the field to the row
            Object value = TypeUtils.readNativeValue(column.getType(), page.getBlock(channel), position);
            put.addColumn(
                    Bytes.toBytes(column.getFamily().get()),
                    Bytes.toBytes(column.getQualifier().get()),
                    toHbaseBytes(column.getType(), value));
        }

        // Convert row to a Mutation, writing and indexing it
        puts.add(put);
        ++numRows;

        // TODO Fix arbitrary flush every 10k rows
        if (numRows % MAX_PUT_NUM == 0) {
            flush();
        }
    }

    return NOT_BLOCKED;
}
 

/**
 * Gets the next page of data.  This method is allowed to return null.
 */
@Override
public Page getNextPage()
{
    try {
        batchId++;
        long start = System.nanoTime();

        int batchSize = parquetReader.nextBatch();

        readTimeNanos += System.nanoTime() - start;

        if (closed || batchSize <= 0) {
            close();
            return null;
        }

        Block[] blocks = new Block[columnSize];
        for (int fieldId = 0; fieldId < blocks.length; fieldId++) {
            if (constantBlocks[fieldId] != null) {
                blocks[fieldId] = constantBlocks[fieldId].getRegion(0, batchSize);
            }
            else {
                Type type = types.get(fieldId);
                int fieldIndex = getFieldIndex(fileSchema, columnNames.get(fieldId));
                if (fieldIndex == -1) {
                    blocks[fieldId] = RunLengthEncodedBlock.create(type, null, batchSize);
                    continue;
                }

                List<String> path = new ArrayList<>();
                path.add(fileSchema.getFields().get(fieldIndex).getName());
                if (StandardTypes.ROW.equals(type.getTypeSignature().getBase())) {
                    blocks[fieldId] = parquetReader.readStruct(type, path);
                }
                else {
                    Optional<RichColumnDescriptor> descriptor = ParquetTypeUtils.getDescriptor(fileSchema, requestedSchema, path);
                    if (descriptor.isPresent()) {
                        blocks[fieldId] = new LazyBlock(batchSize, new ParquetBlockLoader(descriptor.get(), type));
                    }
                    else {
                        blocks[fieldId] = RunLengthEncodedBlock.create(type, null, batchSize);
                    }
                }
            }
        }
        return new Page(batchSize, blocks);
    }
    catch (IOException e) {
        closeWithSupression(e);
        throw new ParaflowCursorException();
    }
}
 

@Override
public Page getNextPage() {
    return inner.getNextPage();
}