com.facebook.presto.spi.type.Type Java Examples

@Override
public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session,
                                                  ConnectorTableHandle connectorTableHandle) {
    KuduTableHandle tableHandle = fromConnectorTableHandle(session, connectorTableHandle);
    Schema schema = clientSession.getTableSchema(tableHandle);

    ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
    for (int i = 0; i < schema.getColumnCount(); i++) {
        ColumnSchema col = schema.getColumnByIndex(i);
        String name = col.getName();
        Type type = TypeHelper.fromKuduColumn(col);
        KuduColumnHandle columnHandle = new KuduColumnHandle(name, i, type);
        columnHandles.put(name, columnHandle);
    }

    return columnHandles.build();
}
 

private static XContentBuilder getMapping(List<ColumnMetadata> columns)
{
    XContentBuilder mapping = null;
    try {
        mapping = jsonBuilder()
                .startObject().startObject("properties");
        for (ColumnMetadata columnMetadata : columns) {
            String columnName = columnMetadata.getName();
            Type type = columnMetadata.getType();
            if ("@timestamp".equals(columnName)) {    //break @timestamp field
                continue;
            }
            buildFieldType(mapping.startObject(columnName), type).endObject();
        }
        mapping.endObject().endObject();
    }
    catch (IOException e) {
        e.printStackTrace();
    }
    return mapping;
}
 

/**
 * Get all column handles of specified table
 */
public Optional<List<ParaflowColumnHandle>> getTableColumnHandle(String connectorId, String dbName, String tblName)
{
    log.debug("Get list of column handles of table " + formName(dbName, tblName));
    List<ParaflowColumnHandle> columnHandles = new ArrayList<>();
    String colName;
    int colTypeName;
    String dataTypeName;
    MetaProto.StringListType listColumns = metaClient.listColumns(dbName, tblName);
    if (listColumns.getIsEmpty()) {
        log.warn("No col matches!");
        return Optional.empty();
    }
    for (int i = 0; i < listColumns.getStrCount(); i++) {
        colName = listColumns.getStr(i);
        MetaProto.ColParam colParam = metaClient.getColumn(dbName, tblName, colName);
        colTypeName = colParam.getColType();
        dataTypeName = colParam.getDataType();
        // Deal with col type
        ParaflowColumnHandle.ColumnType colType = getColType(colTypeName);
        // Deal with data type
        Type type = getType(dataTypeName);
        columnHandles.add(new ParaflowColumnHandle(colName, type, "", colType, connectorId));
    }
    return Optional.of(columnHandles);
}
 

private static void serializePrimitive(Type type, BlockBuilder builder, Object object) {
    requireNonNull(builder, "parent builder is null");

    if (object == null) {
        builder.appendNull();
        return;
    }

    if (BOOLEAN.equals(type)) {
        BOOLEAN.writeBoolean(builder, (Boolean) object);
    } else if (BIGINT.equals(type) || INTEGER.equals(type) || SMALLINT.equals(type) || TINYINT.equals(type)
            || REAL.equals(type) || DATE.equals(type) || TIMESTAMP.equals(type)) {
        type.writeLong(builder, getLongExpressedValue(object));
    } else if (DOUBLE.equals(type)) {
        DOUBLE.writeDouble(builder, ((Number) object).doubleValue());
    } else if (isVarcharType(type) || VARBINARY.equals(type) || isCharType(type)) {
        type.writeSlice(builder, getSliceExpressedValue(object, type));
    } else {
        throw new UnsupportedOperationException("Unsupported primitive type: " + type);
    }
}
 

private static XContentBuilder getMapping(List<ColumnMetadata> columns)
{
    XContentBuilder mapping = null;
    try {
        mapping = jsonBuilder()
                .startObject().startObject("properties");
        for (ColumnMetadata columnMetadata : columns) {
            String columnName = columnMetadata.getName();
            Type type = columnMetadata.getType();
            if ("@timestamp".equals(columnName)) {    //break @timestamp field
                continue;
            }
            buildFieldType(mapping.startObject(columnName), type).endObject();
        }
        mapping.endObject().endObject();
    }
    catch (IOException e) {
        e.printStackTrace();
    }
    return mapping;
}
 

/**
 * Contributes bindings and other configurations for this module to {@code binder}.
 *
 * @param binder binder
 */
@Override
public void configure(Binder binder)
{
    binder.bind(ParaflowConnectorId.class).toInstance(new ParaflowConnectorId(connectorId));
    binder.bind(TypeManager.class).toInstance(typeManager);

    configBinder(binder).bindConfig(ParaflowPrestoConfig.class);

    binder.bind(ParaflowMetadataFactory.class).in(Scopes.SINGLETON);
    binder.bind(ParaflowMetadata.class).in(Scopes.SINGLETON);
    binder.bind(ParaflowMetaDataReader.class).in(Scopes.SINGLETON);
    binder.bind(FSFactory.class).in(Scopes.SINGLETON);
    binder.bind(ParaflowConnector.class).in(Scopes.SINGLETON);
    binder.bind(ParaflowSplitManager.class).in(Scopes.SINGLETON);
    binder.bind(ParaflowPageSourceProvider.class).in(Scopes.SINGLETON);
    binder.bind(ClassLoader.class).toInstance(ParaflowPlugin.getClassLoader());

    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
}
 

private static Slice getSliceExpressedValue(Object value, Type type) {
    Slice sliceValue;
    if (value instanceof String) {
        sliceValue = Slices.utf8Slice((String) value);
    } else if (value instanceof byte[]) {
        sliceValue = Slices.wrappedBuffer((byte[]) value);
    } else if (value instanceof Integer) {
        sliceValue = Slices.utf8Slice(value.toString());
    } else {
        throw new IllegalStateException("unsupported string field type: " + value.getClass().getName());
    }
    if (isVarcharType(type)) {
        sliceValue = truncateToLength(sliceValue, type);
    }
    if (isCharType(type)) {
        sliceValue = truncateToLengthAndTrimSpaces(sliceValue, type);
    }

    return sliceValue;
}
 

public Block readPrimitive(ColumnDescriptor columnDescriptor, Type type, IntList offsets)
        throws IOException
{
    ParquetColumnReader columnReader = columnReadersMap.get(columnDescriptor);
    if (columnReader.getPageReader() == null) {
        validateParquet(currentBlockMetadata.getRowCount() > 0, "Row group has 0 rows");
        ColumnChunkMetaData metadata = getColumnChunkMetaData(columnDescriptor);
        long startingPosition = metadata.getStartingPos();
        int totalSize = checkedCast(metadata.getTotalSize());
        byte[] buffer = new byte[totalSize];
        dataSource.readFully(startingPosition, buffer);
        ParquetColumnChunkDescriptor descriptor = new ParquetColumnChunkDescriptor(columnDescriptor, metadata, totalSize);
        ParquetColumnChunk columnChunk = new ParquetColumnChunk(descriptor, buffer, 0);
        columnReader.setPageReader(columnChunk.readAllPages());
    }
    return columnReader.readPrimitive(type, offsets);
}
 

@Override
public void configure(Binder binder)
{
    // Note: handle resolver handled separately, along with several other classes.
    binder.bind(KinesisConnector.class).in(Scopes.SINGLETON);

    binder.bind(KinesisMetadata.class).in(Scopes.SINGLETON);
    binder.bind(KinesisSplitManager.class).in(Scopes.SINGLETON);
    binder.bind(KinesisRecordSetProvider.class).in(Scopes.SINGLETON);
    binder.bind(S3TableConfigClient.class).in(Scopes.SINGLETON);

    configBinder(binder).bindConfig(KinesisConnectorConfig.class);

    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
    jsonCodecBinder(binder).bindJsonCodec(KinesisStreamDescription.class);

    binder.install(new KinesisDecoderModule());

    for (KinesisInternalFieldDescription internalFieldDescription : KinesisInternalFieldDescription.getInternalFields()) {
        bindInternalColumn(binder, internalFieldDescription);
    }
}
 

@JsonCreator
public KinesisStreamFieldDescription(
        @JsonProperty("name") String name,
        @JsonProperty("type") Type type,
        @JsonProperty("mapping") String mapping,
        @JsonProperty("comment") String comment,
        @JsonProperty("dataFormat") String dataFormat,
        @JsonProperty("formatHint") String formatHint,
        @JsonProperty("hidden") boolean hidden)
{
    checkArgument(!isNullOrEmpty(name), "name is null or is empty");
    this.name = name;
    this.type = checkNotNull(type, "type is null");
    this.mapping = mapping;
    this.comment = comment;
    this.dataFormat = dataFormat;
    this.formatHint = formatHint;
    this.hidden = hidden;
}
 

public static Object getTypeValue(Type type, Object value)
{
    Object toEncode;
    if (Types.isArrayType(type)) {
        throw new UnsupportedOperationException("Unsupported type " + type);
    }
    else if (Types.isMapType(type)) {
        throw new UnsupportedOperationException("Unsupported type " + type);
    }
    else if (type.equals(VARBINARY)) {
        return ((Slice) value).getBytes();
    }
    else if (type instanceof VarcharType) {
        return ((Slice) value).toStringUtf8();
    }
    else {
        return value;
    }
}
 

void accept()
{
    for (int i = 1; i < rowlisteMap.size(); i++) {
        pageBuilder.declarePosition();
        for (int column = 0; column < columnTypes.size(); column++) {
            BlockBuilder output = pageBuilder.getBlockBuilder(column);
            Type type = columnTypes.get(column);
            Object value = docMap.get(columnNames.get(column));
            if (column == rowlisteMapColumn) {
                appendTo(type, rowlisteMap.get(i), output);
            }
            else {
                appendTo(type, value, output);
            }
        }
    }
}
 

/**
 * Recursive helper function used by {@link this#getArrayFromBlock} and
 * {@link this#getMapFromBlock} to add the given object to the given block
 * builder. Supports nested complex types!
 *
 * @param builder Block builder
 * @param type Presto type
 * @param obj Object to write to the block builder
 */
private static void writeObject(BlockBuilder builder, Type type, Object obj)
{
    if (Types.isArrayType(type)) {
        BlockBuilder arrayBldr = builder.beginBlockEntry();
        Type elementType = Types.getElementType(type);
        for (Object item : (List<?>) obj) {
            writeObject(arrayBldr, elementType, item);
        }
        builder.closeEntry();
    }
    else if (Types.isMapType(type)) {
        BlockBuilder mapBlockBuilder = builder.beginBlockEntry();
        for (Map.Entry<?, ?> entry : ((Map<?, ?>) obj).entrySet()) {
            writeObject(mapBlockBuilder, Types.getKeyType(type), entry.getKey());
            writeObject(mapBlockBuilder, Types.getValueType(type), entry.getValue());
        }
        builder.closeEntry();
    }
    else {
        TypeUtils.writeNativeValue(type, builder, obj);
    }
}
 

/**
 * Encodes the given map into a Block.
 *
 * @param mapType Presto type of the map
 * @param bytes hbase byte[]
 * @return Presto Block
 */
public static Block getBlockFromMap(Type mapType, byte[] bytes)
        throws IOException
{
    Type keyType = mapType.getTypeParameters().get(0);
    Type valueType = mapType.getTypeParameters().get(1);
    Map<?, ?> map = MAPPER.readValue(bytes, new MyTypeReference(keyType.getJavaType(), valueType.getJavaType()));

    BlockBuilder mapBlockBuilder = mapType.createBlockBuilder(null, 1);
    BlockBuilder builder = mapBlockBuilder.beginBlockEntry();

    for (Map.Entry<?, ?> entry : map.entrySet()) {
        writeObject(builder, keyType, entry.getKey());
        writeObject(builder, valueType, entry.getValue());
    }

    mapBlockBuilder.closeEntry();
    return (Block) mapType.getObject(mapBlockBuilder, 0);
}
 

/**
 * @param type Presto type
 * @param block Block to decode
 * @param position Position in the block to get
 * @return Java object from the Block
 */
private static Object readObject(Type type, Block block, int position)
{
    if (Types.isArrayType(type)) {
        Type elementType = Types.getElementType(type);
        return getArrayFromBlock(elementType, block.getObject(position, Block.class));
    }
    else if (Types.isMapType(type)) {
        return getMapFromBlock(type, block.getObject(position, Block.class));
    }
    else {
        if (type.getJavaType() == Slice.class) {
            Slice slice = (Slice) TypeUtils.readNativeValue(type, block, position);
            return type.equals(VarcharType.VARCHAR) ? slice.toStringUtf8() : slice.getBytes();
        }

        return TypeUtils.readNativeValue(type, block, position);
    }
}
 

public static long getLong(Type type, RowResult row, int field) {
    if (type == TimestampType.TIMESTAMP) {
        return row.getLong(field) / 1000;
    } else if (type == BigintType.BIGINT) {
        return row.getLong(field);
    } else if (type == IntegerType.INTEGER) {
        return row.getInt(field);
    } else if (type == SmallintType.SMALLINT) {
        return row.getShort(field);
    } else if (type == TinyintType.TINYINT) {
        return row.getByte(field);
    } else if (type == RealType.REAL) {
        return floatToRawIntBits(row.getFloat(field));
    } else if (type instanceof DecimalType) {
        DecimalType dtype = (DecimalType) type;
        if (dtype.isShort()) {
            return row.getDecimal(field).unscaledValue().longValue();
        } else {
            throw new IllegalStateException("getLong not supported for long decimal: " + type);
        }
    } else {
        throw new IllegalStateException("getLong not implemented for " + type);
    }
}
 

@JsonCreator
public HbaseColumnHandle(
        @JsonProperty("name") String name,
        @JsonProperty("family") Optional<String> family,
        @JsonProperty("qualifier") Optional<String> qualifier,
        @JsonProperty("type") Type type,
        @JsonProperty("ordinal") int ordinal,
        @JsonProperty("comment") String comment,
        @JsonProperty("indexed") boolean indexed)
{
    this.name = requireNonNull(name, "columnName is null");
    this.family = requireNonNull(family, "family is null");
    this.qualifier = requireNonNull(qualifier, "qualifier is null");
    this.type = requireNonNull(type, "type is null");
    this.ordinal = requireNonNull(ordinal, "type is null");
    checkArgument(ordinal >= 0, "ordinal must be >= zero");

    this.comment = requireNonNull(comment, "comment is null");
    this.indexed = requireNonNull(indexed, "indexed is null");
}
 

public KuduOutputTableHandle(String connectorId, SchemaTableName schemaTableName,
                             List<Type> originalColumnTypes,
                             List<String> columnNames, List<Type> columnTypes,
                             boolean generateUUID, KuduTable table) {
    super(connectorId, schemaTableName, columnNames, columnTypes, table);
    this.originalColumnTypes = ImmutableList.copyOf(originalColumnTypes);
    this.generateUUID = generateUUID;
}
 

public ParaflowPageSource(
        ParquetReader parquetReader,
        ParquetDataSource dataSource,
        MessageType fileSchema,
        MessageType requestedSchema,
        long totalBytes,
        List<ParaflowColumnHandle> columns,
        TypeManager typeManager)
{
    checkArgument(totalBytes >= 0, "totalBytes is negative");

    this.parquetReader = requireNonNull(parquetReader, "parquetReader is null");
    this.dataSource = requireNonNull(dataSource, "dataSource is null");
    this.fileSchema = requireNonNull(fileSchema, "fileSchema is null");
    this.requestedSchema = requireNonNull(requestedSchema, "requestedSchema is null");
    this.totalBytes = totalBytes;

    this.columnSize = columns.size();
    this.constantBlocks = new Block[columnSize];
    ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
    for (int columnIndex = 0; columnIndex < columnSize; columnIndex++) {
        ParaflowColumnHandle column = columns.get(columnIndex);
        String name = column.getName();
        Type type = typeManager.getType(column.getType().getTypeSignature());

        namesBuilder.add(name);
        typesBuilder.add(type);

        if (getParquetType(column, fileSchema) == null) {
            constantBlocks[columnIndex] = RunLengthEncodedBlock.create(type, null, MAX_VECTOR_LENGTH);
        }
    }
    columnNames = namesBuilder.build();
    types = typesBuilder.build();
}
 

private static Block serializeMap(Type type, BlockBuilder builder, Object object) {
    Map<?, ?> map = (Map) object;
    if (map == null) {
        requireNonNull(builder, "parent builder is null").appendNull();
        return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter");
    Type keyType = typeParameters.get(0);
    Type valueType = typeParameters.get(1);
    boolean builderSynthesized = false;

    if (builder == null) {
        builderSynthesized = true;
        builder = type.createBlockBuilder(new BlockBuilderStatus(), 1);
    }
    BlockBuilder currentBuilder = builder.beginBlockEntry();

    for (Map.Entry<?, ?> entry : map.entrySet()) {
        // Hive skips map entries with null keys
        if (entry.getKey() != null) {
            serializeObject(keyType, currentBuilder, entry.getKey());
            serializeObject(valueType, currentBuilder, entry.getValue());
        }
    }

    builder.closeEntry();
    if (builderSynthesized) {
        return (Block) type.getObject(builder, 0);
    } else {
        return null;
    }
}
 

private parquet.schema.Type getParquetType(ParaflowColumnHandle column, MessageType messageType)
{
    if (messageType.containsField(column.getName())) {
        return messageType.getType(column.getName());
    }
    // parquet is case-insensitive, all hdfs-columns get converted to lowercase
    for (parquet.schema.Type type : messageType.getFields()) {
        if (type.getName().equalsIgnoreCase(column.getName())) {
            return type;
        }
    }
    return null;
}
 

@Override
public long getLong(int field)
{
    checkFieldType(field, BIGINT, DATE, INTEGER, REAL, SMALLINT, TIME, TIMESTAMP, TINYINT);
    Type type = getType(field);
    byte[] bytes = getValue(field);
    if (type.equals(BIGINT)) {
        return Bytes.toLong(bytes);
    }
    else if (type.equals(DATE)) {
        return Bytes.toLong(bytes);
    }
    else if (type.equals(INTEGER)) {
        return Bytes.toLong(bytes);
    }
    else if (type.equals(REAL)) {
        return Bytes.toLong(bytes);
    }
    else if (type.equals(SMALLINT)) {
        return Bytes.toLong(bytes);
    }
    else if (type.equals(TIME)) {
        return Bytes.toLong(bytes);
    }
    else if (type.equals(TIMESTAMP)) {
        return Bytes.toLong(bytes);
    }
    else if (type.equals(TINYINT)) {
        return Bytes.toLong(bytes);
    }
    else {
        throw new PrestoException(NOT_SUPPORTED, "Unsupported type " + getType(field));
    }
}
 

private static Block serializeStruct(Type type, BlockBuilder builder, Object object) {
    if (object == null) {
        requireNonNull(builder, "parent builder is null").appendNull();
        return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    EthBlock.TransactionObject structData = (EthBlock.TransactionObject) object;
    boolean builderSynthesized = false;
    if (builder == null) {
        builderSynthesized = true;
        builder = type.createBlockBuilder(new BlockBuilderStatus(), 1);
    }
    BlockBuilder currentBuilder = builder.beginBlockEntry();

    ImmutableList.Builder<Supplier> lstBuilder = ImmutableList.builder();
    lstBuilder.add(structData::getHash);
    lstBuilder.add(structData::getNonce);
    lstBuilder.add(structData::getBlockHash);
    lstBuilder.add(structData::getBlockNumber);
    lstBuilder.add(structData::getTransactionIndex);
    lstBuilder.add(structData::getFrom);
    lstBuilder.add(structData::getTo);
    lstBuilder.add(structData::getValue);
    lstBuilder.add(structData::getGas);
    lstBuilder.add(structData::getGasPrice);
    lstBuilder.add(structData::getInput);
    ImmutableList<Supplier> txColumns = lstBuilder.build();

    for (int i = 0; i < typeParameters.size(); i++) {
        serializeObject(typeParameters.get(i), currentBuilder, txColumns.get(i).get());
    }

    builder.closeEntry();
    if (builderSynthesized) {
        return (Block) type.getObject(builder, 0);
    } else {
        return null;
    }
}
 

@Override
public void configure(Binder binder) {
    binder.bind(EthereumConnector.class).in(Scopes.SINGLETON);
    binder.bind(EthereumMetadata.class).in(Scopes.SINGLETON);
    binder.bind(EthereumWeb3jProvider.class).in(Scopes.SINGLETON);

    binder.bind(EthereumSplitManager.class).in(Scopes.SINGLETON);
    binder.bind(EthereumRecordSetProvider.class).in(Scopes.SINGLETON);

    configBinder(binder).bindConfig(EthereumConnectorConfig.class);
    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
}
 

KinesisInternalFieldDescription(
            String name,
            Type type,
            String comment)
{
    checkArgument(!isNullOrEmpty(name), "name is null or is empty");
    this.name = name;
    this.type = checkNotNull(type, "type is null");
    this.comment = checkNotNull(comment, "comment is null");
}
 

/**
 * Given the map type and Presto Block, decodes the Block into a map of values.
 *
 * @param type Map type
 * @param block Map block
 * @return List of values
 */
private static Map<Object, Object> getMapFromBlock(Type type, Block block)
{
    Map<Object, Object> map = new HashMap<>(block.getPositionCount() / 2);
    Type keyType = Types.getKeyType(type);
    Type valueType = Types.getValueType(type);
    for (int i = 0; i < block.getPositionCount(); i += 2) {
        map.put(readObject(keyType, block, i), readObject(valueType, block, i + 1));
    }
    return map;
}
 

/**
 * Given the array element type and Presto Block, decodes the Block into a list of values.
 *
 * @param elementType Array element type
 * @param block Array block
 * @return List of values
 */
private static List<Object> getArrayFromBlock(Type elementType, Block block)
{
    ImmutableList.Builder<Object> arrayBuilder = ImmutableList.builder();
    for (int i = 0; i < block.getPositionCount(); ++i) {
        arrayBuilder.add(readObject(elementType, block, i));
    }
    return arrayBuilder.build();
}
 

public KuduExtendedTableHandle(String connectorId, SchemaTableName schemaTableName,
                               List<String> columnNames, List<Type> columnTypes,
                               KuduTable table) {
    super(connectorId, schemaTableName, table);

    requireNonNull(columnNames, "columnNames is null");
    requireNonNull(columnTypes, "columnTypes is null");
    checkArgument(columnNames.size() == columnTypes.size(), "columnNames and columnTypes sizes don't match");
    this.columnNames = ImmutableList.copyOf(columnNames);
    this.columnTypes = ImmutableList.copyOf(columnTypes);
}
 

public KuduRecordCursorWithVirtualRowId(KuduScanner scanner, KuduTable table,
                                        List<Type> columnTypes,
                                        Map<Integer, Integer> fieldMapping) {
    super(scanner, columnTypes);
    this.table = table;
    this.fieldMapping = fieldMapping;
}
 

@Override
protected Type _deserialize(String value, DeserializationContext context)
{
    Type type = typeManager.getType(parseTypeSignature(value));
    checkArgument(type != null, "Unknown type %s", value);
    return type;
}