Java Code Examples for org.apache.kudu.Type

The following examples show how to use org.apache.kudu.Type. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: KuduTableProperties.java    License: Apache License 2.0 6 votes vote down vote up
private static long toUnixTimeMicros(Object obj, Type type, String name)
{
    if (Number.class.isAssignableFrom(obj.getClass())) {
        return ((Number) obj).longValue();
    }
    else if (obj instanceof String) {
        String s = (String) obj;
        s = s.trim().replace(' ', 'T');
        long millis = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC).parseMillis(s);
        return millis * 1000;
    }
    else {
        handleInvalidValue(name, type, obj);
        return 0;
    }
}
 
Example 2
Source Project: bahir-flink   Source File: KuduCatalogTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTimestamp() throws Exception {
    tableEnv.sqlUpdate("CREATE TABLE TestTableTsC (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");
    tableEnv.sqlUpdate("INSERT INTO TestTableTsC values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')");
    tableEnv.execute("test");

    KuduTable kuduTable = harness.getClient().openTable("TestTableTsC");
    assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString(0));
    assertEquals(Timestamp.valueOf("2020-01-01 12:12:12.123"), rows.get(0).getTimestamp(1));
}
 
Example 3
private void createAndFillSchemasTable(KuduClient client)
        throws KuduException
{
    List<String> existingSchemaNames = listSchemaNamesFromTablets(client);
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            insert.getRow().addString(0, schemaName);
            session.apply(insert);
        }
    }
    finally {
        session.close();
    }
}
 
Example 4
Source Project: canal   Source File: KuduTemplate.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * kudu数据类型映射
 *
 * @param
 */
private Type toKuduType(String mysqlType) throws IllegalArgumentException {

    switch (mysqlType) {
        case "varchar":
            return Type.STRING;
        case "int":
            return Type.INT8;
        case "decimal":
            return Type.DOUBLE;
        case "double":
            return Type.DOUBLE;
        case "datetime":
            return Type.STRING;
        case "timestamp":
            return Type.STRING;
        default:
            throw new IllegalArgumentException("The provided data type doesn't map to know any known one.");
    }
}
 
Example 5
Source Project: DataflowTemplates   Source File: KuduToBigQueryTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * The schemas for the main input of the kudu rows transformation. Checking all possible supported
 * data types
 */
private Schema createTestKuduSchema() {
  List<ColumnSchema> columns =
      ImmutableList.<ColumnSchema>builder()
          .add(new ColumnSchema.ColumnSchemaBuilder("INT32", Type.INT32).key(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("BOOL", Type.BOOL).nullable(false).build())
          .add(
              new ColumnSchema.ColumnSchemaBuilder("DOUBLE", Type.DOUBLE).nullable(false).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("FLOAT", Type.FLOAT).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT8", Type.INT8).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT16", Type.INT16).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT64", Type.INT64).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("STRING", Type.STRING).nullable(true).build())
          .add(
              new ColumnSchema.ColumnSchemaBuilder("UNIXTIME_MICROS", Type.UNIXTIME_MICROS)
                  .nullable(true)
                  .build())
          .build();

  return new Schema(columns);
}
 
Example 6
Source Project: syndesis   Source File: AbstractKuduTest.java    License: Apache License 2.0 6 votes vote down vote up
protected void createTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final List<ColumnSchema> columns = new ArrayList<>(5);
        final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

        for (int i = 0; i < columnNames.size(); i++) {
            final Type type = i == 0 ? Type.INT32 : Type.STRING;
            columns.add(
                new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), type)
                    .key(i == 0)
                    .build());
        }

        final List<String> rangeKeys = new ArrayList<>();
        rangeKeys.add("id");

        client.createTable(tableName,
            new Schema(columns),
            new CreateTableOptions().setRangePartitionColumns(rangeKeys));
    }
}
 
Example 7
Source Project: presto-kudu   Source File: NativeKuduClientSession.java    License: Apache License 2.0 6 votes vote down vote up
private void createAndFillSchemasTable() throws KuduException {
    List<String> existingSchemaNames = listSchemaNamesFromTablets();
    ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING)
            .key(true).build();
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.setNumReplicas(1); // TODO config
    options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            fillSchemaRow(insert.getRow(), schemaName);
            session.apply(insert);
        }
    } finally {
        session.close();
    }
}
 
Example 8
Source Project: presto-kudu   Source File: KuduTableProperties.java    License: Apache License 2.0 6 votes vote down vote up
private static Object toValue(Schema schema, PartialRow bound, Integer idx) {
    Type type = schema.getColumnByIndex(idx).getType();
    switch (type) {
        case UNIXTIME_MICROS:
            long millis = bound.getLong(idx) / 1000;
            return ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC).print(millis);
        case STRING:
            return bound.getString(idx);
        case INT64:
            return bound.getLong(idx);
        case INT32:
            return bound.getInt(idx);
        case INT16:
            return bound.getShort(idx);
        case INT8:
            short s = bound.getByte(idx);
            return s;
        case BOOL:
            return bound.getBoolean(idx);
        case BINARY:
            return bound.getBinaryCopy(idx);
        default:
            throw new IllegalStateException("Unhandled type " + type + " for range partition");
    }
}
 
Example 9
Source Project: datacollector   Source File: KuduUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Convert from Kudu type to SDC Field type
 * @param kuduType
 * @return Field.Type
 */
public static Field.Type convertFromKuduType(Type kuduType){
  switch(kuduType) {
    case BINARY: return Field.Type.BYTE_ARRAY;
    case BOOL: return Field.Type.BOOLEAN;
    case DOUBLE: return Field.Type.DOUBLE;
    case FLOAT: return Field.Type.FLOAT;
    case INT8: return Field.Type.BYTE;
    case INT16: return Field.Type.SHORT;
    case INT32: return Field.Type.INTEGER;
    case INT64: return Field.Type.LONG;
    case STRING: return  Field.Type.STRING;
    case UNIXTIME_MICROS: return Field.Type.DATETIME;
    default:
      if ("DECIMAL".equals(kuduType.name())) {
        return Field.Type.DECIMAL;
      }
      throw new UnsupportedOperationException("Unknown data type: " + kuduType.getName());
  }
}
 
Example 10
Source Project: datacollector   Source File: TestKuduTarget.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that if given field is null and column doesn't support that, the record will
 * end up in error stream rather then terminating whole pipeline execution.
 */
@Test
public void testNullColumnWillEndInError() throws Exception{
  TargetRunner targetRunner = getTargetRunner(tableName, KuduOperationType.INSERT, UnsupportedOperationAction.SEND_TO_ERROR);
  targetRunner.runInit();

  Record record =  RecordCreator.create();
  LinkedHashMap<String, Field> field = new LinkedHashMap<>();
  field.put("key", Field.create(1));
  field.put("value", Field.create(Field.Type.STRING, null));
  field.put("name", Field.create(Field.Type.STRING, null));
  record.set(Field.createListMap(field));

  try {
    targetRunner.runWrite(ImmutableList.of(record));

    List<Record> errors = targetRunner.getErrorRecords();
    Assert.assertEquals(1, errors.size());
  } finally {
    targetRunner.runDestroy();
  }
}
 
Example 11
Source Project: bahir-flink   Source File: KuduTestBase.java    License: Apache License 2.0 6 votes vote down vote up
public static KuduTableInfo booksTableInfo(String tableName, boolean createIfNotExist) {

        KuduTableInfo tableInfo = KuduTableInfo.forTable(tableName);

        if (createIfNotExist) {
            ColumnSchemasFactory schemasFactory = () -> {
                List<ColumnSchema> schemas = new ArrayList<>();
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("title", Type.STRING).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("author", Type.STRING).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("price", Type.DOUBLE).nullable(true).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("quantity", Type.INT32).nullable(true).build());
                return schemas;
            };

            tableInfo.createTableIfNotExists(
                    schemasFactory,
                    () -> new CreateTableOptions()
                            .setNumReplicas(1)
                            .addHashPartitions(Lists.newArrayList("id"), 2));
        }

        return tableInfo;
    }
 
Example 12
Source Project: nifi   Source File: TestPutKudu.java    License: Apache License 2.0 5 votes vote down vote up
private PartialRow buildPartialRow(Long id, String name, Short age, String kuduIdName, String recordIdName, String airport_code, Boolean lowercaseFields) {
    final Schema kuduSchema = new Schema(Arrays.asList(
        new ColumnSchema.ColumnSchemaBuilder(kuduIdName, Type.INT64).key(true).build(),
        new ColumnSchema.ColumnSchemaBuilder("name", Type.STRING).nullable(true).build(),
        new ColumnSchema.ColumnSchemaBuilder("age", Type.INT16).nullable(false).build(),
        new ColumnSchema.ColumnSchemaBuilder("updated_at", Type.UNIXTIME_MICROS).nullable(false).build(),
        new ColumnSchema.ColumnSchemaBuilder("score", Type.DECIMAL).nullable(true).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().precision(9).scale(0).build()
        ).build(),
        new ColumnSchema.ColumnSchemaBuilder("airport_code", Type.VARCHAR).nullable(true).typeAttributes(
                new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(3).build()
        ).build()));

    final RecordSchema schema = new SimpleRecordSchema(Arrays.asList(
        new RecordField(recordIdName, RecordFieldType.BIGINT.getDataType()),
        new RecordField("name", RecordFieldType.STRING.getDataType()),
        new RecordField("age", RecordFieldType.SHORT.getDataType()),
        new RecordField("updated_at", RecordFieldType.TIMESTAMP.getDataType()),
        new RecordField("score", RecordFieldType.LONG.getDataType()),
        new RecordField("airport_code", RecordFieldType.STRING.getDataType())));

    Map<String, Object> values = new HashMap<>();
    PartialRow row = kuduSchema.newPartialRow();
    values.put(recordIdName, id);
    values.put("name", name);
    values.put("age", age);
    values.put("updated_at", new Timestamp(System.currentTimeMillis()));
    values.put("score", 10000L);
    values.put("airport_code", airport_code);
    processor.buildPartialRow(
        kuduSchema,
        row,
        new MapRecord(schema, values),
        schema.getFieldNames(),
            true,
        lowercaseFields
    );
    return row;
}
 
Example 13
Source Project: presto   Source File: KuduTableProperties.java    License: Apache License 2.0 5 votes vote down vote up
private static byte[] toByteArray(Object obj, Type type, String name)
{
    if (obj instanceof byte[]) {
        return (byte[]) obj;
    }
    else if (obj instanceof String) {
        return Base64.getDecoder().decode((String) obj);
    }
    else {
        handleInvalidValue(name, type, obj);
        return null;
    }
}
 
Example 14
Source Project: presto   Source File: KuduTableProperties.java    License: Apache License 2.0 5 votes vote down vote up
private static Number toNumber(Object obj, Type type, String name)
{
    if (Number.class.isAssignableFrom(obj.getClass())) {
        return (Number) obj;
    }
    else if (obj instanceof String) {
        String s = (String) obj;
        BigDecimal d = new BigDecimal((String) obj);
        return d;
    }
    else {
        handleInvalidValue(name, type, obj);
        return 0;
    }
}
 
Example 15
Source Project: presto   Source File: KuduClientSession.java    License: Apache License 2.0 5 votes vote down vote up
public void addColumn(SchemaTableName schemaTableName, ColumnMetadata column)
{
    try {
        String rawName = schemaEmulation.toRawName(schemaTableName);
        AlterTableOptions alterOptions = new AlterTableOptions();
        Type type = TypeHelper.toKuduClientType(column.getType());
        alterOptions.addNullableColumn(column.getName(), type);
        client.alterTable(rawName, alterOptions);
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 
Example 16
Source Project: geowave   Source File: KuduMetadataRow.java    License: Apache License 2.0 5 votes vote down vote up
KuduMetadataField(
    final String fieldName,
    final Type dataType,
    final KuduColumnType columnType) {
  this.fieldName = fieldName;
  this.dataType = dataType;
  this.columnType = columnType;
}
 
Example 17
Source Project: presto   Source File: KuduClientSession.java    License: Apache License 2.0 5 votes vote down vote up
private KuduPredicate createComparisonPredicate(ColumnSchema columnSchema, KuduPredicate.ComparisonOp op, Object value)
{
    io.prestosql.spi.type.Type type = TypeHelper.fromKuduColumn(columnSchema);
    Object javaValue = TypeHelper.getJavaValue(type, value);
    if (javaValue instanceof Long) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Long) javaValue);
    }
    if (javaValue instanceof BigDecimal) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (BigDecimal) javaValue);
    }
    if (javaValue instanceof Integer) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Integer) javaValue);
    }
    if (javaValue instanceof Short) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Short) javaValue);
    }
    if (javaValue instanceof Byte) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Byte) javaValue);
    }
    if (javaValue instanceof String) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (String) javaValue);
    }
    if (javaValue instanceof Double) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Double) javaValue);
    }
    if (javaValue instanceof Float) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Float) javaValue);
    }
    if (javaValue instanceof Boolean) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Boolean) javaValue);
    }
    if (javaValue instanceof byte[]) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (byte[]) javaValue);
    }
    if (javaValue == null) {
        throw new IllegalStateException("Unexpected null java value for column " + columnSchema.getName());
    }
    throw new IllegalStateException("Unexpected java value for column "
            + columnSchema.getName() + ": " + javaValue + "(" + javaValue.getClass() + ")");
}
 
Example 18
Source Project: datacollector   Source File: TestKuduRecordConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDelete() throws Exception {
  record.set("/str", Field.create("primary key"));
  record.set("/long", Field.create((long)10));
  record.set("/short1", Field.create(Field.Type.SHORT, null));
  kuduRecordConverter.convert(record, partialRow, KuduOperationType.DELETE.code); // must not throw NPE
  Assert.assertTrue(Utils.format("Message: {}", partialRow.stringifyRowKey()), partialRow.stringifyRowKey().contains("primary key"));
}
 
Example 19
Source Project: flink-learning   Source File: KuduColumnInfo.java    License: Apache License 2.0 5 votes vote down vote up
private KuduColumnInfo(String name, Type type) {
    this.name = name;
    this.type = type;
    this.blockSize = 0;
    this.key = false;
    this.rangeKey = false;
    this.hashKey = false;
    this.nullable = false;
    this.defaultValue = null;
    this.encoding = Encoding.AUTO;
    this.compression = Compression.DEFAULT;
}
 
Example 20
Source Project: datacollector   Source File: KuduRecordConverter.java    License: Apache License 2.0 5 votes vote down vote up
public KuduRecordConverter(Map<String, Field.Type> columnsToFieldTypes, Map<String, String> fieldsToColumns,
                           Schema schema, FieldPathConverter converter) {
  this.columnsToFieldTypes = ImmutableMap.copyOf(columnsToFieldTypes);
  this.fieldsToColumns = ImmutableMap.copyOf(fieldsToColumns);
  this.schema = schema;
  this.fieldConverter = converter;
}
 
Example 21
Source Project: DataLink   Source File: KuduUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Type getKuduType(String typeName) {
    if (typeName == null || typeName.trim() == "") {
        return null;
    }

    String lowercaseTypeNmae = typeName.toLowerCase();

    if (lowercaseTypeNmae.startsWith("char") || lowercaseTypeNmae.startsWith("varchar") || lowercaseTypeNmae.contains("text") || lowercaseTypeNmae.contains("blob")) {
        return Type.STRING;
    } else if (lowercaseTypeNmae.equals("tinyint")) {
        return Type.INT8;
    } else if (lowercaseTypeNmae.equals("smallint")) {
        return Type.INT16;
    } else if (lowercaseTypeNmae.equals("mediumint") || lowercaseTypeNmae.equals("int")) {
        return Type.INT32;
    } else if (lowercaseTypeNmae.equals("bigint")) {
        return Type.INT64;
    } else if (lowercaseTypeNmae.equals("bigint")) {
        return Type.INT64;
    } else if (lowercaseTypeNmae.contains("float")) {
        return Type.FLOAT;
    } else if (lowercaseTypeNmae.contains("double")) {
        return Type.DOUBLE;
    } else if (lowercaseTypeNmae.contains("double")) {
        return Type.DOUBLE;
    } else if (lowercaseTypeNmae.contains("date") || lowercaseTypeNmae.contains("time") || lowercaseTypeNmae.contains("datetime") || lowercaseTypeNmae.contains("timestamp")) {
        return Type.STRING;
    } else {
        return null;
    }
}
 
Example 22
Source Project: datacollector   Source File: TestKuduRecordConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUpdate() throws Exception {
  record.set("/str", Field.create("val1"));
  record.set("/long", Field.create((long)10));
  record.set("/short1", Field.create(Field.Type.SHORT, null));
  kuduRecordConverter.convert(record, partialRow, KuduOperationType.UPDATE.code); // must not throw NPE
}
 
Example 23
Source Project: flink-learning   Source File: KuduColumnInfo.java    License: Apache License 2.0 5 votes vote down vote up
private KuduColumnInfo(String name, Type type) {
    this.name = name;
    this.type = type;
    this.blockSize = 0;
    this.key = false;
    this.rangeKey = false;
    this.hashKey = false;
    this.nullable = false;
    this.defaultValue = null;
    this.encoding = Encoding.AUTO;
    this.compression = Compression.DEFAULT;
}
 
Example 24
Source Project: nifi   Source File: KuduLookupService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Optional<Record> lookup(Map<String, Object> coordinates) {

    //Scanner
    KuduScanner.KuduScannerBuilder builder = kuduClient.newScannerBuilder(table);

    builder.setProjectedColumnNames(columnNames);
    builder.replicaSelection(replicaSelection);

    //Only expecting one match
    builder.limit(1);

    coordinates.forEach((key,value)->
            builder.addPredicate(KuduPredicate.newComparisonPredicate(tableSchema.getColumn(key), KuduPredicate.ComparisonOp.EQUAL, value))
    );

    KuduScanner kuduScanner = builder.build();

    //Run lookup
    for ( RowResult row : kuduScanner){
        final Map<String, Object> values = new HashMap<>();
        for(String columnName : columnNames){
            Object object;
            if(row.getColumnType(columnName) == Type.BINARY){
                object = Base64.getEncoder().encodeToString(row.getBinaryCopy(columnName));
            } else {
                object = row.getObject(columnName);
            }
            values.put(columnName, object);
        }
        return Optional.of(new MapRecord(resultSchema, values));
    }

    //No match
    return Optional.empty();
}
 
Example 25
Source Project: bahir-flink   Source File: KuduTableFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTimestamp() throws Exception {
    // Timestamp should be bridged to sql.Timestamp
    // Test it when creating the table...
    tableEnv.sqlUpdate("CREATE TABLE TestTableTs (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTableTs', 'kudu.masters'='" + kuduMasters + "', " +
            "'kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");
    tableEnv.sqlUpdate("INSERT INTO TestTableTs values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')");
    tableEnv.execute("test");

    // And also when inserting into existing table
    tableEnv.sqlUpdate("CREATE TABLE TestTableTsE (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTableTs', 'kudu.masters'='" + kuduMasters + "')");

    tableEnv.sqlUpdate("INSERT INTO TestTableTsE values ('s', TIMESTAMP '2020-02-02 23:23:23')");
    tableEnv.execute("test");

    KuduTable kuduTable = harness.getClient().openTable("TestTableTs");
    assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    HashSet<Timestamp> results = new HashSet<>();
    scanner.forEach(sc -> results.add(sc.getTimestamp("second")));

    assertEquals(2, results.size());
    List<Timestamp> expected = Lists.newArrayList(
            Timestamp.valueOf("2020-01-01 12:12:12.123"),
            Timestamp.valueOf("2020-02-02 23:23:23"));
    assertEquals(new HashSet<>(expected), results);
}
 
Example 26
Source Project: datacollector   Source File: KuduUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a field and assign a value off of RowResult.
 * @param result Result obtained from scan
 * @param fieldName Field name to create
 * @param type Kudu Type for the field
 * @return Generated field
 * @throws StageException
 */
public static Field createField(RowResult result, String fieldName, Type type) throws StageException {
  switch (type) {
    case INT8:
      return Field.create(Field.Type.BYTE, result.getByte(fieldName));
    case INT16:
      return Field.create(Field.Type.SHORT, result.getShort(fieldName));
    case INT32:
      return Field.create(Field.Type.INTEGER, result.getInt(fieldName));
    case INT64:
      return Field.create(Field.Type.LONG, result.getLong(fieldName));
    case BINARY:
      try {
        ByteBuffer bBuffer = result.getBinary(fieldName);
        byte[] bArray = new byte[bBuffer.remaining()];
        bBuffer.get(bArray);

        return Field.create(Field.Type.BYTE_ARRAY, bArray);
      } catch (IllegalArgumentException ex) {
        throw new OnRecordErrorException(Errors.KUDU_35, fieldName);
      }
    case STRING:
      return Field.create(Field.Type.STRING, result.getString(fieldName));
    case BOOL:
      return Field.create(Field.Type.BOOLEAN, result.getBoolean(fieldName));
    case FLOAT:
      return Field.create(Field.Type.FLOAT, result.getFloat(fieldName));
    case DOUBLE:
      return Field.create(Field.Type.DOUBLE, result.getDouble(fieldName));
    case UNIXTIME_MICROS:
      //UNIXTIME_MICROS is in microsecond
      return Field.create(Field.Type.DATETIME, new Date(result.getLong(fieldName)/1000L));
    default:
      if ("DECIMAL".equals(type.name())) {
        return Field.create(Field.Type.DECIMAL, result.getDecimal(fieldName));
      }
      throw new StageException(Errors.KUDU_10, fieldName, type.getName());
  }
}
 
Example 27
Source Project: syndesis   Source File: KuduProducerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Ignore
public void createTable() throws InterruptedException, KuduException {
    deleteTestTable(TABLE, HOST + ":" + PORT);

    errorEndpoint.expectedMessageCount(0);
    successEndpoint.expectedMessageCount(1);

    final Map<String, Object> headers = new HashMap<>();

    final List<ColumnSchema> columns = new ArrayList<>(5);
    final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

    for (int i = 0; i < columnNames.size(); i++) {
        columns.add(
            new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), Type.STRING)
                .key(i == 0)
                .build());
    }

    final List<String> rangeKeys = new ArrayList<>();
    rangeKeys.add("id");

    headers.put("Schema", new Schema(columns));
    headers.put("TableOptions", new CreateTableOptions().setRangePartitionColumns(rangeKeys));

    requestBodyAndHeaders("direct://create", null, headers);

    errorEndpoint.assertIsSatisfied();
    successEndpoint.assertIsSatisfied();
}
 
Example 28
Source Project: presto-kudu   Source File: NativeKuduClientSession.java    License: Apache License 2.0 5 votes vote down vote up
private ColumnSchema toColumnSchema(ColumnMetadata columnMetadata, Map<String, ColumnDesign> columnDesignMap) {
    String name = columnMetadata.getName();
    ColumnDesign design = columnDesignMap.getOrDefault(name, ColumnDesign.DEFAULT);
    Type ktype = TypeHelper.toKuduClientType(columnMetadata.getType());
    ColumnSchema.ColumnSchemaBuilder builder = new ColumnSchema.ColumnSchemaBuilder(name, ktype);
    builder.key(design.isKey()).nullable(design.isNullable());
    setEncoding(name, builder, design);
    setCompression(name, builder, design);
    setTypeAttributes(columnMetadata, builder);
    return builder.build();
}
 
Example 29
Source Project: presto-kudu   Source File: NativeKuduClientSession.java    License: Apache License 2.0 5 votes vote down vote up
private KuduPredicate createComparisonPredicate(ColumnSchema columnSchema,
                                                KuduPredicate.ComparisonOp op,
                                                Object value) {
    com.facebook.presto.spi.type.Type type = TypeHelper.fromKuduColumn(columnSchema);
    Object javaValue = TypeHelper.getJavaValue(type, value);
    if (javaValue instanceof Long) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Long) javaValue);
    } else if (javaValue instanceof Integer) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Integer) javaValue);
    } else if (javaValue instanceof Short) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Short) javaValue);
    } else if (javaValue instanceof Byte) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Byte) javaValue);
    } else if (javaValue instanceof String) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (String) javaValue);
    } else if (javaValue instanceof Double) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Double) javaValue);
    } else if (javaValue instanceof Float) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Float) javaValue);
    } else if (javaValue instanceof Boolean) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Boolean) javaValue);
    } else if (javaValue instanceof byte[]) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (byte[]) javaValue);
    } else if (javaValue == null) {
        throw new IllegalStateException("Unexpected null java value for column " + columnSchema.getName());
    } else {
        throw new IllegalStateException("Unexpected java value for column "
                + columnSchema.getName() + ": " + javaValue + "(" + javaValue.getClass() + ")");
    }
}
 
Example 30
Source Project: presto-kudu   Source File: KuduTableProperties.java    License: Apache License 2.0 5 votes vote down vote up
private static byte[] toByteArray(Object obj, Type type, String name) {
    if (obj instanceof byte[]) {
        return (byte[]) obj;
    } else if (obj instanceof String) {
        return Base64.getDecoder().decode((String) obj);
    } else {
        handleInvalidValue(name, type, obj);
        return null;
    }
}