org.apache.kudu.Type Java Examples

The following examples show how to use org.apache.kudu.Type. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractKuduTest.java    From syndesis with Apache License 2.0 6 votes vote down vote up
protected void createTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final List<ColumnSchema> columns = new ArrayList<>(5);
        final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

        for (int i = 0; i < columnNames.size(); i++) {
            final Type type = i == 0 ? Type.INT32 : Type.STRING;
            columns.add(
                new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), type)
                    .key(i == 0)
                    .build());
        }

        final List<String> rangeKeys = new ArrayList<>();
        rangeKeys.add("id");

        client.createTable(tableName,
            new Schema(columns),
            new CreateTableOptions().setRangePartitionColumns(rangeKeys));
    }
}
 
Example #2
Source File: KuduTableProperties.java    From presto with Apache License 2.0 6 votes vote down vote up
private static long toUnixTimeMicros(Object obj, Type type, String name)
{
    if (Number.class.isAssignableFrom(obj.getClass())) {
        return ((Number) obj).longValue();
    }
    else if (obj instanceof String) {
        String s = (String) obj;
        s = s.trim().replace(' ', 'T');
        long millis = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC).parseMillis(s);
        return millis * 1000;
    }
    else {
        handleInvalidValue(name, type, obj);
        return 0;
    }
}
 
Example #3
Source File: KuduTestBase.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
public static KuduTableInfo booksTableInfo(String tableName, boolean createIfNotExist) {

        KuduTableInfo tableInfo = KuduTableInfo.forTable(tableName);

        if (createIfNotExist) {
            ColumnSchemasFactory schemasFactory = () -> {
                List<ColumnSchema> schemas = new ArrayList<>();
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("title", Type.STRING).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("author", Type.STRING).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("price", Type.DOUBLE).nullable(true).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("quantity", Type.INT32).nullable(true).build());
                return schemas;
            };

            tableInfo.createTableIfNotExists(
                    schemasFactory,
                    () -> new CreateTableOptions()
                            .setNumReplicas(1)
                            .addHashPartitions(Lists.newArrayList("id"), 2));
        }

        return tableInfo;
    }
 
Example #4
Source File: KuduUtils.java    From datacollector with Apache License 2.0 6 votes vote down vote up
/**
 * Convert from Kudu type to SDC Field type
 * @param kuduType
 * @return Field.Type
 */
public static Field.Type convertFromKuduType(Type kuduType){
  switch(kuduType) {
    case BINARY: return Field.Type.BYTE_ARRAY;
    case BOOL: return Field.Type.BOOLEAN;
    case DOUBLE: return Field.Type.DOUBLE;
    case FLOAT: return Field.Type.FLOAT;
    case INT8: return Field.Type.BYTE;
    case INT16: return Field.Type.SHORT;
    case INT32: return Field.Type.INTEGER;
    case INT64: return Field.Type.LONG;
    case STRING: return  Field.Type.STRING;
    case UNIXTIME_MICROS: return Field.Type.DATETIME;
    default:
      if ("DECIMAL".equals(kuduType.name())) {
        return Field.Type.DECIMAL;
      }
      throw new UnsupportedOperationException("Unknown data type: " + kuduType.getName());
  }
}
 
Example #5
Source File: KuduCatalogTest.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testTimestamp() throws Exception {
    tableEnv.sqlUpdate("CREATE TABLE TestTableTsC (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");
    tableEnv.sqlUpdate("INSERT INTO TestTableTsC values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')");
    tableEnv.execute("test");

    KuduTable kuduTable = harness.getClient().openTable("TestTableTsC");
    assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString(0));
    assertEquals(Timestamp.valueOf("2020-01-01 12:12:12.123"), rows.get(0).getTimestamp(1));
}
 
Example #6
Source File: SchemaEmulationByTableNameConvention.java    From presto with Apache License 2.0 6 votes vote down vote up
private void createAndFillSchemasTable(KuduClient client)
        throws KuduException
{
    List<String> existingSchemaNames = listSchemaNamesFromTablets(client);
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            insert.getRow().addString(0, schemaName);
            session.apply(insert);
        }
    }
    finally {
        session.close();
    }
}
 
Example #7
Source File: TestKuduTarget.java    From datacollector with Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that if given field is null and column doesn't support that, the record will
 * end up in error stream rather then terminating whole pipeline execution.
 */
@Test
public void testNullColumnWillEndInError() throws Exception{
  TargetRunner targetRunner = getTargetRunner(tableName, KuduOperationType.INSERT, UnsupportedOperationAction.SEND_TO_ERROR);
  targetRunner.runInit();

  Record record =  RecordCreator.create();
  LinkedHashMap<String, Field> field = new LinkedHashMap<>();
  field.put("key", Field.create(1));
  field.put("value", Field.create(Field.Type.STRING, null));
  field.put("name", Field.create(Field.Type.STRING, null));
  record.set(Field.createListMap(field));

  try {
    targetRunner.runWrite(ImmutableList.of(record));

    List<Record> errors = targetRunner.getErrorRecords();
    Assert.assertEquals(1, errors.size());
  } finally {
    targetRunner.runDestroy();
  }
}
 
Example #8
Source File: KuduTableProperties.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
private static Object toValue(Schema schema, PartialRow bound, Integer idx) {
    Type type = schema.getColumnByIndex(idx).getType();
    switch (type) {
        case UNIXTIME_MICROS:
            long millis = bound.getLong(idx) / 1000;
            return ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC).print(millis);
        case STRING:
            return bound.getString(idx);
        case INT64:
            return bound.getLong(idx);
        case INT32:
            return bound.getInt(idx);
        case INT16:
            return bound.getShort(idx);
        case INT8:
            short s = bound.getByte(idx);
            return s;
        case BOOL:
            return bound.getBoolean(idx);
        case BINARY:
            return bound.getBinaryCopy(idx);
        default:
            throw new IllegalStateException("Unhandled type " + type + " for range partition");
    }
}
 
Example #9
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
private void createAndFillSchemasTable() throws KuduException {
    List<String> existingSchemaNames = listSchemaNamesFromTablets();
    ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING)
            .key(true).build();
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.setNumReplicas(1); // TODO config
    options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            fillSchemaRow(insert.getRow(), schemaName);
            session.apply(insert);
        }
    } finally {
        session.close();
    }
}
 
Example #10
Source File: KuduTemplate.java    From canal with Apache License 2.0 6 votes vote down vote up
/**
 * kudu数据类型映射
 *
 * @param
 */
private Type toKuduType(String mysqlType) throws IllegalArgumentException {

    switch (mysqlType) {
        case "varchar":
            return Type.STRING;
        case "int":
            return Type.INT8;
        case "decimal":
            return Type.DOUBLE;
        case "double":
            return Type.DOUBLE;
        case "datetime":
            return Type.STRING;
        case "timestamp":
            return Type.STRING;
        default:
            throw new IllegalArgumentException("The provided data type doesn't map to know any known one.");
    }
}
 
Example #11
Source File: KuduToBigQueryTest.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up
/**
 * The schemas for the main input of the kudu rows transformation. Checking all possible supported
 * data types
 */
private Schema createTestKuduSchema() {
  List<ColumnSchema> columns =
      ImmutableList.<ColumnSchema>builder()
          .add(new ColumnSchema.ColumnSchemaBuilder("INT32", Type.INT32).key(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("BOOL", Type.BOOL).nullable(false).build())
          .add(
              new ColumnSchema.ColumnSchemaBuilder("DOUBLE", Type.DOUBLE).nullable(false).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("FLOAT", Type.FLOAT).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT8", Type.INT8).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT16", Type.INT16).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT64", Type.INT64).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("STRING", Type.STRING).nullable(true).build())
          .add(
              new ColumnSchema.ColumnSchemaBuilder("UNIXTIME_MICROS", Type.UNIXTIME_MICROS)
                  .nullable(true)
                  .build())
          .build();

  return new Schema(columns);
}
 
Example #12
Source File: TestKuduRecordConverter.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testDelete() throws Exception {
  record.set("/str", Field.create("primary key"));
  record.set("/long", Field.create((long)10));
  record.set("/short1", Field.create(Field.Type.SHORT, null));
  kuduRecordConverter.convert(record, partialRow, KuduOperationType.DELETE.code); // must not throw NPE
  Assert.assertTrue(Utils.format("Message: {}", partialRow.stringifyRowKey()), partialRow.stringifyRowKey().contains("primary key"));
}
 
Example #13
Source File: KuduUtils.java    From DataLink with Apache License 2.0 5 votes vote down vote up
public static Type getKuduType(String typeName) {
    if (typeName == null || typeName.trim() == "") {
        return null;
    }

    String lowercaseTypeNmae = typeName.toLowerCase();

    if (lowercaseTypeNmae.startsWith("char") || lowercaseTypeNmae.startsWith("varchar") || lowercaseTypeNmae.contains("text") || lowercaseTypeNmae.contains("blob")) {
        return Type.STRING;
    } else if (lowercaseTypeNmae.equals("tinyint")) {
        return Type.INT8;
    } else if (lowercaseTypeNmae.equals("smallint")) {
        return Type.INT16;
    } else if (lowercaseTypeNmae.equals("mediumint") || lowercaseTypeNmae.equals("int")) {
        return Type.INT32;
    } else if (lowercaseTypeNmae.equals("bigint")) {
        return Type.INT64;
    } else if (lowercaseTypeNmae.equals("bigint")) {
        return Type.INT64;
    } else if (lowercaseTypeNmae.contains("float")) {
        return Type.FLOAT;
    } else if (lowercaseTypeNmae.contains("double")) {
        return Type.DOUBLE;
    } else if (lowercaseTypeNmae.contains("double")) {
        return Type.DOUBLE;
    } else if (lowercaseTypeNmae.contains("date") || lowercaseTypeNmae.contains("time") || lowercaseTypeNmae.contains("datetime") || lowercaseTypeNmae.contains("timestamp")) {
        return Type.STRING;
    } else {
        return null;
    }
}
 
Example #14
Source File: TestKuduRecordConverter.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testUpdate() throws Exception {
  record.set("/str", Field.create("val1"));
  record.set("/long", Field.create((long)10));
  record.set("/short1", Field.create(Field.Type.SHORT, null));
  kuduRecordConverter.convert(record, partialRow, KuduOperationType.UPDATE.code); // must not throw NPE
}
 
Example #15
Source File: TestKuduLookup.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
  // Sample table and schema
  List<ColumnSchema> columns = new ArrayList(2);
  columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
  columns.add(new ColumnSchema.ColumnSchemaBuilder("value", Type.STRING).build());
  columns.add(new ColumnSchema.ColumnSchemaBuilder("name", Type.STRING).build());
  final Schema schema = new Schema(columns);

  // Mock KuduTable class
  KuduTable table = PowerMockito.mock(KuduTable.class);
  PowerMockito.doReturn(schema).when(table).getSchema();

  // Mock KuduSession class
  PowerMockito.suppress(PowerMockito.method(
      AsyncKuduSession.class,
      "apply",
      Operation.class
  ));
  PowerMockito.suppress(PowerMockito.method(
      AsyncKuduSession.class,
      "flush"
  ));
  PowerMockito.suppress(PowerMockito.method(
      KuduLookupProcessor.class,
      "destroy"
  ));
}
 
Example #16
Source File: KuduMetadataRow.java    From geowave with Apache License 2.0 5 votes vote down vote up
KuduMetadataField(
    final String fieldName,
    final Type dataType,
    final KuduColumnType columnType) {
  this.fieldName = fieldName;
  this.dataType = dataType;
  this.columnType = columnType;
}
 
Example #17
Source File: KuduTableProperties.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
private static Number toNumber(Object obj, Type type, String name) {
    if (Number.class.isAssignableFrom(obj.getClass())) {
        return (Number) obj;
    } else if (obj instanceof String) {
        String s = (String) obj;
        BigDecimal d = new BigDecimal((String) obj);
        return d;
    } else {
        handleInvalidValue(name, type, obj);
        return 0;
    }
}
 
Example #18
Source File: KuduTableProperties.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
private static long toUnixTimeMicros(Object obj, Type type, String name) {
    if (Number.class.isAssignableFrom(obj.getClass())) {
        return ((Number) obj).longValue();
    } else if (obj instanceof String) {
        String s = (String) obj;
        s = s.trim().replace(' ', 'T');
        long millis = ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC).parseMillis(s);
        return millis * 1000;
    } else {
        handleInvalidValue(name, type, obj);
        return 0;
    }
}
 
Example #19
Source File: KuduLookupService.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public Optional<Record> lookup(Map<String, Object> coordinates) {

    //Scanner
    KuduScanner.KuduScannerBuilder builder = kuduClient.newScannerBuilder(table);

    builder.setProjectedColumnNames(columnNames);
    builder.replicaSelection(replicaSelection);

    //Only expecting one match
    builder.limit(1);

    coordinates.forEach((key,value)->
            builder.addPredicate(KuduPredicate.newComparisonPredicate(tableSchema.getColumn(key), KuduPredicate.ComparisonOp.EQUAL, value))
    );

    KuduScanner kuduScanner = builder.build();

    //Run lookup
    for ( RowResult row : kuduScanner){
        final Map<String, Object> values = new HashMap<>();
        for(String columnName : columnNames){
            Object object;
            if(row.getColumnType(columnName) == Type.BINARY){
                object = Base64.getEncoder().encodeToString(row.getBinaryCopy(columnName));
            } else {
                object = row.getObject(columnName);
            }
            values.put(columnName, object);
        }
        return Optional.of(new MapRecord(resultSchema, values));
    }

    //No match
    return Optional.empty();
}
 
Example #20
Source File: KuduTableProperties.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
private static boolean toBoolean(Object obj, Type type, String name) {
    if (obj instanceof Boolean) {
        return (Boolean) obj;
    } else if (obj instanceof String) {
        return Boolean.valueOf((String) obj);
    } else {
        handleInvalidValue(name, type, obj);
        return false;
    }
}
 
Example #21
Source File: KuduTableProperties.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
private static byte[] toByteArray(Object obj, Type type, String name) {
    if (obj instanceof byte[]) {
        return (byte[]) obj;
    } else if (obj instanceof String) {
        return Base64.getDecoder().decode((String) obj);
    } else {
        handleInvalidValue(name, type, obj);
        return null;
    }
}
 
Example #22
Source File: KuduTableProperties.java    From presto with Apache License 2.0 5 votes vote down vote up
private static Number toNumber(Object obj, Type type, String name)
{
    if (Number.class.isAssignableFrom(obj.getClass())) {
        return (Number) obj;
    }
    else if (obj instanceof String) {
        String s = (String) obj;
        BigDecimal d = new BigDecimal((String) obj);
        return d;
    }
    else {
        handleInvalidValue(name, type, obj);
        return 0;
    }
}
 
Example #23
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
private KuduPredicate createComparisonPredicate(ColumnSchema columnSchema,
                                                KuduPredicate.ComparisonOp op,
                                                Object value) {
    com.facebook.presto.spi.type.Type type = TypeHelper.fromKuduColumn(columnSchema);
    Object javaValue = TypeHelper.getJavaValue(type, value);
    if (javaValue instanceof Long) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Long) javaValue);
    } else if (javaValue instanceof Integer) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Integer) javaValue);
    } else if (javaValue instanceof Short) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Short) javaValue);
    } else if (javaValue instanceof Byte) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Byte) javaValue);
    } else if (javaValue instanceof String) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (String) javaValue);
    } else if (javaValue instanceof Double) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Double) javaValue);
    } else if (javaValue instanceof Float) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Float) javaValue);
    } else if (javaValue instanceof Boolean) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (Boolean) javaValue);
    } else if (javaValue instanceof byte[]) {
        return KuduPredicate.newComparisonPredicate(columnSchema, op, (byte[]) javaValue);
    } else if (javaValue == null) {
        throw new IllegalStateException("Unexpected null java value for column " + columnSchema.getName());
    } else {
        throw new IllegalStateException("Unexpected java value for column "
                + columnSchema.getName() + ": " + javaValue + "(" + javaValue.getClass() + ")");
    }
}
 
Example #24
Source File: TestPutKudu.java    From nifi with Apache License 2.0 5 votes vote down vote up
private PartialRow buildPartialRow(Long id, String name, Short age, String kuduIdName, String recordIdName, String airport_code, Boolean lowercaseFields) {
    final Schema kuduSchema = new Schema(Arrays.asList(
        new ColumnSchema.ColumnSchemaBuilder(kuduIdName, Type.INT64).key(true).build(),
        new ColumnSchema.ColumnSchemaBuilder("name", Type.STRING).nullable(true).build(),
        new ColumnSchema.ColumnSchemaBuilder("age", Type.INT16).nullable(false).build(),
        new ColumnSchema.ColumnSchemaBuilder("updated_at", Type.UNIXTIME_MICROS).nullable(false).build(),
        new ColumnSchema.ColumnSchemaBuilder("score", Type.DECIMAL).nullable(true).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().precision(9).scale(0).build()
        ).build(),
        new ColumnSchema.ColumnSchemaBuilder("airport_code", Type.VARCHAR).nullable(true).typeAttributes(
                new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(3).build()
        ).build()));

    final RecordSchema schema = new SimpleRecordSchema(Arrays.asList(
        new RecordField(recordIdName, RecordFieldType.BIGINT.getDataType()),
        new RecordField("name", RecordFieldType.STRING.getDataType()),
        new RecordField("age", RecordFieldType.SHORT.getDataType()),
        new RecordField("updated_at", RecordFieldType.TIMESTAMP.getDataType()),
        new RecordField("score", RecordFieldType.LONG.getDataType()),
        new RecordField("airport_code", RecordFieldType.STRING.getDataType())));

    Map<String, Object> values = new HashMap<>();
    PartialRow row = kuduSchema.newPartialRow();
    values.put(recordIdName, id);
    values.put("name", name);
    values.put("age", age);
    values.put("updated_at", new Timestamp(System.currentTimeMillis()));
    values.put("score", 10000L);
    values.put("airport_code", airport_code);
    processor.buildPartialRow(
        kuduSchema,
        row,
        new MapRecord(schema, values),
        schema.getFieldNames(),
            true,
        lowercaseFields
    );
    return row;
}
 
Example #25
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
private ColumnSchema toColumnSchema(ColumnMetadata columnMetadata, Map<String, ColumnDesign> columnDesignMap) {
    String name = columnMetadata.getName();
    ColumnDesign design = columnDesignMap.getOrDefault(name, ColumnDesign.DEFAULT);
    Type ktype = TypeHelper.toKuduClientType(columnMetadata.getType());
    ColumnSchema.ColumnSchemaBuilder builder = new ColumnSchema.ColumnSchemaBuilder(name, ktype);
    builder.key(design.isKey()).nullable(design.isNullable());
    setEncoding(name, builder, design);
    setCompression(name, builder, design);
    setTypeAttributes(columnMetadata, builder);
    return builder.build();
}
 
Example #26
Source File: KuduTableFactoryTest.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTimestamp() throws Exception {
    // Timestamp should be bridged to sql.Timestamp
    // Test it when creating the table...
    tableEnv.sqlUpdate("CREATE TABLE TestTableTs (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTableTs', 'kudu.masters'='" + kuduMasters + "', " +
            "'kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");
    tableEnv.sqlUpdate("INSERT INTO TestTableTs values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')");
    tableEnv.execute("test");

    // And also when inserting into existing table
    tableEnv.sqlUpdate("CREATE TABLE TestTableTsE (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTableTs', 'kudu.masters'='" + kuduMasters + "')");

    tableEnv.sqlUpdate("INSERT INTO TestTableTsE values ('s', TIMESTAMP '2020-02-02 23:23:23')");
    tableEnv.execute("test");

    KuduTable kuduTable = harness.getClient().openTable("TestTableTs");
    assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    HashSet<Timestamp> results = new HashSet<>();
    scanner.forEach(sc -> results.add(sc.getTimestamp("second")));

    assertEquals(2, results.size());
    List<Timestamp> expected = Lists.newArrayList(
            Timestamp.valueOf("2020-01-01 12:12:12.123"),
            Timestamp.valueOf("2020-02-02 23:23:23"));
    assertEquals(new HashSet<>(expected), results);
}
 
Example #27
Source File: KuduTableProperties.java    From presto with Apache License 2.0 5 votes vote down vote up
private static byte[] toByteArray(Object obj, Type type, String name)
{
    if (obj instanceof byte[]) {
        return (byte[]) obj;
    }
    else if (obj instanceof String) {
        return Base64.getDecoder().decode((String) obj);
    }
    else {
        handleInvalidValue(name, type, obj);
        return null;
    }
}
 
Example #28
Source File: KuduUtils.java    From datacollector with Apache License 2.0 5 votes vote down vote up
/**
 * Create a field and assign a value off of RowResult.
 * @param result Result obtained from scan
 * @param fieldName Field name to create
 * @param type Kudu Type for the field
 * @return Generated field
 * @throws StageException
 */
public static Field createField(RowResult result, String fieldName, Type type) throws StageException {
  switch (type) {
    case INT8:
      return Field.create(Field.Type.BYTE, result.getByte(fieldName));
    case INT16:
      return Field.create(Field.Type.SHORT, result.getShort(fieldName));
    case INT32:
      return Field.create(Field.Type.INTEGER, result.getInt(fieldName));
    case INT64:
      return Field.create(Field.Type.LONG, result.getLong(fieldName));
    case BINARY:
      try {
        ByteBuffer bBuffer = result.getBinary(fieldName);
        byte[] bArray = new byte[bBuffer.remaining()];
        bBuffer.get(bArray);

        return Field.create(Field.Type.BYTE_ARRAY, bArray);
      } catch (IllegalArgumentException ex) {
        throw new OnRecordErrorException(Errors.KUDU_35, fieldName);
      }
    case STRING:
      return Field.create(Field.Type.STRING, result.getString(fieldName));
    case BOOL:
      return Field.create(Field.Type.BOOLEAN, result.getBoolean(fieldName));
    case FLOAT:
      return Field.create(Field.Type.FLOAT, result.getFloat(fieldName));
    case DOUBLE:
      return Field.create(Field.Type.DOUBLE, result.getDouble(fieldName));
    case UNIXTIME_MICROS:
      //UNIXTIME_MICROS is in microsecond
      return Field.create(Field.Type.DATETIME, new Date(result.getLong(fieldName)/1000L));
    default:
      if ("DECIMAL".equals(type.name())) {
        return Field.create(Field.Type.DECIMAL, result.getDecimal(fieldName));
      }
      throw new StageException(Errors.KUDU_10, fieldName, type.getName());
  }
}
 
Example #29
Source File: KuduProducerTest.java    From syndesis with Apache License 2.0 5 votes vote down vote up
@Ignore
public void createTable() throws InterruptedException, KuduException {
    deleteTestTable(TABLE, HOST + ":" + PORT);

    errorEndpoint.expectedMessageCount(0);
    successEndpoint.expectedMessageCount(1);

    final Map<String, Object> headers = new HashMap<>();

    final List<ColumnSchema> columns = new ArrayList<>(5);
    final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

    for (int i = 0; i < columnNames.size(); i++) {
        columns.add(
            new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), Type.STRING)
                .key(i == 0)
                .build());
    }

    final List<String> rangeKeys = new ArrayList<>();
    rangeKeys.add("id");

    headers.put("Schema", new Schema(columns));
    headers.put("TableOptions", new CreateTableOptions().setRangePartitionColumns(rangeKeys));

    requestBodyAndHeaders("direct://create", null, headers);

    errorEndpoint.assertIsSatisfied();
    successEndpoint.assertIsSatisfied();
}
 
Example #30
Source File: KuduColumnInfo.java    From flink-learning with Apache License 2.0 5 votes vote down vote up
private KuduColumnInfo(String name, Type type) {
    this.name = name;
    this.type = type;
    this.blockSize = 0;
    this.key = false;
    this.rangeKey = false;
    this.hashKey = false;
    this.nullable = false;
    this.defaultValue = null;
    this.encoding = Encoding.AUTO;
    this.compression = Compression.DEFAULT;
}