Java Code Examples for org.apache.kudu.client.KuduClient#openTable()

public static List<ColumnMeta> getColumns(MediaSourceInfo info, String tableName) {
    checkKudu(info);

    KuduMediaSrcParameter parameter = info.getParameterObj();
    List<String> hostList = parameter.getKuduMasterConfigs().stream().
            map(config->{return MessageFormat.format("{0}:{1}",config.getHost(),String.valueOf(config.getPort()));}).
            collect(Collectors.toList());

    KuduClient client = createClient(hostList);
    try {
        KuduTable kuduTable = client.openTable(tableName);
        return getColumnMetaList(kuduTable);
    } catch (KuduException e){
        LOGGER.info("执行kudu查询时报错:",e);
        throw new RuntimeException(e);
    } finally{
        closeClient(client);
    }
}
 

@Test
public void test01() {
    List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
    KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
        .defaultSocketReadTimeoutMs(30000)
        .defaultAdminOperationTimeoutMs(60000)
        .build();
    try {
        List<String> tablesList = kuduClient.getTablesList().getTablesList();
        System.out.println(tablesList.toString());
        KuduTable web_white_list = kuduClient.openTable("web_white_list");

    } catch (KuduException e) {
        e.printStackTrace();
    }
}
 

private KuduTable getSchemasTable(KuduClient client)
        throws KuduException
{
    if (rawSchemasTable == null) {
        rawSchemasTable = client.openTable(rawSchemasTableName);
    }
    return rawSchemasTable;
}
 

public static MetaTable getMetaTable(List<String> masterAddresses, String tableName, String[] columns) throws Exception {
    KuduClient client = null;
    try {
        client = createClient(masterAddresses);
        KuduTable kuduTable = client.openTable(tableName);
        Schema schema = kuduTable.getSchema();
        List<ColumnSchema> columnSchemas = schema.getColumns();
        Set<String> realColumnNames = new HashSet<String>();
        for (ColumnSchema cs : columnSchemas) {
            realColumnNames.add(cs.getName());
        }
        Set<String> inputColumnNames = new HashSet<String>(Arrays.asList(columns));
        LOG.info("config column:"  + ArrayUtils.toString(inputColumnNames.toString()));
        LOG.info("kudu column:"  + ArrayUtils.toString(realColumnNames));
        inputColumnNames.removeAll(realColumnNames);

        //如果kudu不包含输入字段,抛出异常
        if (inputColumnNames.size() != 0) {
        	ErrorRecord.addError(String.format("字段配置与kudu实际不相符{%s}!",ArrayUtils.toString(inputColumnNames.toString())));
            throw new Exception(String.format("字段配置与kudu实际不相符{%s}!",ArrayUtils.toString(inputColumnNames.toString())));
        }

        return new MetaTable(columnSchemas);
    } catch (KuduException e) {
        throw e;
    } finally {
        closeClient(client);
    }
}
 

private static List<MediaMeta> execute(KuduClient client) throws KuduException {
    List<String> tables = client.getTablesList().getTablesList();
    List<MediaMeta> list = Lists.newArrayList();
    for (String tableName:tables){
        KuduTable kuduTable = client.openTable(tableName);
        list.add(parseKuduTable(kuduTable));
    }
    return list;
}
 

private static KuduTable getKuduTable(KuduClient client, KuduMediaSrcParameter kuduMediaSrcParameter, MediaMappingInfo mediaMappingInfo) throws KuduException {
    String database = kuduMediaSrcParameter.getDatabase();
    String tableName = mediaMappingInfo.getTargetMediaName();
    KuduTable kuduTable = client.openTable(String.format("%s.%s", database, tableName));
    return kuduTable;
}
 

private static ObjectSchema createSpec(KuduMetaData kuduMetaData) {
    // build the input and output schemas
    ObjectSchema spec = new ObjectSchema();

    spec.set$schema(JSON_SCHEMA_ORG_SCHEMA);
    spec.setTitle("KUDU_INSERT");

    Map<String, Object> options = new HashMap<>();
    options.put("host", kuduMetaData.getHost());
    options.put("port", kuduMetaData.getPort());
    KuduClient client = KuduSupport.createConnection(options);

    try {
        KuduTable table = client.openTable(kuduMetaData.getTableName());
        Iterator<ColumnSchema> columns = table.getSchema().getColumns().iterator();

        while (columns.hasNext()) {
            ColumnSchema column = columns.next();

            switch (column.getType().getName()) {
                case "string":
                    spec.putProperty(column.getName(), new JsonSchemaFactory().stringSchema());
                    break;
                case "bool":
                    spec.putProperty(column.getName(), new JsonSchemaFactory().booleanSchema());
                    break;
                case "float":
                case "double":
                case "int8":
                case "int16":
                case "int32":
                case "int64":
                    spec.putProperty(column.getName(), new JsonSchemaFactory().integerSchema());
                    break;
                default:
                    throw new SyndesisServerException("The column schema type " + column.getType().getName()
                            + " for column " + column.getName()
                            + " is not supported at the moment");
            }
        }
    } catch (KuduException e) {
        throw new SyndesisServerException("Unable to connect to kudu schema " + kuduMetaData.getTableName(), e);
    }

    return spec;
}
 

protected KuduTable getKuduTable(KuduClient client, String tableName) throws KuduException {
    return client.openTable(tableName);
}
 

@Before
public void init() throws Exception {
    testRunner = TestRunners.newTestRunner(SampleProcessor.class);
    testRunner.setValidateExpressionUsage(false);
    final String tableName = "table1";

    KuduClient client =  harness.getClient();
    List<ColumnSchema> columns = new ArrayList<>();
    columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build());
    columns.add(new ColumnSchema
            .ColumnSchemaBuilder("decimal", Type.DECIMAL)
            .typeAttributes(DecimalUtil.typeAttributes(DecimalUtil.MAX_DECIMAL64_PRECISION, 1))
            .build()
    );
    columns.add(new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int8", Type.INT8).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int16", Type.INT16).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int32", Type.INT32).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int64", Type.INT64).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("unixtime_micros", Type.UNIXTIME_MICROS).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("varchar_3", Type.VARCHAR).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(3).build()
    ).build());
    Schema schema = new Schema(columns);

    CreateTableOptions opts = new CreateTableOptions().setRangePartitionColumns(Collections.singletonList("string"));
    client.createTable(tableName, schema, opts);

    KuduTable table = client.openTable(tableName);
    KuduSession session = client.newSession();

    Insert insert = table.newInsert();
    PartialRow row = insert.getRow();
    row.addString("string", "string1");
    row.addBinary("binary", "binary1".getBytes());
    row.addBoolean("bool",true);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",0.2);
    row.addFloat("float",0.3f);
    row.addByte("int8", (byte) 1);
    row.addShort("int16", (short) 2);
    row.addInt("int32",3);
    row.addLong("int64",4L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis));
    row.addVarchar("varchar_3", "SFO");
    session.apply(insert);

    insert = table.newInsert();
    row = insert.getRow();
    row.addString("string", "string2");
    row.addBinary("binary", "binary2".getBytes());
    row.addBoolean("bool",false);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",1.2);
    row.addFloat("float",1.3f);
    row.addByte("int8", (byte) 11);
    row.addShort("int16", (short) 12);
    row.addInt("int32",13);
    row.addLong("int64",14L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis+(1000L * 60 * 60 * 24 * 365))); //+ 1 year
    row.addVarchar("varchar_3", "SJC");
    session.apply(insert);

    session.close();

    kuduLookupService = new KuduLookupService();
    testRunner.addControllerService("kuduLookupService", kuduLookupService);
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_MASTERS, "testLocalHost:7051");
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_REPLICA_SELECTION, KuduLookupService.LEADER_ONLY);
    testRunner.setProperty(kuduLookupService, KuduLookupService.TABLE_NAME, tableName);
    kuduLookupService.kuduClient = client;
}
 

@Test
public void testWriteKudu() throws IOException, InitializationException {
    final int recordCount = 100;
    final int numFlowFiles = 5;
    createRecordReader(recordCount);

    final String filename = "testWriteKudu-" + System.currentTimeMillis();

    final Map<String,String> flowFileAttributes = new HashMap<>();
    flowFileAttributes.put(CoreAttributes.FILENAME.key(), filename);

    // Use values to ensure multiple batches and multiple flow files per-trigger
    testRunner.setProperty(PutKudu.BATCH_SIZE, "10");
    testRunner.setProperty(PutKudu.FLOWFILE_BATCH_SIZE, "2");

    // Set the operation type.
    flowFileAttributes.put("kudu.operation.type", "upsert");
    testRunner.setProperty(PutKudu.INSERT_OPERATION, "${kudu.operation.type}");

    // Don't ignore null values.
    flowFileAttributes.put("kudu.ignore.null", "false");
    testRunner.setProperty(PutKudu.IGNORE_NULL, "${kudu.ignore.null}");

    // Enable lowercase handling.
    flowFileAttributes.put("kudu.lowercase.field.names", "true");
    testRunner.setProperty(PutKudu.LOWERCASE_FIELD_NAMES, "${kudu.lowercase.field.names}");

    // Enable schema drift handling.
    flowFileAttributes.put("kudu.handle.schema.drift", "true");
    testRunner.setProperty(PutKudu.HANDLE_SCHEMA_DRIFT, "${kudu.handle.schema.drift}");

    // Increase the thread count to better simulate a production environment
    testRunner.setThreadCount(4);

    // Trigger the flow
    IntStream.range(0, numFlowFiles).forEach(i ->
        testRunner.enqueue("trigger", flowFileAttributes));
    testRunner.run(numFlowFiles);
    testRunner.assertAllFlowFilesTransferred(PutKudu.REL_SUCCESS, numFlowFiles);

    // verify the successful flow file has the expected content & attributes
    final MockFlowFile mockFlowFile =
        testRunner.getFlowFilesForRelationship(PutKudu.REL_SUCCESS).get(0);
    mockFlowFile.assertAttributeEquals(CoreAttributes.FILENAME.key(), filename);
    mockFlowFile.assertAttributeEquals(PutKudu.RECORD_COUNT_ATTR, "100");
    mockFlowFile.assertContentEquals("trigger");

    // verify we generated provenance events
    final List<ProvenanceEventRecord> provEvents = testRunner.getProvenanceEvents();
    Assert.assertEquals(numFlowFiles, provEvents.size());

    // verify it was a SEND event with the correct URI
    final ProvenanceEventRecord provEvent = provEvents.get(0);
    Assert.assertEquals(ProvenanceEventType.SEND, provEvent.getEventType());

    KuduClient client = harness.getClient();
    KuduTable kuduTable = client.openTable(DEFAULT_TABLE_NAME);

    // Verify the extra field was added.
    Assert.assertEquals(7, kuduTable.getSchema().getColumnCount());
    Assert.assertTrue(kuduTable.getSchema().hasColumn("doubleval"));
    Assert.assertTrue(kuduTable.getSchema().hasColumn("floatval"));

    // Verify Kudu record count.
    KuduScanner scanner = client.newScannerBuilder(kuduTable).build();
    int count = 0;
    for (RowResult row : scanner) {
        Assert.assertEquals(NOW, row.getTimestamp("timestampval"));
        count++;
    }
    Assert.assertEquals(recordCount, count);
}