org.apache.kudu.client.AsyncKuduClient Java Examples

@Override
public boolean start() throws IOException {
  LOG.debug("Starting Kudu reader");
  client =
      new AsyncKuduClient.AsyncKuduClientBuilder(source.spec.getMasterAddresses())
          .build()
          .syncClient();

  if (source.serializedToken != null) {
    // tokens available if the source is already split
    scanner = KuduScanToken.deserializeIntoScanner(source.serializedToken, client);
  } else {
    KuduTable table = client.openTable(source.spec.getTable());
    KuduScanner.KuduScannerBuilder builder =
        table.getAsyncClient().syncClient().newScannerBuilder(table);

    configureBuilder(source.spec, table.getSchema(), builder);
    scanner = builder.build();
  }

  return advance();
}
 

@BeforeClass
public static void setUp() throws KuduException {
  PipelineOptionsFactory.register(KuduPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(KuduPipelineOptions.class);

  // synchronous operations
  client =
      new AsyncKuduClient.AsyncKuduClientBuilder(options.getKuduMasterAddresses())
          .build()
          .syncClient();

  if (client.tableExists(options.getKuduTable())) {
    client.deleteTable(options.getKuduTable());
  }

  kuduTable =
      client.createTable(options.getKuduTable(), KuduTestUtils.SCHEMA, createTableOptions());
}
 

Tagsets(AsyncKuduClient client, Tags tags, KuduTable tagsetsTable) {
  this.client = client;
  this.tagsetsTable = tagsetsTable;
  this.tags = tags;
  this.columnIndexes = ImmutableList.of(Tables.TAGSETS_ID_INDEX,
                                        Tables.TAGSETS_TAGSET_INDEX);
  this.tagsets = CacheBuilder.newBuilder()
      .maximumSize(1024 * 1024)
      .build(new CacheLoader<SerializedTagset, Deferred<Integer>>() {
        @Override
        public Deferred<Integer> load(SerializedTagset tagset) {
          return lookupOrInsertTagset(tagset, hashForTesting == null ?
                                              tagset.hashCode() : hashForTesting);
        }
      });
}
 

@Test
public void testConnectionFailure() throws Exception{
  // Mock connection refused
  PowerMockito.stub(
      PowerMockito.method(AsyncKuduClient.class, "getTablesList"))
      .toThrow(PowerMockito.mock(KuduException.class));
  ProcessorRunner runner = getProcessorRunner(tableName);

  try {
    List<Stage.ConfigIssue> issues = runner.runValidateConfigs();
    Assert.assertEquals(1, issues.size());
    Stage.ConfigIssue issue = issues.get(0);
    assertThat(issue.toString(), Matchers.containsIgnoringCase(KuduException.class.getSimpleName()));
  } catch (StageException e) {
    Assert.fail("should not throw StageException");
  }
}
 

/**
 * Check network connection to the kudu master.
 * @param kuduClient AsyncKuduClient
 * @param context
 * @param KUDU_MASTER
 * @param issues
 */
public static void checkConnection(AsyncKuduClient kuduClient,
                                   Context context,
                                   String KUDU_MASTER,
                                   final List<Stage.ConfigIssue> issues
){
  try {
    kuduClient.getTablesList().join();
  } catch (Exception ex) {
    issues.add(
        context.createConfigIssue(
            Groups.KUDU.name(),
            KuduLookupConfig.CONF_PREFIX + KUDU_MASTER ,
            Errors.KUDU_00,
            ex.toString(),
            ex
        )
    );
  }
}
 

/**
 * Opens a Kudu TS instance on a Kudu cluster.
 *
 * @param kuduMasterAddressess list of "host:port" pair master addresses
 * @param name the name of the Kudu timeseries store. Multiple instances of
 *             Kudu TS can occupy the same Kudu cluster by using a different name.
 * @return the opened {@code KuduTS}.
 * @throws Exception on error
 */
public static KuduTS open(List<String> kuduMasterAddressess, String name) throws Exception {
  AsyncKuduClient client = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasterAddressess).build();

  Deferred<KuduTable> metricsDeferred = client.openTable(Tables.metricsTableName(name));
  Deferred<KuduTable> tagsetsDeferred = client.openTable(Tables.tagsetsTableName(name));
  Deferred<KuduTable> tagsDeferred = client.openTable(Tables.tagsTableName(name));
  KuduTable metricsTable = metricsDeferred.join(client.getDefaultAdminOperationTimeoutMs());
  KuduTable tagsetsTable = tagsetsDeferred.join(client.getDefaultAdminOperationTimeoutMs());
  KuduTable tagsTable = tagsDeferred.join(client.getDefaultAdminOperationTimeoutMs());

  Tags tags = new Tags(client, tagsTable);
  Tagsets tagsets = new Tagsets(client, tags, tagsetsTable);
  Metrics metrics = new Metrics(client, metricsTable, tagsets);
  return new KuduTS(client, name, metrics, tagsets, tags);
}
 

private static Deferred<KuduTable> openOrCreateTable(final AsyncKuduClient client,
                                                     final String table,
                                                     final Schema schema,
                                                     final CreateTableOptions options) throws Exception {
  class CreateTableErrback implements Callback<Deferred<KuduTable>, Exception> {
    @Override
    public Deferred<KuduTable> call(Exception e) throws Exception {
      // TODO(danburkert): we should only do this if the error is "not found"
      LOG.debug("Creating table {}", table);
      return client.createTable(table, schema, options);
    }
    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this).add("table", table).toString();
    }
  }

  return client.openTable(table).addErrback(new CreateTableErrback());
}
 

public synchronized AsyncKuduClient getClient(final String kuduMaster) {
  if (kuduMaster == null) {
    LOGGER.error("Kudu Master server must be set for Kudu");
    return null;
  }
  return sessionCache.get(kuduMaster);
}
 

WriterImpl(KuduIO.Write<T> spec) throws KuduException {
  checkNotNull(spec.masterAddresses(), "masterAddresses cannot be null");
  checkNotNull(spec.table(), "table cannot be null");
  this.formatFunction = checkNotNull(spec.formatFn(), "formatFn cannot be null");
  client =
      new AsyncKuduClient.AsyncKuduClientBuilder(spec.masterAddresses()).build().syncClient();
  table = client.openTable(spec.table());
}
 

@Test
public void testCheckConnection() throws Exception{
  PowerMockito.stub(
      PowerMockito.method(AsyncKuduClient.class, "getTablesList"))
      .toThrow(PowerMockito.mock(KuduException.class));

  List<Stage.ConfigIssue> issues = new ArrayList<>();
  Context context = getContext();
  KuduUtils.checkConnection(client, context, KUDU_MASTER, issues);
  Assert.assertEquals(1, issues.size());
  Stage.ConfigIssue issue = issues.get(0);
  Assert.assertTrue(issues.get(0).toString().contains(Errors.KUDU_00.name()));
}
 

public KuduLookupLoader(Stage.Context context,
                        AsyncKuduClient kuduClient,
                        List<String> keyColumns,
                        Map<String, String> columnToField,
                        KuduLookupConfig conf
) {
  this.selectMeter = context.createMeter("Select Queries");
  this.selectTimer = context.createTimer("Select Queries");
  this.kuduClient = kuduClient;
  this.keyColumns = keyColumns;
  this.columnToField = columnToField;
  this.conf = conf;

  // output fields
  for (KuduOutputColumnMapping columnConfig : conf.outputColumnMapping) {
    String columnName = conf.caseSensitive ? columnConfig.columnName : columnConfig.columnName.toLowerCase();
    projectColumns.add(columnName);
    outputColumnToField.put(columnName, columnConfig.field);
    outputDefault.put(columnName, columnConfig.defaultValue);
  }
  // Build table cache.
  CacheBuilder cacheBuilder;
  if (conf.enableTableCache) {
    cacheBuilder = CacheBuilder.newBuilder().maximumSize(0);
  } else {
    cacheBuilder = CacheBuilder.newBuilder().maximumSize(conf.cacheSize);
  }

  tableCache = cacheBuilder.build(new CacheLoader<String, KuduTable>() {
    @Override
    public KuduTable load(String tableName) throws Exception {
      return kuduClient.openTable(tableName).join();
    }
  });
}
 

/**
 * Creates (if necessary) and opens a Kudu TS instance on a Kudu cluster.
 *
 * @param kuduMasterAddressess list of "host:port" pair master addresses
 * @param name the name of the Kudu timeseries store. Multiple instances of
 *             Kudu TS can occupy the same Kudu cluster by using a different name.
 * @return the opened {@code KuduTS}.
 * @throws Exception on error
 */
public static KuduTS openOrCreate(List<String> kuduMasterAddressess,
                                  String name,
                                  CreateOptions options) throws Exception {

  AsyncKuduClient client = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasterAddressess).build();

  int numTabletServers = client.listTabletServers()
                               .joinUninterruptibly(client.getDefaultAdminOperationTimeoutMs())
                               .getTabletServersCount();

  Deferred<KuduTable> metricsDeferred = openOrCreateTable(client,
                                                          Tables.metricsTableName(name),
                                                          Tables.METRICS_SCHEMA,
                                                          Tables.metricsCreateTableOptions(options, numTabletServers));
  Deferred<KuduTable> tagsetsDeferred = openOrCreateTable(client,
                                                          Tables.tagsetsTableName(name),
                                                          Tables.TAGSETS_SCHEMA,
                                                          Tables.tagsetsCreateTableOptions(options, numTabletServers));
  Deferred<KuduTable> tagsDeferred = openOrCreateTable(client,
                                                       Tables.tagsTableName(name),
                                                       Tables.TAGS_SCHEMA,
                                                       Tables.tagsCreateTableOptions(options, numTabletServers));
  KuduTable metricsTable = metricsDeferred.join(client.getDefaultAdminOperationTimeoutMs());
  KuduTable tagsetsTable = tagsetsDeferred.join(client.getDefaultAdminOperationTimeoutMs());
  KuduTable tagsTable = tagsDeferred.join(client.getDefaultAdminOperationTimeoutMs());

  Tags tags = new Tags(client, tagsTable);
  Tagsets tagsets = new Tagsets(client, tags, tagsetsTable);
  Metrics metrics = new Metrics(client, metricsTable, tagsets);
  return new KuduTS(client, name, metrics, tagsets, tags);
}
 

private KuduTS(AsyncKuduClient client,
               String name,
               Metrics metrics,
               Tagsets tagsets,
               Tags tags) {
  this.client = client;
  this.name = name;
  this.metrics = metrics;
  this.tags = tags;
  this.tagsets = tagsets;
}
 

public Metrics(AsyncKuduClient client, KuduTable table, Tagsets tagsets) {
  this.client = client;
  this.table = table;
  this.tagsets = tagsets;
}
 

@Before
public void setup() {
  client = new AsyncKuduClient.AsyncKuduClientBuilder(KUDU_MASTER).build();
}
 

public Tags(AsyncKuduClient client, KuduTable table) {
  this.client = client;
  this.table = table;
}
 

/** Creates a new synchronous client. */
private synchronized KuduClient getKuduClient(List<String> masterAddresses) {
  return new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses).build().syncClient();
}
 

/**
 * Returns the {@link AsyncKuduClient} being used by this {@code KuduTS}.
 * Package private because the client is internal.
 * @return the client instance
 */
AsyncKuduClient getClient() {
  return client;
}