org.apache.kudu.client.Insert Java Examples

private void createAndFillSchemasTable(KuduClient client)
        throws KuduException
{
    List<String> existingSchemaNames = listSchemaNamesFromTablets(client);
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            insert.getRow().addString(0, schemaName);
            session.apply(insert);
        }
    }
    finally {
        session.close();
    }
}
 

protected void insertRowInTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final KuduTable table = client.openTable(tableName);

        final Insert insert = table.newInsert();
        final PartialRow row = insert.getRow();

        row.addInt("id", ThreadLocalRandom.current().nextInt(1, 99));
        row.addString("title", "Mr.");
        row.addString("name", "Samuel");
        row.addString("lastname", "Smith");
        row.addString("address", "4359  Plainfield Avenue");

        client.newSession().apply(insert);
    }
}
 

private void createAndFillSchemasTable() throws KuduException {
    List<String> existingSchemaNames = listSchemaNamesFromTablets();
    ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING)
            .key(true).build();
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.setNumReplicas(1); // TODO config
    options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            fillSchemaRow(insert.getRow(), schemaName);
            session.apply(insert);
        }
    } finally {
        session.close();
    }
}
 

private LinkedList<OperationResponse> queueInsert(MockPutKudu putKudu, KuduSession session, boolean sync, ResultCode... results) throws Exception {
    LinkedList<OperationResponse> responses = new LinkedList<>();
    for (ResultCode result : results) {
        boolean ok = result == OK;
        Tuple<Insert, OperationResponse> tuple = insert(ok);
        putKudu.queue(tuple.getKey());

        if (result == EXCEPTION) {
            when(session.apply(tuple.getKey())).thenThrow(mock(KuduException.class));
            // Stop processing the rest of the records on the first exception
            break;
        } else {
            responses.add(tuple.getValue());
            if (sync) {
                when(session.apply(tuple.getKey())).thenReturn(ok ? null : tuple.getValue());

                // In AUTO_FLUSH_SYNC mode, PutKudu immediately knows when an operation has failed.
                // In that case, it does not process the rest of the records in the FlowFile.
                if (result == FAIL) break;
            }
        }
    }
    return responses;
}
 

/**
 * Attempts to insert the provided tagset and ID. Returns {@code true} if the
 * write was successful, or {@code false} if the write failed due to a tagset
 * with the same ID already existing in the table.
 *
 * @param tagset the tagset to insert
 * @param id     the ID to insert the tagset with
 * @return whether the write succeeded
 */
private Deferred<Boolean> insertTagset(final SerializedTagset tagset, final int id) throws KuduException {
  final class InsertTagsetCB implements Callback<Deferred<Boolean>, OperationResponse> {
    @Override
    public Deferred<Boolean> call(OperationResponse response) {
      if (response.hasRowError()) {
        if (response.getRowError().getErrorStatus().isAlreadyPresent()) {
          LOG.info("Attempted to insert duplicate tagset; id: {}, tagset: {}", id, tagset);
          // TODO: Consider adding a backoff with jitter before attempting
          //       the insert again (if the lookup fails).
          return Deferred.fromResult(false);
        }
        return Deferred.fromError(new RuntimeException(
            String.format("Unable to insert tagset; id: %s, tagset: %s, error: %s",
                          id, tagset, response.getRowError())));
      } else {
        return Deferred.fromResult(true);
      }
    }
    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this).toString();
    }
  }

  LOG.debug("Inserting tagset; id: {}, tags: {}", id, tagset);
  final AsyncKuduSession session = client.newSession();
  try {
    // We don't have to handle PleaseThrottleException because we are only
    // inserting a single row.
    final Insert insert = tagsetsTable.newInsert();
    insert.getRow().addInt(Tables.TAGSETS_ID_INDEX, id);
    insert.getRow().addBinary(Tables.TAGSETS_TAGSET_INDEX, tagset.getBytes());
    return session.apply(insert).addCallbackDeferring(new InsertTagsetCB());
  } finally {
    session.close();
  }
}
 

@Override
public void write(final GeoWaveMetadata metadata) {
  try {
    final Insert insert = operations.getTable(tableName).newInsert();
    final PartialRow partialRow = insert.getRow();
    final KuduMetadataRow row = new KuduMetadataRow(metadata);
    row.populatePartialRow(partialRow);
    final OperationResponse resp = session.apply(insert);
    if (resp.hasRowError()) {
      LOGGER.error("Encountered error while writing metadata: {}", resp.getRowError());
    }
  } catch (final KuduException e) {
    LOGGER.error("Kudu error when writing metadata", e);
  }
}
 

@Override
protected Insert insertRecordToKudu(KuduTable kuduTable, Record record, List<String> fieldNames) throws IllegalStateException, Exception {
    Insert insert = kuduTable.newInsert();
    this.insert(kuduTable, insert, record, fieldNames);
    return insert;
}
 

/**
 * Insert a tagset into the {@code tags} table.
 * @param id the tagset ID.
 * @param tagset the tagset.
 * @return The tagset ID.
 */
public Deferred<Integer> insertTagset(final int id, final SortedMap<String, String> tagset)
    throws KuduException {
  if (tagset.isEmpty()) { return Deferred.fromResult(id); }
  LOG.debug("Inserting tags; tagsetID: {}, tags: {}", id, tagset);
  final AsyncKuduSession session = client.newSession();

  class InsertTagsetCB implements Callback<Deferred<Integer>, List<OperationResponse>> {
    @Override
    public Deferred<Integer> call(List<OperationResponse> responses) {
      try {
        for (OperationResponse response : responses) {
          if (response.hasRowError()) {
            return Deferred.fromError(new RuntimeException(
                String.format("Unable to insert tag: %s", response.getRowError())));
          }
        }
        return Deferred.fromResult(id);
      } finally {
        session.close();
      }
    }
    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this)
                        .add("id", id)
                        .add("tags", tagset)
                        .toString();
    }
  }

  if (tagset.size() > 1000) {
    session.setMutationBufferSpace(tagset.size());
  }
  session.setMutationBufferLowWatermark(1.0f);

  // buffer all of the tags into the session, and ensure that we don't get
  // a PleaseThrottleException. In practice the number of tags should be
  // small.
  session.setMutationBufferSpace(tagset.size());
  session.setMutationBufferLowWatermark(1.0f);
  session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
  for (Map.Entry<String, String> tag : tagset.entrySet()) {
    Insert insert = table.newInsert();
    // TODO: check with JD that if the inserts below fail, the error will
    // also be returned in the flush call.
    insert.getRow().addString(Tables.TAGS_KEY_INDEX, tag.getKey());
    insert.getRow().addString(Tables.TAGS_VALUE_INDEX, tag.getValue());
    insert.getRow().addInt(Tables.TAGS_TAGSET_ID_INDEX, id);
    session.apply(insert);
  }
  return session.flush().addCallbackDeferring(new InsertTagsetCB());
}
 

protected void processForInsert(KuduExecutionContext kuduExecutionContext)
{
  Insert thisInsert = kuduTable.newInsert();
  performCommonProcessing(thisInsert,kuduExecutionContext);
}
 

@Before
public void init() throws Exception {
    testRunner = TestRunners.newTestRunner(SampleProcessor.class);
    testRunner.setValidateExpressionUsage(false);
    final String tableName = "table1";

    KuduClient client =  harness.getClient();
    List<ColumnSchema> columns = new ArrayList<>();
    columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build());
    columns.add(new ColumnSchema
            .ColumnSchemaBuilder("decimal", Type.DECIMAL)
            .typeAttributes(DecimalUtil.typeAttributes(DecimalUtil.MAX_DECIMAL64_PRECISION, 1))
            .build()
    );
    columns.add(new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int8", Type.INT8).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int16", Type.INT16).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int32", Type.INT32).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int64", Type.INT64).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("unixtime_micros", Type.UNIXTIME_MICROS).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("varchar_3", Type.VARCHAR).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(3).build()
    ).build());
    Schema schema = new Schema(columns);

    CreateTableOptions opts = new CreateTableOptions().setRangePartitionColumns(Collections.singletonList("string"));
    client.createTable(tableName, schema, opts);

    KuduTable table = client.openTable(tableName);
    KuduSession session = client.newSession();

    Insert insert = table.newInsert();
    PartialRow row = insert.getRow();
    row.addString("string", "string1");
    row.addBinary("binary", "binary1".getBytes());
    row.addBoolean("bool",true);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",0.2);
    row.addFloat("float",0.3f);
    row.addByte("int8", (byte) 1);
    row.addShort("int16", (short) 2);
    row.addInt("int32",3);
    row.addLong("int64",4L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis));
    row.addVarchar("varchar_3", "SFO");
    session.apply(insert);

    insert = table.newInsert();
    row = insert.getRow();
    row.addString("string", "string2");
    row.addBinary("binary", "binary2".getBytes());
    row.addBoolean("bool",false);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",1.2);
    row.addFloat("float",1.3f);
    row.addByte("int8", (byte) 11);
    row.addShort("int16", (short) 12);
    row.addInt("int32",13);
    row.addLong("int64",14L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis+(1000L * 60 * 60 * 24 * 365))); //+ 1 year
    row.addVarchar("varchar_3", "SJC");
    session.apply(insert);

    session.close();

    kuduLookupService = new KuduLookupService();
    testRunner.addControllerService("kuduLookupService", kuduLookupService);
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_MASTERS, "testLocalHost:7051");
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_REPLICA_SELECTION, KuduLookupService.LEADER_ONLY);
    testRunner.setProperty(kuduLookupService, KuduLookupService.TABLE_NAME, tableName);
    kuduLookupService.kuduClient = client;
}
 

protected Insert insertRecordToKudu(KuduTable kuduTable, Record record, List<String> fieldNames, Boolean ignoreNull, Boolean lowercaseFields) {
    Insert insert = kuduTable.newInsert();
    buildPartialRow(kuduTable.getSchema(), insert.getRow(), record, fieldNames, ignoreNull, lowercaseFields);
    return insert;
}
 

public void queue(Insert... operations) {
    insertQueue.addAll(Arrays.asList(operations));
}
 

@Override
protected Insert insertRecordToKudu(KuduTable kuduTable, Record record, List<String> fieldNames, Boolean ignoreNull, Boolean lowercaseFields) {
    Insert insert = insertQueue.poll();
    return insert != null ? insert : mock(Insert.class);
}
 

protected abstract Insert insertRecordToKudu(final KuduTable table, final Record record, final List<String> fields) throws Exception;