org.apache.kudu.client.RowResult Java Examples

The following examples show how to use org.apache.kudu.client.RowResult. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KuduOutput.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Override
public Iterable<Row> getExistingForFilters(Iterable<Row> filters) throws Exception {
  List<Row> existingForFilters = Lists.newArrayList();

  if (!filters.iterator().hasNext()) {
    return existingForFilters;
  }

  KuduTable table = getConnection().getTable(config.getString(TABLE_CONFIG_NAME));
  KuduScanner scanner = scannerForFilters(filters, table);

  long startTime = System.nanoTime();
  while (scanner.hasMoreRows()) {
    for (RowResult rowResult : scanner.nextRows()) {
      Row existing = resultAsRow(rowResult, table);

      existingForFilters.add(existing);
    }
  }
  long endTime = System.nanoTime();
  if (hasAccumulators()) {
    accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_SCANNING).add((endTime - startTime) / 1000.0 / 1000.0 / 1000.0);
  }

  return existingForFilters;
}
 
Example #2
Source File: KuduIOIT.java    From beam with Apache License 2.0 6 votes vote down vote up
private void runReadWithPredicates() {
  PCollection<String> output =
      readPipeline.apply(
          "Read with predicates",
          KuduIO.<String>read()
              .withMasterAddresses(options.getKuduMasterAddresses())
              .withTable(options.getKuduTable())
              .withParseFn(
                  (SerializableFunction<RowResult, String>) input -> input.getString(COL_NAME))
              .withPredicates(
                  Arrays.asList(
                      KuduPredicate.newComparisonPredicate(
                          SCHEMA.getColumn(COL_ID), KuduPredicate.ComparisonOp.GREATER_EQUAL, 2),
                      KuduPredicate.newComparisonPredicate(
                          SCHEMA.getColumn(COL_ID), KuduPredicate.ComparisonOp.LESS, 7)))
              .withCoder(StringUtf8Coder.of()));

  output.apply(Count.globally());

  PAssert.thatSingleton(output.apply("Count", Count.globally())).isEqualTo((long) 5);

  readPipeline.run().waitUntilFinish();
}
 
Example #3
Source File: KuduConsumerTest.java    From syndesis with Apache License 2.0 6 votes vote down vote up
@Ignore
public void insertRow() throws KuduException, InterruptedException {
    MockEndpoint mock = getMockEndpoint("mock:result");
    mock.expectedMessageCount(1);

    assertMockEndpointsSatisfied();

    List<Exchange> exchanges = mock.getReceivedExchanges();
    assertEquals(1, exchanges.size());

    KuduScanner scanner = exchanges.get(0).getIn().getBody(KuduScanner.class);

    RowResultIterator results = scanner.nextRows();
    RowResult result = results.next();

    ColumnSchema columnByIndex = result.getSchema().getColumnByIndex(0);
    String name = columnByIndex.getName();

    assertEquals("id", name);
    assertEquals(46, result.getInt(0));
    assertEquals(10, result.getInt(1));
}
 
Example #4
Source File: TypeHelper.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
public static long getLong(Type type, RowResult row, int field) {
    if (type == TimestampType.TIMESTAMP) {
        return row.getLong(field) / 1000;
    } else if (type == BigintType.BIGINT) {
        return row.getLong(field);
    } else if (type == IntegerType.INTEGER) {
        return row.getInt(field);
    } else if (type == SmallintType.SMALLINT) {
        return row.getShort(field);
    } else if (type == TinyintType.TINYINT) {
        return row.getByte(field);
    } else if (type == RealType.REAL) {
        return floatToRawIntBits(row.getFloat(field));
    } else if (type instanceof DecimalType) {
        DecimalType dtype = (DecimalType) type;
        if (dtype.isShort()) {
            return row.getDecimal(field).unscaledValue().longValue();
        } else {
            throw new IllegalStateException("getLong not supported for long decimal: " + type);
        }
    } else {
        throw new IllegalStateException("getLong not implemented for " + type);
    }
}
 
Example #5
Source File: KuduToBigQuery.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up
@Override
public TableRow apply(RowResult input) {
  TableRow outputTableRow = new TableRow();
  Schema kuduSchema = input.getSchema();

  for (int i = 0; i < kuduSchema.getColumnCount(); i++) {
    String columnName = kuduSchema.getColumnByIndex(i).getName();

    if (kuduSchema.getColumnByIndex(i).getType() == Type.UNIXTIME_MICROS) {
      outputTableRow.set(columnName, convertNanoSecondsToMilliSeconds(input, i));
    } else {
      // For other datatypes return the object value
      outputTableRow.set(columnName, input.getObject(i));
    }
  }
  return outputTableRow;
}
 
Example #6
Source File: KuduInputOperatorCommons.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
public long countNumRowsInTable() throws Exception
{
  List<String> allProjectedCols = new ArrayList<>(
      unitTestStepwiseScanInputOperator.getKuduColNameToSchemaMapping().keySet());
  KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable)
      .setProjectedColumnNames(allProjectedCols)
      .build();
  long counter = 0;
  while (scanner.hasMoreRows()) {
    RowResultIterator rowResultItr = scanner.nextRows();
    while (rowResultItr.hasNext()) {
      RowResult thisRow = rowResultItr.next();
      counter += 1;
    }
  }
  return counter;
}
 
Example #7
Source File: KuduClientTestCommons.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
protected void lookUpAndPopulateRecord(UnitTestTablePojo keyInfo) throws Exception
{
  KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable)
      .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("introwkey"),
      KuduPredicate.ComparisonOp.EQUAL,keyInfo.getIntrowkey()))
      .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("stringrowkey"),
      KuduPredicate.ComparisonOp.EQUAL,keyInfo.getStringrowkey()))
      .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("timestamprowkey"),
      KuduPredicate.ComparisonOp.EQUAL,keyInfo.getTimestamprowkey()))
      .build();
  RowResultIterator rowResultItr = scanner.nextRows();
  while (rowResultItr.hasNext()) {
    RowResult thisRow = rowResultItr.next();
    keyInfo.setFloatdata(thisRow.getFloat("floatdata"));
    keyInfo.setBooldata(thisRow.getBoolean("booldata"));
    keyInfo.setBinarydata(thisRow.getBinary("binarydata"));
    keyInfo.setLongdata(thisRow.getLong("longdata"));
    keyInfo.setTimestampdata(thisRow.getLong("timestampdata"));
    keyInfo.setStringdata("stringdata");
    break;
  }
}
 
Example #8
Source File: SchemaEmulationByTableNameConvention.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public List<String> listSchemaNames(KuduClient client)
{
    try {
        if (rawSchemasTable == null) {
            if (!client.tableExists(rawSchemasTableName)) {
                createAndFillSchemasTable(client);
            }
            rawSchemasTable = getSchemasTable(client);
        }

        KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build();
        RowResultIterator iterator = scanner.nextRows();
        ArrayList<String> result = new ArrayList<>();
        while (iterator != null) {
            for (RowResult row : iterator) {
                result.add(row.getString(0));
            }
            iterator = scanner.nextRows();
        }
        return result;
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 
Example #9
Source File: KuduCatalogTest.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
private void validateMultiKey(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(2, schema.getPrimaryKeyColumnCount());
    assertEquals(3, schema.getColumnCount());

    assertTrue(schema.getColumn("first").isKey());
    assertTrue(schema.getColumn("second").isKey());

    assertFalse(schema.getColumn("third").isKey());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals(2, rows.get(0).getInt("second"));
    assertEquals("t", rows.get(0).getString("third"));
}
 
Example #10
Source File: KuduOperations.java    From geowave with Apache License 2.0 6 votes vote down vote up
public List<Delete> getDeletions(
    final KuduTable table,
    final List<KuduPredicate> predicates,
    final Function<RowResult, PersistentKuduRow> adapter) throws KuduException {
  // TODO: Kudu Java API does not support deleting with predicates, so we first perform a scan and
  // then perform individual row deletions with the full primary key. This is inefficient, because
  // we need to read in entire rows in order to perform deletions.
  final KuduScannerBuilder scannerBuilder = getScannerBuilder(table);
  for (final KuduPredicate pred : predicates) {
    scannerBuilder.addPredicate(pred);
  }
  final KuduScanner scanner = scannerBuilder.build();
  final List<RowResultIterator> allResults = new ArrayList<>();
  while (scanner.hasMoreRows()) {
    allResults.add(scanner.nextRows());
  }
  final Iterator<Delete> deletions =
      Streams.stream(Iterators.concat(allResults.iterator())).map(result -> {
        final PersistentKuduRow row = adapter.apply(result);
        final Delete delete = table.newDelete();
        row.populatePartialRowPrimaryKey(delete.getRow());
        return delete;
      }).iterator();
  return Lists.newArrayList(deletions);
}
 
Example #11
Source File: KuduCatalogTest.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testTimestamp() throws Exception {
    tableEnv.sqlUpdate("CREATE TABLE TestTableTsC (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");
    tableEnv.sqlUpdate("INSERT INTO TestTableTsC values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')");
    tableEnv.execute("test");

    KuduTable kuduTable = harness.getClient().openTable("TestTableTsC");
    assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString(0));
    assertEquals(Timestamp.valueOf("2020-01-01 12:12:12.123"), rows.get(0).getTimestamp(1));
}
 
Example #12
Source File: KuduTestBase.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
protected void validateSingleKey(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(1, schema.getPrimaryKeyColumnCount());
    assertEquals(2, schema.getColumnCount());

    assertTrue(schema.getColumn("first").isKey());
    assertFalse(schema.getColumn("second").isKey());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals("s", rows.get(0).getString("second"));
}
 
Example #13
Source File: KuduTableFactoryTest.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testExistingTable() throws Exception {
    // Creating a table
    tableEnv.sqlUpdate("CREATE TABLE TestTable12 (`first` STRING, `second` STRING) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTable12', 'kudu.masters'='" + kuduMasters + "', " +
            "'kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");

    tableEnv.sqlUpdate("INSERT INTO TestTable12 values ('f', 's')");
    tableEnv.execute("test");

    // Then another one in SQL that refers to the previously created one
    tableEnv.sqlUpdate("CREATE TABLE TestTable12b (`first` STRING, `second` STRING) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTable12', 'kudu.masters'='" + kuduMasters + "')");
    tableEnv.sqlUpdate("INSERT INTO TestTable12b values ('f2','s2')");
    tableEnv.execute("test2");

    // Validate that both insertions were into the same table
    KuduTable kuduTable = harness.getClient().openTable("TestTable12");
    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(2, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals("s", rows.get(0).getString("second"));
    assertEquals("f2", rows.get(1).getString("first"));
    assertEquals("s2", rows.get(1).getString("second"));
}
 
Example #14
Source File: Tagsets.java    From kudu-ts with Apache License 2.0 5 votes vote down vote up
@Override
public Deferred<TagsetLookupResult> call(RowResultIterator rows) {
  for (RowResult row : rows) {
    int id = row.getInt(Tables.TAGSETS_ID_INDEX);
    Preconditions.checkState(id >= probe);
    if (id != probe) {
      // We found a hole in the table where we expected the tagset.
      return Deferred.fromResult(new TagsetLookupResult(false, probe));
    }

    if (tagset.equals(row.getBinary(Tables.TAGSETS_TAGSET_INDEX))) {
      return Deferred.fromResult(new TagsetLookupResult(true, id));
    }

    probe++;
  }

  // We probed through the entire RowResult and didn't find the tagset.
  if (!scanner.hasMoreRows()) {
    if (probe <= Ints.saturatedCast((long) id + TAGSETS_PER_SCAN)) {
      // We found a hole at the end of the scan.
      return Deferred.fromResult(new TagsetLookupResult(false, probe));
    }
    // The current scanner has been exhausted; create a new scanner from the
    // latest probe point.
    scanner = tagsetScanner(probe);
    id = probe;
  }
  return scanner.nextRows().addCallbackDeferring(this);
}
 
Example #15
Source File: KuduCatalogTest.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up
private void validateManyTypes(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(Type.STRING, schema.getColumn("first").getType());
    assertEquals(Type.BOOL, schema.getColumn("second").getType());
    assertEquals(Type.BINARY, schema.getColumn("third").getType());
    assertEquals(Type.INT8, schema.getColumn("fourth").getType());
    assertEquals(Type.INT16, schema.getColumn("fifth").getType());
    assertEquals(Type.INT32, schema.getColumn("sixth").getType());
    assertEquals(Type.INT64, schema.getColumn("seventh").getType());
    assertEquals(Type.FLOAT, schema.getColumn("eighth").getType());
    assertEquals(Type.DOUBLE, schema.getColumn("ninth").getType());
    assertEquals(Type.UNIXTIME_MICROS, schema.getColumn("tenth").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString(0));
    assertEquals(false, rows.get(0).getBoolean(1));
    assertEquals(ByteBuffer.wrap("bbbb".getBytes()), rows.get(0).getBinary(2));
    assertEquals(12, rows.get(0).getByte(3));
    assertEquals(34, rows.get(0).getShort(4));
    assertEquals(56, rows.get(0).getInt(5));
    assertEquals(78, rows.get(0).getLong(6));
    assertEquals(3.14, rows.get(0).getFloat(7), 0.01);
    assertEquals(1.2345, rows.get(0).getDouble(8), 0.0001);
    assertEquals(Timestamp.valueOf("2020-04-15 12:34:56.123"), rows.get(0).getTimestamp(9));
}
 
Example #16
Source File: KuduIOTest.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Tests the read path using a {@link FakeReader}. The {@link KuduService} is mocked to simulate 4
 * tablets and fake the encoding of a scanner for each tablet. The test verifies that the {@link
 * KuduIO} correctly splits into 4 sources and instantiates a reader for each, and that the
 * correct number of records are read.
 */
@Test
public void testRead() throws KuduException {
  when(mockReadService.createReader(any())).thenAnswer(new FakeReaderAnswer());
  // Simulate the equivalent of Kudu providing an encoded scanner per tablet. Here we encode
  // a range which the fake reader will use to simulate a single tablet read.
  List<byte[]> fakeScanners =
      Arrays.asList(
          ByteBuffer.allocate(8).putInt(0).putInt(25).array(),
          ByteBuffer.allocate(8).putInt(25).putInt(50).array(),
          ByteBuffer.allocate(8).putInt(50).putInt(75).array(),
          ByteBuffer.allocate(8).putInt(75).putInt(100).array());
  when(mockReadService.createTabletScanners(any())).thenReturn(fakeScanners);

  PCollection<Integer> output =
      readPipeline.apply(
          KuduIO.<Integer>read()
              .withMasterAddresses("mock")
              .withTable("Table")
              // the fake reader only deals with a single int
              .withParseFn(
                  (SerializableFunction<RowResult, Integer>) input -> input.getInt(COL_ID))
              .withKuduService(mockReadService)
              .withCoder(BigEndianIntegerCoder.of()));

  PAssert.thatSingleton(output.apply("Count", Count.globally())).isEqualTo((long) 100);

  readPipeline.run().waitUntilFinish();

  // check that the fake tablet ranges were read
  expectedReadLogs.verifyDebug(String.format(FakeReader.LOG_SET_RANGE, 0, 25));
  expectedReadLogs.verifyDebug(String.format(FakeReader.LOG_SET_RANGE, 25, 50));
  expectedReadLogs.verifyDebug(String.format(FakeReader.LOG_SET_RANGE, 50, 75));
  expectedReadLogs.verifyDebug(String.format(FakeReader.LOG_SET_RANGE, 75, 100));
}
 
Example #17
Source File: KuduIOIT.java    From beam with Apache License 2.0 5 votes vote down vote up
private void runReadAll() {
  // Lambdas erase too much type information so specify the coder
  PCollection<String> output =
      readPipeline.apply(
          KuduIO.<String>read()
              .withMasterAddresses(options.getKuduMasterAddresses())
              .withTable(options.getKuduTable())
              .withParseFn(
                  (SerializableFunction<RowResult, String>) input -> input.getString(COL_NAME))
              .withCoder(StringUtf8Coder.of()));
  PAssert.thatSingleton(output.apply("Count", Count.globally()))
      .isEqualTo((long) options.getNumberOfRecords());

  readPipeline.run().waitUntilFinish();
}
 
Example #18
Source File: KuduIOIT.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the projected columns are passed down to the Kudu scanner by attempting to read the
 * {@value KuduTestUtils#COL_NAME} in the parse function when it is omitted.
 */
private void runReadProjectedColumns() {
  thrown.expect(IllegalArgumentException.class);
  readPipeline
      .apply(
          "Read with projected columns",
          KuduIO.<String>read()
              .withMasterAddresses(options.getKuduMasterAddresses())
              .withTable(options.getKuduTable())
              .withParseFn(
                  (SerializableFunction<RowResult, String>) input -> input.getString(COL_NAME))
              .withProjectedColumns(Collections.singletonList(COL_ID))) // COL_NAME excluded
      .setCoder(StringUtf8Coder.of());
  readPipeline.run().waitUntilFinish();
}
 
Example #19
Source File: Tags.java    From kudu-ts with Apache License 2.0 5 votes vote down vote up
/**
 * Retrieves the tagset IDs of all tagsets which contain the specified tag.
 * The tagset IDs are returned in sorted order.
 *
 * @param key the tag key
 * @param value the tag value
 * @return the sorted tagset IDs
 */
public Deferred<IntVec> getTagsetIDsForTag(final String key, final String value) {
  AsyncKuduScanner.AsyncKuduScannerBuilder scan = client.newScannerBuilder(table);
  scan.addPredicate(KuduPredicate.newComparisonPredicate(Tables.TAGS_KEY_COLUMN,
                                                         ComparisonOp.EQUAL, key));
  scan.addPredicate(KuduPredicate.newComparisonPredicate(Tables.TAGS_VALUE_COLUMN,
                                                         ComparisonOp.EQUAL, value));
  scan.setProjectedColumnIndexes(TAGSET_ID_PROJECTION);
  final AsyncKuduScanner scanner = scan.build();

  class GetTagCB implements Callback<Deferred<IntVec>, RowResultIterator> {
    private final IntVec tagsetIDs = IntVec.create();
    @Override
    public Deferred<IntVec> call(RowResultIterator results) {
      for (RowResult result : results) {
        tagsetIDs.push(result.getInt(0));
      }
      if (scanner.hasMoreRows()) {
        return scanner.nextRows().addCallbackDeferring(this);
      }
      // The Kudu java client doesn't yet allow us to specify a sorted
      // (fault-tolerant) scan, so have to sort manually.
      tagsetIDs.sort();
      return Deferred.fromResult(tagsetIDs);
    }
    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this).add("key", key).add("value", value).toString();
    }
  }

  return scanner.nextRows().addCallbackDeferring(new GetTagCB());
}
 
Example #20
Source File: TypeHelper.java    From presto with Apache License 2.0 5 votes vote down vote up
public static Object getObject(Type type, RowResult row, int field)
{
    if (row.isNull(field)) {
        return null;
    }
    if (type instanceof VarcharType) {
        return row.getString(field);
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return row.getLong(field) / 1000;
    }
    if (type == BigintType.BIGINT) {
        return row.getLong(field);
    }
    if (type == IntegerType.INTEGER) {
        return row.getInt(field);
    }
    if (type == SmallintType.SMALLINT) {
        return row.getShort(field);
    }
    if (type == TinyintType.TINYINT) {
        return row.getByte(field);
    }
    if (type == DoubleType.DOUBLE) {
        return row.getDouble(field);
    }
    if (type == RealType.REAL) {
        return row.getFloat(field);
    }
    if (type == BooleanType.BOOLEAN) {
        return row.getBoolean(field);
    }
    if (type instanceof VarbinaryType) {
        return Slices.wrappedBuffer(row.getBinary(field));
    }
    if (type instanceof DecimalType) {
        return row.getDecimal(field);
    }
    throw new IllegalStateException("getObject not implemented for " + type);
}
 
Example #21
Source File: KuduInputOperatorCommons.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
public void truncateTable() throws Exception
{
  AbstractKuduPartitionScanner<UnitTestTablePojo,InputOperatorControlTuple> scannerForDeletingRows =
      unitTestStepwiseScanInputOperator.getScanner();
  List<KuduScanToken> scansForAllTablets = unitTestStepwiseScanInputOperator
      .getPartitioner().getKuduScanTokensForSelectAllColumns();
  ApexKuduConnection aCurrentConnection = scannerForDeletingRows.getConnectionPoolForThreads().get(0);
  KuduSession aSessionForDeletes = aCurrentConnection.getKuduClient().newSession();
  KuduTable currentTable = aCurrentConnection.getKuduTable();
  for ( KuduScanToken aTabletScanToken : scansForAllTablets) {
    KuduScanner aScanner = aTabletScanToken.intoScanner(aCurrentConnection.getKuduClient());
    while ( aScanner.hasMoreRows()) {
      RowResultIterator itrForRows = aScanner.nextRows();
      while ( itrForRows.hasNext()) {
        RowResult aRow = itrForRows.next();
        int intRowKey = aRow.getInt("introwkey");
        String stringRowKey = aRow.getString("stringrowkey");
        long timestampRowKey = aRow.getLong("timestamprowkey");
        Delete aDeleteOp = currentTable.newDelete();
        aDeleteOp.getRow().addInt("introwkey",intRowKey);
        aDeleteOp.getRow().addString("stringrowkey", stringRowKey);
        aDeleteOp.getRow().addLong("timestamprowkey",timestampRowKey);
        aSessionForDeletes.apply(aDeleteOp);
      }
    }
  }
  aSessionForDeletes.close();
  Thread.sleep(2000); // Sleep to allow for scans to complete
}
 
Example #22
Source File: KuduUtils.java    From datacollector with Apache License 2.0 5 votes vote down vote up
/**
 * Create a field and assign a value off of RowResult.
 * @param result Result obtained from scan
 * @param fieldName Field name to create
 * @param type Kudu Type for the field
 * @return Generated field
 * @throws StageException
 */
public static Field createField(RowResult result, String fieldName, Type type) throws StageException {
  switch (type) {
    case INT8:
      return Field.create(Field.Type.BYTE, result.getByte(fieldName));
    case INT16:
      return Field.create(Field.Type.SHORT, result.getShort(fieldName));
    case INT32:
      return Field.create(Field.Type.INTEGER, result.getInt(fieldName));
    case INT64:
      return Field.create(Field.Type.LONG, result.getLong(fieldName));
    case BINARY:
      try {
        ByteBuffer bBuffer = result.getBinary(fieldName);
        byte[] bArray = new byte[bBuffer.remaining()];
        bBuffer.get(bArray);

        return Field.create(Field.Type.BYTE_ARRAY, bArray);
      } catch (IllegalArgumentException ex) {
        throw new OnRecordErrorException(Errors.KUDU_35, fieldName);
      }
    case STRING:
      return Field.create(Field.Type.STRING, result.getString(fieldName));
    case BOOL:
      return Field.create(Field.Type.BOOLEAN, result.getBoolean(fieldName));
    case FLOAT:
      return Field.create(Field.Type.FLOAT, result.getFloat(fieldName));
    case DOUBLE:
      return Field.create(Field.Type.DOUBLE, result.getDouble(fieldName));
    case UNIXTIME_MICROS:
      //UNIXTIME_MICROS is in microsecond
      return Field.create(Field.Type.DATETIME, new Date(result.getLong(fieldName)/1000L));
    default:
      if ("DECIMAL".equals(type.name())) {
        return Field.create(Field.Type.DECIMAL, result.getDecimal(fieldName));
      }
      throw new StageException(Errors.KUDU_10, fieldName, type.getName());
  }
}
 
Example #23
Source File: KuduRow.java    From geowave with Apache License 2.0 5 votes vote down vote up
public KuduRow(final RowResult row) {
  super(getFieldValues(row));
  partitionKey = row.getBinaryCopy(KuduField.GW_PARTITION_ID_KEY.getFieldName());
  adapterId = row.getShort(KuduField.GW_ADAPTER_ID_KEY.getFieldName());
  sortKey = row.getBinaryCopy(KuduField.GW_SORT_KEY.getFieldName());
  dataId = row.getBinaryCopy(KuduField.GW_DATA_ID_KEY.getFieldName());
  fieldVisibility = row.getBinaryCopy(KuduField.GW_FIELD_VISIBILITY_KEY.getFieldName());
  nanoTime = row.getBinaryCopy(KuduField.GW_NANO_TIME_KEY.getFieldName());
  fieldMask = row.getBinaryCopy(KuduField.GW_FIELD_MASK_KEY.getFieldName());
  value = row.getBinaryCopy(KuduField.GW_VALUE_KEY.getFieldName());
  numDuplicates = row.getByte(KuduField.GW_NUM_DUPLICATES_KEY.getFieldName());
}
 
Example #24
Source File: KuduRow.java    From geowave with Apache License 2.0 5 votes vote down vote up
private static GeoWaveValue[] getFieldValues(final RowResult row) {
  final byte[] fieldMask = row.getBinaryCopy(KuduField.GW_FIELD_MASK_KEY.getFieldName());
  final byte[] value = row.getBinaryCopy(KuduField.GW_VALUE_KEY.getFieldName());
  final byte[] visibility = row.getBinaryCopy(KuduField.GW_FIELD_VISIBILITY_KEY.getFieldName());

  return new GeoWaveValueImpl[] {new GeoWaveValueImpl(fieldMask, visibility, value)};
}
 
Example #25
Source File: KuduMetadataRow.java    From geowave with Apache License 2.0 5 votes vote down vote up
public KuduMetadataRow(final RowResult result) {
  primaryId = result.getBinaryCopy(KuduMetadataField.GW_PRIMARY_ID_KEY.getFieldName());
  secondaryId = result.getBinaryCopy(KuduMetadataField.GW_SECONDARY_ID_KEY.getFieldName());
  visibility = result.getBinaryCopy(KuduMetadataField.GW_VISIBILITY_KEY.getFieldName());
  value = result.getBinaryCopy(KuduMetadataField.GW_VALUE_KEY.getFieldName());
  timestamp = result.getBinaryCopy(KuduMetadataField.GW_TIMESTAMP_KEY.getFieldName());
}
 
Example #26
Source File: KuduDataIndexRow.java    From geowave with Apache License 2.0 5 votes vote down vote up
public static GeoWaveRow deserializeDataIndexRow(
    final RowResult row,
    final boolean isVisibilityEnabled) {
  return DataIndexUtils.deserializeDataIndexRow(
      row.getBinaryCopy(KuduDataIndexField.GW_PARTITION_ID_KEY.getFieldName()),
      row.getShort(KuduDataIndexField.GW_ADAPTER_ID_KEY.getFieldName()),
      row.getBinaryCopy(KuduDataIndexField.GW_VALUE_KEY.getFieldName()),
      isVisibilityEnabled);
}
 
Example #27
Source File: KuduLookupService.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public Optional<Record> lookup(Map<String, Object> coordinates) {

    //Scanner
    KuduScanner.KuduScannerBuilder builder = kuduClient.newScannerBuilder(table);

    builder.setProjectedColumnNames(columnNames);
    builder.replicaSelection(replicaSelection);

    //Only expecting one match
    builder.limit(1);

    coordinates.forEach((key,value)->
            builder.addPredicate(KuduPredicate.newComparisonPredicate(tableSchema.getColumn(key), KuduPredicate.ComparisonOp.EQUAL, value))
    );

    KuduScanner kuduScanner = builder.build();

    //Run lookup
    for ( RowResult row : kuduScanner){
        final Map<String, Object> values = new HashMap<>();
        for(String columnName : columnNames){
            Object object;
            if(row.getColumnType(columnName) == Type.BINARY){
                object = Base64.getEncoder().encodeToString(row.getBinaryCopy(columnName));
            } else {
                object = row.getObject(columnName);
            }
            values.put(columnName, object);
        }
        return Optional.of(new MapRecord(resultSchema, values));
    }

    //No match
    return Optional.empty();
}
 
Example #28
Source File: TypeHelper.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
public static Slice getSlice(Type type, RowResult row, int field) {
    if (type instanceof VarcharType) {
        return Slices.utf8Slice(row.getString(field));
    } else if (type instanceof VarbinaryType) {
        return Slices.wrappedBuffer(row.getBinary(field));
    } else if (type instanceof DecimalType) {
        BigDecimal dec = row.getDecimal(field);
        return Decimals.encodeScaledValue(dec);
    } else {
        throw new IllegalStateException("getSlice not implemented for " + type);
    }
}
 
Example #29
Source File: TypeHelper.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
public static double getDouble(Type type, RowResult row, int field) {
    if (type == DoubleType.DOUBLE) {
        return row.getDouble(field);
    } else {
        throw new IllegalStateException("getDouble not implemented for " + type);
    }
}
 
Example #30
Source File: TypeHelper.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
public static boolean getBoolean(Type type, RowResult row, int field) {
    if (type == BooleanType.BOOLEAN) {
        return row.getBoolean(field);
    } else {
        throw new IllegalStateException("getBoolean not implemented for " + type);
    }
}