com.google.cloud.bigtable.hbase.BigtableConfiguration Java Examples

public static void writeIncrement(String projectId, String instanceId, String tableId) {
  // String projectId = "my-project-id";
  // String instanceId = "my-instance-id";
  // String tableId = "mobile-time-series";

  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(tableId)));

    String rowKey = "phone#4c410523#20190501";

    table.incrementColumnValue(
        Bytes.toBytes(rowKey), COLUMN_FAMILY_NAME, Bytes.toBytes("connected_cell"), -1);

    System.out.printf("Successfully updated row %s", rowKey);

  } catch (Exception e) {
    System.out.println("Error during WriteIncrement: \n" + e.toString());
  }
}
 

@BeforeClass
public static void beforeClass() {
  projectId = requireEnv("GOOGLE_CLOUD_PROJECT");
  instanceId = requireEnv(INSTANCE_ENV);
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Admin admin = connection.getAdmin();
    HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(TABLE_ID));
    descriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY_NAME));
    admin.createTable(descriptor);

    Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(TABLE_ID)));

    String rowKey = "phone#4c410523#20190401";
    Put put = new Put(Bytes.toBytes(rowKey));

    put.addColumn(
        Bytes.toBytes(COLUMN_FAMILY_NAME), Bytes.toBytes("os_name"), Bytes.toBytes("android"));
    table.put(put);

  } catch (Exception e) {
    System.out.println("Error during beforeClass: \n" + e.toString());
  }
}
 

@Test
public void testWrite() {
  HelloWorldWrite.main(
      new String[] {
        "--bigtableProjectId=" + projectId,
        "--bigtableInstanceId=" + instanceId,
        "--bigtableTableId=" + TABLE_ID
      });

  long count = 0;
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(TABLE_ID));
    Scan scan = new Scan();

    ResultScanner rows = table.getScanner(scan);

    for (Result row : rows) {
      count++;
    }
  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
  assertThat(count).isGreaterThan(0);
}
 

private Connection connectToBigtable(List<ConfigIssue> issues) {
  Connection conn;
  try {
    conn = BigtableConfiguration.connect(conf.bigtableProjectID, conf.bigtableInstanceID);

  } catch (Exception ex) {
    LOG.error(Errors.BIGTABLE_01.getMessage(), conf.bigtableProjectID, ex.toString(), ex);
    issues.add(getContext().createConfigIssue(Groups.BIGTABLE.name(),
        CONFIG,
        Errors.BIGTABLE_01,
        conf.bigtableProjectID,
        ex.toString()
    ));
    return null;
  }
  return conn;
}
 

@Before
public void prepare() throws Exception {
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableId));
    descriptor.addFamily(new HColumnDescriptor(columnFamilyName));
    try (Admin admin = connection.getAdmin()) {
      admin.createTable(descriptor);
      // Retrieve the table we just created so we can do some reads and writes
      try (Table table = connection.getTable(TableName.valueOf(tableId))) {

        String rowKey = "r1";

        Put put = new Put(Bytes.toBytes(rowKey));
        put.addColumn(Bytes.toBytes(columnFamilyName), Bytes.toBytes(columnName),
            Bytes.toBytes(data));
        table.put(put);
      }
    }
  }
}
 

public static void readPrefix(String projectId, String instanceId, String tableId) {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(tableId));
    Scan prefixScan = new Scan().setRowPrefixFilter(Bytes.toBytes("phone"));
    ResultScanner rows = table.getScanner(prefixScan);

    for (Result row : rows) {
      printRow(row);
    }
  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
}
 

public static void readRowRange(String projectId, String instanceId, String tableId) {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(tableId));

    Scan rangeQuery =
        new Scan()
            .withStartRow(Bytes.toBytes("phone#4c410523#20190501"))
            .withStopRow(Bytes.toBytes("phone#4c410523#201906201"));

    ResultScanner rows = table.getScanner(rangeQuery);

    for (Result row : rows) {
      printRow(row);
    }

  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
}
 

public static void readRows(String projectId, String instanceId, String tableId) {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(tableId));
    List<Get> queryRowList = new ArrayList<Get>();
    queryRowList.add(new Get(Bytes.toBytes("phone#4c410523#20190501")));
    queryRowList.add(new Get(Bytes.toBytes("phone#4c410523#20190502")));

    Result[] rows = table.get(queryRowList);

    for (Result row : rows) {
      printRow(row);
    }
  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
}
 

public static void readRowPartial(String projectId, String instanceId, String tableId) {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(tableId));
    byte[] rowkey = Bytes.toBytes("phone#4c410523#20190501");

    Result row =
        table.get(
            new Get(rowkey).addColumn(Bytes.toBytes("stats_summary"), Bytes.toBytes("os_build")));
    printRow(row);

  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
}
 

public static void readRow(String projectId, String instanceId, String tableId) {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(tableId));

    byte[] rowkey = Bytes.toBytes("phone#4c410523#20190501");

    Result row = table.get(new Get(rowkey));
    printRow(row);

  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
}
 

public static void readWithFilter(
    String projectId, String instanceId, String tableId, Scan scan) {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(tableId));

    ResultScanner rows = table.getScanner(scan);

    for (Result row : rows) {
      printRow(row);
    }
  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
}
 

public static Connection createBigTableConnection(Config config) {
  final String projectId = config.getString(BIGTABLE_PROJECT_ID);
  final String instanceId = config.getString(BIGTABLE_INSTANCE_ID);

  LOG.info("Creating Bigtable connection for project:{}, instance:{}",
           projectId, instanceId);

  final Configuration bigtableConfiguration = new Configuration();
  bigtableConfiguration.set("google.bigtable.project.id", projectId);
  bigtableConfiguration.set("google.bigtable.instance.id", instanceId);
  bigtableConfiguration.setBoolean("google.bigtable.rpc.use.timeouts", true);

  return BigtableConfiguration.connect(bigtableConfiguration);
}
 

/**
 * Connect will establish the connection to Cloud Bigtable.
 **/
public static void connect() throws IOException {
  if (PROJECT_ID == null || INSTANCE_ID == null) {
    sc.log(
        "environment variables "
            + "BIGTABLE_PROJECT, and INSTANCE_ID "
            + "need to be defined.");
    return;
  }

  connection = BigtableConfiguration.connect(PROJECT_ID, INSTANCE_ID);
}
 

public synchronized Connection getConnection(final String projectId, final String instanceId)
    throws IOException {
  final String key = projectId + "_" + instanceId;
  Connection connection = connectorCache.get(key);
  if (connection == null) {
    final Configuration config = BigtableConfiguration.configure(projectId, instanceId);

    config.setInt(HBASE_CONFIGURATION_TIMEOUT, 120000);

    connection = BigtableConfiguration.connect(config);
    connectorCache.put(key, connection);
  }

  return connection;
}
 

public static void writeTestData(String projectId, String instanceId,
    TableName tableName, long rowCount, int valueSize) throws IOException {
  System.out.println("Writing to table: " + tableName);
  try (Connection conn = BigtableConfiguration.connect(projectId, instanceId)) {
    BigtableUtilities.createTable(tableName, conn);
    runMutationTests(conn, tableName, rowCount, valueSize);
  }
  System.out.println("Closed the connection");
}
 

/**
* Connect will establish the connection to Cloud Bigtable.
**/
 public static void connect() throws IOException {

   if (PROJECT_ID == null || INSTANCE_ID == null ) {
     if (sc != null) {
       sc.log("environment variables BIGTABLE_PROJECT, and BIGTABLE_INSTANCE need to be defined.");
     }
     return;
   }

   connection = BigtableConfiguration.connect(PROJECT_ID, INSTANCE_ID);
 }
 

private static void dropTable() {
  Connection conn = BigtableConfiguration.connect(projectID, instanceID);
  try {
    Admin admin = conn.getAdmin();
    admin.disableTable(TableName.valueOf(tableName));
    admin.deleteTable(TableName.valueOf(tableName));
  } catch (Exception ex) {
    LOG.info("dropTable(): exception {} ", ex.toString());
  }
}
 

private Table getTable() {
  Connection conn = BigtableConfiguration.connect(projectID, instanceID);
  Table tab;
  try {
    tab = conn.getTable(TableName.valueOf(tableName));

  } catch (Exception ex) {
    tab = null;
  }
  return tab;
}
 

/** Connect will establish the connection to Cloud Bigtable. */
public static void connect() throws IOException {

  if (PROJECT_ID == null || INSTANCE_ID == null) {
    if (sc != null) {
      sc.log("environment variables BIGTABLE_PROJECT, and BIGTABLE_INSTANCE need to be defined.");
    }
    return;
  }

  connection = BigtableConfiguration.connect(PROJECT_ID, INSTANCE_ID);
}
 

@AfterClass
public static void afterClass() {
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Admin admin = connection.getAdmin();
    Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(TABLE_ID)));
    admin.disableTable(table.getName());
    admin.deleteTable(table.getName());
  } catch (Exception e) {
    System.out.println("Error during afterClass: \n" + e.toString());
  }
}
 

@AfterClass
public static void afterClass() {
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Admin admin = connection.getAdmin();
    Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(TABLE_ID)));
    admin.disableTable(table.getName());
    admin.deleteTable(table.getName());
  } catch (Exception e) {
    System.out.println("Error during afterClass: \n" + e.toString());
  }
}
 

@BeforeClass
public static void beforeClass() {
  projectId = requireEnv("GOOGLE_CLOUD_PROJECT");
  instanceId = requireEnv(INSTANCE_ENV);
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Admin admin = connection.getAdmin();
    HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(TABLE_ID));
    descriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY_NAME));
    admin.createTable(descriptor);
  } catch (Exception e) {
    System.out.println("Error during beforeClass: \n" + e.toString());
  }
}
 

@AfterClass
public static void afterClass() {
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Admin admin = connection.getAdmin();
    Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(TABLE_ID)));
    admin.disableTable(table.getName());
    admin.deleteTable(table.getName());
  } catch (Exception e) {
    System.out.println("Error during afterClass: \n" + e.toString());
  }
}
 

@BeforeClass
public static void beforeClass() {
  projectId = requireEnv("GOOGLE_CLOUD_PROJECT");
  instanceId = requireEnv(INSTANCE_ENV);
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Admin admin = connection.getAdmin();
    HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(TABLE_ID));
    descriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY_NAME));
    admin.createTable(descriptor);
  } catch (Exception e) {
    System.out.println("Error during beforeClass: \n" + e.toString());
  }
}
 

public static void writeBatch(String projectId, String instanceId, String tableId) {
  // String projectId = "my-project-id";
  // String instanceId = "my-instance-id";
  // String tableId = "mobile-time-series";

  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    final Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(tableId)));
    long timestamp = System.currentTimeMillis();
    byte[] one = new byte[]{0, 0, 0, 0, 0, 0, 0, 1};

    List<Put> puts = new ArrayList<Put>();
    puts.add(new Put(Bytes.toBytes("tablet#a0b81f74#20190501")));
    puts.add(new Put(Bytes.toBytes("tablet#a0b81f74#20190502")));

    puts.get(0).addColumn(COLUMN_FAMILY_NAME, Bytes.toBytes("connected_wifi"), timestamp, one);
    puts.get(0)
        .addColumn(
            COLUMN_FAMILY_NAME,
            Bytes.toBytes("os_build"),
            timestamp,
            Bytes.toBytes("12155.0.0-rc1"));

    puts.get(1).addColumn(COLUMN_FAMILY_NAME, Bytes.toBytes("connected_wifi"), timestamp, one);
    puts.get(1)
        .addColumn(
            COLUMN_FAMILY_NAME,
            Bytes.toBytes("os_build"),
            timestamp,
            Bytes.toBytes("12145.0.0-rc6"));

    table.put(puts);

    System.out.print("Successfully wrote 2 rows");
  } catch (Exception e) {
    System.out.println("Error during WriteBatch: \n" + e.toString());
  }
}
 

public static void writeConditionally(String projectId, String instanceId, String tableId) {
  // String projectId = "my-project-id";
  // String instanceId = "my-instance-id";
  // String tableId = "mobile-time-series";

  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(tableId)));
    long timestamp = System.currentTimeMillis();

    String rowKey = "phone#4c410523#20190501";
    RowMutations mutations = new RowMutations(Bytes.toBytes(rowKey));

    Put put = new Put(Bytes.toBytes(rowKey));
    put.addColumn(
        COLUMN_FAMILY_NAME, Bytes.toBytes("os_name"), timestamp, Bytes.toBytes("android"));
    mutations.add(put);

    table.checkAndMutate(
        Bytes.toBytes(rowKey),
        COLUMN_FAMILY_NAME,
        Bytes.toBytes("os_build"),
        CompareOp.GREATER_OR_EQUAL,
        Bytes.toBytes("PQ2A.190405"),
        mutations);

    System.out.print("Successfully updated row's os_name");

  } catch (Exception e) {
    System.out.println("Error during WriteConditionally: \n" + e.toString());
    e.printStackTrace();
  }
}
 

public static void writeSimple(String projectId, String instanceId, String tableId) {
  // String projectId = "my-project-id";
  // String instanceId = "my-instance-id";
  // String tableId = "mobile-time-series";

  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    final Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(tableId)));
    long timestamp = System.currentTimeMillis();
    byte[] one = new byte[]{0, 0, 0, 0, 0, 0, 0, 1};

    String rowKey = "phone#4c410523#20190501";
    Put put = new Put(Bytes.toBytes(rowKey));

    put.addColumn(COLUMN_FAMILY_NAME, Bytes.toBytes("connected_cell"), timestamp, one);
    put.addColumn(COLUMN_FAMILY_NAME, Bytes.toBytes("connected_wifi"), timestamp, one);
    put.addColumn(
        COLUMN_FAMILY_NAME,
        Bytes.toBytes("os_build"),
        timestamp,
        Bytes.toBytes("PQ2A.190405.003"));
    table.put(put);

    System.out.printf("Successfully wrote row %s", rowKey);

  } catch (Exception e) {
    System.out.println("Error during WriteSimple: \n" + e.toString());
  }
}
 

public static void readFilter(String projectId, String instanceId, String tableId) {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(tableId));

    ValueFilter valueFilter =
        new ValueFilter(CompareOp.EQUAL, new RegexStringComparator("PQ2A.*"));
    Scan scan = new Scan().setFilter(valueFilter);

    ResultScanner rows = table.getScanner(scan);

    for (Result row : rows) {
      printRow(row);
    }
  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
}
 

public static void readRowRanges(String projectId, String instanceId, String tableId) {
  // Initialize client that will be used to send requests. This client only needs to be created
  // once, and can be reused for multiple requests. After completing all of your requests, call
  // the "close" method on the client to safely clean up any remaining background resources.
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(tableId));
    List<RowRange> ranges = new ArrayList<>();

    ranges.add(
        new RowRange(
            Bytes.toBytes("phone#4c410523#20190501"),
            true,
            Bytes.toBytes("phone#4c410523#20190601"),
            false));
    ranges.add(
        new RowRange(
            Bytes.toBytes("phone#5c10102#20190501"),
            true,
            Bytes.toBytes("phone#5c10102#20190601"),
            false));
    Filter filter = new MultiRowRangeFilter(ranges);
    Scan scan = new Scan().setFilter(filter);

    ResultScanner rows = table.getScanner(scan);

    for (Result row : rows) {
      printRow(row);
    }
  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }
}
 

@Test
public void testWriteAndRead() {
  LoadData.main(
      new String[]{
          "--bigtableProjectId=" + projectId,
          "--bigtableInstanceId=" + instanceId,
          "--bigtableTableId=" + TABLE_ID,
          "--gigabytesWritten=" + GIGABYTES_WRITTEN,
          "--megabytesPerRow=" + MEGABYTES_PER_ROW
      });

  long count = 0;
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Table table = connection.getTable(TableName.valueOf(TABLE_ID));
    Scan scan = new Scan();

    ResultScanner rows = table.getScanner(scan);

    for (Result row : rows) {
      count++;
    }
  } catch (IOException e) {
    System.out.println(
        "Unable to initialize service client, as a network error occurred: \n" + e.toString());
  }

  assertEquals(10, count);

  ReadDataOptions options =
      PipelineOptionsFactory.fromArgs("--bigtableProjectId=" + projectId,
          "--bigtableInstanceId=" + instanceId,
          "--bigtableTableId=" + TABLE_ID,
          "--gigabytesWritten=" + GIGABYTES_WRITTEN,
          "--megabytesPerRow=" + MEGABYTES_PER_ROW,
          "--filePath=gs://keyviz-art/maxgrid.txt").withValidation().as(ReadDataOptions.class);
  Pipeline p = Pipeline.create(options);
  CloudBigtableTableConfiguration bigtableTableConfig =
      new CloudBigtableTableConfiguration.Builder()
          .withProjectId(options.getBigtableProjectId())
          .withInstanceId(options.getBigtableInstanceId())
          .withTableId(options.getBigtableTableId())
          .build();

  // Initiates a new pipeline every second
  p.apply(Create.of(1L))
      .apply(ParDo.of(new ReadFromTableFn(bigtableTableConfig, options)));
  p.run().waitUntilFinish();

  String output = bout.toString();
  assertThat(output).contains("got 10 rows");

  options =
      PipelineOptionsFactory.fromArgs("--bigtableProjectId=" + projectId,
          "--bigtableInstanceId=" + instanceId,
          "--bigtableTableId=" + TABLE_ID,
          "--gigabytesWritten=" + GIGABYTES_WRITTEN,
          "--megabytesPerRow=" + MEGABYTES_PER_ROW,
          "--filePath=gs://keyviz-art/halfgrid.txt").withValidation().as(ReadDataOptions.class);
  p = Pipeline.create(options);

  // Initiates a new pipeline every second
  p.apply(Create.of(1L))
      .apply(ParDo.of(new ReadFromTableFn(bigtableTableConfig, options)));
  p.run().waitUntilFinish();

  output = bout.toString();
  assertThat(output).contains("got 5 rows");
}