Java Code Examples for com.google.api.services.bigquery.model.TableRow#keySet()

The following examples show how to use com.google.api.services.bigquery.model.TableRow#keySet() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BigQueryConverters.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up 
/**
 * Returns {@code String} using Key/Value style formatting.
 *
 * @param formatTemplate a String with bracketed keys to apply "I am a {key}"
 * @param row is a TableRow object which is used to supply key:values to the template
 *
 * <p> Extracts TableRow fields and applies values to the formatTemplate.
 * ie. formatStringTemplate("I am {key}"{"key": "formatted"}) -> "I am formatted"
 */
public static String formatStringTemplate(String formatTemplate, TableRow row) {
    // Key/Value Map used to replace values in template
    Map<String, String> values = new HashMap<>();

    // Put all column/value pairs into key/value map
    Set<String> rowKeys = row.keySet();
    for (String rowKey : rowKeys) {
      // Only String types can be used in comparison
      if(row.get(rowKey) instanceof String) {
        values.put(rowKey, (String) row.get(rowKey));
      }
    }
    // Substitute any templated values in the template
    String result = StringSubstitutor.replace(formatTemplate, values, "{", "}");
    return result;
}
Example 2
Source File: BigQueryMapper.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up 
/**
 * Extracts and applies new column information to BigQuery by comparing the TableRow against the
 * BigQuery Table.
 *
 * @param tableId a TableId referencing the BigQuery table to be loaded to.
 * @param row a TableRow with the raw data to be loaded into BigQuery.
 * @param inputSchema The source schema lookup to be used in mapping.
 */
private void updateTableIfRequired(
    TableId tableId, TableRow row, Map<String, LegacySQLTypeName> inputSchema) {
  Table table = getOrCreateBigQueryTable(tableId);
  FieldList tableFields = table.getDefinition().getSchema().getFields();

  Set<String> rowKeys = row.keySet();
  Boolean tableWasUpdated = false;
  List<Field> newFieldList = new ArrayList<Field>();
  for (String rowKey : rowKeys) {
    // Check if rowKey (column from data) is in the BQ Table
    try {
      Field tableField = tableFields.get(rowKey);
    } catch (IllegalArgumentException e) {
      tableWasUpdated = addNewTableField(tableId, row, rowKey, newFieldList, inputSchema);
    }
  }

  if (tableWasUpdated) {
    LOG.info("Updating Table");
    updateBigQueryTable(tableId, table, tableFields, newFieldList);
  }
}
Example 3
Source File: BigQueryConverters.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up 
/**
 * Return a formatted String Using Key/Value Style formatting
 * from the TableRow applied to the Format Template.
 * ie. formatStringTemplate("I am {key}"{"key": "formatted"}) -> "I am formatted"
 */
public static String formatStringTemplate(String formatTemplate, TableRow row) {
    // Key/Value Map used to replace values in template
    Map<String, String> values = new HashMap<>();

    // Put all column/value pairs into key/value map
    Set<String> rowKeys = row.keySet();
    for (String rowKey : rowKeys) {
      // Only String types can be used in comparison
      if(row.get(rowKey) instanceof String) {
        values.put(rowKey, (String) row.get(rowKey));
      }
    }
    // Substitute any templated values in the template
    String result = StringSubstitutor.replace(formatTemplate, values, "{", "}");
    return result;
}
Example 4
Source File: BigQueryMapper.java    From DataflowTemplates with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns {@code TableRow} after cleaning each field according to
 * the data type found in BigQuery.
 *
 * @param tableId a TableId referencing the BigQuery table to be loaded to.
 * @param row a TableRow with the raw data to be loaded into BigQuery.
 */
public TableRow getCleanedTableRow(TableId tableId, TableRow row) {
  TableRow cleanRow = row.clone();

  Table table = this.tableCache.get(tableId);
  FieldList tableFields = table.getDefinition().getSchema().getFields();

  Set<String> rowKeys = cleanRow.keySet();
  for (String rowKey : rowKeys) {
    this.bqTableRowCleaner.cleanTableRowField(cleanRow, tableFields, rowKey);
  }

  return cleanRow;
}
Example 5
Source File: BigQueryMapper.java    From DataflowTemplates with Apache License 2.0 5 votes vote down vote up 
private void updateTableIfRequired(TableId tableId, TableRow row,
    Map<String, LegacySQLTypeName> inputSchema) {
  // Ensure Instance of BigQuery Exists
  if (this.bigquery == null) {
    this.bigquery =
        BigQueryOptions.newBuilder()
            .setProjectId(getProjectId())
            .build()
            .getService();
  }

  // Get BigQuery Table for Given Row
  Table table = getBigQueryTable(tableId);

  // Validate Table Schema
  FieldList tableFields = table.getDefinition().getSchema().getFields();

  Set<String> rowKeys = row.keySet();
  Boolean tableWasUpdated = false;
  List<Field> newFieldList = new ArrayList<Field>();
  for (String rowKey : rowKeys) {
    // Check if rowKey (column from data) is in the BQ Table
    try {
      Field tableField = tableFields.get(rowKey);
    } catch (IllegalArgumentException e) {
      tableWasUpdated = addNewTableField(tableId, row, rowKey, newFieldList, inputSchema);
    }
  }

  if (tableWasUpdated) {
    LOG.info("Updating Table");
    updateBigQueryTable(tableId, table, tableFields, newFieldList);
  }
}