Java Code Examples for com.google.api.services.bigquery.model.TableFieldSchema#setName()

The following examples show how to use com.google.api.services.bigquery.model.TableFieldSchema#setName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FieldSchemaListBuilder.java From bigquery-etl-dataflow-sample with Apache License 2.0

5 votes

/**
 *  Returns a new repeated record field.
 *  @param name the name of the field.
 */
public TableFieldSchema repeatedRecord(String name) {
  TableFieldSchema tfs = fieldSchema(this);
  tfs.setName(name);
  tfs.setMode("REPEATED");
  return tfs;
}

Example 2

Source File: BeamBQOutputTransform.java From hop with Apache License 2.0

4 votes

@Override public PDone expand( PCollection<HopRow> input ) {

    try {
      // Only initialize once on this node/vm
      //
      BeamHop.init( transformPluginClasses, xpPluginClasses );

      // Inflate the metadata on the node where this is running...
      //
      IRowMeta rowMeta = JsonRowMeta.fromJson( rowMetaJson );


      // Which table do we write to?
      //
      TableReference tableReference = new TableReference();
      if ( StringUtils.isNotEmpty( projectId ) ) {
        tableReference.setProjectId( projectId );
      }
      tableReference.setDatasetId( datasetId );
      tableReference.setTableId( tableId );

      TableSchema tableSchema = new TableSchema();
      List<TableFieldSchema> schemaFields = new ArrayList<>();
      for ( IValueMeta valueMeta : rowMeta.getValueMetaList() ) {
        TableFieldSchema schemaField = new TableFieldSchema();
        schemaField.setName( valueMeta.getName() );
        switch(valueMeta.getType()){
          case IValueMeta.TYPE_STRING: schemaField.setType( "STRING" ); break;
          case IValueMeta.TYPE_INTEGER: schemaField.setType( "INTEGER" ); break;
          case IValueMeta.TYPE_DATE: schemaField.setType( "DATETIME" ); break;
          case IValueMeta.TYPE_BOOLEAN: schemaField.setType( "BOOLEAN" ); break;
          case IValueMeta.TYPE_NUMBER: schemaField.setType( "FLOAT" ); break;
          default:
            throw new RuntimeException( "Conversion from Hop value "+valueMeta.toString()+" to BigQuery TableRow isn't supported yet" );
        }
        schemaFields.add(schemaField);
      }
      tableSchema.setFields( schemaFields );

      SerializableFunction<HopRow, TableRow> formatFunction = new HopToBQTableRowFn( transformName, rowMetaJson, transformPluginClasses, xpPluginClasses );

      BigQueryIO.Write.CreateDisposition createDisposition;
      if (createIfNeeded) {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED;
      }  else {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_NEVER;
      }

      BigQueryIO.Write.WriteDisposition writeDisposition;
      if (truncateTable) {
        writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
      } else {
        if (failIfNotEmpty) {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_EMPTY;
        } else {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
        }
      }

      BigQueryIO.Write<HopRow> bigQueryWrite = BigQueryIO
        .<HopRow>write()
        .to( tableReference )
        .withSchema( tableSchema )
        .withCreateDisposition( createDisposition )
        .withWriteDisposition( writeDisposition )
        .withFormatFunction( formatFunction );

      // TODO: pass the results along the way at some point
      //
      input.apply( transformName, bigQueryWrite );

      // End of the line
      //
      return PDone.in( input.getPipeline() );

    } catch ( Exception e ) {
      numErrors.inc();
      LOG.error( "Error in Beam BigQuery output transform", e );
      throw new RuntimeException( "Error in Beam BigQuery output transform", e );
    }
  }

Example 3

Source File: BeamBQOutputTransform.java From kettle-beam with Apache License 2.0

4 votes

@Override public PDone expand( PCollection<KettleRow> input ) {

    try {
      // Only initialize once on this node/vm
      //
      BeamKettle.init( stepPluginClasses, xpPluginClasses );

      // Inflate the metadata on the node where this is running...
      //
      RowMetaInterface rowMeta = JsonRowMeta.fromJson( rowMetaJson );


      // Which table do we write to?
      //
      TableReference tableReference = new TableReference();
      if ( StringUtils.isNotEmpty( projectId ) ) {
        tableReference.setProjectId( projectId );
      }
      tableReference.setDatasetId( datasetId );
      tableReference.setTableId( tableId );

      TableSchema tableSchema = new TableSchema();
      List<TableFieldSchema> schemaFields = new ArrayList<>();
      for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
        TableFieldSchema schemaField = new TableFieldSchema();
        schemaField.setName( valueMeta.getName() );
        switch(valueMeta.getType()){
          case ValueMetaInterface.TYPE_STRING: schemaField.setType( "STRING" ); break;
          case ValueMetaInterface.TYPE_INTEGER: schemaField.setType( "INTEGER" ); break;
          case ValueMetaInterface.TYPE_DATE: schemaField.setType( "DATETIME" ); break;
          case ValueMetaInterface.TYPE_BOOLEAN: schemaField.setType( "BOOLEAN" ); break;
          case ValueMetaInterface.TYPE_NUMBER: schemaField.setType( "FLOAT" ); break;
          default:
            throw new RuntimeException( "Conversion from Kettle value "+valueMeta.toString()+" to BigQuery TableRow isn't supported yet" );
        }
        schemaFields.add(schemaField);
      }
      tableSchema.setFields( schemaFields );

      SerializableFunction<KettleRow, TableRow> formatFunction = new KettleToBQTableRowFn( stepname, rowMetaJson, stepPluginClasses, xpPluginClasses );

      BigQueryIO.Write.CreateDisposition createDisposition;
      if (createIfNeeded) {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED;
      }  else {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_NEVER;
      }

      BigQueryIO.Write.WriteDisposition writeDisposition;
      if (truncateTable) {
        writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
      } else {
        if (failIfNotEmpty) {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_EMPTY;
        } else {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
        }
      }

      BigQueryIO.Write<KettleRow> bigQueryWrite = BigQueryIO
        .<KettleRow>write()
        .to( tableReference )
        .withSchema( tableSchema )
        .withCreateDisposition( createDisposition )
        .withWriteDisposition( writeDisposition )
        .withFormatFunction( formatFunction );

      // TODO: pass the results along the way at some point
      //
      input.apply( stepname, bigQueryWrite );

      // End of the line
      //
      return PDone.in( input.getPipeline() );

    } catch ( Exception e ) {
      numErrors.inc();
      LOG.error( "Error in Beam BigQuery output transform", e );
      throw new RuntimeException( "Error in Beam BigQuery output transform", e );
    }
  }

Example 4

Source File: BigQueryUtils.java From hadoop-connectors with Apache License 2.0

4 votes

/**
 * Parses the given JSON string and returns the extracted schema.
 *
 * @param fields a string to read the TableSchema from.
 * @return the List of TableFieldSchema described by the string fields.
 */
public static List<TableFieldSchema> getSchemaFromString(String fields) {
  logger.atFine().log("getSchemaFromString('%s')", fields);

  // Parse the output schema for Json from fields.
  JsonParser jsonParser = new JsonParser();
  JsonArray json = jsonParser.parse(fields).getAsJsonArray();
  List<TableFieldSchema> fieldsList = new ArrayList<>();

  // For each item in the list of fields.
  for (JsonElement jsonElement : json) {
    checkArgument(
        jsonElement.isJsonObject(), "Expected JsonObject for element, got '%s'.", jsonElement);
    JsonObject jsonObject = jsonElement.getAsJsonObject();

    // Set the name and type.
    checkArgument(
        jsonObject.get("name") != null,
        "Expected non-null entry for key 'name' in JsonObject '%s'", jsonObject);
    checkArgument(
        jsonObject.get("type") != null,
        "Expected non-null entry for key 'type' in JsonObject '%s'", jsonObject);
    TableFieldSchema fieldDef = new TableFieldSchema();
    fieldDef.setName(jsonObject.get("name").getAsString());
    fieldDef.setType(jsonObject.get("type").getAsString());

    // If mode is not null, set mode.
    if (jsonObject.get("mode") != null) {
      fieldDef.setMode(jsonObject.get("mode").getAsString());
    }

    // If the type is RECORD set the fields.
    if (jsonObject.get("type").getAsString().equals("RECORD")) {
      checkArgument(
          jsonObject.get("fields") != null,
          "Expected non-null entry for key 'fields' in JsonObject of type RECORD: '%s'",
          jsonObject);
      fieldDef.setFields(getSchemaFromString(jsonObject.get("fields").toString()));
    }

    fieldsList.add(fieldDef);
  }
  // Return list of TableFieldSchema.
  return fieldsList;
}

Example 5

Source File: BigQueryLoader.java From quetzal with Eclipse Public License 2.0

4 votes

public static TableFieldSchema field(String name, String type) {
	TableFieldSchema x = new TableFieldSchema();
	x.setName(name);
	x.setType(type);
	return x;
}

Example 6

Source File: FieldSchemaListBuilder.java From bigquery-etl-dataflow-sample with Apache License 2.0

3 votes

/**
 * Creates a TableFieldSchema with all of the parameters
 * @param type - the datatype @see https://cloud.google.com/bigquery/data-types
 * @param name - the name of the field
 * @param mode - the mode of the field
 * @param description - a description of the field to create.
 * @see https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html
 * @return
 */
public TableFieldSchema fieldSchema(String type, String name, String mode, String description) {
  TableFieldSchema tfs = new TableFieldSchema();
  tfs.setType(type);
  tfs.setName(name);
  tfs.setMode(mode);
  tfs.setDescription(description);
  return tfs;
}