Java Code Examples for com.google.api.services.bigquery.model.TableFieldSchema#getType()

private static Field convertField(TableFieldSchema bigQueryField) {
  ImmutableCollection<Type> avroTypes = BIG_QUERY_TO_AVRO_TYPES.get(bigQueryField.getType());
  if (avroTypes.isEmpty()) {
    throw new IllegalArgumentException(
        "Unable to map BigQuery field type " + bigQueryField.getType() + " to avro type.");
  }

  Type avroType = avroTypes.iterator().next();
  Schema elementSchema;
  if (avroType == Type.RECORD) {
    elementSchema = toGenericAvroSchema(bigQueryField.getName(), bigQueryField.getFields());
  } else {
    elementSchema = Schema.create(avroType);
  }
  Schema fieldSchema;
  if (bigQueryField.getMode() == null || "NULLABLE".equals(bigQueryField.getMode())) {
    fieldSchema = Schema.createUnion(Schema.create(Type.NULL), elementSchema);
  } else if ("REQUIRED".equals(bigQueryField.getMode())) {
    fieldSchema = elementSchema;
  } else if ("REPEATED".equals(bigQueryField.getMode())) {
    fieldSchema = Schema.createArray(elementSchema);
  } else {
    throw new IllegalArgumentException(
        String.format("Unknown BigQuery Field Mode: %s", bigQueryField.getMode()));
  }
  return new Field(
      bigQueryField.getName(),
      fieldSchema,
      bigQueryField.getDescription(),
      (Object) null /* Cast to avoid deprecated JsonNode constructor. */);
}
 

/**
 * Converts the value of a BigQuery column in Avro format into the value of a Datastore Entity
 * property.
 */
public static Value columnToValue(TableFieldSchema column, Object columnValue)
    throws IllegalArgumentException {
  String columnName = column.getName();
  if (columnValue == null) {
    return Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build();
  } else {
    Value.Builder valueBuilder = Value.newBuilder();
    switch (column.getType()) {
      case "STRING":
        // Datastore string properties greater than 1500 bytes will not be indexed in order to
        // respect the limit imposed on the maximum size of index-able string properties. See
        // https://cloud.google.com/datastore/docs/concepts/limits
        String strValue = columnValue.toString();
        valueBuilder.setStringValue(strValue);
        boolean excludeFromIndexes = strValue.getBytes().length > MAX_STRING_SIZE_BYTES;
        valueBuilder.setExcludeFromIndexes(excludeFromIndexes);
        break;
      case "INTEGER":
      case "INT64":
        valueBuilder.setIntegerValue((long) columnValue);
        break;
      case "FLOAT":
      case "FLOAT64":
        valueBuilder.setDoubleValue((double) columnValue);
        break;
      case "BOOLEAN":
      case "BOOL":
        valueBuilder.setBooleanValue((boolean) columnValue);
        break;
      case "TIMESTAMP":
        // Convert into milliseconds from the BigQuery timestamp, which is in micro seconds
        long timeInMillis = ((long) columnValue) / 1000;
        valueBuilder.setTimestampValue(Timestamps.fromMillis(timeInMillis));
        break;
      case "DATE":
      case "TIME":
      case "DATETIME":
        // Handle these types as STRING by default, no dedicated type exists in Datastore
        valueBuilder.setStringValue(columnValue.toString());
        break;
      default:
        // TODO: handle nested fields (types "RECORD" or "STRUCT")
        throw new IllegalArgumentException(
            String.format(
                "Column [%s] of type [%s] not supported.", column.getName(), column.getType()));
    }
    return valueBuilder.build();
  }
}
 

private static Object convertRequiredField(
    Type avroType, LogicalType avroLogicalType, TableFieldSchema fieldSchema, Object v) {
  // REQUIRED fields are represented as the corresponding Avro types. For example, a BigQuery
  // INTEGER type maps to an Avro LONG type.
  checkNotNull(v, "REQUIRED field %s should not be null", fieldSchema.getName());
  // Per https://cloud.google.com/bigquery/docs/reference/v2/tables#schema, the type field
  // is required, so it may not be null.
  String bqType = fieldSchema.getType();
  ImmutableCollection<Type> expectedAvroTypes = BIG_QUERY_TO_AVRO_TYPES.get(bqType);
  verifyNotNull(expectedAvroTypes, "Unsupported BigQuery type: %s", bqType);
  verify(
      expectedAvroTypes.contains(avroType),
      "Expected Avro schema types %s for BigQuery %s field %s, but received %s",
      expectedAvroTypes,
      bqType,
      fieldSchema.getName(),
      avroType);
  // For historical reasons, don't validate avroLogicalType except for with NUMERIC.
  // BigQuery represents NUMERIC in Avro format as BYTES with a DECIMAL logical type.
  switch (bqType) {
    case "STRING":
    case "DATETIME":
    case "GEOGRAPHY":
      // Avro will use a CharSequence to represent String objects, but it may not always use
      // java.lang.String; for example, it may prefer org.apache.avro.util.Utf8.
      verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass());
      return v.toString();
    case "DATE":
      if (avroType == Type.INT) {
        verify(v instanceof Integer, "Expected Integer, got %s", v.getClass());
        verifyNotNull(avroLogicalType, "Expected Date logical type");
        verify(avroLogicalType instanceof LogicalTypes.Date, "Expected Date logical type");
        return formatDate((Integer) v);
      } else {
        verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass());
        return v.toString();
      }
    case "TIME":
      if (avroType == Type.LONG) {
        verify(v instanceof Long, "Expected Long, got %s", v.getClass());
        verifyNotNull(avroLogicalType, "Expected TimeMicros logical type");
        verify(
            avroLogicalType instanceof LogicalTypes.TimeMicros,
            "Expected TimeMicros logical type");
        return formatTime((Long) v);
      } else {
        verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass());
        return v.toString();
      }
    case "INTEGER":
      verify(v instanceof Long, "Expected Long, got %s", v.getClass());
      return ((Long) v).toString();
    case "FLOAT":
      verify(v instanceof Double, "Expected Double, got %s", v.getClass());
      return v;
    case "NUMERIC":
      // NUMERIC data types are represented as BYTES with the DECIMAL logical type. They are
      // converted back to Strings with precision and scale determined by the logical type.
      verify(v instanceof ByteBuffer, "Expected ByteBuffer, got %s", v.getClass());
      verifyNotNull(avroLogicalType, "Expected Decimal logical type");
      verify(avroLogicalType instanceof LogicalTypes.Decimal, "Expected Decimal logical type");
      BigDecimal numericValue =
          new Conversions.DecimalConversion()
              .fromBytes((ByteBuffer) v, Schema.create(avroType), avroLogicalType);
      return numericValue.toString();
    case "BOOLEAN":
      verify(v instanceof Boolean, "Expected Boolean, got %s", v.getClass());
      return v;
    case "TIMESTAMP":
      // TIMESTAMP data types are represented as Avro LONG types, microseconds since the epoch.
      // Values may be negative since BigQuery timestamps start at 0001-01-01 00:00:00 UTC.
      verify(v instanceof Long, "Expected Long, got %s", v.getClass());
      return formatTimestamp((Long) v);
    case "RECORD":
      verify(v instanceof GenericRecord, "Expected GenericRecord, got %s", v.getClass());
      return convertGenericRecordToTableRow((GenericRecord) v, fieldSchema.getFields());
    case "BYTES":
      verify(v instanceof ByteBuffer, "Expected ByteBuffer, got %s", v.getClass());
      ByteBuffer byteBuffer = (ByteBuffer) v;
      byte[] bytes = new byte[byteBuffer.limit()];
      byteBuffer.get(bytes);
      return BaseEncoding.base64().encode(bytes);
    default:
      throw new UnsupportedOperationException(
          String.format(
              "Unexpected BigQuery field schema type %s for field named %s",
              fieldSchema.getType(), fieldSchema.getName()));
  }
}