Java Code Examples for org.apache.kafka.connect.data.SchemaBuilder#optional()

The following examples show how to use org.apache.kafka.connect.data.SchemaBuilder#optional() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CellData.java    From debezium-incubator with Apache License 2.0 6 votes vote down vote up
static Schema cellSchema(ColumnMetadata cm, boolean optional) {
    AbstractType<?> convertedType = CassandraTypeConverter.convert(cm.getType());
    Schema valueSchema = CassandraTypeDeserializer.getSchemaBuilder(convertedType).optional().build();
    if (valueSchema != null) {
        SchemaBuilder schemaBuilder = SchemaBuilder.struct().name(cm.getName())
                .field(CELL_VALUE_KEY, valueSchema)
                .field(CELL_DELETION_TS_KEY, Schema.OPTIONAL_INT64_SCHEMA)
                .field(CELL_SET_KEY, Schema.BOOLEAN_SCHEMA);
        if (optional) {
            schemaBuilder.optional();
        }
        return schemaBuilder.build();
    }
    else {
        return null;
    }
}
 
Example 2
Source File: LogFieldConverterFactory.java    From kafka-connect-spooldir with Apache License 2.0 6 votes vote down vote up
static Schema schema(Class<?> logClass, String logFieldName) {
  final SchemaBuilder builder;
  if (LocalDate.class.equals(logClass)) {
    builder = Date.builder();
  } else if (LocalTime.class.equals(logClass)) {
    builder = Time.builder();
  } else if (Integer.class.equals(logClass)) {
    builder = SchemaBuilder.int32();
  } else if (Long.class.equals(logClass)) {
    builder = SchemaBuilder.int64();
  } else if (String.class.equals(logClass)) {
    builder = SchemaBuilder.string();
  } else {
    throw new UnsupportedOperationException(
        String.format("%s is not a supported type.", logClass.getName())
    );
  }
  builder.optional();


  return builder.build();
}
 
Example 3
Source File: CamelTypeConverterTransform.java    From camel-kafka-connector with Apache License 2.0 5 votes vote down vote up
private Schema getOrBuildRecordSchema(final Schema originalSchema, final Object value) {
    final SchemaBuilder builder = SchemaUtil.copySchemaBasics(originalSchema, SchemaHelper.buildSchemaBuilderForType(value));

    if (originalSchema.isOptional()) {
        builder.optional();
    }
    if (originalSchema.defaultValue() != null) {
        builder.defaultValue(convertValueWithCamelTypeConverter(originalSchema.defaultValue()));
    }

    return builder.build();
}
 
Example 4
Source File: DataConverter.java    From jkes with Apache License 2.0 5 votes vote down vote up
private static SchemaBuilder copySchemaBasics(Schema source, SchemaBuilder target) {
  if (source.isOptional()) {
    target.optional();
  }
  if (source.defaultValue() != null && source.type() != Schema.Type.STRUCT) {
    final Object preProcessedDefaultValue = preProcessValue(source.defaultValue(), source, target);
    target.defaultValue(preProcessedDefaultValue);
  }
  return target;
}
 
Example 5
Source File: PatternRename.java    From kafka-connect-transform-common with Apache License 2.0 5 votes vote down vote up
@Override
protected SchemaAndValue processStruct(R record, Schema inputSchema, Struct inputStruct) {
  final SchemaBuilder outputSchemaBuilder = SchemaBuilder.struct();
  outputSchemaBuilder.name(inputSchema.name());
  outputSchemaBuilder.doc(inputSchema.doc());
  if (null != inputSchema.defaultValue()) {
    outputSchemaBuilder.defaultValue(inputSchema.defaultValue());
  }
  if (null != inputSchema.parameters() && !inputSchema.parameters().isEmpty()) {
    outputSchemaBuilder.parameters(inputSchema.parameters());
  }
  if (inputSchema.isOptional()) {
    outputSchemaBuilder.optional();
  }
  Map<String, String> fieldMappings = new HashMap<>(inputSchema.fields().size());
  for (final Field inputField : inputSchema.fields()) {
    log.trace("process() - Processing field '{}'", inputField.name());
    final Matcher fieldMatcher = this.config.pattern.matcher(inputField.name());
    final String outputFieldName;
    if (fieldMatcher.find()) {
      outputFieldName = fieldMatcher.replaceAll(this.config.replacement);
    } else {
      outputFieldName = inputField.name();
    }
    log.trace("process() - Mapping field '{}' to '{}'", inputField.name(), outputFieldName);
    fieldMappings.put(inputField.name(), outputFieldName);
    outputSchemaBuilder.field(outputFieldName, inputField.schema());
  }
  final Schema outputSchema = outputSchemaBuilder.build();
  final Struct outputStruct = new Struct(outputSchema);
  for (Map.Entry<String, String> entry : fieldMappings.entrySet()) {
    final String inputField = entry.getKey(), outputField = entry.getValue();
    log.trace("process() - Copying '{}' to '{}'", inputField, outputField);
    final Object value = inputStruct.get(inputField);
    outputStruct.put(outputField, value);
  }
  return new SchemaAndValue(outputSchema, outputStruct);
}
 
Example 6
Source File: SchemaBuilders.java    From connect-utils with Apache License 2.0 5 votes vote down vote up
public static SchemaBuilder of(Schema schema, Collection<String> excludeFields) {

    Set<String> exclude = null != excludeFields ? ImmutableSet.copyOf(excludeFields) : ImmutableSet.of();
    SchemaBuilder builder;

    if (Schema.Type.ARRAY == schema.type()) {
      builder = SchemaBuilder.array(schema.valueSchema());
    } else if (Schema.Type.MAP == schema.type()) {
      builder = SchemaBuilder.map(schema.keySchema(), schema.valueSchema());
    } else {
      builder = SchemaBuilder.type(schema.type());
    }

    if (schema.isOptional()) {
      builder.optional();
    }
    if (!Strings.isNullOrEmpty(schema.name())) {
      builder.name(schema.name());
    }
    if (!Strings.isNullOrEmpty(schema.doc())) {
      builder.doc(schema.doc());
    }
    builder.version(schema.version());

    if (null != schema.parameters()) {
      builder.parameters(schema.parameters());
    }

    if (Schema.Type.STRUCT == schema.type()) {
      schema.fields()
          .stream()
          .filter(field -> !exclude.contains(field.name()))
          .forEach(field -> builder.field(field.name(), field.schema()));
    }

    return builder;
  }
 
Example 7
Source File: MsSqlTableMetadataProvider.java    From kafka-connect-cdc-mssql with Apache License 2.0 4 votes vote down vote up
Schema generateSchema(ResultSet resultSet,
                      final ChangeKey changeKey,
                      final String columnName) throws SQLException {
  boolean optional = resultSet.getBoolean(2);
  String dataType = resultSet.getString(3);
  int scale = resultSet.getInt(4);
  SchemaBuilder builder;

  log.trace("{}: columnName='{}' dataType='{}' scale={} optional={}", changeKey, columnName, dataType, scale, optional);

  switch (dataType) {
    case "bigint":
      builder = SchemaBuilder.int64();
      break;
    case "bit":
      builder = SchemaBuilder.bool();
      break;
    case "char":
    case "varchar":
    case "text":
    case "nchar":
    case "nvarchar":
    case "ntext":
    case "uniqueidentifier":
      builder = SchemaBuilder.string();
      break;
    case "smallmoney":
    case "money":
    case "decimal":
    case "numeric":
      builder = Decimal.builder(scale);
      break;
    case "binary":
    case "image":
    case "varbinary":
      builder = SchemaBuilder.bytes();
      break;
    case "date":
      builder = Date.builder();
      break;
    case "datetime":
    case "datetime2":
    case "smalldatetime":
      builder = Timestamp.builder();
      break;
    case "time":
      builder = Time.builder();
      break;
    case "int":
      builder = SchemaBuilder.int32();
      break;
    case "smallint":
      builder = SchemaBuilder.int16();
      break;
    case "tinyint":
      builder = SchemaBuilder.int8();
      break;
    case "real":
      builder = SchemaBuilder.float32();
      break;
    case "float":
      builder = SchemaBuilder.float64();
      break;

    default:
      throw new DataException(
          String.format("Could not process (dataType = '%s', optional = %s, scale = %d) for %s.",
              dataType, optional, scale, changeKey
          )
      );
  }

  log.trace("{}: columnName='{}' schema.type='{}' schema.name='{}'", changeKey, columnName, builder.type(), builder.name());

  builder.parameters(
      ImmutableMap.of(Change.ColumnValue.COLUMN_NAME, columnName)
  );

  if (optional) {
    builder.optional();
  }

  return builder.build();
}
 
Example 8
Source File: SchemaSerializationModule.java    From connect-utils with Apache License 2.0 4 votes vote down vote up
public Schema build() {
  log.trace(this.toString());
  SchemaBuilder builder;

  switch (this.type) {
    case MAP:
      Preconditions.checkNotNull(this.keySchema, "keySchema cannot be null.");
      Preconditions.checkNotNull(this.valueSchema, "valueSchema cannot be null.");
      builder = SchemaBuilder.map(this.keySchema, this.valueSchema);
      break;
    case ARRAY:
      Preconditions.checkNotNull(this.valueSchema, "valueSchema cannot be null.");
      builder = SchemaBuilder.array(this.valueSchema);
      break;
    default:
      builder = SchemaBuilder.type(this.type);
      break;
  }

  if (Schema.Type.STRUCT == this.type) {
    for (Map.Entry<String, Schema> kvp : this.fieldSchemas.entrySet()) {
      builder.field(kvp.getKey(), kvp.getValue());
    }
  }

  if (!Strings.isNullOrEmpty(this.name)) {
    builder.name(this.name);
  }

  if (!Strings.isNullOrEmpty(this.doc)) {
    builder.doc(this.doc);
  }

  if (null != this.defaultValue) {
    Object value;
    switch (this.type) {
      case INT8:
        value = ((Number) this.defaultValue).byteValue();
        break;
      case INT16:
        value = ((Number) this.defaultValue).shortValue();
        break;
      case INT32:
        value = ((Number) this.defaultValue).intValue();
        break;
      case INT64:
        value = ((Number) this.defaultValue).longValue();
        break;
      case FLOAT32:
        value = ((Number) this.defaultValue).floatValue();
        break;
      case FLOAT64:
        value = ((Number) this.defaultValue).doubleValue();
        break;
      default:
        value = this.defaultValue;
        break;
    }
    builder.defaultValue(value);
  }

  if (null != this.parameters) {
    builder.parameters(this.parameters);
  }

  if (this.isOptional) {
    builder.optional();
  }

  if (null != this.version) {
    builder.version(this.version);
  }

  return builder.build();
}