Java Code Examples for org.apache.kafka.connect.data.Schema#isOptional()

/**
 * MapEntry types in connect Schemas are represented as Arrays of record.
 * Return the array type from the union instead of the union itself.
 */
private static org.apache.avro.Schema avroSchemaForUnderlyingMapEntryType(
    Schema schema,
    org.apache.avro.Schema avroSchema) {

    if (schema != null && schema.isOptional()) {
        if (avroSchema.getType() == org.apache.avro.Schema.Type.UNION) {
            for (org.apache.avro.Schema typeSchema : avroSchema.getTypes()) {
                if (!typeSchema.getType().equals(org.apache.avro.Schema.Type.NULL)
                    && Schema.Type.ARRAY.getName().equals(typeSchema.getType().getName())) {
                    return typeSchema;
                }
            }
        } else {
            throw new DataException(
                "An optional schema should have an Avro Union type, not "
                + schema.type());
        }
    }
    return avroSchema;
}
 

/**
 * Connect optional fields are represented as a unions (null & type) in Avro
 * Return the Avro schema of the actual type in the Union (instead of the union itself)
 */
private static org.apache.avro.Schema avroSchemaForUnderlyingTypeIfOptional(
    Schema schema, org.apache.avro.Schema avroSchema) {

    if (schema != null && schema.isOptional()) {
        if (avroSchema.getType() == org.apache.avro.Schema.Type.UNION) {
            for (org.apache.avro.Schema typeSchema : avroSchema
                .getTypes()) {
                if (!typeSchema.getType().equals(org.apache.avro.Schema.Type.NULL)
                    && crossReferenceSchemaNames(schema, typeSchema)) {
                    return typeSchema;
                }
            }
        } else {
            throw new DataException(
                "An optinal schema should have an Avro Union type, not "
                + schema.type());
        }
    }
    return avroSchema;
}
 

public BsonValue toBson(final Object data, final Schema fieldSchema) {
  if (!fieldSchema.isOptional()) {
    if (data == null) {
      throw new DataException("Error: schema not optional but data was null");
    }
    LOGGER.trace("field not optional and data is '{}'", data.toString());
    return toBson(data);
  }

  if (data != null) {
    LOGGER.trace("field optional and data is '{}'", data.toString());
    return toBson(data);
  }

  if (fieldSchema.defaultValue() != null) {
    LOGGER.trace(
        "field optional and no data but default value is '{}'",
        fieldSchema.defaultValue().toString());
    return toBson(fieldSchema.defaultValue());
  }

  LOGGER.trace("field optional, no data and no default value thus '{}'", BsonNull.VALUE);
  return BsonNull.VALUE;
}
 

public BsonValue toBson(Object data, Schema fieldSchema) {
    if(!fieldSchema.isOptional()) {

        if(data == null)
            throw new DataException("error: schema not optional but data was null");

        logger.trace("field not optional and data is '{}'",data.toString());
        return toBson(data);
    }

    if(data != null) {
        logger.trace("field optional and data is '{}'",data.toString());
        return toBson(data);
    }

    if(fieldSchema.defaultValue() != null) {
        logger.trace("field optional and no data but default value is '{}'",fieldSchema.defaultValue().toString());
        return toBson(fieldSchema.defaultValue());
    }

    logger.trace("field optional, no data and no default value thus '{}'", BsonNull.VALUE);
    return BsonNull.VALUE;
}
 

Storage(Schema schema) {
  this.name = schema.name();
  this.doc = schema.doc();
  this.type = schema.type();
  this.defaultValue = schema.defaultValue();
  this.version = schema.version();
  this.parameters = schema.parameters();
  this.isOptional = schema.isOptional();

  if (Schema.Type.MAP == this.type) {
    this.keySchema = schema.keySchema();
    this.valueSchema = schema.valueSchema();
  } else if (Schema.Type.ARRAY == this.type) {
    this.keySchema = null;
    this.valueSchema = schema.valueSchema();
  } else if (Schema.Type.STRUCT == this.type) {
    this.fieldSchemas = new LinkedHashMap<>();
    for (Field field : schema.fields()) {
      this.fieldSchemas.put(field.name(), field.schema());
    }
  }
}
 

private void addAvroRecordField(
    List<org.apache.avro.Schema.Field> fields,
    String fieldName, Schema fieldSchema,
    FromConnectContext fromConnectContext) {

    Object defaultVal = null;
    if (fieldSchema.defaultValue() != null) {
        defaultVal = fieldSchema.defaultValue();

        // If this is a logical, convert to the primitive form for the Avro default value
        defaultVal = toAvroLogical(fieldSchema, defaultVal);

        // Avro doesn't handle a few types that Connect uses, so convert those explicitly here
        if (defaultVal instanceof Byte) {
            // byte are mapped to integers in Avro
            defaultVal = ((Byte) defaultVal).intValue();
        } else if (defaultVal instanceof Short) {
            // Shorts are mapped to integers in Avro
            defaultVal = ((Short) defaultVal).intValue();
        } else if (defaultVal instanceof ByteBuffer) {
            // Avro doesn't handle ByteBuffer directly, but does handle 'byte[]'
            // Copy the contents of the byte buffer without side effects on the buffer
            ByteBuffer buffer = (ByteBuffer) defaultVal;
            byte[] bytes = new byte[buffer.remaining()];
            buffer.duplicate().get(bytes);
            defaultVal = bytes;
        }
    } else if (fieldSchema.isOptional()) {
        defaultVal = JsonProperties.NULL_VALUE;
    }
    org.apache.avro.Schema.Field field = new org.apache.avro.Schema.Field(
        fieldName,
        fromConnectSchema(fieldSchema, fromConnectContext, false),
        fieldSchema.doc(),
        defaultVal);
    fields.add(field);
}
 

private Schema getOrBuildRecordSchema(final Schema originalSchema, final Object value) {
    final SchemaBuilder builder = SchemaUtil.copySchemaBasics(originalSchema, SchemaHelper.buildSchemaBuilderForType(value));

    if (originalSchema.isOptional()) {
        builder.optional();
    }
    if (originalSchema.defaultValue() != null) {
        builder.defaultValue(convertValueWithCamelTypeConverter(originalSchema.defaultValue()));
    }

    return builder.build();
}
 

private static SchemaBuilder copySchemaBasics(Schema source, SchemaBuilder target) {
  if (source.isOptional()) {
    target.optional();
  }
  if (source.defaultValue() != null && source.type() != Schema.Type.STRUCT) {
    final Object preProcessedDefaultValue = preProcessValue(source.defaultValue(), source, target);
    target.defaultValue(preProcessedDefaultValue);
  }
  return target;
}
 

@Override
protected SchemaAndValue processStruct(R record, Schema inputSchema, Struct inputStruct) {
  final SchemaBuilder outputSchemaBuilder = SchemaBuilder.struct();
  outputSchemaBuilder.name(inputSchema.name());
  outputSchemaBuilder.doc(inputSchema.doc());
  if (null != inputSchema.defaultValue()) {
    outputSchemaBuilder.defaultValue(inputSchema.defaultValue());
  }
  if (null != inputSchema.parameters() && !inputSchema.parameters().isEmpty()) {
    outputSchemaBuilder.parameters(inputSchema.parameters());
  }
  if (inputSchema.isOptional()) {
    outputSchemaBuilder.optional();
  }
  Map<String, String> fieldMappings = new HashMap<>(inputSchema.fields().size());
  for (final Field inputField : inputSchema.fields()) {
    log.trace("process() - Processing field '{}'", inputField.name());
    final Matcher fieldMatcher = this.config.pattern.matcher(inputField.name());
    final String outputFieldName;
    if (fieldMatcher.find()) {
      outputFieldName = fieldMatcher.replaceAll(this.config.replacement);
    } else {
      outputFieldName = inputField.name();
    }
    log.trace("process() - Mapping field '{}' to '{}'", inputField.name(), outputFieldName);
    fieldMappings.put(inputField.name(), outputFieldName);
    outputSchemaBuilder.field(outputFieldName, inputField.schema());
  }
  final Schema outputSchema = outputSchemaBuilder.build();
  final Struct outputStruct = new Struct(outputSchema);
  for (Map.Entry<String, String> entry : fieldMappings.entrySet()) {
    final String inputField = entry.getKey(), outputField = entry.getValue();
    log.trace("process() - Copying '{}' to '{}'", inputField, outputField);
    final Object value = inputStruct.get(inputField);
    outputStruct.put(outputField, value);
  }
  return new SchemaAndValue(outputSchema, outputStruct);
}
 

public static SchemaBuilder of(Schema schema, Collection<String> excludeFields) {

    Set<String> exclude = null != excludeFields ? ImmutableSet.copyOf(excludeFields) : ImmutableSet.of();
    SchemaBuilder builder;

    if (Schema.Type.ARRAY == schema.type()) {
      builder = SchemaBuilder.array(schema.valueSchema());
    } else if (Schema.Type.MAP == schema.type()) {
      builder = SchemaBuilder.map(schema.keySchema(), schema.valueSchema());
    } else {
      builder = SchemaBuilder.type(schema.type());
    }

    if (schema.isOptional()) {
      builder.optional();
    }
    if (!Strings.isNullOrEmpty(schema.name())) {
      builder.name(schema.name());
    }
    if (!Strings.isNullOrEmpty(schema.doc())) {
      builder.doc(schema.doc());
    }
    builder.version(schema.version());

    if (null != schema.parameters()) {
      builder.parameters(schema.parameters());
    }

    if (Schema.Type.STRUCT == schema.type()) {
      schema.fields()
          .stream()
          .filter(field -> !exclude.contains(field.name()))
          .forEach(field -> builder.field(field.name(), field.schema()));
    }

    return builder;
  }
 

String builderOfTestCaseName(Schema schema) {
  StringBuilder builder = new StringBuilder();
  builder.append(schema.type());
  if (!Strings.isNullOrEmpty(schema.name())) {
    builder.append("(");
    builder.append(schema.name());
    builder.append(")");
  }
  switch (schema.type()) {
    case ARRAY:
      builder.append("[");
      builder.append(builderOfTestCaseName(schema.valueSchema()));
      builder.append("]");
      break;
    case MAP:
      builder.append("[");
      builder.append(builderOfTestCaseName(schema.valueSchema()));
      builder.append(",");
      builder.append(builderOfTestCaseName(schema.valueSchema()));
      builder.append("]");
      break;
  }

  if (schema.isOptional()) {
    builder.append(":optional");
  }
  if (null != schema.version()) {
    builder.append(":version(");
    builder.append(schema.version());
    builder.append(")");
  }
  if (null != schema.parameters() && !schema.parameters().isEmpty()) {
    builder.append(":parameters(");
    builder.append(schema.parameters());
    builder.append(")");
  }


  return builder.toString();
}
 

private static void validateSchemaValue(Schema schema, Object value) throws DataException {
    if (value == null && schema != null && !schema.isOptional()) {
        throw new DataException("Found null value for non-optional schema");
    }
}
 

public CyclicSchemaWrapper(Schema schema) {
    this(schema, schema.isOptional());
}
 

@Override
protected SchemaAndValue processBytes(R record, Schema inputSchema, byte[] input) {
  final Schema outputSchema = inputSchema.isOptional() ? Schema.OPTIONAL_STRING_SCHEMA : Schema.STRING_SCHEMA;
  final String output = new String(input, this.config.charset);
  return new SchemaAndValue(outputSchema, output);
}
 

void checkSchemaAndInput(Schema schema, Object input) {
  Preconditions.checkNotNull(schema, "schema cannot be null");
  if (!schema.isOptional()) {
    Preconditions.checkNotNull(input, "schema is not optional so input cannot be null.");
  }
}