org.apache.kafka.connect.data.Field Java Examples

@Override
protected Object convertTinyInt(Column column, Field fieldDefn, Object data) {
    return convertValue(column, fieldDefn, data, BYTE_FALSE, (r) -> {
        if (data instanceof Byte) {
            r.deliver(data);
        }
        else if (data instanceof Number) {
            Number value = (Number) data;
            r.deliver(value.byteValue());
        }
        else if (data instanceof Boolean) {
            r.deliver(NumberConversions.getByte((boolean) data));
        }
        else if (data instanceof String) {
            r.deliver(Byte.parseByte((String) data));
        }
    });
}
 

Plugin.SchemaInput buildSchemaInput(Schema schema, String fieldName) {
  ImmutableSchemaInput.Builder schemaInput = ImmutableSchemaInput.builder()
      .name(schema.name())
      .doc(schema.doc())
      .type(schema.type())
      .fieldName(fieldName)
      .isOptional(schema.isOptional());

  if (Schema.Type.STRUCT == schema.type()) {
    for (Field field : schema.fields()) {
      Plugin.SchemaInput fieldSchema = buildSchemaInput(field.schema(), field.name());
      schemaInput.addFields(fieldSchema);
    }
  } else if (Schema.Type.MAP == schema.type()) {
    schemaInput.key(buildSchemaInput(schema.keySchema()));
    schemaInput.value(buildSchemaInput(schema.valueSchema()));
  } else if (Schema.Type.ARRAY == schema.type()) {
    schemaInput.value(buildSchemaInput(schema.valueSchema()));
  }

  return schemaInput.build();
}
 

public static String getSchemaDefinitionString(Schema schema) {
  StringBuilder stringBuilder = new StringBuilder("[");
  boolean addComma = false;
  for (Field field : schema.fields()) {
    if (addComma) {
      stringBuilder.append(" , ");
    } else {
      addComma = true;
    }
    stringBuilder.append(field.name())
        .append(" : ")
        .append(field.schema().type());
  }
  stringBuilder.append("]");
  return stringBuilder.toString();
}
 

public static int getFieldIndexByName(final Schema schema, final String fieldName) {
  if (schema.fields() == null) {
    return -1;
  }
  for (int i = 0; i < schema.fields().size(); i++) {
    Field field = schema.fields().get(i);
    int dotIndex = field.name().indexOf('.');
    if (dotIndex == -1) {
      if (field.name().equals(fieldName)) {
        return i;
      }
    } else {
      if (dotIndex < fieldName.length()) {
        String
            fieldNameWithDot =
            fieldName.substring(0, dotIndex) + "." + fieldName.substring(dotIndex + 1);
        if (field.name().equals(fieldNameWithDot)) {
          return i;
        }
      }
    }

  }
  return -1;
}
 

void assertFoo(int count, Struct struct) {
  assertEquals(SCHEMA_NAME, struct.schema().name(), "struct.schema().name() does not match.");
  assertNotNull(struct, "struct should not be null");

  assertEquals(count, struct.schema().fields().size(), "struct.schema().fields().size() does not match.");
  for (int i = 1; i <= count; i++) {
    final String fieldName = String.format("f%s", i);
    Field field = struct.schema().field(fieldName);
    assertNotNull(field, "schema should have field " + fieldName);
    assertEquals(Type.INT32, field.schema().type(), "schema().type() for " + fieldName + " does not match.");

    final Integer expectedValue = i;
    final Integer actualValue = struct.getInt32(fieldName);
    assertEquals(
        expectedValue,
        actualValue,
        String.format("value for field %s does not match", fieldName)
    );
  }

}
 

@Override
public void serialize(Struct struct, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
  struct.validate();
  Storage result = new Storage();
  result.schema = struct.schema();
  result.fieldValues = new ArrayList<>();
  for (Field field : struct.schema().fields()) {
    log.trace("serialize() - Processing field '{}'", field.name());
    KeyValue keyValue = new KeyValue();
    keyValue.name = field.name();
    keyValue.schema = field.schema();
    keyValue.value(struct.get(field));
    result.fieldValues.add(keyValue);
  }
  jsonGenerator.writeObject(result);
}
 

@Override
protected Object convertDecimal(Column column, Field fieldDefn, Object data) {
    if (data instanceof NUMBER) {
        try {
            data = ((NUMBER) data).bigDecimalValue();
        }
        catch (SQLException e) {
            throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
        }
    }

    // adjust scale to column's scale if the column's scale is larger than the one from
    // the value (e.g. 4.4444 -> 4.444400)
    if (data instanceof BigDecimal) {
        data = withScaleAdjustedIfNeeded(column, (BigDecimal) data);
    }

    return super.convertDecimal(column, fieldDefn, data);
}
 

public SchemaKStream(
    final Schema schema,
    final KStream<String, GenericRow> kstream,
    final Field keyField,
    final List<SchemaKStream> sourceSchemaKStreams,
    final Type type,
    final FunctionRegistry functionRegistry,
    final SchemaRegistryClient schemaRegistryClient
) {
  this.schema = schema;
  this.kstream = kstream;
  this.keyField = keyField;
  this.sourceSchemaKStreams = sourceSchemaKStreams;
  this.genericRowValueTypeEnforcer = new GenericRowValueTypeEnforcer(schema);
  this.type = type;
  this.functionRegistry = functionRegistry;
  this.schemaRegistryClient = schemaRegistryClient;
}
 

public SchemaKStream select(final Schema selectSchema) {
  final KStream<String, GenericRow>
      projectedKStream =
      kstream.mapValues(row -> {
        List<Object> newColumns = new ArrayList<>();
        for (Field schemaField : selectSchema.fields()) {
          newColumns.add(extractColumn(schemaField, row)
          );
        }
        return new GenericRow(newColumns);
      });

  return new SchemaKStream(
      selectSchema,
      projectedKStream,
      keyField,
      Collections.singletonList(this),
      Type.PROJECT,
      functionRegistry,
      schemaRegistryClient
  );
}
 

public static Map<String, Object> asMap(Struct struct) {
  Preconditions.checkNotNull(struct, "struct cannot be null.");
  Map<String, Object> result = new LinkedHashMap<>(struct.schema().fields().size());

  for (Field field : struct.schema().fields()) {
    final Object value;
    if (Schema.Type.STRUCT == field.schema().type()) {
      Struct s = struct.getStruct(field.name());
      value = asMap(s);
    } else {
      value = struct.get(field);
    }
    result.put(field.name(), value);
  }

  return result;
}
 

@Override
protected Object convertFloat(Column column, Field fieldDefn, Object data) {
    if (data instanceof Float) {
        return data;
    }
    else if (data instanceof NUMBER) {
        return ((NUMBER) data).floatValue();
    }
    else if (data instanceof BINARY_FLOAT) {
        try {
            return ((BINARY_FLOAT) data).floatValue();
        }
        catch (SQLException e) {
            throw new RuntimeException("Couldn't convert value for column " + column.name(), e);
        }
    }

    return super.convertFloat(column, fieldDefn, data);
}
 

R filter(R record, Struct struct) {
  for (Field field : struct.schema().fields()) {
    if (this.config.fields.contains(field.name())) {
      if (field.schema().type() == Schema.Type.STRING) {
        String input = struct.getString(field.name());
        if (null != input) {
          Matcher matcher = this.config.pattern.matcher(input);
          if (matcher.matches()) {
            return null;
          }
        }
      }
    }
  }
  return record;
}
 

protected Object convertIntervalYearMonth(Column column, Field fieldDefn, Object data) {
    return convertValue(column, fieldDefn, data, NumberConversions.LONG_FALSE, (r) -> {
        if (data instanceof Number) {
            // we expect to get back from the plugin a double value
            r.deliver(((Number) data).longValue());
        }
        else if (data instanceof INTERVALYM) {
            final String interval = ((INTERVALYM) data).stringValue();
            int sign = 1;
            int start = 0;
            if (interval.charAt(0) == '-') {
                sign = -1;
                start = 1;
            }
            for (int i = 1; i < interval.length(); i++) {
                if (interval.charAt(i) == '-') {
                    final int year = sign * Integer.parseInt(interval.substring(start, i));
                    final int month = sign * Integer.parseInt(interval.substring(i + 1, interval.length()));
                    r.deliver(MicroDuration.durationMicros(year, month, 0, 0,
                            0, 0, MicroDuration.DAYS_PER_MONTH_AVG));
                }
            }
        }
    });
}
 

Storage(Schema schema) {
  this.name = schema.name();
  this.doc = schema.doc();
  this.type = schema.type();
  this.defaultValue = schema.defaultValue();
  this.version = schema.version();
  this.parameters = schema.parameters();
  this.isOptional = schema.isOptional();

  if (Schema.Type.MAP == this.type) {
    this.keySchema = schema.keySchema();
    this.valueSchema = schema.valueSchema();
  } else if (Schema.Type.ARRAY == this.type) {
    this.keySchema = null;
    this.valueSchema = schema.valueSchema();
  } else if (Schema.Type.STRUCT == this.type) {
    this.fieldSchemas = new LinkedHashMap<>();
    for (Field field : schema.fields()) {
      this.fieldSchemas.put(field.name(), field.schema());
    }
  }
}
 

public SchemaKTable(
    final Schema schema,
    final KTable ktable,
    final Field keyField,
    final List<SchemaKStream> sourceSchemaKStreams,
    boolean isWindowed,
    Type type,
    final FunctionRegistry functionRegistry,
    final SchemaRegistryClient schemaRegistryClient
) {
  super(
      schema,
      null,
      keyField,
      sourceSchemaKStreams,
      type,
      functionRegistry,
      schemaRegistryClient
  );
  this.ktable = ktable;
  this.isWindowed = isWindowed;
}
 

public KsqlTable(
    String sqlExpression,
    final String datasourceName,
    final Schema schema,
    final Field keyField,
    final TimestampExtractionPolicy timestampExtractionPolicy,
    final KsqlTopic ksqlTopic,
    final String stateStoreName,
    boolean isWindowed
) {
  super(
      sqlExpression,
      datasourceName,
      schema,
      keyField,
      timestampExtractionPolicy,
      DataSourceType.KTABLE,
      ksqlTopic
  );
  this.stateStoreName = stateStoreName;
  this.isWindowed = isWindowed;
}
 

private AbstractStreamCreateStatement addAvroFields(
    final AbstractStreamCreateStatement abstractStreamCreateStatement,
    final Schema schema,
    int schemaId
) {
  List<TableElement> elements = new ArrayList<>();
  for (Field field : schema.fields()) {
    TableElement tableElement = new TableElement(field.name().toUpperCase(), SchemaUtil
        .getSqlTypeName(field.schema()));
    elements.add(tableElement);
  }
  StringLiteral schemaIdLiteral = new StringLiteral(String.format("%d", schemaId));
  Map<String, Expression> properties =
      new HashMap<>(abstractStreamCreateStatement.getProperties());
  if (!abstractStreamCreateStatement.getProperties().containsKey(KsqlConstants.AVRO_SCHEMA_ID)) {
    properties.put(KsqlConstants.AVRO_SCHEMA_ID, schemaIdLiteral);
  }

  return abstractStreamCreateStatement.copyWith(elements, properties);
}
 

@Override
public ValueConverter converter(Column column, Field fieldDefn) {
    switch (column.jdbcType()) {
        // Numeric integers
        case Types.TINYINT:
            // values are an 8-bit unsigned integer value between 0 and 255, we thus need to store it in short int
            return (data) -> convertSmallInt(column, fieldDefn, data);
        default:
            return super.converter(column, fieldDefn);
    }
}
 

public SourceDescription(
    PersistentQueryMetadata queryMetadata,
    KafkaTopicClient topicClient
) {
  this(
      queryMetadata.getStatementString(),
      queryMetadata.getStatementString(),
      Collections.EMPTY_LIST,
      Collections.EMPTY_LIST,
      queryMetadata.getResultSchema().fields().stream().map(
          field ->
            new FieldSchemaInfo(field.name(), SchemaUtil
                .getSchemaFieldName(field))
          ).collect(Collectors.toList()),
      "QUERY",
      Optional.ofNullable(outputNodeFromMetadata(queryMetadata)
          .getKeyField()).map(Field::name).orElse(""),
      Optional.ofNullable(outputNodeFromMetadata(queryMetadata)
          .getTimestampExtractionPolicy()).map(TimestampExtractionPolicy::timestampField)
          .orElse(""),
      MetricCollectors.getStatsFor(
          outputNodeFromMetadata(queryMetadata).getKafkaTopicName(), false),
      MetricCollectors.getStatsFor(
          outputNodeFromMetadata(queryMetadata).getKafkaTopicName(), true),
      true,
      outputNodeFromMetadata(queryMetadata).getTopicSerde().getSerDe().name(),
      outputNodeFromMetadata(queryMetadata).getKafkaTopicName(),
      queryMetadata.getTopologyDescription(),
      queryMetadata.getExecutionPlan(),
      getPartitions(topicClient, outputNodeFromMetadata(queryMetadata).getKafkaTopicName()),
      getReplication(topicClient, outputNodeFromMetadata(queryMetadata).getKafkaTopicName()),
      queryMetadata.getOverriddenProperties()
  );
}
 

@Override
protected Object visitQualifiedNameReference(QualifiedNameReference node, Object context) {
  Optional<Field> schemaField = SchemaUtil.getFieldByName(schema, node.getName().getSuffix());
  if (!schemaField.isPresent()) {
    throw new RuntimeException(
        "Cannot find the select field in the available fields: " + node.getName().getSuffix());
  }
  parameterMap.put(
      schemaField.get().name().replace(".", "_"),
      SchemaUtil.getJavaType(schemaField.get().schema())
  );
  return null;
}
 

/**
 * Remove the alias when reading/writing from outside
 */
public static Schema getSchemaWithNoAlias(Schema schema) {
  SchemaBuilder schemaBuilder = SchemaBuilder.struct();
  for (Field field : schema.fields()) {
    String name = getFieldNameWithNoAlias(field);
    schemaBuilder.field(name, field.schema());
  }
  return schemaBuilder.build();
}
 

@Override
protected Object visitDereferenceExpression(DereferenceExpression node, Object context) {
  Optional<Field> schemaField = SchemaUtil.getFieldByName(schema, node.toString());
  if (!schemaField.isPresent()) {
    throw new RuntimeException(
        "Cannot find the select field in the available fields: " + node.toString());
  }
  parameterMap.put(
      schemaField.get().name().replace(".", "_"),
      SchemaUtil.getJavaType(schemaField.get().schema())
  );
  return null;
}
 

public static int getIndexInSchema(final String fieldName, final Schema schema) {
  List<Field> fields = schema.fields();
  for (int i = 0; i < fields.size(); i++) {
    Field field = fields.get(i);
    if (field.name().equals(fieldName)) {
      return i;
    }
  }
  throw new KsqlException(
      "Couldn't find field with name="
          + fieldName
          + " in schema. fields="
          + fields
  );
}
 

private long processStruct(SchemaAndValue schemaAndValue) {
  final Struct inputStruct = (Struct) schemaAndValue.value();
  final Field inputField = schemaAndValue.schema().field(this.config.fieldName);

  if (null == inputField) {
    throw new DataException(
        String.format("Schema does not have field '{}'", this.config.fieldName)
    );
  }

  final Schema fieldSchema = inputField.schema();
  final long result;
  if (Schema.Type.INT64 == fieldSchema.type()) {
    final Object fieldValue = inputStruct.get(inputField);

    if (null == fieldValue) {
      throw new DataException(
          String.format("Field '%s' cannot be null.", this.config.fieldName)
      );
    }

    if (Timestamp.LOGICAL_NAME.equals(fieldSchema.name())) {
      final Date date = (Date) fieldValue;
      result = date.getTime();
    } else {
      final long timestamp = (long) fieldValue;
      result = timestamp;
    }
  } else {
    throw new DataException(
        String.format("Schema '{}' is not supported.", inputField.schema())
    );
  }

  return result;
}
 

@Override
protected Pair<String, Schema> visitDereferenceExpression(
    DereferenceExpression node,
    Boolean unmangleNames
) {
  String fieldName = node.toString();
  Optional<Field> schemaField = SchemaUtil.getFieldByName(schema, fieldName);
  if (!schemaField.isPresent()) {
    throw new KsqlException("Field not found: " + fieldName);
  }
  return new Pair<>(fieldName.replace(".", "_"), schemaField.get().schema());
}
 

@Override
protected Pair<String, Schema> visitSymbolReference(
    SymbolReference node,
    Boolean context
) {
  String fieldName = formatIdentifier(node.getName());
  Optional<Field> schemaField = SchemaUtil.getFieldByName(schema, fieldName);
  if (!schemaField.isPresent()) {
    throw new KsqlException("Field not found: " + fieldName);
  }
  return new Pair<>(fieldName, schemaField.get().schema());
}
 

@Override
protected Pair<String, Schema> visitQualifiedNameReference(
    QualifiedNameReference node,
    Boolean unmangleNames
) {
  String fieldName = formatQualifiedName(node.getName());
  Optional<Field> schemaField = SchemaUtil.getFieldByName(schema, fieldName);
  if (!schemaField.isPresent()) {
    throw new KsqlException("Field not found: " + fieldName);
  }
  return new Pair<>(fieldName.replace(".", "_"), schemaField.get().schema());
}
 

@Test
public void shouldGetTheCorrectFieldName() {
  Optional<Field> field = SchemaUtil.getFieldByName(schema, "orderid".toUpperCase());
  Assert.assertTrue(field.isPresent());
  assertThat(field.get().schema(), sameInstance(Schema.INT64_SCHEMA));
  assertThat("", field.get().name().toLowerCase(), equalTo("orderid"));

  Optional<Field> field1 = SchemaUtil.getFieldByName(schema, "orderid");
  Assert.assertFalse(field1.isPresent());
}
 

public static Optional<Field> getFieldByName(final Schema schema, final String fieldName) {
  if (schema.fields() != null) {
    for (Field field : schema.fields()) {
      if (field.name().equals(fieldName)) {
        return Optional.of(field);
      } else if (field.name().equals(fieldName.substring(fieldName.indexOf(".") + 1))) {
        return Optional.of(field);
      }
    }
  }
  return Optional.empty();
}
 

public static List<Field> fields(Schema schema) {
    Schema.Type type = schema.type();
    if (Schema.Type.STRUCT.equals(type)) {
        return schema.fields();
    } else {
        return null;
    }
}