org.apache.avro.SchemaBuilder Java Examples

The following examples show how to use org.apache.avro.SchemaBuilder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SnowflakeRuntimeIT.java    From components with Apache License 2.0 7 votes vote down vote up
public Schema getMakeRowSchema() {
    SchemaBuilder.FieldAssembler<Schema> fa = SchemaBuilder.builder().record("MakeRowRecord").fields() //
            .name("ID").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "ID").type(AvroUtils._decimal()).noDefault() //
            .name("C1").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "C1").type().nullable().stringType().noDefault() //
            .name("C2").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "C2").type().nullable().booleanType().noDefault() //
            .name("C3").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "C3").type().nullable().doubleType().noDefault() //
            // date
            .name("C4").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "C4").type(AvroUtils._logicalDate()).noDefault() //
            // time
            .name("C5").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "C5").type(AvroUtils._logicalTime()).noDefault() //
            // timestamp
            .name("C6").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "C6").type(AvroUtils._logicalTimestamp())
            .noDefault() //
            // variant
            .name("C7").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "C7").type().nullable().stringType().noDefault();
    return fa.endRecord();
}
 
Example #2
Source File: SnowflakeReaderTest.java    From components with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
    schema = SchemaBuilder.builder().record("Schema").fields()
            .requiredString("field")
            .requiredString("column")
            .endRecord();
    schema.getField("field").addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "field");
    schema.getField("column").addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "column");

    TSnowflakeInputProperties tSnowflakeInputProperties = new TSnowflakeInputProperties("test");
    tSnowflakeInputProperties.setupProperties();

    tSnowflakeInputProperties.table.main.schema.setValue(schema);
    Mockito.when(snowflakeSourceMock.getRuntimeSchema(Mockito.any(SchemaResolver.class))).thenReturn(schema);
    tSnowflakeInputProperties.table.tableName.setValue("Table");
    Mockito.doCallRealMethod().when(snowflakeSourceMock).initialize(Mockito.any(), Mockito.eq(tSnowflakeInputProperties));
    snowflakeSourceMock.initialize(null, tSnowflakeInputProperties);
    snowflakeReader = new SnowflakeReader(runtimeContainerMock, snowflakeSourceMock, tSnowflakeInputProperties);
}
 
Example #3
Source File: TestAvroDecoder.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testMapOfMapsWithNulls()
{
    Schema schema = SchemaBuilder.map()
            .values()
            .nullable().map()
            .values()
            .nullable().floatType();

    Map<String, Map<String, Float>> data = buildMapFromKeysAndValues(ImmutableList.of("k1", "k2", "k3"),
            Arrays.asList(buildMapFromKeysAndValues(ImmutableList.of("key1", "key2", "key3"), ImmutableList.of(1.3F, 2.3F, -.5F)),
                    null,
                    buildMapFromKeysAndValues(ImmutableList.of("key10", "key20", "key30"), Arrays.asList(11.3F, null, -1.5F))));
    DecoderTestColumnHandle row = new DecoderTestColumnHandle(0, "row", MAP_OF_REAL_MAP_TYPE, "map_field", null, null, false, false, false);
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = buildAndDecodeColumn(row, "map_field", schema.toString(), data);
    checkMapValue(decodedRow, row, data);
}
 
Example #4
Source File: TestAvroDecoder.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testMapOfArrayOfMapsWithDifferentValues()
{
    Schema schema = SchemaBuilder.map()
            .values()
            .array()
            .items()
            .map()
            .values()
            .floatType();

    Map<String, List<Map<String, Float>>> data = buildMapFromKeysAndValues(ImmutableList.of("k1", "k2"),
            Arrays.asList(Arrays.asList(buildMapFromKeysAndValues(ImmutableList.of("sk1", "sk2", "sk3"), Arrays.asList(1.3F, -5.3F, 2.3F))),
                    Arrays.asList(buildMapFromKeysAndValues(ImmutableList.of("sk11", "sk21", "sk31"), Arrays.asList(11.3F, -1.5F, 12.3F)))));
    Map<String, List<Map<String, Float>>> mismatchedData = buildMapFromKeysAndValues(ImmutableList.of("k1", "k2"),
            Arrays.asList(Arrays.asList(buildMapFromKeysAndValues(ImmutableList.of("sk1", "sk2", "sk3"), Arrays.asList(1.3F, -5.3F, -2.3F))),
                    Arrays.asList(buildMapFromKeysAndValues(ImmutableList.of("sk11", "sk21", "sk31"), Arrays.asList(11.3F, -1.5F, 12.3F)))));

    DecoderTestColumnHandle row = new DecoderTestColumnHandle(0, "row", MAP_OF_ARRAY_OF_MAP_TYPE, "map_field", null, null, false, false, false);
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = buildAndDecodeColumn(row, "map_field", schema.toString(), data);
    assertThrows(AssertionError.class, () -> checkArrayValue(decodedRow, row, mismatchedData));
}
 
Example #5
Source File: TestKiteProcessorsCluster.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaFromDistributedFileSystem() throws IOException {
    Schema expected = SchemaBuilder.record("Test").fields()
            .requiredLong("id")
            .requiredString("color")
            .optionalDouble("price")
            .endRecord();

    Path schemaPath = new Path("hdfs:/tmp/schema.avsc");
    FileSystem fs = schemaPath.getFileSystem(DefaultConfiguration.get());
    OutputStream out = fs.create(schemaPath);
    out.write(bytesFor(expected.toString(), Charset.forName("utf8")));
    out.close();

    Schema schema = AbstractKiteProcessor.getSchema(
            schemaPath.toString(), DefaultConfiguration.get());

    Assert.assertEquals("Schema from file should match", expected, schema);
}
 
Example #6
Source File: MarketoConstants.java    From components with Apache License 2.0 6 votes vote down vote up
public static Schema getRESTSchemaForGetLeadOrGetMultipleLeads() {
    return SchemaBuilder.builder().record("getLeadOrGetMultipleLeadsREST").fields() //
            .name("id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().nullable().intType().noDefault() //
            .name("email").type().nullable().stringType().noDefault() //
            .name("firstName").type().nullable().stringType().noDefault() //
            .name("lastName").type().nullable().stringType().noDefault() //
            .name(FIELD_CREATED_AT)//
            .prop(SchemaConstants.TALEND_COLUMN_PATTERN, DATETIME_PATTERN_REST)//
            .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) //
            .type(AvroUtils._date()).noDefault()//
            .name(FIELD_UPDATED_AT)//
            .prop(SchemaConstants.TALEND_COLUMN_PATTERN, DATETIME_PATTERN_REST)//
            .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) //
            .type(AvroUtils._date()).noDefault()//
            .endRecord();
}
 
Example #7
Source File: TestSchemaManager.java    From kite with Apache License 2.0 6 votes vote down vote up
@Test(expected = IncompatibleSchemaException.class)
public void testIndirectIncompatibleUpdate() {
  SchemaManager manager = SchemaManager.create(getConfiguration(), testDirectory);

  // Write two schemas that are compatible since they use optional fields.
  manager.writeSchema(SchemaBuilder.record("test")
      .fields()
      .optionalString("foo")
      .endRecord());

  manager.writeSchema(SchemaBuilder.record("test")
      .fields()
      .optionalString("bar")
      .endRecord());

  // This schema creates a schema compatible with the immediately previous
  // version, but incompatible with the original.
  manager.writeSchema(SchemaBuilder.record("test")
      .fields()
      .optionalInt("foo")
      .endRecord());
}
 
Example #8
Source File: TestJsonUtil.java    From kite with Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaInferenceNullableMap() throws Exception {
  Schema recordSchema = SchemaBuilder.record("Test").fields()
      .requiredString("aString")
      .name("aMap").type().map().values()
          .unionOf().nullType().and().stringType().endUnion().noDefault()
      .endRecord();

  String jsonSample = "{" +
      "\"aString\": \"triangle\"," +
      "\"aMap\": { \"left\": null, \"right\": \"dictionary\" }" +
      "}";

  JsonNode datum = JsonUtil.parse(jsonSample);
  Assert.assertEquals("Should produce expected schema",
      recordSchema, JsonUtil.inferSchemaWithMaps(datum, "Test"));

  Map<String, Object> aMap = Maps.newLinkedHashMap();
  aMap.put("left", null);
  aMap.put("right", "dictionary");
  GenericData.Record expected = new GenericData.Record(recordSchema);
  expected.put("aString", "triangle");
  expected.put("aMap", aMap);
  Assert.assertEquals("Should convert to record",
      expected, convertGeneric(datum, recordSchema));
}
 
Example #9
Source File: ConfluentSchemaRegistryCoderTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testSpecificRecordWithConfluentSchemaRegistry() throws Exception {
	MockSchemaRegistryClient client = new MockSchemaRegistryClient();

	Schema schema = SchemaBuilder.record("testRecord")
		.fields()
		.optionalString("testField")
		.endRecord();
	int schemaId = client.register("testTopic", schema);

	ConfluentSchemaRegistryCoder registryCoder = new ConfluentSchemaRegistryCoder(client);
	ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
	DataOutputStream dataOutputStream = new DataOutputStream(byteOutStream);
	dataOutputStream.writeByte(0);
	dataOutputStream.writeInt(schemaId);
	dataOutputStream.flush();

	ByteArrayInputStream byteInStream = new ByteArrayInputStream(byteOutStream.toByteArray());
	Schema readSchema = registryCoder.readSchema(byteInStream);

	assertEquals(schema, readSchema);
	assertEquals(0, byteInStream.available());
}
 
Example #10
Source File: KsqlEngineTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldCleanUpInternalTopicSchemasFromSchemaRegistry() throws Exception {
  final List<QueryMetadata> queries
      = ksqlEngine.buildMultipleQueries(
      "create stream s1  with (value_format = 'avro') as select * from test1;"
      + "create table t1 as select col1, count(*) from s1 group by col1;",
      Collections.emptyMap());
  Schema schema = SchemaBuilder
      .record("Test").fields()
      .name("clientHash").type().fixed("MD5").size(16).noDefault()
      .endRecord();
  ksqlEngine.getSchemaRegistryClient().register
      ("_confluent-ksql-default_query_CTAS_T1-KSTREAM-AGGREGATE-STATE-STORE-0000000006"
       + "-changelog-value", schema);
  ksqlEngine.getSchemaRegistryClient().register("_confluent-ksql-default_query_CTAS_T1-KSTREAM-AGGREGATE-STATE-STORE-0000000006-repartition-value", schema);

  assertThat(schemaRegistryClient.getAllSubjects().contains("_confluent-ksql-default_query_CTAS_T1-KSTREAM-AGGREGATE-STATE-STORE-0000000006-changelog-value"), equalTo(true));
  assertThat(schemaRegistryClient.getAllSubjects().contains("_confluent-ksql-default_query_CTAS_T1-KSTREAM-AGGREGATE-STATE-STORE-0000000006-repartition-value"), equalTo(true));
  ksqlEngine.terminateQuery(new QueryId("CTAS_T1"), true);
  assertThat(schemaRegistryClient.getAllSubjects().contains("_confluent-ksql-default_query_CTAS_T1-KSTREAM-AGGREGATE-STATE-STORE-0000000006-changelog-value"), equalTo(false));
  assertThat(schemaRegistryClient.getAllSubjects().contains("_confluent-ksql-default_query_CTAS_T1-KSTREAM-AGGREGATE-STATE-STORE-0000000006-repartition-value"), equalTo(false));
}
 
Example #11
Source File: MarketoInputReaderTestIT.java    From components with Apache License 2.0 6 votes vote down vote up
@Test
public void testCustomObjectDynamicSchema() throws Exception {
    TMarketoInputProperties props = getRESTProperties();
    String coName = "smartphone_c";
    String brand = "Apple";
    String models = "iPhone 7";
    props.inputOperation.setValue(CustomObject);
    props.customObjectAction.setValue(CustomObjectAction.get);
    props.batchSize.setValue(1);
    props.afterCustomObjectAction();
    props.customObjectName.setValue(coName);
    props.customObjectFilterType.setValue("model");
    props.customObjectFilterValues.setValue(models);
    Schema design = SchemaBuilder.builder().record("test").prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true").fields()
            .endRecord();
    design.addProp(SchemaConstants.INCLUDE_ALL_FIELDS, "true");
    props.schemaInput.schema.setValue(design);
    reader = getReader(props);
    assertTrue(reader.start());
    IndexedRecord r = reader.getCurrent();
    assertNotNull(r);
    assertTrue(r.getSchema().getFields().size() > 6);
    assertFalse(reader.advance());
}
 
Example #12
Source File: TestJsonUtil.java    From kite with Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaInferenceNullablePrimitiveArray() throws Exception {
  Schema recordSchema = SchemaBuilder.record("Test").fields()
      .requiredString("aString")
      .name("anArray").type().array().items()
          .unionOf().nullType().and().intType().endUnion().noDefault()
      .endRecord();

  String jsonSample = "{" +
      "\"aString\": \"triangle\"," +
      "\"anArray\": [ null, 1, 2, 3, 4 ]" +
      "}";

  JsonNode datum = JsonUtil.parse(jsonSample);
  Assert.assertEquals("Should produce expected schema",
      recordSchema, JsonUtil.inferSchema(datum, "Test"));

  GenericData.Record expected = new GenericData.Record(recordSchema);
  expected.put("aString", "triangle");
  expected.put("anArray", Lists.newArrayList(null, 1, 2, 3, 4));
  Assert.assertEquals("Should convert to record",
      expected, convertGeneric(datum, recordSchema));
}
 
Example #13
Source File: AvroCoderTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testAvroCoderNestedRecords() {
  // Nested Record
  assertDeterministic(
      AvroCoder.of(
          SchemaBuilder.record("nestedRecord")
              .fields()
              .name("subRecord")
              .type()
              .record("subRecord")
              .fields()
              .name("innerField")
              .type()
              .stringType()
              .noDefault()
              .endRecord()
              .noDefault()
              .endRecord()));
}
 
Example #14
Source File: GoogleDriveDeleteProperties.java    From components with Apache License 2.0 6 votes vote down vote up
public void setupProperties() {
    super.setupProperties();

    Schema schema = SchemaBuilder.builder().record(GoogleDriveDeleteDefinition.COMPONENT_NAME).fields() //
            .name(GoogleDriveDeleteDefinition.RETURN_FILE_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .endRecord();
    schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    schemaMain.schema.setValue(schema);

    deleteMode.setPossibleValues(AccessMethod.values());
    deleteMode.setValue(AccessMethod.Name);
    file.setValue("");
    useTrash.setValue(true);
}
 
Example #15
Source File: SnowflakeRowReaderTest.java    From components with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException, SQLException {
    source = Mockito.mock(SnowflakeRowSource.class);

    connection = Mockito.mock(Connection.class);
    Mockito.when(source.createConnection(Mockito.any())).thenReturn(connection);
    Mockito.doNothing().when(source).closeConnection(Mockito.any(), Mockito.any(Connection.class));
    properties = new TSnowflakeRowProperties("rowProperties");
    schema = SchemaBuilder.builder().record("test").fields().requiredString("name").endRecord();
    query = "SELECT id, name from " + TABLE_NAME;
    properties.query.setValue(query);
    properties.schemaFlow.schema.setValue(schema);
    properties.setupProperties();
    properties.table.tableName.setValue(TABLE_NAME);
    Mockito.when(source.getRowProperties()).thenReturn(properties);
    Mockito.when(source.getRuntimeSchema(null)).thenReturn(schema);
    reader = new SnowflakeRowReader(null, source);
    Mockito.doCallRealMethod().when(source).getQuery();
}
 
Example #16
Source File: RootSchemaUtilsTest.java    From components with Apache License 2.0 6 votes vote down vote up
/**
 * Checks {@link RootSchemaUtils#getMainSchema(Schema)} returns correct Main schema retrieved from incoming
 * Root schema
 */
@Test
public void testGetMainSchema() {
    Schema mainSchema = SchemaBuilder.builder().record("Main").fields() //
            .name("id").type().intType().noDefault() //
            .endRecord(); //

    Schema outOfBandSchema = SchemaBuilder.builder().record("EmptySchema").fields().endRecord(); //$NON-NLS-1$

    Schema rootSchema = SchemaBuilder.record("Root").fields() //$NON-NLS-1$
            .name("Main").type(mainSchema).noDefault() // $NON-NLS-1$
            .name("OutOfBand").type(outOfBandSchema).noDefault() // $NON-NLS-1$
            .endRecord(); //

    Schema actualMainSchema = RootSchemaUtils.getMainSchema(rootSchema);
    assertEquals(mainSchema, actualMainSchema);
}
 
Example #17
Source File: TestAvroUtils.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Test
public void toDataTypeRecordNested() {
  Schema inner = SchemaBuilder.record("inner").fields()
      .name("field1").type().intType().noDefault()
      .endRecord();

  Schema outer = SchemaBuilder.record("outer").fields()
      .name("inner").type(inner).noDefault()
      .endRecord();

  assertEquals("Invalid DataType",
      DataTypes.createStructType(Lists.newArrayList(
        // Outer
        DataTypes.createStructField("inner",
          // Inner
          DataTypes.createStructType(Lists.newArrayList(
            DataTypes.createStructField("field1", DataTypes.IntegerType, false)
          )), false))
      ),
      AvroUtils.dataTypeFor(outer));
}
 
Example #18
Source File: TollboothSchemaStoreClientTest.java    From data-highway with Apache License 2.0 6 votes vote down vote up
@Test(expected = InvalidPiiAnnotationException.class)
public void addPiiToExistingFieldFails() throws Exception {
  Schema newSchema = SchemaBuilder
      .record("r")
      .fields()
      .name("f")
      .prop("sensitivity", "PII")
      .type(SchemaBuilder.builder().stringType())
      .noDefault()
      .endRecord();

  Schema currentSchema = SchemaBuilder
      .record("r")
      .fields()
      .name("f")
      .type(SchemaBuilder.builder().stringType())
      .noDefault()
      .endRecord();
  client.registerSchema("road1", currentSchema);
  client.registerSchema("road1", newSchema);
}
 
Example #19
Source File: SalesforceGetDeletedUpdatedReaderTestIT.java    From components with Apache License 2.0 6 votes vote down vote up
@Override
public Schema getMakeRowSchema(boolean isDynamic) {
    SchemaBuilder.FieldAssembler<Schema> fa = SchemaBuilder.builder().record("MakeRowRecord").fields() //
            .name("Id").type(AvroUtils._string()).noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("ShippingStreet").type(AvroUtils._string()).noDefault() //
            .name("ShippingPostalCode").type(AvroUtils._int()).noDefault() //
            .name("BillingStreet").type(AvroUtils._string()).noDefault() //
            .name("BillingState").type(AvroUtils._string()).noDefault() //
            .name("BillingPostalCode").type(AvroUtils._string()).noDefault();
    if (isDynamic) {
        fa = fa.name("ShippingState").type(AvroUtils._string()).noDefault();
    }

    return fa.endRecord();
}
 
Example #20
Source File: TestReflectLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadUUIDArray() throws IOException {
  Schema uuidArraySchema = SchemaBuilder.record(RecordWithUUIDArray.class.getName())
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  LogicalTypes.uuid().addToSchema(
      uuidArraySchema.getField("uuids").schema().getElementType());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  GenericRecord r = new GenericData.Record(uuidArraySchema);
  r.put("uuids", Arrays.asList(u1.toString(), u2.toString()));

  RecordWithUUIDArray expected = new RecordWithUUIDArray();
  expected.uuids = new UUID[] {u1, u2};

  File test = write(uuidArraySchema, r);

  Assert.assertEquals("Should convert Strings to UUIDs",
      expected,
      read(REFLECT, uuidArraySchema, test).get(0));
}
 
Example #21
Source File: MarketoConstants.java    From components with Apache License 2.0 5 votes vote down vote up
public static Schema getListOperationFlowRESTSchema() {
    return SchemaBuilder.builder().record("REST").fields() //
            .name(FIELD_LIST_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_LEAD_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_STATUS)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .endRecord();
}
 
Example #22
Source File: TestNiFiOrcUtils.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void test_getOrcField_map() throws Exception {
    final SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
    builder.name("map").type().map().values().doubleType().noDefault();
    Schema testSchema = builder.endRecord();
    TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("map").schema());
    assertEquals(
            TypeInfoFactory.getMapTypeInfo(
                    TypeInfoCreator.createString(),
                    TypeInfoCreator.createDouble()),
            orcType);
}
 
Example #23
Source File: TSalesforceInputPropertiesIT.java    From components with Apache License 2.0 5 votes vote down vote up
/**
 * Checks {@link TSalesforceInputProperties#guessQuery} returns empty {@link java.lang.String}
 * when schema does not include any fields
 */
@Test
public void testValidateGuessQueryEmptySchema() throws Exception {
    final String field1 = "Id";
    final String field2 = "Name";
    final String moduleName = "Module";

    String expectedQuery = "";

    Schema schema = SchemaBuilder.record("Result").fields()
            .endRecord();

    SalesforceModuleProperties salesforceModuleProperties = new SalesforceModuleProperties("properties");
    salesforceModuleProperties.moduleName.setValue(moduleName);
    salesforceModuleProperties.main.schema.setValue(schema);

    properties.module = salesforceModuleProperties;

    ValidationResult.Result resultStatus = properties.validateGuessQuery().getStatus();
    String expectedMessage = properties.validateGuessQuery().getMessage();

    LOGGER.debug("validation result status: " + resultStatus);
    Assert.assertEquals(ValidationResult.Result.ERROR, resultStatus);
    Assert.assertNotNull(expectedMessage);
    Assert.assertEquals(expectedMessage, "Schema does not contain any field. Query cannot be guessed.");

    String resultQuery = properties.query.getValue();
    LOGGER.debug("result query: " + resultQuery);
    Assert.assertNotNull(resultQuery);
    Assert.assertEquals(expectedQuery, resultQuery);
}
 
Example #24
Source File: TestAvroUtils.java    From envelope with Apache License 2.0 5 votes vote down vote up
@Test
public void toDataTypeArrayNested() {
  Schema inner = SchemaBuilder.array().items().intType();
  Schema outer = SchemaBuilder.array().items(inner);
  assertEquals("Invalid DataType",
      // Outer
      DataTypes.createArrayType(
          // Inner
          DataTypes.createArrayType(DataTypes.IntegerType, false),
          false
      ),
      AvroUtils.dataTypeFor(outer));
}
 
Example #25
Source File: RootRecordUtilsTest.java    From components with Apache License 2.0 5 votes vote down vote up
/**
 * Checks {@link RootRecordUtils#createRootRecord(Schema, Schema)} throws {@link IllegalArgumentException} in case of
 * out of band schema is null with following message "Input schemas should be not null"
 */
@Test
public void testCreateRootRecordOutOfBandNull() {
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage("Input schemas should be not null");

    Schema notNullMainSchema = SchemaBuilder.record("Main").fields() //
            .name("name").type().stringType().noDefault().endRecord(); //

    RootRecordUtils.createRootRecord(notNullMainSchema, null);
}
 
Example #26
Source File: GoogleDriveDatasetProperties.java    From components with Apache License 2.0 5 votes vote down vote up
@Override
public void setupProperties() {
    Schema s = SchemaBuilder.builder().record(GoogleDriveListDefinition.COMPONENT_NAME).fields() //
            .name(GoogleDriveListDefinition.RETURN_ID).prop(SchemaConstants.TALEND_IS_LOCKED, "true").type().nullable()
            .stringType().noDefault()//
            .name(GoogleDriveListDefinition.RETURN_NAME).prop(SchemaConstants.TALEND_IS_LOCKED, "true").type().nullable()
            .stringType().noDefault()//
            .name(GoogleDriveListDefinition.RETURN_MIME_TYPE).prop(SchemaConstants.TALEND_IS_LOCKED, "true").type().nullable()
            .stringType().noDefault()//
            .name(GoogleDriveListDefinition.RETURN_MODIFIED_TIME).prop(SchemaConstants.TALEND_IS_LOCKED, "true")
            .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy'-'MM'-'dd'T'HH':'mm':'ss'Z'")//
            .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) //
            .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255")//
            .type(AvroUtils._logicalTimestamp()).noDefault() //
            .name(GoogleDriveListDefinition.RETURN_SIZE).prop(SchemaConstants.TALEND_IS_LOCKED, "true").type().nullable()
            .longType().noDefault() //
            .name(GoogleDriveListDefinition.RETURN_KIND).prop(SchemaConstants.TALEND_IS_LOCKED, "true").type().nullable()
            .stringType().noDefault() //
            .name(GoogleDriveListDefinition.RETURN_TRASHED).prop(SchemaConstants.TALEND_IS_LOCKED, "true").type().nullable()
            .booleanType().noDefault() //
            // TODO This should be a List<String>
            .name(GoogleDriveListDefinition.RETURN_PARENTS).prop(SchemaConstants.TALEND_IS_LOCKED, "true").type().nullable()
            .stringType().noDefault() //
            .name(GoogleDriveListDefinition.RETURN_WEB_VIEW_LINK).prop(SchemaConstants.TALEND_IS_LOCKED, "true").type()
            .nullable().stringType().noDefault() //
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    main.schema.setValue(s);
    folder.setValue("root");
    listMode.setPossibleValues(ListMode.values());
    listMode.setValue(Both);
    includeSubDirectories.setValue(true);
    includeTrashedFiles.setValue(false);
}
 
Example #27
Source File: TestReflectLogicalTypes.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testReflectedSchema() {
  Schema expected = SchemaBuilder.record(RecordWithUUIDList.class.getName())
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  expected.getField("uuids").schema().addProp(
      SpecificData.CLASS_PROP, List.class.getName());
  LogicalTypes.uuid().addToSchema(
      expected.getField("uuids").schema().getElementType());

  Schema actual = REFLECT.getSchema(RecordWithUUIDList.class);

  Assert.assertEquals("Should use the UUID logical type", expected, actual);
}
 
Example #28
Source File: TestSchemaCommandCluster.java    From kite with Apache License 2.0 5 votes vote down vote up
@Test
public void testHDFSCSVSchemaToHDFSFile() throws Exception {
  String csvSample = "hdfs:/tmp/sample/users.csv";
  FSDataOutputStream out = getDFS()
      .create(new Path(csvSample), true /* overwrite */);
  OutputStreamWriter writer = new OutputStreamWriter(out, "utf8");
  writer.append("id, username, email\n");
  writer.append("1, test, [email protected]\n");
  writer.close();

  Schema schema = SchemaBuilder.record("User").fields()
      .optionalLong("id")
      .optionalString("username")
      .optionalString("email")
      .endRecord();

  String hdfsSchemaPath = "hdfs:/tmp/schemas/csv2.avsc";
  CSVSchemaCommand command = new CSVSchemaCommand(console);
  command.setConf(getConfiguration());
  command.samplePaths = Lists.newArrayList(csvSample);
  command.outputPath = hdfsSchemaPath;
  command.recordName = "User";
  int rc = command.run();
  Assert.assertEquals("Should return success code", 0, rc);
  String fileContent = CharStreams.toString(
      new InputStreamReader(getDFS().open(new Path(hdfsSchemaPath)), "utf8"));
  Assert.assertTrue("File should contain pretty printed schema",
      TestUtil.matchesSchema(schema).matches(fileContent));
  verifyNoMoreInteractions(console);
}
 
Example #29
Source File: RootRecordUtilsTest.java    From components with Apache License 2.0 5 votes vote down vote up
/**
 * Checks {@link RootRecordUtils#createRootRecord(Schema, Schema)} throws {@link IndexOutOfBoundsException} in case
 * one tries to add value by incorrect index. The only correct indexes are 0 and 1
 */
@Test(expected = IndexOutOfBoundsException.class)
public void testCreatRootRecordOutOfBound1() {
    Schema mainSchema = SchemaBuilder.record("Main").fields() //
            .name("name").type().stringType().noDefault().endRecord(); //

    Schema outOfBandSchema = SchemaBuilder.record("OutOfBand").fields() //
            .name("id").type().intType().noDefault().endRecord(); //

    IndexedRecord rootRecord = RootRecordUtils.createRootRecord(mainSchema, outOfBandSchema);
    rootRecord.put(-1, "someValue");
}
 
Example #30
Source File: DynamicSchemaUtilsTest.java    From components with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetCommonFieldsWithCompletlyDifferentSchemas() {
    exception.expect(ComponentException.class);
    exception.expectMessage(I18N_MESSAGES.getMessage("error.message.differentSchema"));
    Schema remoteSchema = SchemaBuilder.builder().record("record").fields().requiredInt("age").endRecord();
    Schema inputSchema = SchemaBuilder.builder().record("record").fields().requiredInt("year").endRecord();

    DynamicSchemaUtils.getCommonFieldsForDynamicSchema(remoteSchema, inputSchema);
}