Java Code Examples for org.apache.avro.generic.IndexedRecord#getSchema()

The following examples show how to use org.apache.avro.generic.IndexedRecord#getSchema() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NetSuiteMockTestBase.java    From components with Apache License 2.0 6 votes vote down vote up
public static <T> List<IndexedRecord> makeIndexedRecords(NetSuiteClientService<?> clientService, Schema schema,
        ObjectComposer<T> objectComposer, int count) throws Exception {

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(clientService, schema, schema.getName());

    List<IndexedRecord> recordList = new ArrayList<>();

    while (count > 0) {
        T nsRecord = objectComposer.composeObject();

        IndexedRecord convertedRecord = transducer.read(nsRecord);
        Schema recordSchema = convertedRecord.getSchema();

        GenericRecord record = new GenericData.Record(recordSchema);
        for (Schema.Field field : schema.getFields()) {
            Object value = convertedRecord.get(field.pos());
            record.put(field.pos(), value);
        }

        recordList.add(record);

        count--;
    }

    return recordList;
}
 
Example 2
Source File: BulkResultAdapterFactoryTest.java    From components with Apache License 2.0 6 votes vote down vote up
@Test
public void testDynamicSchema(){
    Schema designSchema = SchemaBuilder.builder().record("Schema").prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true").fields() //
            .name("salesforce_id").type().intType().noDefault() //
            .name("salesforce_created").type().booleanType().noDefault() //
            .endRecord();

    converter.setSchema(designSchema);

    BulkResult result = new BulkResult();
    result.setValue("Created", "true");
    result.setValue("Error", "");
    result.setValue("Id", "a0M2v00000JSnn6EAD");
    result.setValue("Success", "true");
    result.setValue("field_1__c", "vlaue 1");
    result.setValue("field_2__c", "vlaue 2");
    result.setValue("test_uk__c", "0012v00002OGo4JAAT");

    IndexedRecord indexedRecord = converter.convertToAvro(result);

    Schema runtimSchema = indexedRecord.getSchema();
    Assert.assertEquals(5,runtimSchema.getFields().size());
    Assert.assertNotNull(runtimSchema.getField("field_1__c"));
    Assert.assertNotNull(runtimSchema.getField("field_2__c"));
    Assert.assertNotNull(runtimSchema.getField("test_uk__c"));
}
 
Example 3
Source File: MarketoCampaignWriter.java    From components with Apache License 2.0 6 votes vote down vote up
@Override
public void write(Object object) throws IOException {
    if (object == null) {
        return;
    }
    //
    successfulWrites.clear();
    rejectedWrites.clear();
    //
    inputRecord = (IndexedRecord) object;
    result.totalCount++;
    // This for dynamic which would get schema from the first record
    if (inputSchema == null) {
        inputSchema = inputRecord.getSchema();
    }
    //
    recordsToProcess.add(inputRecord);
    if (recordsToProcess.size() >= batchSize) {
        processResult(((MarketoRESTClient) client).requestCampaign(properties, recordsToProcess));
        recordsToProcess.clear();
    }
}
 
Example 4
Source File: MarkLogicRowProcessor.java    From components with Apache License 2.0 6 votes vote down vote up
@Override
public void write(Object indexedRecordDatum) throws IOException {
    if (indexedRecordDatum == null) {
        return;
    }
    cleanWrites();
    IndexedRecord indexedRecord = (IndexedRecord) indexedRecordDatum;
    Schema indexedRecordSchema = indexedRecord.getSchema();

    int docIdFieldIndex = indexedRecordSchema.getFields().indexOf(indexedRecordSchema.getField(inputProperties.docIdColumn.getStringValue()));
    if (docIdFieldIndex == -1) {
        throw new MarkLogicException(new MarkLogicErrorCode("Can't find docId column "
                + inputProperties.docIdColumn.getStringValue() + " in input row"));
    }
    String docId = (String) indexedRecord.get(docIdFieldIndex);
    GenericData.Record matchedDocument = docContentReader.readDocument(docId);
    totalCounter++;

    documents.add(matchedDocument);
}
 
Example 5
Source File: SalesforceInputReaderTestIT.java    From components with Apache License 2.0 5 votes vote down vote up
@Test
public void testManualQuery() throws Throwable {
    TSalesforceInputProperties props = createTSalesforceInputProperties(false, false);
    props.manualQuery.setValue(true);
    props.query.setValue("select Id from Account WHERE Name = '" + randomizedValue + "'");
    List<IndexedRecord> outputRows = readRows(props);
    assertEquals(100, outputRows.size());
    props.module.main.schema.setValue(SchemaBuilder.builder().record("MakeRowRecord").fields()//
            .name("Id").type().nullable().stringType().noDefault() //
            .name("Name").type().nullable().stringType().noDefault() //
            .name("Owner_Name").type().nullable().stringType().noDefault() //
            .name("Owner_Id").type().nullable().stringType().noDefault().endRecord());
    props.query
            .setValue("SELECT Id, Name, Owner.Name ,Owner.Id FROM Account WHERE Name = '" + randomizedValue + "'");
    List<IndexedRecord> rowsWithForeignKey = readRows(props);

    props.module.main.schema.setValue(SchemaBuilder.builder().record("MakeRowRecord").fields()//
            .name("Id").type().nullable().stringType().noDefault() //
            .name("Name").type().nullable().stringType().noDefault() //
            .name("OwnerId").type().nullable().stringType().noDefault().endRecord());
    props.query.setValue("SELECT Id, Name, OwnerId FROM Account WHERE Name = '" + randomizedValue + "'");
    outputRows = readRows(props);

    assertEquals(rowsWithForeignKey.size(), outputRows.size());
    assertEquals(100, rowsWithForeignKey.size());
    IndexedRecord fkRecord = rowsWithForeignKey.get(0);
    IndexedRecord commonRecord = outputRows.get(0);
    assertNotNull(fkRecord);
    assertNotNull(commonRecord);
    Schema schemaFK = fkRecord.getSchema();
    Schema schemaCommon = commonRecord.getSchema();

    assertNotNull(schemaFK);
    assertNotNull(schemaCommon);
    assertEquals(commonRecord.get(schemaCommon.getField("OwnerId").pos()), fkRecord.get(schemaFK.getField("Owner_Id").pos()));

}
 
Example 6
Source File: NetSuiteMockTestBase.java    From components with Apache License 2.0 5 votes vote down vote up
public static <T> List<IndexedRecord> makeIndexedRecords(
        NetSuiteClientService<?> clientService, Schema schema,
        ObjectComposer<T> objectComposer, int count) throws Exception {

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(clientService, schema, schema.getName());

    List<IndexedRecord> recordList = new ArrayList<>();

    while (count > 0) {
        T nsRecord = objectComposer.composeObject();

        IndexedRecord convertedRecord = transducer.read(nsRecord);
        Schema recordSchema = convertedRecord.getSchema();

        GenericRecord record = new GenericData.Record(recordSchema);
        for (Schema.Field field : schema.getFields()) {
            Object value = convertedRecord.get(field.pos());
            record.put(field.pos(), value);
        }

        recordList.add(record);

        count--;
    }

    return recordList;
}
 
Example 7
Source File: SalesforceInputReaderTestIT.java    From components with Apache License 2.0 5 votes vote down vote up
@Test
public void testComplexSOQLQuery() throws Throwable {
    TSalesforceInputProperties props = createTSalesforceInputProperties(false, false);
    props.manualQuery.setValue(true);
    // Manual query with foreign key
    // Need to specify where clause to be sure that this record exists and has parent-to-child relation.
    props.query.setValue("Select Id, Name,(Select Contact.Id,Contact.Name from Account.Contacts) from Account WHERE Name = 'United Oil & Gas, UK' Limit 1");
    props.validateGuessSchema();
    List<IndexedRecord> rows = readRows(props);

    if (rows.size() > 0) {
        for (IndexedRecord row : rows) {
            Schema schema = row.getSchema();
            assertNotNull(schema.getField("Id"));
            assertNotNull(schema.getField("Name"));
            assertNotNull(schema.getField("Account_Contacts_records_Contact_Id"));
            assertNotNull(schema.getField("Account_Contacts_records_Contact_Name"));

            assertNotNull(row.get(schema.getField("Id").pos()));
            assertNotNull(row.get(schema.getField("Name").pos()));
            assertNotNull(row.get(schema.getField("Account_Contacts_records_Contact_Id").pos()));
            assertNotNull(row.get(schema.getField("Account_Contacts_records_Contact_Name").pos()));
        }
    } else {
        LOGGER.warn("Query result is empty!");
    }
}
 
Example 8
Source File: NsObjectInputTransducerIT.java    From components with Apache License 2.0 5 votes vote down vote up
@Test
public void testIncludeAllFields() throws Exception {
    NetSuiteClientService<?> connection = webServiceTestFixture.getClientService();

    connection.login();

    TypeDesc basicTypeDesc = connection.getBasicMetaData().getTypeInfo("Opportunity");
    Schema schema = getDynamicSchema();

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(connection, schema, basicTypeDesc.getTypeName());

    SearchResultSet<Record> rs = connection.newSearch()
            .target(basicTypeDesc.getTypeName())
            .search();

    TypeDesc typeDesc = connection.getMetaDataSource().getTypeInfo(basicTypeDesc.getTypeName());

    int count = 0;
    while (count++ < connection.getSearchPageSize() && rs.next()) {
        Record record = rs.get();
        IndexedRecord indexedRecord = transducer.read(record);
        logger.debug("Indexed record: {}", indexedRecord);

        Schema recordSchema = indexedRecord.getSchema();
        assertEquals(typeDesc.getFields().size(), recordSchema.getFields().size());

        for (FieldDesc fieldDesc : typeDesc.getFields()) {
            String fieldName = fieldDesc.getName();
            Schema.Field field = recordSchema.getField(fieldName);
            assertNotNull(field);

            Object value = indexedRecord.get(field.pos());
        }
    }
    if (count == 0) {
        throw new IllegalStateException("No records");
    }
}
 
Example 9
Source File: AvroRecord.java    From component-runtime with Apache License 2.0 5 votes vote down vote up
public AvroRecord(final IndexedRecord record) {
    this.schema = new AvroSchema(record.getSchema());
    this.delegate = record;
    // dirty fix for Avro DateTime related logicalTypes converted to org.joda.time.DateTime
    this.delegate
            .getSchema()
            .getFields()
            .stream()
            .filter(f -> org.joda.time.DateTime.class.isInstance(this.delegate.get(f.pos())))
            .forEach(f -> this.delegate
                    .put(f.pos(), org.joda.time.DateTime.class.cast(this.delegate.get(f.pos())).getMillis()));
}
 
Example 10
Source File: JiraInsertWriter.java    From components with Apache License 2.0 5 votes vote down vote up
/**
 * Inserts resources into Jira server formed from incoming data input <br>
 * Method should be called only after {@link JiraInsertWriter#open(String)}
 * 
 * @param datum input data
 */
@Override
public void write(Object datum) throws IOException {
    if (!opened) {
        throw new IOException(MESSAGES.getMessage("error.writerNotOpened"));
    }
    result.totalCount++;
    if (datum == null) {
        return;
    }
    IndexedRecord record = getFactory(datum).convertToAvro(datum);

    if (dataSchema == null) {
        dataSchema = record.getSchema();
        Field jsonField = dataSchema.getField("json");
        if (jsonField == null) {
            throw new IOException(MESSAGES.getMessage("error.schemaNotContainJson"));
        }
        jsonPos = jsonField.pos();
    }

    String json = (String) record.get(jsonPos);
    validateRequestBody(json);

    JiraResponse response = getConnection().post(resource, json);
    handleResponse(response, json, record);
}
 
Example 11
Source File: CouchbaseWriter.java    From components with Apache License 2.0 5 votes vote down vote up
@Override
public void write(Object datum) throws IOException {
    if (!opened) {
        throw new IOException("Writer is not opened");
    }

    result.totalCount++;
    if (datum == null) {
        return;
    }

    // Data object is always IndexedRecord
    IndexedRecord record = (IndexedRecord) datum;

    Schema schema = record.getSchema();
    Schema.Field idField = schema.getField(idFieldName);
    if (idField == null) {
        throw new IOException("Schema does not contain ID field: " + idFieldName);
    }

    int idPos = idField.pos();
    Object id = record.get(idPos);
    if (id == null) {
        handleException("Record is not processed. ID is null.", new IllegalArgumentException("ID field should not be null"));
        return;
    }
    try {
        if (containsJson){
            connection.insertJsonDocument(id.toString(), createHierarchicalJson(schema, record, idPos));
        } else {
            connection.upsert(id.toString(), datum.toString());
        }
        result.successCount++;
    } catch (Exception e) {
        handleException("Record is not processed. Failed to upsert value - " + datum.toString(), e);
    }
}
 
Example 12
Source File: RootRecordUtilsTest.java    From components with Apache License 2.0 5 votes vote down vote up
/**
 * Checks {@link RootRecordUtils#createRootRecord(Schema, Schema)} returns Root record, which schema is Root schema
 */
@Test
public void testCreateRootRecord() {
    Schema mainSchema = SchemaBuilder.record("Main").fields() //
            .name("name").type().stringType().noDefault().endRecord(); //

    Schema outOfBandSchema = SchemaBuilder.record("OutOfBand").fields() //
            .name("id").type().intType().noDefault().endRecord(); //

    IndexedRecord rootRecord = RootRecordUtils.createRootRecord(mainSchema, outOfBandSchema);

    assertNotNull(rootRecord);
    Schema rootSchema = rootRecord.getSchema();
    assertTrue(RootSchemaUtils.isRootSchema(rootSchema));
}
 
Example 13
Source File: RootRecordUtils.java    From components with Apache License 2.0 5 votes vote down vote up
/**
 * Checks whether input {@link IndexedRecord} is Root record
 * It checks its {@link Schema}, it should be Root schema
 * 
 * @param data data to check
 * @return true if incoming {@link IndexedRecord} is Root record; false otherwise
 */
public static boolean isRootRecord(Object data) {
    if (data == null) {
        return false;
    }
    if (!(data instanceof IndexedRecord)) {
        return false;
    }
    IndexedRecord record = (IndexedRecord) data;
    Schema recordSchema = record.getSchema();
    boolean result = RootSchemaUtils.isRootSchema(recordSchema);
    return result;
}
 
Example 14
Source File: DBTestUtils.java    From components with Apache License 2.0 5 votes vote down vote up
private static IndexedRecord copyValueFrom(IndexedRecord record) {
    Schema schema = record.getSchema();
    IndexedRecord result = new GenericData.Record(schema);
    List<Schema.Field> fields = schema.getFields();
    for (int i = 0; i < fields.size(); i++) {
        result.put(i, record.get(i));
    }

    return result;
}
 
Example 15
Source File: AvroRelConverter.java    From samza with Apache License 2.0 5 votes vote down vote up
public static void fetchFieldNamesAndValuesFromIndexedRecord(IndexedRecord record, List<String> fieldNames,
    List<Object> fieldValues, Schema cachedSchema) {
  // Please note that record schema and cached schema could be different due to schema evolution.
  // Always represent record schema in the form of cached schema. This approach has the side-effect
  // of dropping the newly added fields in the scenarios where the record schema has newer version
  // than the cached schema. [TODO: SAMZA-1679]
  Schema recordSchema = record.getSchema();
  fieldNames.addAll(cachedSchema.getFields().stream().map(Schema.Field::name).collect(Collectors.toList()));
  fieldValues.addAll(fieldNames.stream()
      .map(f -> convertToJavaObject(
          recordSchema.getField(f) != null ? record.get(recordSchema.getField(f).pos()) : null,
          getNonNullUnionSchema(cachedSchema.getField(f).schema()))) // get schema from cachedSchema
      .collect(Collectors.toList()));
}
 
Example 16
Source File: TestAzureBlobAvroWriter.java    From samza with Apache License 2.0 5 votes vote down vote up
private byte[] encodeRecord(IndexedRecord record) throws Exception {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = record.getSchema();
  EncoderFactory encoderfactory = new EncoderFactory();
  BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null);
  DatumWriter<IndexedRecord> writer;
  if (record instanceof SpecificRecord) {
    writer = new SpecificDatumWriter<>(schema);
  } else {
    writer = new GenericDatumWriter<>(schema);
  }
  writer.write(record, encoder);
  encoder.flush(); //encoder may buffer
  return out.toByteArray();
}
 
Example 17
Source File: NsObjectInputTransducerIT.java    From components with Apache License 2.0 5 votes vote down vote up
@Test
public void testIncludeAllFields() throws Exception {
    NetSuiteClientService<?> connection = webServiceTestFixture.getClientService();

    connection.login();

    TypeDesc basicTypeDesc = connection.getBasicMetaData().getTypeInfo("Opportunity");
    Schema schema = getDynamicSchema();

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(connection, schema, basicTypeDesc.getTypeName());

    SearchResultSet<Record> rs = connection.newSearch()
            .target(basicTypeDesc.getTypeName())
            .search();

    TypeDesc typeDesc = connection.getMetaDataSource().getTypeInfo(basicTypeDesc.getTypeName());

    int count = 0;
    while (count++ < connection.getSearchPageSize() && rs.next()) {
        Record record = rs.get();
        IndexedRecord indexedRecord = transducer.read(record);
        logger.debug("Indexed record: {}", indexedRecord);

        Schema recordSchema = indexedRecord.getSchema();
        assertEquals(typeDesc.getFields().size(), recordSchema.getFields().size());

        for (FieldDesc fieldDesc : typeDesc.getFields()) {
            String fieldName = fieldDesc.getName();
            Schema.Field field = recordSchema.getField(fieldName);
            assertNotNull(field);

            Object value = indexedRecord.get(field.pos());
        }
    }
    if (count == 0) {
        throw new IllegalStateException("No records");
    }
}
 
Example 18
Source File: FileDelimitedAvroRegistry.java    From components with Apache License 2.0 4 votes vote down vote up
private Schema inferSchemaRecord(IndexedRecord in) {
    return in.getSchema();
}
 
Example 19
Source File: MarketoInputWriter.java    From components with Apache License 2.0 4 votes vote down vote up
@Override
public void write(Object object) throws IOException {
    if (object == null) {
        return;
    }
    cleanWrites();
    //
    inputRecord = (IndexedRecord) object;
    result.totalCount++;
    // This for dynamic which would get schema from the first record
    if (inputSchema == null) {
        inputSchema = inputRecord.getSchema();
        if (isDynamic) {
            adaptSchemaToDynamic();
        }
    }
    // switch between column name in design and column value for runtime
    properties.leadKeyValues.setValue(String.valueOf(inputRecord.get(inputSchema.getField(leadKeyColumn).pos())));
    //
    for (int i = 0; i < getRetryAttemps(); i++) {
        result.apiCalls++;
        MarketoRecordResult mktoResult = client.getMultipleLeads(properties, null);
        //
        if (!mktoResult.isSuccess()) {
            if (dieOnError) {
                throw new IOException(mktoResult.getErrorsString());
            }
            // is recoverable error
            if (client.isErrorRecoverable(mktoResult.getErrors())) {
                LOG.debug("Recoverable error during operation : `{}`. Retrying...", mktoResult.getErrorsString());
                waitForRetryAttempInterval();
                continue;
            } else {
                LOG.error("Unrecoverable error : `{}`.", mktoResult.getErrorsString());
                break;
            }
        } else {
            for (IndexedRecord record : mktoResult.getRecords()) {
                if (record != null) {
                    this.result.successCount++;
                    successfulWrites.add(record);
                }
            }
            break;
        }
    }
}
 
Example 20
Source File: JDBCSPWriter.java    From components with Apache License 2.0 3 votes vote down vote up
public void write(Object datum) throws IOException {
    result.totalCount++;

    cleanWrites();
    
    IndexedRecord inputRecord = this.getGenericIndexedRecordConverter(datum).convertToAvro(datum);

    Schema inputSchema = inputRecord.getSchema();

    try {

        sink.fillParameters(cs, componentSchema, inputSchema, inputRecord, setting);

        cs.execute();

        if (indexedRecordCreator == null) {
            indexedRecordCreator = new JDBCSPIndexedRecordCreator();
            indexedRecordCreator.init(componentSchema, outputSchema, setting);
        }

        IndexedRecord outputRecord = indexedRecordCreator.createOutputIndexedRecord(cs, inputRecord);

        successfulWrites.add(outputRecord);
    } catch (Exception e) {
        throw CommonUtils.newComponentException(e);
    }

}