Java Code Examples for org.apache.avro.reflect.ReflectData

The following examples show how to use org.apache.avro.reflect.ReflectData. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testWriteParquetAvroReflect() throws Exception {

	final File folder = TEMPORARY_FOLDER.newFolder();

	final List<Datum> data = Arrays.asList(
			new Datum("a", 1), new Datum("b", 2), new Datum("c", 3));

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Datum> stream = env.addSource(
			new FiniteTestSource<>(data), TypeInformation.of(Datum.class));

	stream.addSink(
			StreamingFileSink.forBulkFormat(
					Path.fromLocalFile(folder),
					ParquetAvroWriters.forReflectRecord(Datum.class))
					.build());

	env.execute();

	validateResults(folder, ReflectData.get(), data);
}
 
Example 2
Source Project: flink   Source File: ParquetStreamingFileSinkITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteParquetAvroReflect() throws Exception {

	final File folder = TEMPORARY_FOLDER.newFolder();

	final List<Datum> data = Arrays.asList(
			new Datum("a", 1), new Datum("b", 2), new Datum("c", 3));

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Datum> stream = env.addSource(
			new FiniteTestSource<>(data), TypeInformation.of(Datum.class));

	stream.addSink(
			StreamingFileSink.forBulkFormat(
					Path.fromLocalFile(folder),
					ParquetAvroWriters.forReflectRecord(Datum.class))
					.build());

	env.execute();

	validateResults(folder, ReflectData.get(), data);
}
 
Example 3
Source Project: hadoop   Source File: AvroTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void testReflect(Object value, Type type, String schema)
  throws Exception {

  // check that schema matches expected
  Schema s = ReflectData.get().getSchema(type);
  assertEquals(Schema.parse(schema), s);

  // check that value is serialized correctly
  ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(s);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  writer.write(value, EncoderFactory.get().directBinaryEncoder(out, null));
  ReflectDatumReader<Object> reader = new ReflectDatumReader<Object>(s);
  Object after =
    reader.read(null,
                DecoderFactory.get().binaryDecoder(out.toByteArray(), null));
  assertEquals(value, after);
}
 
Example 4
Source Project: schema-evolution-samples   Source File: AvroCodecTests.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void dynamicReflectEncoderReflectDecoder() throws Exception{
	Schema schema = ReflectData.get().getSchema(Account.class);
	SchemaRegistryClient client = mock(SchemaRegistryClient.class);
	when(client.register(any())).thenReturn(10);
	when(client.fetch(eq(10))).thenReturn(schema);
	AvroCodec codec = new AvroCodec();
	AvroCodecProperties properties = new AvroCodecProperties();
	properties.setDynamicSchemaGenerationEnabled(true);
	codec.setProperties(properties);
	codec.setSchemaRegistryClient(client);
	codec.setResolver(new PathMatchingResourcePatternResolver(new AnnotationConfigApplicationContext()));
	codec.init();
	Account account = new Account();
	account.setCreatedAt(System.currentTimeMillis());
	account.setId(1L);
	byte[] results = codec.encode(account);
	Account decoded = codec.decode(results,Account.class);
	Assert.assertEquals(account.getId(), decoded.getId());
}
 
Example 5
Source Project: kite   Source File: CSVAppender.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns a the value as the first matching schema type or null.
 *
 * Note that if the value may be null even if the schema does not allow the
 * value to be null.
 *
 * @param value a value
 * @param schema a Schema
 * @return a String representation of the value according to the Schema type
 */
private static String valueString(Object value, Schema schema) {
  if (value == null || schema.getType() == Schema.Type.NULL) {
    return null;
  }

  switch (schema.getType()) {
    case BOOLEAN:
    case FLOAT:
    case DOUBLE:
    case INT:
    case LONG:
    case STRING:
      return value.toString();
    case ENUM:
      // serialize as the ordinal from the schema
      return String.valueOf(schema.getEnumOrdinal(value.toString()));
    case UNION:
      int index = ReflectData.get().resolveUnion(schema, value);
      return valueString(value, schema.getTypes().get(index));
    default:
      // FIXED, BYTES, MAP, ARRAY, RECORD are not supported
      throw new DatasetOperationException(
          "Unsupported field type:" + schema.getType());
  }
}
 
Example 6
Source Project: beam   Source File: AvroSourceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreationWithSchema() throws Exception {
  List<Bird> expected = createRandomRecords(100);
  String filename =
      generateTestFile(
          "tmp.avro",
          expected,
          SyncBehavior.SYNC_DEFAULT,
          0,
          AvroCoder.of(Bird.class),
          DataFileConstants.NULL_CODEC);

  // Create a source with a schema object
  Schema schema = ReflectData.get().getSchema(Bird.class);
  AvroSource<GenericRecord> source = AvroSource.from(filename).withSchema(schema);
  List<GenericRecord> records = SourceTestUtils.readFromSource(source, null);
  assertEqualsWithGeneric(expected, records);

  // Create a source with a JSON schema
  String schemaString = ReflectData.get().getSchema(Bird.class).toString();
  source = AvroSource.from(filename).withSchema(schemaString);
  records = SourceTestUtils.readFromSource(source, null);
  assertEqualsWithGeneric(expected, records);
}
 
Example 7
Source Project: beam   Source File: PubsubIO.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns a {@link PTransform} that continuously reads binary encoded Avro messages of the
 * specific type.
 *
 * <p>Beam will infer a schema for the Avro schema. This allows the output to be used by SQL and
 * by the schema-transform library.
 */
@Experimental(Kind.SCHEMAS)
public static <T> Read<T> readAvrosWithBeamSchema(Class<T> clazz) {
  if (clazz.equals(GenericRecord.class)) {
    throw new IllegalArgumentException("For GenericRecord, please call readAvroGenericRecords");
  }
  org.apache.avro.Schema avroSchema = ReflectData.get().getSchema(clazz);
  AvroCoder<T> coder = AvroCoder.of(clazz);
  Schema schema = AvroUtils.getSchema(clazz, null);
  return Read.newBuilder(parsePayloadUsingCoder(coder))
      .setCoder(
          SchemaCoder.of(
              schema,
              TypeDescriptor.of(clazz),
              AvroUtils.getToRowFunction(clazz, avroSchema),
              AvroUtils.getFromRowFunction(clazz)))
      .build();
}
 
Example 8
Source Project: beam   Source File: ParquetIOTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = org.apache.beam.sdk.Pipeline.PipelineExecutionException.class)
public void testWriteAndReadUsingReflectDataSchemaWithoutDataModelThrowsException() {
  Schema testRecordSchema = ReflectData.get().getSchema(TestRecord.class);

  List<GenericRecord> records = generateGenericRecords(1000);
  mainPipeline
      .apply(Create.of(records).withCoder(AvroCoder.of(testRecordSchema)))
      .apply(
          FileIO.<GenericRecord>write()
              .via(ParquetIO.sink(testRecordSchema))
              .to(temporaryFolder.getRoot().getAbsolutePath()));
  mainPipeline.run().waitUntilFinish();

  PCollection<GenericRecord> readBack =
      readPipeline.apply(
          ParquetIO.read(testRecordSchema)
              .from(temporaryFolder.getRoot().getAbsolutePath() + "/*"));

  PAssert.that(readBack).containsInAnyOrder(records);
  readPipeline.run().waitUntilFinish();
}
 
Example 9
Source Project: beam   Source File: ParquetIOTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteAndReadUsingReflectDataSchemaWithDataModel() {
  Schema testRecordSchema = ReflectData.get().getSchema(TestRecord.class);

  List<GenericRecord> records = generateGenericRecords(1000);
  mainPipeline
      .apply(Create.of(records).withCoder(AvroCoder.of(testRecordSchema)))
      .apply(
          FileIO.<GenericRecord>write()
              .via(ParquetIO.sink(testRecordSchema))
              .to(temporaryFolder.getRoot().getAbsolutePath()));
  mainPipeline.run().waitUntilFinish();

  PCollection<GenericRecord> readBack =
      readPipeline.apply(
          ParquetIO.read(testRecordSchema)
              .withAvroDataModel(GenericData.get())
              .from(temporaryFolder.getRoot().getAbsolutePath() + "/*"));

  PAssert.that(readBack).containsInAnyOrder(records);
  readPipeline.run().waitUntilFinish();
}
 
Example 10
Source Project: kite   Source File: TestFileSystemDatasetRepository.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadNullsWithPrimitivesAllowNullSchema() {
  final String name = "allowNullPrimitives";
  try {
    repo.create(NAMESPACE, name, new DatasetDescriptor.Builder()
        .schema(ReflectData.AllowNull.get().getSchema(ObjectPoJo.class))
        .build(), ObjectPoJo.class);

    // should load the dataset because PrimitivePoJo can be used to write
    final Dataset<PrimitivePoJo> dataset = repo.load(
        NAMESPACE, name, PrimitivePoJo.class);
    TestHelpers.assertThrows("AllowNull primitives cannot read nullable type",
        IncompatibleSchemaException.class, new Runnable() {
          @Override
          public void run() {
            dataset.newReader();
          }
        });

  } catch (RuntimeException e) {
    throw e;
  } finally {
    repo.delete(NAMESPACE, name);
  }
}
 
Example 11
Source Project: kite   Source File: TestSchemaCommandCluster.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testObjSchemaToHDFSFile() throws Exception {
  Schema schema = ReflectData.get().getSchema(User.class);

  String hdfsSchemaPath = "hdfs:/tmp/schemas/obj.avsc";
  ObjectSchemaCommand command = new ObjectSchemaCommand(console);
  command.setConf(getConfiguration());
  command.classNames = Lists.newArrayList("org.kitesdk.cli.example.User");
  command.outputPath = hdfsSchemaPath;
  int rc = command.run();
  Assert.assertEquals("Should return success code", 0, rc);
  String fileContent = CharStreams.toString(
      new InputStreamReader(getDFS().open(new Path(hdfsSchemaPath)), "utf8"));
  Assert.assertTrue("File should contain pretty printed schema",
      TestUtil.matchesSchema(schema).matches(fileContent));
  verifyNoMoreInteractions(console);
}
 
Example 12
Source Project: pulsar   Source File: AvroSchema.java    License: Apache License 2.0 6 votes vote down vote up
public static void addLogicalTypeConversions(ReflectData reflectData, boolean jsr310ConversionEnabled) {
    reflectData.addLogicalTypeConversion(new Conversions.DecimalConversion());
    reflectData.addLogicalTypeConversion(new TimeConversions.DateConversion());
    reflectData.addLogicalTypeConversion(new TimeConversions.TimeMillisConversion());
    reflectData.addLogicalTypeConversion(new TimeConversions.TimeMicrosConversion());
    reflectData.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion());
    if (jsr310ConversionEnabled) {
        reflectData.addLogicalTypeConversion(new TimeConversions.TimestampMillisConversion());
    } else {
        try {
            Class.forName("org.joda.time.DateTime");
            reflectData.addLogicalTypeConversion(new JodaTimeConversions.TimestampConversion());
        } catch (ClassNotFoundException e) {
            // Skip if have not provide joda-time dependency.
        }
    }
}
 
Example 13
Source Project: pulsar   Source File: AvroSchemaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void discardBufferIfBadAvroData() {
    AvroWriter<NasaMission> avroWriter = new AvroWriter<>(
            ReflectData.AllowNull.get().getSchema(NasaMission.class));

    NasaMission badNasaMissionData = new NasaMission();
    badNasaMissionData.setId(1);
    // set null in the non-null field. The java set will accept it but going ahead, the avro encode will crash.
    badNasaMissionData.setName(null);

    // Because data does not conform to schema expect a crash
    Assert.assertThrows( SchemaSerializationException.class, () -> avroWriter.write(badNasaMissionData));

    // Get the buffered data using powermock
    BinaryEncoder encoder = Whitebox.getInternalState(avroWriter, "encoder");

    // Assert that the buffer position is reset to zero
    Assert.assertEquals(((BufferedBinaryEncoder)encoder).bytesBuffered(), 0);
}
 
Example 14
Source Project: flink   Source File: ParquetStreamingFileSinkITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteParquetAvroReflect() throws Exception {

	final File folder = TEMPORARY_FOLDER.newFolder();

	final List<Datum> data = Arrays.asList(
			new Datum("a", 1), new Datum("b", 2), new Datum("c", 3));

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Datum> stream = env.addSource(
			new FiniteTestSource<>(data), TypeInformation.of(Datum.class));

	stream.addSink(
			StreamingFileSink.forBulkFormat(
					Path.fromLocalFile(folder),
					ParquetAvroWriters.forReflectRecord(Datum.class))
					.build());

	env.execute();

	validateResults(folder, ReflectData.get(), data);
}
 
Example 15
Source Project: kite   Source File: TestDataModelUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateRecord() {
  assertNull("createRecord should not create Specific instances",
      DataModelUtil.createRecord(StandardEvent.class, StandardEvent.getClassSchema()));

  assertNull("createRecord should not create Reflect instances",
      DataModelUtil.createRecord(ReflectStandardEvent.class,
          ReflectData.get().getSchema(ReflectStandardEvent.class)));

  assertNotNull("createRecord should create Generic instances",
      DataModelUtil.createRecord(GenericData.Record.class,
          StandardEvent.getClassSchema()));

  assertEquals("createRecord did not return the expected class",
      TestGenericRecord.class,
      DataModelUtil.createRecord(TestGenericRecord.class,
          StandardEvent.getClassSchema()).getClass());
}
 
Example 16
Source Project: parquet-mr   Source File: TestReflectReadWrite.java    License: Apache License 2.0 6 votes vote down vote up
private Path writePojosToParquetFile( int num, CompressionCodecName compression,
                                      boolean enableDictionary) throws IOException {
  File tmp = File.createTempFile(getClass().getSimpleName(), ".tmp");
  tmp.deleteOnExit();
  tmp.delete();
  Path path = new Path(tmp.getPath());

  Pojo object = getPojo();

  Schema schema = ReflectData.get().getSchema(object.getClass());
  try(ParquetWriter<Pojo> writer = AvroParquetWriter.<Pojo>builder(path)
      .withSchema(schema)
      .withCompressionCodec(compression)
      .withDataModel(ReflectData.get())
      .withDictionaryEncoding(enableDictionary)
      .build()) {
    for (int i = 0; i < num; i++) {
      writer.write(object);
    }
  }
  return path;
}
 
Example 17
Source Project: apicurio-registry   Source File: AvroSchemaUtils.java    License: Apache License 2.0 5 votes vote down vote up
static Schema getReflectSchema(Object object) {
    Class<?> clazz = (object instanceof Class) ? (Class) object : object.getClass();
    Schema schema = ReflectData.get().getSchema(clazz);
    if (schema == null) {
        throw new SerializationException("No schema for class: " + clazz.getName());
    }
    return schema;
}
 
Example 18
Source Project: Flink-CEPplus   Source File: AvroFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static <T> AvroFactory<T> fromReflective(Class<T> type, ClassLoader cl, Optional<Schema> previousSchema) {
	ReflectData reflectData = new ReflectData(cl);
	Schema newSchema = reflectData.getSchema(type);

	return new AvroFactory<>(
		reflectData,
		newSchema,
		new ReflectDatumReader<>(previousSchema.orElse(newSchema), newSchema, reflectData),
		new ReflectDatumWriter<>(newSchema, reflectData)
	);
}
 
Example 19
Source Project: kite   Source File: CSVRecordBuilder.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private E newRecordInstance() {
  if (recordClass != GenericData.Record.class && !recordClass.isInterface()) {
    E record = (E) ReflectData.newInstance(recordClass, schema);
    if (record != null) {
      return record;
    }
  }
  return (E) new GenericData.Record(schema);
}
 
Example 20
Source Project: kite   Source File: TestLog4jAppenderWithAvro.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAvroReflect() throws IOException {
  loadProperties("flume-log4jtest-avro-reflect.properties");
  PropertyConfigurator.configure(props);
  Logger logger = LogManager.getLogger(TestLog4jAppenderWithAvro.class);
  String msg = "This is log message number " + String.valueOf(0);

  AppEvent appEvent = new AppEvent();
  appEvent.setMessage(msg);

  logger.info(appEvent);

  Transaction transaction = ch.getTransaction();
  transaction.begin();
  Event event = ch.take();
  Assert.assertNotNull(event);

  Schema schema = ReflectData.get().getSchema(appEvent.getClass());

  ReflectDatumReader<AppEvent> reader = new ReflectDatumReader<AppEvent>(AppEvent.class);
  BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null);
  AppEvent recordFromEvent = reader.read(null, decoder);
  Assert.assertEquals(msg, recordFromEvent.getMessage());

  Map<String, String> hdrs = event.getHeaders();

  Assert.assertNull(hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString()));

  Assert.assertNull("Schema URL should not be set",
      hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_URL.toString()));
  Assert.assertEquals("Schema string should be set", schema.toString(),
      hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_LITERAL.toString()));

  transaction.commit();
  transaction.close();

}
 
Example 21
Source Project: flink   Source File: AvroOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	super.open(taskNumber, numTasks);

	DatumWriter<E> datumWriter;
	Schema schema;
	if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) {
		datumWriter = new SpecificDatumWriter<E>(avroValueType);
		try {
			schema = ((org.apache.avro.specific.SpecificRecordBase) avroValueType.newInstance()).getSchema();
		} catch (InstantiationException | IllegalAccessException e) {
			throw new RuntimeException(e.getMessage());
		}
	} else if (org.apache.avro.generic.GenericRecord.class.isAssignableFrom(avroValueType)) {
		if (userDefinedSchema == null) {
			throw new IllegalStateException("Schema must be set when using Generic Record");
		}
		datumWriter = new GenericDatumWriter<E>(userDefinedSchema);
		schema = userDefinedSchema;
	} else {
		datumWriter = new ReflectDatumWriter<E>(avroValueType);
		schema = ReflectData.get().getSchema(avroValueType);
	}
	dataFileWriter = new DataFileWriter<E>(datumWriter);
	if (codec != null) {
		dataFileWriter.setCodec(codec.getCodecFactory());
	}
	if (userDefinedSchema == null) {
		dataFileWriter.create(schema, stream);
	} else {
		dataFileWriter.create(userDefinedSchema, stream);
	}
}
 
Example 22
Source Project: kite   Source File: DataModelUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Resolves the type based on the given schema. In most cases, the type should
 * stay as is. However, if the type is Object, then that means that the old
 * default behavior of determining the class from ReflectData#getClass(Schema)
 * should be used. If a class can't be found, it will default to
 * GenericData.Record.
 *
 * @param <E> The entity type
 * @param type The Java class of the entity type
 * @param schema The {@link Schema} for the entity
 * @return The resolved Java class object
 */
@SuppressWarnings("unchecked")
public static <E> Class<E> resolveType(Class<E> type, Schema schema) {
  if (type == Object.class) {
    type = ReflectData.get().getClass(schema);
  }

  if (type == null) {
    type = (Class<E>) GenericData.Record.class;
  }

  return type;
}
 
Example 23
Source Project: iceberg   Source File: Pair.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("deprecation")
public Schema load(Pair<Class<?>, Class<?>> key) {
  Schema xSchema = ReflectData.get().getSchema(key.first);
  Schema ySchema = ReflectData.get().getSchema(key.second);
  return Schema.createRecord("pair", null, null, false, Lists.newArrayList(
      new Schema.Field("x", xSchema, null, (Object) null),
      new Schema.Field("y", ySchema, null, (Object) null)
  ));
}
 
Example 24
Source Project: iceberg   Source File: Pair.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("deprecation")
public Schema load(Pair<Class<?>, Class<?>> key) {
  Schema xSchema = ReflectData.get().getSchema(key.x);
  Schema ySchema = ReflectData.get().getSchema(key.y);
  return Schema.createRecord("pair", null, null, false, Lists.newArrayList(
      new Schema.Field("x", xSchema, null, null),
      new Schema.Field("y", ySchema, null, null)
  ));
}
 
Example 25
Source Project: kite   Source File: AvroConversions.java    License: Apache License 2.0 5 votes vote down vote up
public static Object toAvro(Object item, Field field) {
  if (item == null && !nullOk(field.schema())) {
    try { // this will fail if there is no default value
      return ReflectData.get().getDefaultValue(field);
    } catch (AvroRuntimeException e) {
      return ERROR;
    }
  }
  Object result = toAvro(item, field.schema());
  return result;
}
 
Example 26
Source Project: kite   Source File: CSVRecordBuilder.java    License: Apache License 2.0 5 votes vote down vote up
private void fillReflect(Object record, String[] data) {
  for (int i = 0; i < indexes.length; i += 1) {
    Schema.Field field = fields[i];
    int index = indexes[i];
    Object value = makeValue(index < data.length ? data[index] : null, field);
    ReflectData.get().setField(record, field.name(), i, value);
  }
}
 
Example 27
Source Project: beam   Source File: AvroUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns an {@code SchemaCoder} instance for the provided element type.
 *
 * @param <T> the element type
 */
public static <T> SchemaCoder<T> schemaCoder(TypeDescriptor<T> type) {
  @SuppressWarnings("unchecked")
  Class<T> clazz = (Class<T>) type.getRawType();
  org.apache.avro.Schema avroSchema = new ReflectData(clazz.getClassLoader()).getSchema(clazz);
  Schema beamSchema = toBeamSchema(avroSchema);
  return SchemaCoder.of(
      beamSchema, type, getToRowFunction(clazz, avroSchema), getFromRowFunction(clazz));
}
 
Example 28
Source Project: beam   Source File: AvroIO.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads records of the given type from an Avro file (or multiple Avro files matching a pattern).
 *
 * <p>The schema must be specified using one of the {@code withSchema} functions.
 */
public static <T> Read<T> read(Class<T> recordClass) {
  return new AutoValue_AvroIO_Read.Builder<T>()
      .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW))
      .setRecordClass(recordClass)
      .setSchema(ReflectData.get().getSchema(recordClass))
      .setInferBeamSchema(false)
      .setHintMatchesManyFiles(false)
      .build();
}
 
Example 29
Source Project: beam   Source File: AvroIO.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Like {@link #read}, but reads each filepattern in the input {@link PCollection}.
 *
 * @deprecated You can achieve The functionality of {@link #readAll} using {@link FileIO} matching
 *     plus {@link #readFiles(Class)}. This is the preferred method to make composition explicit.
 *     {@link ReadAll} will not receive upgrades and will be removed in a future version of Beam.
 */
@Deprecated
public static <T> ReadAll<T> readAll(Class<T> recordClass) {
  return new AutoValue_AvroIO_ReadAll.Builder<T>()
      .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD))
      .setRecordClass(recordClass)
      .setSchema(ReflectData.get().getSchema(recordClass))
      .setInferBeamSchema(false)
      .setDesiredBundleSizeBytes(DEFAULT_BUNDLE_SIZE_BYTES)
      .build();
}
 
Example 30
Source Project: beam   Source File: AvroIO.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Writes a {@link PCollection} to an Avro file (or multiple Avro files matching a sharding
 * pattern).
 */
public static <T> Write<T> write(Class<T> recordClass) {
  return new Write<>(
      AvroIO.<T, T>defaultWriteBuilder()
          .setGenericRecords(false)
          .setSchema(ReflectData.get().getSchema(recordClass))
          .build());
}