org.apache.avro.io.ResolvingDecoder Java Examples

The following examples show how to use org.apache.avro.io.ResolvingDecoder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PigAvroDatumReader.java    From spork with Apache License 2.0 6 votes vote down vote up 
/**
 * Called to read a record instance. Overridden to read a pig tuple.
 */
@Override
protected Object readRecord(Object old, Schema expected, ResolvingDecoder in) throws IOException {

    // find out the order in which we will receive fields from the ResolvingDecoder
    Field[] readOrderedFields = in.readFieldOrder();

    /* create an empty tuple */
    Tuple tuple = TupleFactory.getInstance().newTuple(readOrderedFields.length);

    /* read fields and put in output order in tuple
     * The ResolvingDecoder figures out the writer schema to reader schema mapping for us
     */
    for (Field f : readOrderedFields) {
        tuple.set(f.pos(), read(old, f.schema(), in));
    }

    return tuple;
}
Example #2
Source File: PigAvroDatumReader.java    From Cubert with Apache License 2.0 6 votes vote down vote up 
/**
 * Called to read a record instance. Overridden to read a pig tuple.
 */
@Override
protected Object readRecord(Object old, Schema expected, ResolvingDecoder in) throws IOException {

    // find out the order in which we will receive fields from the ResolvingDecoder
    Field[] readOrderedFields = in.readFieldOrder();

    /* create an empty tuple */
    Tuple tuple = TupleFactory.getInstance().newTuple(readOrderedFields.length);

    /* read fields and put in output order in tuple
     * The ResolvingDecoder figures out the writer schema to reader schema mapping for us
     */
    for (Field f : readOrderedFields) {
        tuple.set(f.pos(), read(old, f.schema(), in));
    }

    return tuple;
}
Example #3
Source File: SparkAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
Example #4
Source File: ReadAvroContainerBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up 
public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {   
  super(builder, config, parent, child, context);

  String schemaString = getConfigs().getString(config, "readerSchemaString", null);
  if (schemaString != null) {
    this.readerSchema = new Parser().parse(schemaString);
  } else {        
    String schemaFile = getConfigs().getString(config, "readerSchemaFile", null);
    if (schemaFile != null) {
      try { 
        this.readerSchema = new Parser().parse(new File(schemaFile));
      } catch (IOException e) {
        throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e);
      }
    } else {
      this.readerSchema = null;
    }
  }
  
  if (getClass() == ReadAvroContainer.class) {
    resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>(
        getConfigs().getInt(config, "schemaCacheCapacity", 100));
    
    validateArguments();
  } else {
    resolverCache = null;
  }
}
Example #5
Source File: PigAvroDatumReader.java    From spork with Apache License 2.0 5 votes vote down vote up 
/**
 * Called to read a map instance. Overridden to read a pig map.
 */
protected Object readMap(Object old, Schema expected, ResolvingDecoder in) throws IOException {
    Schema eValue = expected.getValueType();
    long l = in.readMapStart();
    Object map = newMap(old, (int) l);
    if (l > 0) {
        do {
            for (int i = 0; i < l; i++) {
                addToMap(map, readString(null, AvroStorageUtils.StringSchema, in),
                                  read(null, eValue, in));
            }
        } while ((l = in.mapNext()) > 0);
    }
    return map;
}
Example #6
Source File: PigAvroDatumReader.java    From Cubert with Apache License 2.0 5 votes vote down vote up 
/**
 * Called to read a map instance. Overridden to read a pig map.
 */
protected Object readMap(Object old, Schema expected, ResolvingDecoder in) throws IOException {
    Schema eValue = expected.getValueType();
    long l = in.readMapStart();
    Object map = newMap(old, (int) l);
    if (l > 0) {
        do {
            for (int i = 0; i < l; i++) {
                addToMap(map, readString(null, AvroStorageUtils.StringSchema, in),
                                  read(null, eValue, in));
            }
        } while ((l = in.mapNext()) > 0);
    }
    return map;
}
Example #7
Source File: GenericAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
Example #8
Source File: GenericAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
Example #9
Source File: GenericAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
Example #10
Source File: SparkAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
Example #11
Source File: SparkAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
Example #12
Source File: SparkAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
@Override
public InternalRow read(InternalRow reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  InternalRow row = reader.read(resolver, reuse);
  resolver.drain();
  return row;
}
Example #13
Source File: DataReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
Example #14
Source File: DataReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
Example #15
Source File: DataReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
Example #16
Source File: GenericAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
Example #17
Source File: GenericAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
Example #18
Source File: GenericAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
Example #19
Source File: DataReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
Example #20
Source File: DataReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
Example #21
Source File: DataReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
Example #22
Source File: SparkAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
Example #23
Source File: SparkAvroReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
@Override
public InternalRow read(InternalRow reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  InternalRow row = reader.read(resolver, reuse);
  resolver.drain();
  return row;
}
Example #24
Source File: ReadAvroContainerBuilder.java    From kite with Apache License 2.0 4 votes vote down vote up 
protected ResolvingDecoder createResolver(Schema writerSchema, Schema readerSchema) throws IOException {
  return DecoderFactory.get().resolvingDecoder(
      Schema.applyAliases(writerSchema, readerSchema), readerSchema, null);
}
Example #25
Source File: FastGenericDatumReader.java    From kite with Apache License 2.0 4 votes vote down vote up 
/** Call this whenever appropriate */
public void setResolver(ResolvingDecoder resolver) {
  this.resolver = resolver;
}