Java Code Examples for org.apache.avro.io.Decoder

The following examples show how to use org.apache.avro.io.Decoder. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: iceberg   Source File: SparkValueReaders.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericArrayData read(Decoder decoder, Object reuse) throws IOException {
  reusedList.clear();
  long chunkLength = decoder.readArrayStart();

  while (chunkLength > 0) {
    for (int i = 0; i < chunkLength; i += 1) {
      reusedList.add(elementReader.read(decoder, null));
    }

    chunkLength = decoder.arrayNext();
  }

  // this will convert the list to an array so it is okay to reuse the list
  return new GenericArrayData(reusedList.toArray());
}
 
Example 2
Source Project: iceberg   Source File: SparkValueReaders.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ArrayBasedMapData read(Decoder decoder, Object reuse) throws IOException {
  reusedKeyList.clear();
  reusedValueList.clear();

  long chunkLength = decoder.readArrayStart();

  while (chunkLength > 0) {
    for (int i = 0; i < chunkLength; i += 1) {
      reusedKeyList.add(keyReader.read(decoder, null));
      reusedValueList.add(valueReader.read(decoder, null));
    }

    chunkLength = decoder.arrayNext();
  }

  return new ArrayBasedMapData(
      new GenericArrayData(reusedKeyList.toArray()),
      new GenericArrayData(reusedValueList.toArray()));
}
 
Example 3
public List<Integer> deserialize(List<Integer> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveIntList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveIntList) {
            array0 = ((PrimitiveIntList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveIntArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readInt()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveIntArrayList(((int) chunkLen0));
    }
    return array0;
}
 
Example 4
public List<Boolean> deserialize(List<Boolean> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveBooleanList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveBooleanList) {
            array0 = ((PrimitiveBooleanList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readBoolean()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
    }
    return array0;
}
 
Example 5
public List<Integer> deserialize(List<Integer> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveIntList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveIntList) {
            array0 = ((PrimitiveIntList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveIntArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readInt()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveIntArrayList(((int) chunkLen0));
    }
    return array0;
}
 
Example 6
public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == mapMapValueSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(mapMapValueSchema0);
    }
    int unionIndex0 = (decoder.readIndex());
    if (unionIndex0 == 0) {
        decoder.readNull();
    }
    if (unionIndex0 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 
Example 7
public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == mapValueOptionSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(mapValueOptionSchema0);
    }
    int unionIndex1 = (decoder.readIndex());
    if (unionIndex1 == 0) {
        decoder.readNull();
    }
    if (unionIndex1 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 
Example 8
Source Project: flume-elasticsearch-sink   Source File: AvroSerializer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Converts the avro binary data to the json format
 */
@Override
public XContentBuilder serialize(Event event) {
    XContentBuilder builder = null;
    try {
        if (datumReader != null) {
            Decoder decoder = new DecoderFactory().binaryDecoder(event.getBody(), null);
            GenericRecord data = datumReader.read(null, decoder);
            logger.trace("Record in event " + data);
            XContentParser parser = XContentFactory
                    .xContent(XContentType.JSON)
                    .createParser(NamedXContentRegistry.EMPTY,
                            DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                            data.toString());
            builder = jsonBuilder().copyCurrentStructure(parser);
            parser.close();
        } else {
            logger.error("Schema File is not configured");
        }
    } catch (IOException e) {
        logger.error("Exception in parsing avro format data but continuing serialization to process further records",
                e.getMessage(), e);
    }
    return builder;
}
 
Example 9
public List<Boolean> deserialize(List<Boolean> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveBooleanList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveBooleanList) {
            array0 = ((PrimitiveBooleanList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readBoolean()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
    }
    return array0;
}
 
Example 10
private JMethod createMethod(final Schema schema, boolean read) {
  if (!Schema.Type.RECORD.equals(schema.getType())) {
    throw new FastDeserializerGeneratorException("Methods are defined only for records, not for " + schema.getType());
  }
  if (methodAlreadyDefined(schema, read)) {
    throw new FastDeserializerGeneratorException("Method already exists for: " + schema.getFullName());
  }

  JClass schemaClass = schemaAssistant.classFromSchema(schema);
  JMethod method = generatedClass.method(JMod.PUBLIC, read ? schemaClass : codeModel.VOID,
      getUniqueName("deserialize" + schema.getName()));

  method._throws(IOException.class);
  method.param(Object.class, VAR_NAME_FOR_REUSE);
  method.param(Decoder.class, DECODER);

  (read ? deserializeMethodMap : skipMethodMap).put(schema.getFullName(), method);

  return method;
}
 
Example 11
public <T> Decoder dataAsBinaryDecoder(T data, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null); //new BinaryEncoder(baos);

  try {
    FastGenericSerializerGenerator<T> fastGenericSerializerGenerator =
        new FastGenericSerializerGenerator<>(schema, tempDir, classLoader, null);
    FastSerializer<T> fastSerializer = fastGenericSerializerGenerator.generateSerializer();
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 
Example 12
public List<Double> deserialize(List<Double> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveDoubleList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveDoubleList) {
            array0 = ((PrimitiveDoubleList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveDoubleArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readDouble()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveDoubleArrayList(((int) chunkLen0));
    }
    return array0;
}
 
Example 13
public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == mapValueOptionSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(mapValueOptionSchema0);
    }
    int unionIndex1 = (decoder.readIndex());
    if (unionIndex1 == 0) {
        decoder.readNull();
    }
    if (unionIndex1 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 
Example 14
public List<Long> deserialize(List<Long> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveLongList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveLongList) {
            array0 = ((PrimitiveLongList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveLongArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readLong()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveLongArrayList(((int) chunkLen0));
    }
    return array0;
}
 
Example 15
public <T> Decoder dataAsDecoder(T data, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator =
        new FastSpecificSerializerGenerator<>(schema, tempDir, classLoader, null);
    FastSerializer<T> fastSerializer = fastSpecificSerializerGenerator.generateSerializer();
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 
Example 16
public <T> Decoder writeWithFastAvro(T data, Schema schema, boolean specific) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    FastSerializer<T> fastSerializer;
    if (specific) {
      FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator = new FastSpecificSerializerGenerator<>(schema, tempDir, classLoader, null);
      fastSerializer = fastSpecificSerializerGenerator.generateSerializer();
    } else {
      FastGenericSerializerGenerator<T> fastGenericSerializerGenerator = new FastGenericSerializerGenerator<>(schema, tempDir, classLoader, null);
      fastSerializer = fastGenericSerializerGenerator.generateSerializer();
    }
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 
Example 17
public List<Integer> deserialize(List<Integer> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveIntList array0 = null;
    long chunkLen0 = (decoder.readArrayStart());
    if (chunkLen0 > 0) {
        if ((reuse) instanceof PrimitiveIntList) {
            array0 = ((PrimitiveIntList)(reuse));
            array0 .clear();
        } else {
            array0 = new PrimitiveIntArrayList(((int) chunkLen0));
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                array0 .addPrimitive((decoder.readInt()));
            }
            chunkLen0 = (decoder.arrayNext());
        } while (chunkLen0 > 0);
    } else {
        array0 = new PrimitiveIntArrayList(((int) chunkLen0));
    }
    return array0;
}
 
Example 18
Source Project: iceberg   Source File: SparkAvroReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public InternalRow read(InternalRow reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  InternalRow row = reader.read(resolver, reuse);
  resolver.drain();
  return row;
}
 
Example 19
/**
 * Parse Avro record with schema
 *
 * @param bytes  avro data
 * @param schema avro schema
 * @return JsonNode  array
 */
private JsonNode parseAvroWithSchema(final byte[] bytes, Schema schema)
{
  GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
  InputStream input = new ByteArrayInputStream(bytes);
  Decoder decoder = DecoderFactory.get().binaryDecoder(input, null);
  try
  {
    GenericRecord datum = reader.read(null, decoder);
    return mapper.readTree(datum.toString());
  } catch (IOException e)
  {
    throw SnowflakeErrors.ERROR_0010.getException("Failed to parse AVRO " +
      "record\n" + e.toString());
  }
}
 
Example 20
public Map<Utf8, IndexedRecord> deserialize(Map<Utf8, IndexedRecord> reuse, Decoder decoder)
    throws IOException
{
    Map<Utf8, IndexedRecord> map0 = null;
    long chunkLen0 = (decoder.readMapStart());
    if (chunkLen0 > 0) {
        Map<Utf8, IndexedRecord> mapReuse0 = null;
        if ((reuse) instanceof Map) {
            mapReuse0 = ((Map)(reuse));
        }
        if (mapReuse0 != (null)) {
            mapReuse0 .clear();
            map0 = mapReuse0;
        } else {
            map0 = new HashMap<Utf8, IndexedRecord>();
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                Utf8 key0 = (decoder.readString(null));
                map0 .put(key0, deserializerecord0(null, (decoder)));
            }
            chunkLen0 = (decoder.mapNext());
        } while (chunkLen0 > 0);
    } else {
        map0 = Collections.emptyMap();
    }
    return map0;
}
 
Example 21
Source Project: iceberg   Source File: ValueReaders.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public Collection<T> read(Decoder decoder, Object reused) throws IOException {
  LinkedList<T> resultList;
  if (lastList != null) {
    lastList.clear();
    resultList = (LinkedList<T>) lastList;
  } else {
    resultList = Lists.newLinkedList();
  }

  if (reused instanceof LinkedList) {
    this.lastList = (LinkedList<?>) reused;
  } else {
    this.lastList = null;
  }

  long chunkLength = decoder.readArrayStart();
  Iterator<?> elIter = lastList != null ? lastList.iterator() : emptyIterator();

  while (chunkLength > 0) {
    for (long i = 0; i < chunkLength; i += 1) {
      Object lastValue = elIter.hasNext() ? elIter.next() : null;
      resultList.addLast(elementReader.read(decoder, lastValue));
    }

    chunkLength = decoder.arrayNext();
  }

  return resultList;
}
 
Example 22
Source Project: avro-fastserde   Source File: FastSerdeTestsSupport.java    License: Apache License 2.0 5 votes vote down vote up
public static <T> T deserializeGeneric(Schema schema, Decoder decoder) {
    GenericDatumReader<T> datumReader = new GenericDatumReader<>(schema);
    try {
        return datumReader.read(null, decoder);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example 23
Source Project: iceberg   Source File: SparkAvroReader.java    License: Apache License 2.0 5 votes vote down vote up
private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
 
Example 24
public List<Float> deserialize(List<Float> reuse, Decoder decoder)
    throws IOException
{
    PrimitiveFloatList array0 = null;
    array0 = ((PrimitiveFloatList) ByteBufferBackedPrimitiveFloatList.readPrimitiveFloatArray((reuse), (decoder)));
    return array0;
}
 
Example 25
Source Project: sqoop-on-spark   Source File: AVROIntermediateDataFormat.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void read(DataInput in) throws IOException {
  DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(avroSchema);
  Decoder decoder = DecoderFactory.get().binaryDecoder((InputStream) in, null);
  data = reader.read(null, decoder);
}
 
Example 26
Source Project: incubator-gobblin   Source File: EventUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parses a {@link org.apache.gobblin.metrics.GobblinTrackingEvent} from a byte array Avro serialization.
 * @param reuse GobblinTrackingEvent to reuse.
 * @param bytes Input bytes.
 * @param schemaId Expected schemaId.
 * @return GobblinTrackingEvent.
 * @throws java.io.IOException
 */
public synchronized static GobblinTrackingEvent deserializeEventFromAvroSerialization(GobblinTrackingEvent reuse, byte[] bytes, @Nullable String schemaId)
    throws IOException {
  if (!reader.isPresent()) {
    reader = Optional.of(new SpecificDatumReader<>(GobblinTrackingEvent.class));
  }

  Closer closer = Closer.create();

  try {
    DataInputStream inputStream = closer.register(new DataInputStream(new ByteArrayInputStream(bytes)));

    if (schemaId != null) {
      MetricReportUtils.readAndVerifySchemaId(inputStream, schemaId);
    } else {
      MetricReportUtils.readAndVerifySchemaVersion(inputStream);
    }

    // Decode the rest
    Decoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
    return reader.get().read(reuse, decoder);
  } catch(Throwable t) {
    throw closer.rethrow(t);
  } finally {
    closer.close();
  }
}
 
Example 27
public Map<Utf8, IndexedRecord> deserialize(Map<Utf8, IndexedRecord> reuse, Decoder decoder)
    throws IOException
{
    Map<Utf8, IndexedRecord> map0 = null;
    long chunkLen0 = (decoder.readMapStart());
    if (chunkLen0 > 0) {
        Map<Utf8, IndexedRecord> mapReuse0 = null;
        if ((reuse) instanceof Map) {
            mapReuse0 = ((Map)(reuse));
        }
        if (mapReuse0 != (null)) {
            mapReuse0 .clear();
            map0 = mapReuse0;
        } else {
            map0 = new HashMap<Utf8, IndexedRecord>();
        }
        do {
            for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
                Utf8 key0 = (decoder.readString(null));
                map0 .put(key0, deserializerecord0(null, (decoder)));
            }
            chunkLen0 = (decoder.mapNext());
        } while (chunkLen0 > 0);
    } else {
        map0 = Collections.emptyMap();
    }
    return map0;
}
 
Example 28
Source Project: incubator-gobblin   Source File: EventUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parses a {@link org.apache.gobblin.metrics.MetricReport} from a byte array representing a json input.
 * @param reuse MetricReport to reuse.
 * @param bytes Input bytes.
 * @return MetricReport.
 * @throws java.io.IOException
 */
public synchronized static GobblinTrackingEvent deserializeReportFromJson(GobblinTrackingEvent reuse, byte[] bytes) throws IOException {
  if (!reader.isPresent()) {
    reader = Optional.of(new SpecificDatumReader<>(GobblinTrackingEvent.class));
  }

  Closer closer = Closer.create();

  try {
    DataInputStream inputStream = closer.register(new DataInputStream(new ByteArrayInputStream(bytes)));

    // Check version byte
    int versionNumber = inputStream.readInt();
    if (versionNumber != SCHEMA_VERSION) {
      throw new IOException(String
          .format("MetricReport schema version not recognized. Found version %d, expected %d.", versionNumber,
              SCHEMA_VERSION));
    }

    // Decode the rest
    Decoder decoder = DecoderFactory.get().jsonDecoder(GobblinTrackingEvent.SCHEMA$, inputStream);
    return reader.get().read(reuse, decoder);
  } catch(Throwable t) {
    throw closer.rethrow(t);
  } finally {
    closer.close();
  }
}
 
Example 29
Source Project: iceberg   Source File: ValueReaders.java    License: Apache License 2.0 5 votes vote down vote up
@Override
    public String read(Decoder decoder, Object ignored) throws IOException {
      // use the decoder's readString(Utf8) method because it may be a resolving decoder
      this.reusedTempUtf8.set(decoder.readString(reusedTempUtf8.get()));
      return reusedTempUtf8.get().toString();
//      int length = decoder.readInt();
//      byte[] bytes = new byte[length];
//      decoder.readFixed(bytes, 0, length);
    }
 
Example 30
Source Project: iceberg   Source File: ValueReaders.java    License: Apache License 2.0 5 votes vote down vote up
@Override
    public Utf8 read(Decoder decoder, Object reuse) throws IOException {
      // use the decoder's readString(Utf8) method because it may be a resolving decoder
      if (reuse instanceof Utf8) {
        return decoder.readString((Utf8) reuse);
      } else {
        return decoder.readString(null);
      }
//      int length = decoder.readInt();
//      byte[] bytes = new byte[length];
//      decoder.readFixed(bytes, 0, length);
    }