org.apache.avro.AvroRuntimeException Java Examples

@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    try {
        if (!reader.hasNext() || reader.pastSync(end)) {
            return false;
        }
        return true;
    } catch (AvroRuntimeException e) {
        if (ignoreBadFiles) {
            // For currupted files, AvroRuntimeException can be thrown.
            // We ignore them if the option 'ignore_bad_files' is enabled.
            LOG.warn("Ignoring bad file '" + path + "'.");
            return false;
        } else {
            throw e;
        }
    }
}
 

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
    throws IOException, InterruptedException {
  if (context.getNumReduceTasks() == 0) {
    context.write(key, NullWritable.get());
  } else {
    populateComparableKeyRecord(key.datum(), this.outKey.datum());
    this.outValue.datum(key.datum());
    try {
      context.write(this.outKey, this.outValue);
    } catch (AvroRuntimeException e) {
      final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
      throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
    }
  }
  context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}
 

@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    try {
        if (!reader.hasNext() || reader.pastSync(end)) {
            return false;
        }
        return true;
    } catch (AvroRuntimeException e) {
        if (ignoreBadFiles) {
            // For currupted files, AvroRuntimeException can be thrown.
            // We ignore them if the option 'ignore_bad_files' is enabled.
            LOG.warn("Ignoring bad file '" + path + "'.");
            return false;
        } else {
            throw e;
        }
    }
}
 

/**
 * Helper method that does the actual work for {@link #getFieldSchema(Schema, String)}
 * @param schema passed from {@link #getFieldSchema(Schema, String)}
 * @param pathList passed from {@link #getFieldSchema(Schema, String)}
 * @param field keeps track of the index used to access the list pathList
 * @return the schema of the field
 */
private static Optional<Schema> getFieldSchemaHelper(Schema schema, List<String> pathList, int field) {
  if (schema.getType() == Type.RECORD && schema.getField(pathList.get(field)) == null) {
    return Optional.absent();
  }
  switch (schema.getType()) {
    case UNION:
      if (AvroSerdeUtils.isNullableType(schema)) {
        return AvroUtils.getFieldSchemaHelper(AvroSerdeUtils.getOtherTypeFromNullableType(schema), pathList, field);
      }
      throw new AvroRuntimeException("Union of complex types cannot be handled : " + schema);
    case MAP:
      if ((field + 1) == pathList.size()) {
        return Optional.fromNullable(schema.getValueType());
      }
      return AvroUtils.getFieldSchemaHelper(schema.getValueType(), pathList, ++field);
    case RECORD:
      if ((field + 1) == pathList.size()) {
        return Optional.fromNullable(schema.getField(pathList.get(field)).schema());
      }
      return AvroUtils.getFieldSchemaHelper(schema.getField(pathList.get(field)).schema(), pathList, ++field);
    default:
      throw new AvroRuntimeException("Invalid type in schema : " + schema);
  }
}
 

/**
 * Helper method that does the actual work for {@link #getField(Schema, String)}
 * @param schema passed from {@link #getFieldSchema(Schema, String)}
 * @param pathList passed from {@link #getFieldSchema(Schema, String)}
 * @param field keeps track of the index used to access the list pathList
 * @return the field
 */
private static Optional<Field> getFieldHelper(Schema schema, List<String> pathList, int field) {
  Field curField = schema.getField(pathList.get(field));
  if (field + 1 == pathList.size()) {
    return Optional.fromNullable(curField);
  }

  Schema fieldSchema = curField.schema();
  switch (fieldSchema.getType()) {
    case UNION:
      throw new AvroRuntimeException("Union of complex types cannot be handled : " + schema);
    case MAP:
      return AvroUtils.getFieldHelper(fieldSchema.getValueType(), pathList, ++field);
    case RECORD:
      return AvroUtils.getFieldHelper(fieldSchema, pathList, ++field);
    case ARRAY:
      return AvroUtils.getFieldHelper(fieldSchema.getElementType(), pathList, ++field);
    default:
      throw new AvroRuntimeException("Invalid type " + fieldSchema.getType() + " in schema : " + schema);
  }
}
 

/**
 * Reads a GenericRecord from the given input stream<br>
 * Emits the FileName,Offset,Exception on the error port if its connected
 *
 * @return GenericRecord
 */
@Override
protected GenericRecord readEntity() throws IOException
{
  GenericRecord record = null;

  record = null;

  try {
    if (avroDataStream != null && avroDataStream.hasNext()) {
      offset++;
      record = avroDataStream.next();
      recordCount++;
      return record;
    }
  } catch (AvroRuntimeException are) {
    LOG.error("Exception in parsing record for file - " + super.currentFile + " at offset - " + offset, are);
    if (errorRecordsPort.isConnected()) {
      errorRecordsPort.emit("FileName:" + super.currentFile + ", Offset:" + offset);
    }
    errorCount++;
    throw new AvroRuntimeException(are);
  }
  return record;
}
 

/**
 * Given a map: key -> value, return a map: key.toString() -> value.toString(). Avro serializer wraps a String
 * into {@link Utf8}. This method helps to restore the original string map object
 *
 * @param map a map object
 * @return a map of strings
 */
@SuppressWarnings("unchecked")
public static Map<String, String> toStringMap(Object map) {
  if (map == null) {
    return null;
  }

  if (map instanceof Map) {
    Map<Object, Object> rawMap = (Map<Object, Object>) map;
    Map<String, String> stringMap = new HashMap<>();
    for (Entry<Object, Object> entry : rawMap.entrySet()) {
      stringMap.put(entry.getKey().toString(), entry.getValue().toString());
    }
    return stringMap;
  } else {
    throw new AvroRuntimeException("value must be a map");
  }
}
 

@Override
public org.apache.gobblin.configuration.State parseJobStatus(byte[] message)
    throws IOException {
  InputStream is = new ByteArrayInputStream(message);
  schemaVersionWriter.readSchemaVersioningInformation(new DataInputStream(is));

  Decoder decoder = DecoderFactory.get().binaryDecoder(is, this.decoder.get());
  try {
    GobblinTrackingEvent decodedMessage = this.reader.get().read(null, decoder);
    return parseJobStatus(decodedMessage);
  } catch (AvroRuntimeException | IOException exc) {
    this.messageParseFailures.mark();
    if (this.messageParseFailures.getFiveMinuteRate() < 1) {
      log.warn("Unable to decode input message.", exc);
    } else {
      log.warn("Unable to decode input message.");
    }
    return null;
  }
}
 

@CliCommand(value = "repair corrupted clean files", help = "repair corrupted clean files")
public void removeCorruptedPendingCleanAction() {

  HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
  HoodieTimeline cleanerTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline().getCleanerTimeline();
  LOG.info("Inspecting pending clean metadata in timeline for corrupted files");
  cleanerTimeline.filterInflightsAndRequested().getInstants().forEach(instant -> {
    try {
      CleanerUtils.getCleanerPlan(client, instant);
    } catch (AvroRuntimeException e) {
      LOG.warn("Corruption found. Trying to remove corrupted clean instant file: " + instant);
      FSUtils.deleteInstantFile(client.getFs(), client.getMetaPath(), instant);
    } catch (IOException ioe) {
      if (ioe.getMessage().contains("Not an Avro data file")) {
        LOG.warn("Corruption found. Trying to remove corrupted clean instant file: " + instant);
        FSUtils.deleteInstantFile(client.getFs(), client.getMetaPath(), instant);
      } else {
        throw new HoodieIOException(ioe.getMessage(), ioe);
      }
    }
  });
}
 

@Override
public Collection<Either<JobSpec, URI>> parseJobSpec(byte[] message)
    throws IOException {

  InputStream is = new ByteArrayInputStream(message);
  this.versionWriter.readSchemaVersioningInformation(new DataInputStream(is));

  Decoder decoder = DecoderFactory.get().binaryDecoder(is, this.decoder.get());
  try {
    T decodedMessage = this.reader.get().read(null, decoder);
    return parseJobSpec(decodedMessage);
  } catch (AvroRuntimeException | IOException exc) {
    this.messageParseFailures.mark();
    if (this.messageParseFailures.getFiveMinuteRate() < 1) {
      log.warn("Unable to decode input message.", exc);
    } else {
      log.warn("Unable to decode input message.");
    }
    return Lists.newArrayList();
  }
}
 

@Test
public void testVanilla14FixedClassesIncompatibleWithAvro17() throws Exception {
  AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
  if (!runtimeVersion.equals(AvroVersion.AVRO_1_7)) {
    throw new SkipException("class only supported under avro 1.7. runtime version detected as " + runtimeVersion);
  }

  String sourceCode = TestUtil.load("Vanilla14Fixed");
  Class clazz = CompilerUtils.CACHED_COMPILER.loadFromJava("com.acme.generatedby14.Vanilla14Fixed", sourceCode);
  try {
    clazz.newInstance();
    Assert.fail("expecting an exception");
  } catch (AvroRuntimeException expected) {
    Assert.assertTrue(expected.getMessage().contains("Not a Specific class")); //fails to find SCHEMA$
  }
}
 

private static SchemaTransformStep findFixFor(AvroRuntimeException issue) {
  String msg = String.valueOf(issue.getMessage());

  //is it an illegal identifier issue? if so, whats the problematic identifier string?
  String illegalIdentifier = tryParseIllegalIdentifier(msg);
  if (illegalIdentifier != null) {
    String fixedIdentifier = escapeIllegalCharacters(illegalIdentifier);
    return new RenameIdentifierStep(issue, illegalIdentifier, fixedIdentifier);
  }

  //is it a duplicate field issue?
  String duplicateFieldName = tryParseDuplicateFieldname(msg);
  if (duplicateFieldName != null) {
    return new RemoveDuplicateStep(issue, duplicateFieldName);
  }

  //is it a case of invalid default field value?
  FixDefaultValueStep.BadDefaultPropertySpec badPropertySpec = tryParseBadPropertyDefault(msg);
  if (badPropertySpec != null) {
    return new FixDefaultValueStep(issue, badPropertySpec);
  }

  return null;
}
 

@Override
public void configure(Context context) {

  int syncIntervalBytes =
      context.getInteger(SYNC_INTERVAL_BYTES, DEFAULT_SYNC_INTERVAL_BYTES);
  String compressionCodec =
      context.getString(COMPRESSION_CODEC, DEFAULT_COMPRESSION_CODEC);

  writer = new ReflectDatumWriter<T>(getSchema());
  dataFileWriter = new DataFileWriter<T>(writer);

  dataFileWriter.setSyncInterval(syncIntervalBytes);

  try {
    CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
    dataFileWriter.setCodec(codecFactory);
  } catch (AvroRuntimeException e) {
    logger.warn("Unable to instantiate avro codec with name (" +
        compressionCodec + "). Compression disabled. Exception follows.", e);
  }
}
 

@Test
public void demonstrateAvro14FixedUnusableUnder16() throws Exception {
  //avro fixed classes extend org.apache.avro.specific.SpecificFixed which, in turn implements
  //org.apache.avro.generic.GenericFixed. in avro 1.5+ GenericFixed extends org.apache.avro.generic.GenericContainer.
  //GenericContainer, in turn, defined method getSchema() that avro-14-generated fixed classes dont implement
  //under 1.6 specifically the failure is a little different - its looking for field SCHEMA$ directly 1st.
  //avro swallows the real root cause (NoSuchFieldException) though - #craftsmanship
  try {
    new by14.SimpleFixed();
    Assert.fail("expected to throw");
  } catch (AvroRuntimeException issue) {
    Throwable root = Throwables.getRootCause(issue);
    Assert.assertTrue(root instanceof AvroRuntimeException);
    Assert.assertTrue(root.getMessage().contains("Not a Specific class"));
  }
}
 

@Test
public void demonstrateAvro14FixedUnusableUnder17() throws Exception {
  //avro fixed classes extend org.apache.avro.specific.SpecificFixed which, in turn implements
  //org.apache.avro.generic.GenericFixed. in avro 1.5+ GenericFixed extends org.apache.avro.generic.GenericContainer.
  //GenericContainer, in turn, defined method getSchema() that avro-14-generated fixed classes dont implement
  //under 1.7 specifically the failure is a little different - its looking for field SCHEMA$ directly 1st.
  //avro swallows the real root cause (NoSuchFieldException) though - #craftsmanship
  try {
    new by14.SimpleFixed();
    Assert.fail("expected to throw");
  } catch (AvroRuntimeException issue) {
    Throwable root = Throwables.getRootCause(issue);
    Assert.assertTrue(root instanceof AvroRuntimeException);
    Assert.assertTrue(root.getMessage().contains("Not a Specific class"));
  }
}
 

Avro18BufferedBinaryEncoder configure(OutputStream out, int bufferSize) {
  if (null == out)
    throw new NullPointerException("OutputStream cannot be null!");
  if (null != this.sink) {
    if ( pos > 0) {
      try {
        flushBuffer();
      } catch (IOException e) {
        throw new AvroRuntimeException("Failure flushing old output", e);
      }
    }
  }
  this.sink = new OutputStreamSink(out);
  pos = 0;
  if (null == buf || buf.length != bufferSize) {
    buf = new byte[bufferSize];
  }
  bulkLimit = buf.length >>> 1;
  if (bulkLimit > 512) {
    bulkLimit = 512;
  }
  return this;
}
 

/**
 * Test null query string.
 */
@Test
public void testNullData() {
  thrown.expect(AvroRuntimeException.class);
  thrown.expectMessage("Field queryString type:STRING pos:3 does not accept null values");
  final String s = "test binary stream data";
  final byte[] b = s.getBytes(StandardCharsets.UTF_8);
  avroRequest = AvroHttpRequest.newBuilder()
      .setRequestUrl("http://localhost:8080/reef/evaluators?id=12&id=34&a=b")
      .setHttpMethod("POST")
      .setQueryString(null)
      .setPathInfo("/reef/evaluators")
      .setHeader(createHeader())
      .setInputStream(ByteBuffer.wrap(b))
      .build();
}
 

Avro18BufferedBinaryEncoder configure(OutputStream out, int bufferSize) {
  if (null == out)
    throw new NullPointerException("OutputStream cannot be null!");
  if (null != this.sink) {
    if ( pos > 0) {
      try {
        flushBuffer();
      } catch (IOException e) {
        throw new AvroRuntimeException("Failure flushing old output", e);
      }
    }
  }
  this.sink = new OutputStreamSink(out);
  pos = 0;
  if (null == buf || buf.length != bufferSize) {
    buf = new byte[bufferSize];
  }
  bulkLimit = buf.length >>> 1;
  if (bulkLimit > 512) {
    bulkLimit = 512;
  }
  return this;
}
 

/**
 * Test null bytes.
 */
@Test
public void testNullBytes() {
  thrown.expect(AvroRuntimeException.class);
  thrown.expectMessage("Field inputStream type:BYTES pos:5 does not accept null values");
  final String s = "test binary stream data";
  final byte[] b = s.getBytes(StandardCharsets.UTF_8);

  avroRequest = AvroHttpRequest.newBuilder()
      .setRequestUrl("http://localhost:8080/reef/evaluators?id=12&id=34&a=b")
      .setHttpMethod("POST")
      .setQueryString("id=12&id=34&a=b")
      .setPathInfo("/reef/evaluators")
      .setInputStream(null)
      .setHeader(createHeader())
      .build();
}
 

public static Object toAvro(Object item, Field field) {
  if (item == null && !nullOk(field.schema())) {
    try { // this will fail if there is no default value
      return ReflectData.get().getDefaultValue(field);
    } catch (AvroRuntimeException e) {
      return ERROR;
    }
  }
  Object result = toAvro(item, field.schema());
  return result;
}
 

private void initialize(Event event) throws IOException {
  Schema schema = null;
  String schemaUrl = event.getHeaders().get(AVRO_SCHEMA_URL_HEADER);
  if (schemaUrl != null) {
    schema = schemaCache.get(schemaUrl);
    if (schema == null) {
      schema = loadFromUrl(schemaUrl);
      schemaCache.put(schemaUrl, schema);
    }
  }
  if (schema == null) {
    String schemaString = event.getHeaders().get(AVRO_SCHEMA_LITERAL_HEADER);
    if (schemaString == null) {
      throw new FlumeException("Could not find schema for event " + event);
    }
    schema = new Schema.Parser().parse(schemaString);
  }

  writer = new GenericDatumWriter<Object>(schema);
  dataFileWriter = new DataFileWriter<Object>(writer);

  dataFileWriter.setSyncInterval(syncIntervalBytes);

  try {
    CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
    dataFileWriter.setCodec(codecFactory);
  } catch (AvroRuntimeException e) {
    logger.warn("Unable to instantiate avro codec with name (" +
        compressionCodec + "). Compression disabled. Exception follows.", e);
  }

  dataFileWriter.create(schema, out);
}
 

/**
 * In case of complex data types in union {@link AvroUtils#getFieldSchema(Schema, String)} should throw {@link AvroRuntimeException}
 * @throws IOException
 */

@Test(expectedExceptions = AvroRuntimeException.class)
public void testComplexTypesInUnionNotSupported()
    throws IOException {
  final String TEST_LOCATION = "TestUnionObject.RecordInUnion";
  String avroFilePath = this.AVRO_DIR + "avroDir/avroUtilsTestFile.avro";
  GenericRecord record = getRecordFromFile(avroFilePath).get(0);

  AvroUtils.getFieldSchema(record.getSchema(), TEST_LOCATION);
}
 

public java.lang.Object get(int field) {
  switch (field) {
    case 0: return userId;
    case 1: return country;
    case 2: return pageId;
    default: throw new AvroRuntimeException("bad index");
  }
}
 

public void put(int field, Object value) {
  switch (field) {
    case 0:
      userId = (String) value; break;
    case 1:
      country = (String) value; break;
    case 2:
      pageId = (String) value; break;
    default:
      throw new AvroRuntimeException("bad index");
  }
}
 

private CodecFactory getCodecFactory(CompressionCodec codec) {
    CompressionCodecName codecName = CompressionCodecName
            .fromCompressionCodec(codec != null ? codec.getClass() : null);
    try {
        return CodecFactory.fromString(codecName.name().toLowerCase());
    } catch (AvroRuntimeException e) {
        LOG.error("Error creating codec factory", e);
    }
    return CodecFactory.fromString("null");
}
 

@Override
public D decode(InputStream stream, D reuse) {
  BinaryDecoder decoder = DecoderFactory.get().directBinaryDecoder(stream, DECODER.get());
  DECODER.set(decoder);
  try {
    return reader.read(reuse, decoder);
  } catch (IOException e) {
    throw new AvroRuntimeException("Decoding datum failed", e);
  }
}
 

@Override
public boolean next(Text key, Text ignore) throws IOException {
    try {
        return super.next(key, ignore);
    } catch (AvroRuntimeException e) {
        log.warn("Cannot get next Key from avro file, may be corrupt: ({})", file);
        log.warn("", e);
        return false;
    }
}
 

/**
 * Test null incomplete request.
 */
@Test
public void testIncompleteData() {
  thrown.expect(AvroRuntimeException.class);
  thrown.expectMessage("Field queryString type:STRING pos:3 not set and has no default value");
  final String s = "test binary stream data";
  final byte[] b = s.getBytes(StandardCharsets.UTF_8);
  avroRequest = AvroHttpRequest.newBuilder()
      .setRequestUrl("http://localhost:8080/reef/evaluators?id=12&id=34&a=b")
      .setHttpMethod("POST")
      .setPathInfo("/reef/evaluators")
      .setInputStream(ByteBuffer.wrap(b))
      .setHeader(createHeader())
      .build();
}
 

private void initialize(Event event) throws IOException {
  Schema schema = null;
  String schemaUrl = event.getHeaders().get(AVRO_SCHEMA_URL_HEADER);
  if (schemaUrl != null) {
    schema = schemaCache.get(schemaUrl);
    if (schema == null) {
      schema = loadFromUrl(schemaUrl);
      schemaCache.put(schemaUrl, schema);
    }
  }
  if (schema == null) {
    String schemaString = event.getHeaders().get(AVRO_SCHEMA_LITERAL_HEADER);
    if (schemaString == null) {
      throw new FlumeException("Could not find schema for event " + event);
    }
    schema = new Schema.Parser().parse(schemaString);
  }

  writer = new GenericDatumWriter<Object>(schema);
  dataFileWriter = new DataFileWriter<Object>(writer);

  dataFileWriter.setSyncInterval(syncIntervalBytes);

  try {
    CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
    dataFileWriter.setCodec(codecFactory);
  } catch (AvroRuntimeException e) {
    logger.warn("Unable to instantiate avro codec with name (" +
        compressionCodec + "). Compression disabled. Exception follows.", e);
  }

  dataFileWriter.create(schema, out);
}
 

@Test(expected = AvroRuntimeException.class)
public void testByteBufferMissingPayload() throws Exception {
  MessageEncoder<Record> encoder = new IcebergEncoder<>(SCHEMA_V2);
  MessageDecoder<Record> decoder = new IcebergDecoder<>(SCHEMA_V2);

  ByteBuffer buffer = encoder.encode(V2_RECORDS.get(0));

  buffer.limit(12);

  decoder.decode(buffer);
}