Java Code Examples for org.apache.avro.AvroTypeException

The following examples show how to use org.apache.avro.AvroTypeException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: TestAvroDecoder.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaEvolutionToIncompatibleType()
{
    byte[] originalIntData = buildAvroData(getFieldBuilder()
                    .name("int_to_string_field").type().intType().noDefault()
                    .endRecord(),
            "int_to_string_field", 100);

    DecoderTestColumnHandle stringColumnReadingIntData = new DecoderTestColumnHandle(0, "row0", VARCHAR, "int_to_string_field", null, null, false, false, false);
    String changedTypeSchema = getFieldBuilder()
            .name("int_to_string_field").type().stringType().noDefault()
            .endRecord()
            .toString();

    assertThatThrownBy(() -> decodeRow(originalIntData, ImmutableSet.of(stringColumnReadingIntData), ImmutableMap.of(DATA_SCHEMA, changedTypeSchema)))
            .isInstanceOf(PrestoException.class)
            .hasCauseExactlyInstanceOf(AvroTypeException.class)
            .hasStackTraceContaining("Found int, expecting string")
            .hasMessageMatching("Decoding Avro record failed.");
}
 
Example 2
Source Project: gcp-ingestion   Source File: GuidedJsonDecoder.java    License: Mozilla Public License 2.0 6 votes vote down vote up
/**
 * Find the index in the union of the current variant.
 *
 * <p>This method only supports a single nullable type. Having more than a single
 * type is invalid in this case and will cause the decoder to panic. This
 * behavior is by design, since BigQuery does not support variant types in
 * columns. It is also inefficient to match sub-documents against various
 * types, given the streaming interface and bias towards performance.
 *
 * <p>Variants of non-null types are invalid. We enforce this by ensuring there
 * are no more than 2 elements and that at least one of them is null if there
 * are 2. Unions are required to be non-empty.
 *
 * <li> Ok: [null], [type], [null, type]
 * <li> Bad: [type, type], [null, type, type]
 */
@Override
public int readIndex() throws IOException {
  parser.advance(Symbol.UNION);
  Symbol.Alternative top = (Symbol.Alternative) parser.popSymbol();

  int nullIndex = top.findLabel("null");
  int typeIndex = nullIndex == 0 ? 1 : 0;

  if ((nullIndex < 0 && top.size() == 2) || (top.size() > 2)) {
    throw new AvroTypeException("Variant types are not supported.");
  }

  int index = in.getCurrentToken() == JsonToken.VALUE_NULL ? nullIndex : typeIndex;
  parser.pushSymbol(top.getSymbol(index));
  return index;
}
 
Example 3
Source Project: avro-util   Source File: SchemaValidator.java    License: BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Override
public void visitField(Schema parent, Schema.Field field) {
  if (grandfathered.contains(parent)) {
    return;
  }
  if (validationSpec.validateNames()) {
    String fieldName = field.name();
    validateName(fieldName, " in field " + parent.getFullName() + "." + fieldName);
  }
  JsonNode defaultValue = field.defaultValue();
  if (validationSpec.validateDefaultValues() && defaultValue != null) {
    Schema fieldSchema = field.schema();
    boolean validDefault = isValidDefault(fieldSchema, defaultValue);
    if (!validDefault) {
      //throw ~the same exception avro would
      String message = "Invalid default for field " + parent.getFullName() + "." + field.name() + ": "
          + defaultValue + " not a " + fieldSchema;
      throw new AvroTypeException(message);
    }
  }
}
 
Example 4
Source Project: kafka-connect-fs   Source File: AvroFileReaderTest.java    License: Apache License 2.0 6 votes vote down vote up
@ParameterizedTest
@MethodSource("fileSystemConfigProvider")
public void readerWithInvalidSchema(ReaderFsTestConfig fsConfig) throws IOException {
    Map<String, Object> readerConfig = getReaderConfig();
    readerConfig.put(AvroFileReader.FILE_READER_AVRO_SCHEMA, Schema.create(Schema.Type.STRING).toString());
    FileSystem testFs = FileSystem.newInstance(fsConfig.getFsUri(), new Configuration());
    fsConfig.setReader(getReader(testFs, fsConfig.getDataFile(), readerConfig));
    assertThrows(ConnectException.class, () -> readAllData(fsConfig));
    assertThrows(AvroTypeException.class, () -> {
        try {
            readAllData(fsConfig);
        } catch (Exception e) {
            throw e.getCause();
        }
    });
}
 
Example 5
Source Project: iow-hadoop-streaming   Source File: IOWJsonDecoder.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int readEnum() throws IOException {
    advance(Symbol.ENUM);
    Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction) parser.popSymbol();
    if (in.getCurrentToken() == JsonToken.VALUE_STRING) {
        in.getText();
        int n = top.findLabel(in.getText());
        if (n >= 0) {
            in.nextToken();
            return n;
        }
        throw new AvroTypeException("Unknown symbol in enum " + in.getText());
    } else {
        throw error("fixed");
    }
}
 
Example 6
Source Project: data-highway   Source File: SchemaDeserializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = AvroTypeException.class)
public void detectsInvalidDefault() throws JsonProcessingException, IOException {
  jsonNode = new ObjectMapper().readTree(
      "{\"type\":\"record\",\"name\":\"mine\",\"fields\":[{\"name\":\"str\",\"type\":\"int\",\"default\":\"0\"}]}");
  // "0" is invalid for int, it should instead be 0

  when(parser.readValueAsTree()).thenReturn(jsonNode);

  new SchemaDeserializer().deserialize(parser, null);
}
 
Example 7
Source Project: gcp-ingestion   Source File: GuidedJsonDecoder.java    License: Mozilla Public License 2.0 5 votes vote down vote up
private void assertCurrentTokenOneOf(JsonToken[] tokens, String type) throws AvroTypeException {
  JsonToken token = in.getCurrentToken();
  for (JsonToken expect : tokens) {
    if (token == expect) {
      return;
    }
  }
  error(token, type);
}
 
Example 8
@Test(expected = AvroTypeException.class)
public void testFormatMissingRequiredFieldThrowsException() {
  PubsubMessageRecordFormatter formatter = new PubsubMessageRecordFormatter();
  byte[] data = Json.createObjectNode().set("unused", NullNode.getInstance()).toString()
      .getBytes(StandardCharsets.UTF_8);
  PubsubMessage message = new PubsubMessage(data, Collections.emptyMap());

  Schema schema = SchemaBuilder.record("root").fields() //
      .name("test").type().booleanType().noDefault() //
      .endRecord();
  formatter.formatRecord(message, schema);
}
 
Example 9
@Test(expectedExceptions = AvroTypeException.class)
public void testParseBadDefaultsFailsOn18() throws Exception {
  AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
  if (runtimeVersion != AvroVersion.AVRO_1_8) {
    throw new SkipException("only supported under " + AvroVersion.AVRO_1_8 + ". runtime version detected as " + runtimeVersion);
  }
  _factory.parse(BAD_DEFAULTS_SCHEMA_JSON, null);
}
 
Example 10
Source Project: pulsar   Source File: StructSchema.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public T decode(byte[] bytes, byte[] schemaVersion) {
    try {
        return schemaVersion == null ? decode(bytes) :
                readerCache.get(BytesSchemaVersion.of(schemaVersion)).read(bytes);
    } catch (ExecutionException | AvroTypeException e) {
        if (e instanceof AvroTypeException) {
            throw new SchemaSerializationException(e);
        }
        LOG.error("Can't get generic schema for topic {} schema version {}",
                schemaInfoProvider.getTopicName(), Hex.encodeHexString(schemaVersion), e);
        throw new RuntimeException("Can't get generic schema for topic " + schemaInfoProvider.getTopicName());
    }
}
 
Example 11
Source Project: parquet-mr   Source File: AvroRecordConverter.java    License: Apache License 2.0 5 votes vote down vote up
private static Map<String, Class<?>> getFieldsByName(Class<?> recordClass,
                                                     boolean excludeJava) {
  Map<String, Class<?>> fields = new LinkedHashMap<String, Class<?>>();

  if (recordClass != null) {
    Class<?> current = recordClass;
    do {
      if (excludeJava && current.getPackage() != null
          && current.getPackage().getName().startsWith("java.")) {
        break; // skip java built-in classes
      }
      for (Field field : current.getDeclaredFields()) {
        if (field.isAnnotationPresent(AvroIgnore.class) ||
            isTransientOrStatic(field)) {
          continue;
        }
        AvroName altName = field.getAnnotation(AvroName.class);
        Class<?> existing = fields.put(
            altName != null ? altName.value() : field.getName(),
            field.getType());
        if (existing != null) {
          throw new AvroTypeException(
              current + " contains two fields named: " + field.getName());
        }
      }
      current = current.getSuperclass();
    } while (current != null);
  }

  return fields;
}
 
Example 12
Source Project: iow-hadoop-streaming   Source File: IOWJsonDecoder.java    License: Apache License 2.0 5 votes vote down vote up
private void checkFixed(int size) throws IOException {
    advance(Symbol.FIXED);
    Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol();
    if (size != top.size) {
        throw new AvroTypeException(
            "Incorrect length for fixed binary: expected " +
        top.size + " but received " + size + " bytes.");
    }
}
 
Example 13
Source Project: iow-hadoop-streaming   Source File: IOWJsonDecoder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readFixed(byte[] bytes, int start, int len) throws IOException {
    checkFixed(len);
    if (in.getCurrentToken() == JsonToken.VALUE_STRING) {
        byte[] result = readByteArray();
        in.nextToken();
        if (result.length != len) {
            throw new AvroTypeException("Expected fixed length " + len
                + ", but got" + result.length);
        }
        System.arraycopy(result, 0, bytes, start, len);
    } else {
        throw error("fixed");
    }
}
 
Example 14
Source Project: iow-hadoop-streaming   Source File: IOWJsonDecoder.java    License: Apache License 2.0 5 votes vote down vote up
private void doSkipFixed(int length) throws IOException {
    if (in.getCurrentToken() == JsonToken.VALUE_STRING) {
        byte[] result = readByteArray();
        in.nextToken();
        if (result.length != length) {
            throw new AvroTypeException("Expected fixed length " + length
                + ", but got" + result.length);
        }
    } else {
        throw error("fixed");
    }
}
 
Example 15
Source Project: data-highway   Source File: SchemaController.java    License: Apache License 2.0 4 votes vote down vote up
@ExceptionHandler(AvroTypeException.class)
public ResponseEntity<StandardResponse> avroTypeExceptionExceptionHandler(HttpServletRequest request, Exception e) {
  return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(
      StandardResponse.failureResponse("Invalid schema. " + e.getMessage()));
}
 
Example 16
Source Project: gcp-ingestion   Source File: GuidedJsonDecoder.java    License: Mozilla Public License 2.0 4 votes vote down vote up
private void error(JsonToken token, String type) throws AvroTypeException {
  throw new AvroTypeException("Expected " + type + ". Got " + token);
}
 
Example 17
Source Project: gcp-ingestion   Source File: GuidedJsonDecoder.java    License: Mozilla Public License 2.0 4 votes vote down vote up
private void assertCurrentToken(JsonToken expect, String type) throws AvroTypeException {
  JsonToken token = in.getCurrentToken();
  if (token != expect) {
    error(token, type);
  }
}
 
Example 18
@Test(expectedExceptions = AvroTypeException.class)
public void demonstrateAvro19ValidatesFieldDefaults() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidFieldDefaultValue.avsc");
  nativeParse(avsc, null, null);
}
 
Example 19
@Test(expectedExceptions = AvroTypeException.class)
public void demonstrateAvro19ValidatesUnionDefaults() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidUnionDefault.avsc");
  nativeParse(avsc, null, null);
}
 
Example 20
@Test(expectedExceptions = AvroTypeException.class)
public void testValidateFieldDefaults() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidFieldDefaultValue.avsc");
  SchemaParseConfiguration conf = new SchemaParseConfiguration(false, true);
  AvroCompatibilityHelper.parse(avsc, conf, null);
}
 
Example 21
@Test(expectedExceptions = AvroTypeException.class)
public void testValidateUnionDefaults() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidUnionDefault.avsc");
  SchemaParseConfiguration conf = new SchemaParseConfiguration(false, true);
  AvroCompatibilityHelper.parse(avsc, conf, null);
}
 
Example 22
@Test(expectedExceptions = AvroTypeException.class)
public void demonstrateAvro18CanValidateFieldDefaults() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidFieldDefaultValue.avsc");
  nativeParse(avsc, null, true);
}
 
Example 23
@Test(expectedExceptions = AvroTypeException.class)
public void demonstrateAvro18CanValidateUnionDefaults() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidUnionDefault.avsc");
  nativeParse(avsc, null, true);
}
 
Example 24
@Test(expectedExceptions = AvroTypeException.class)
public void demonstrateAvro17CanValidateFieldDefaults() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidFieldDefaultValue.avsc");
  nativeParse(avsc, null, true);
}
 
Example 25
@Test(expectedExceptions = AvroTypeException.class)
public void demonstrateAvro17CanValidateUnionDefaults() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidUnionDefault.avsc");
  nativeParse(avsc, null, true);
}
 
Example 26
Source Project: iow-hadoop-streaming   Source File: IOWJsonDecoder.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Symbol doAction(Symbol input, Symbol top) throws IOException {


    if (top instanceof Symbol.FieldAdjustAction) {
        Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction) top;
        String name = fa.fname;
        if (currentReorderBuffer != null) {
            List<JsonElement> node = currentReorderBuffer.savedFields.get(name);
            if (node != null) {
                currentReorderBuffer.savedFields.remove(name);
                currentReorderBuffer.origParser = in;
                in = makeParser(node);
                return null;
            }
        }
        if (in.getCurrentToken() == JsonToken.FIELD_NAME) {
            do {
                String fn = in.getText();
                in.nextToken();
                if (name.equals(fn)) {
                    return null;
                } else {
                    if (currentReorderBuffer == null) {
                        currentReorderBuffer = new ReorderBuffer();
                    }
                    currentReorderBuffer.savedFields.put(fn, getVaueAsTree(in));
                }
            } while (in.getCurrentToken() == JsonToken.FIELD_NAME);
            throw new AvroTypeException("Expected field name not found: " + fa.fname);
        }
    } else if (top == Symbol.FIELD_END) {
        if (currentReorderBuffer != null && currentReorderBuffer.origParser != null) {
            in = currentReorderBuffer.origParser;
            currentReorderBuffer.origParser = null;
        }
    } else if (top == Symbol.RECORD_START) {
        if (in.getCurrentToken() == JsonToken.START_OBJECT) {
            in.nextToken();
            reorderBuffers.push(currentReorderBuffer);
            currentReorderBuffer = null;
        } else {
            throw error("record-start");
        }
    } else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) {
        if (in.getCurrentToken() == JsonToken.END_OBJECT) {
            in.nextToken();
            if (top == Symbol.RECORD_END) {
                if (currentReorderBuffer != null && !currentReorderBuffer.savedFields.isEmpty()) {
                    throw error("Unknown fields: " + currentReorderBuffer.savedFields.keySet());
                }
                currentReorderBuffer = reorderBuffers.pop();
            }
        } else {
            throw error(top == Symbol.RECORD_END ? "record-end" : "union-end");
        }
    } else {
        throw new AvroTypeException("Unknown action symbol " + top);
    }
    return null;
}