Java Code Examples for org.apache.avro.Schema#getFixedSize()

The following examples show how to use org.apache.avro.Schema#getFixedSize() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hudi   File: MercifulJsonConverter.java    License: Apache License 2.0 6 votes vote down vote up 
private static JsonToAvroFieldProcessor generateFixedTypeHandler() {
  return new JsonToAvroFieldProcessor() {
    @Override
    public Pair<Boolean, Object> convert(Object value, String name, Schema schema) {
      // The ObjectMapper use List to represent FixedType
      // eg: "decimal_val": [0, 0, 14, -63, -52] will convert to ArrayList<Integer>
      List<Integer> converval = (List<Integer>) value;
      byte[] src = new byte[converval.size()];
      for (int i = 0; i < converval.size(); i++) {
        src[i] = converval.get(i).byteValue();
      }
      byte[] dst = new byte[schema.getFixedSize()];
      System.arraycopy(src, 0, dst, 0, Math.min(schema.getFixedSize(), src.length));
      return Pair.of(true, new GenericData.Fixed(schema, dst));
    }
  };
}
Example 2
Source Project: registry   File: AvroSchemaValidator.java    License: Apache License 2.0 5 votes vote down vote up 
private SchemaCompatibilityResult checkFixedSize(final Schema reader,
                                                 final Schema writer,
                                                 final Stack<String> location) {
    location.push("size");
    int actual = reader.getFixedSize();
    int expected = writer.getFixedSize();
    if (actual != expected) {
        String message = String.format("expected: %d, found: %d", expected, actual);
        return SchemaCompatibilityResult.incompatible(
                SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, reader,
                writer, message, location);
    }
    location.pop();
    return SchemaCompatibilityResult.compatible();
}
Example 3
Source Project: ksql-fork-with-deep-learning-function   File: Generator.java    License: Apache License 2.0 4 votes vote down vote up 
private GenericFixed generateFixed(Schema schema) {
  byte[] bytes = new byte[schema.getFixedSize()];
  random.nextBytes(bytes);
  return new GenericData.Fixed(schema, bytes);
}
Example 4
Source Project: iceberg   File: ValueReaders.java    License: Apache License 2.0 4 votes vote down vote up 
private GenericFixedReader(Schema schema) {
  this.schema = schema;
  this.length = schema.getFixedSize();
}
Example 5
Source Project: avro-util   File: AvroRandomDataGenerator.java    License: BSD 2-Clause "Simplified" License 4 votes vote down vote up 
private GenericFixed generateFixed(Schema schema) {
  byte[] bytes = new byte[schema.getFixedSize()];
  random.nextBytes(bytes);
  return AvroCompatibilityHelper.newFixedField(schema, bytes);
}
Example 6
Source Project: avro-util   File: AvroAdapter.java    License: BSD 2-Clause "Simplified" License 4 votes vote down vote up 
default GenericData.Fixed newFixedField(Schema ofType) {
  byte[] emptyDataArray = new byte[ofType.getFixedSize()]; //null is probably unsafe
  return newFixedField(ofType, emptyDataArray);
}
Example 7
Source Project: iceberg   File: ValueReaders.java    License: Apache License 2.0 4 votes vote down vote up 
private GenericFixedReader(Schema schema) {
  this.schema = schema;
  this.length = schema.getFixedSize();
}
Example 8
Source Project: avro-random-generator   File: Generator.java    License: Do What The F*ck You Want To Public License 4 votes vote down vote up 
private GenericFixed generateFixed(Schema schema) {
  byte[] bytes = new byte[schema.getFixedSize()];
  random.nextBytes(bytes);
  return new GenericData.Fixed(schema, bytes);
}
Example 9
Source Project: components   File: GeneratorFunctions.java    License: Apache License 2.0 4 votes vote down vote up 
public FixedBytesGenerator(Schema fixedSchema) {
    size = fixedSchema.getFixedSize();
    jsonSchema = fixedSchema.toString();
    this.schema = fixedSchema;
}
Example 10
Source Project: Cubert   File: PigAvroDatumWriter.java    License: Apache License 2.0 4 votes vote down vote up 
/**
 * Recursively check whether "datum" is an instance of "schema" and called 
 * by {@link #resolveUnionSchema(Schema,Object)},
 * {@link #unwrappedInstanceOf(Schema,Object)}. 
 * 
 */
protected boolean instanceOf(Schema schema, Object datum)
                                throws IOException {

    try {
        switch (schema.getType()) {
        case RECORD:
            if (datum instanceof Tuple) {
                Tuple tuple = (Tuple) datum;
                List<Field> fields = schema.getFields();
                if (fields.size() != tuple.size()) {
                    return false;
                }
                for (int i = 0; i < fields.size(); i++) {
                    if (!instanceOf(fields.get(i).schema(), tuple.get(i)))
                        return false;
                }
                return true;
            }
            return false;

        case UNION:
            @SuppressWarnings("unused")
            int index = resolveUnionSchema(schema, datum);
            return true;
        case ENUM:
            return datum instanceof String && schema.hasEnumSymbol(((String) datum))
                        || unwrappedInstanceOf(schema, datum);
        case ARRAY:
            return datum instanceof DataBag
                        ||  unwrappedInstanceOf(schema, datum);
        case MAP:
            return datum instanceof Map
                        || unwrappedInstanceOf(schema, datum);
        case FIXED:
            return datum instanceof DataByteArray && ((DataByteArray) datum).size() == schema.getFixedSize()
                        || unwrappedInstanceOf(schema, datum);
        case STRING:
            return datum instanceof String
                        || unwrappedInstanceOf(schema, datum);
        case BYTES:
            return datum instanceof DataByteArray
                        || unwrappedInstanceOf(schema, datum);
        case INT:
            return datum instanceof Integer
                        || unwrappedInstanceOf(schema, datum);
        case LONG:
            return datum instanceof Long
                        || datum instanceof Integer
                        || unwrappedInstanceOf(schema, datum);
        case FLOAT:
            return datum instanceof Float
                        || datum instanceof Integer
                        || datum instanceof Long
                        || unwrappedInstanceOf(schema, datum);
        case DOUBLE:
            return datum instanceof Double
                        || datum instanceof Float
                        || datum instanceof Integer
                        || datum instanceof Long
                        || unwrappedInstanceOf(schema, datum);
        case BOOLEAN:
            return datum instanceof Boolean
                        || datum instanceof Integer
                        || unwrappedInstanceOf(schema, datum);
        case NULL:
            return datum == null;
        default:
            throw new RuntimeException("Unexpected type: " + schema);
        }
    } catch (ExecException e) {
        e.printStackTrace(System.err);
        throw new RuntimeException(e);
    }
}
Example 11
Source Project: incubator-gobblin   File: AvroSchemaCheckDefaultStrategy.java    License: Apache License 2.0 4 votes vote down vote up 
/**
 * This method will compare the name and types of the two schema
 * @param expected The expected schema
 * @param toValidate The real schema
 * @return true when expected schema and toValidate schema have matching field names and types
 */
public boolean compare(Schema expected, Schema toValidate)
{
  if (toValidate.getType() != expected.getType() || !toValidate.getName().equals(expected.getName())) {return false;}
  else {
    switch (toValidate.getType()) {
      case NULL:
      case BOOLEAN:
      case INT:
      case LONG:
      case FLOAT:
      case DOUBLE:
      case BYTES:
      case STRING: {
        return true;
      }
      case ARRAY: {
        return compare(toValidate.getElementType(), expected.getElementType());
      }
      case MAP: {
        return compare(toValidate.getValueType(), expected.getValueType());
      }
      case FIXED: {
        // fixed size and name must match:
        if (toValidate.getFixedSize() != expected.getFixedSize()) {
          return false;
        }
        return true;
      }
      case ENUM: {
        // expected symbols must contain all toValidate symbols:
        final Set<String> expectedSymbols = new HashSet<>(expected.getEnumSymbols());
        final Set<String> toValidateSymbols = new HashSet<String>(toValidate.getEnumSymbols());
        if (expectedSymbols.size() != toValidateSymbols.size()) {
          return false;
        }
        if (!expectedSymbols.containsAll(toValidateSymbols)) {
          return false;
        }
        return true;
      }

      case RECORD: {
        // Check that each field of toValidate schema is in expected schema
        if (toValidate.getFields().size() != expected.getFields().size()) {
          return false;
        }
        for (final Schema.Field expectedFiled : expected.getFields()) {
          final Schema.Field toValidateField = toValidate.getField(expectedFiled.name());
          if (toValidateField == null) {
            // expected field does not correspond to any field in the toValidate record schema
            return false;
          } else {
            if (!compare(toValidateField.schema(), expectedFiled.schema())) {
              return false;
            }
          }
        }
        return true;
      }
      case UNION: {
        // Check existing schema contains all the type in toValidate schema
        if (toValidate.getTypes().size() != expected.getTypes().size()) {
          return false;
        }
        HashSet<Schema> types = new HashSet<Schema>(expected.getTypes());
        for (Schema toValidateType : toValidate.getTypes()) {
          Schema equalSchema = null;
          for (Schema type : types) {
            if (compare(type, toValidateType)) {
              equalSchema = type;
              break;
            }
          }
          if (equalSchema == null) {
            return false;
          }
          types.remove(equalSchema);
        }
        return true;
      }
      default: {
        throw new AvroRuntimeException("Unknown schema type: " + toValidate.getType());
      }
    }
  }
}
Example 12
Source Project: spork   File: PigAvroDatumWriter.java    License: Apache License 2.0 4 votes vote down vote up 
/**
 * Recursively check whether "datum" is an instance of "schema" and called 
 * by {@link #resolveUnionSchema(Schema,Object)},
 * {@link #unwrappedInstanceOf(Schema,Object)}. 
 * 
 */
protected boolean instanceOf(Schema schema, Object datum)
                                throws IOException {

    try {
        switch (schema.getType()) {
        case RECORD:
            if (datum instanceof Tuple) {
                Tuple tuple = (Tuple) datum;
                List<Field> fields = schema.getFields();
                if (fields.size() != tuple.size()) {
                    return false;
                }
                for (int i = 0; i < fields.size(); i++) {
                    if (!instanceOf(fields.get(i).schema(), tuple.get(i)))
                        return false;
                }
                return true;
            }
            return false;

        case UNION:
            @SuppressWarnings("unused")
            int index = resolveUnionSchema(schema, datum);
            return true;
        case ENUM:
            return datum instanceof String && schema.hasEnumSymbol(((String) datum))
                        || unwrappedInstanceOf(schema, datum);
        case ARRAY:
            return datum instanceof DataBag
                        ||  unwrappedInstanceOf(schema, datum);
        case MAP:
            return datum instanceof Map
                        || unwrappedInstanceOf(schema, datum);
        case FIXED:
            return datum instanceof DataByteArray && ((DataByteArray) datum).size() == schema.getFixedSize()
                        || unwrappedInstanceOf(schema, datum);
        case STRING:
            return datum instanceof String
                        || unwrappedInstanceOf(schema, datum);
        case BYTES:
            return datum instanceof DataByteArray
                        || unwrappedInstanceOf(schema, datum);
        case INT:
            return datum instanceof Integer
                        || unwrappedInstanceOf(schema, datum);
        case LONG:
            return datum instanceof Long
                        || datum instanceof Integer
                        || unwrappedInstanceOf(schema, datum);
        case FLOAT:
            return datum instanceof Float
                        || datum instanceof Integer
                        || datum instanceof Long
                        || unwrappedInstanceOf(schema, datum);
        case DOUBLE:
            return datum instanceof Double
                        || datum instanceof Float
                        || datum instanceof Integer
                        || datum instanceof Long
                        || unwrappedInstanceOf(schema, datum);
        case BOOLEAN:
            return datum instanceof Boolean
                        || datum instanceof Integer
                        || unwrappedInstanceOf(schema, datum);
        case NULL:
            return datum == null;
        default:
            throw new RuntimeException("Unexpected type: " + schema);
        }
    } catch (ExecException e) {
        e.printStackTrace(System.err);
        throw new RuntimeException(e);
    }
}