Java Code Examples for org.apache.avro.generic.GenericArray#size()

@Override
public int compareTo(GenericArray<T> that) {
  if (isInstanceOfCorrectPrimitiveList(that)) {
    L thatPrimitiveList = (L) that;
    if (this.size == that.size()) {
      for (int i = 0; i < this.size; i++) {
        int compare = compareElementAtIndex(thatPrimitiveList, i);
        if (compare != 0) {
          return compare;
        }
      }
      return 0;
    } else if (this.size > that.size()) {
      return 1;
    } else {
      return -1;
    }
  } else {
    // Not our own type of primitive list, so we will delegate to the regular implementation, which will do boxing
    return GenericData.get().compare(this, that, this.getSchema());
  }
}
 

@Test
@SuppressWarnings("unchecked")
public void testEncryptionOfArray()
    throws SchemaConversionException, DataConversionException, IOException {
  AvroStringFieldEncryptorConverter converter = new AvroStringFieldEncryptorConverter();
  WorkUnitState wuState = new WorkUnitState();

  wuState.getJobState().setProp("converter.fieldsToEncrypt", "favorite_quotes");
  wuState.getJobState().setProp("converter.encrypt.algorithm", "insecure_shift");

  converter.init(wuState);
  GenericRecord inputRecord =
      getRecordFromFile(getClass().getClassLoader().getResource("fieldPickInput_arrays.avro").getPath());
  GenericArray origValues = (GenericArray) inputRecord.get("favorite_quotes");
  for (int i = 0; i < origValues.size(); i++) {
    origValues.set(i, origValues.get(i).toString());
  }

  Schema inputSchema = inputRecord.getSchema();
  Schema outputSchema = converter.convertSchema(inputSchema, wuState);

  Iterable<GenericRecord> recordIt = converter.convertRecord(outputSchema, inputRecord, wuState);
  GenericRecord encryptedRecord = recordIt.iterator().next();

  Assert.assertEquals(outputSchema, inputSchema);

  GenericArray<String> encryptedVals = (GenericArray<String>) encryptedRecord.get("favorite_quotes");
  List<String> decryptedValues = Lists.newArrayList();
  for (String encryptedValue: encryptedVals) {
    InsecureShiftCodec codec = new InsecureShiftCodec(Maps.<String, Object>newHashMap());
    InputStream in =
        codec.decodeInputStream(new ByteArrayInputStream(encryptedValue.getBytes(StandardCharsets.UTF_8)));
    byte[] decryptedValue = new byte[in.available()];
    in.read(decryptedValue);
    decryptedValues.add(new String(decryptedValue, StandardCharsets.UTF_8));
  }

  Assert.assertEquals(decryptedValues, origValues);
}
 

@SuppressWarnings("unchecked")
private List convertToList(String fieldName, GenericArray arr) {
  List ret = new ArrayList();
  for (int i = 0; i < arr.size(); i++) {
    ret.add(convertAvroToJava(fieldName + "." + String.valueOf(i), arr.get(i)));
  }

  return ret;
}
 

/**
 * Convert the projected read from delta record into an array writable.
 */
public static Writable avroToArrayWritable(Object value, Schema schema) {

  if (value == null) {
    return null;
  }

  switch (schema.getType()) {
    case STRING:
      return new Text(value.toString());
    case BYTES:
      return new BytesWritable(((ByteBuffer)value).array());
    case INT:
      return new IntWritable((Integer) value);
    case LONG:
      return new LongWritable((Long) value);
    case FLOAT:
      return new FloatWritable((Float) value);
    case DOUBLE:
      return new DoubleWritable((Double) value);
    case BOOLEAN:
      return new BooleanWritable((Boolean) value);
    case NULL:
      return null;
    case RECORD:
      GenericRecord record = (GenericRecord) value;
      Writable[] recordValues = new Writable[schema.getFields().size()];
      int recordValueIndex = 0;
      for (Schema.Field field : schema.getFields()) {
        recordValues[recordValueIndex++] = avroToArrayWritable(record.get(field.name()), field.schema());
      }
      return new ArrayWritable(Writable.class, recordValues);
    case ENUM:
      return new Text(value.toString());
    case ARRAY:
      GenericArray arrayValue = (GenericArray) value;
      Writable[] arrayValues = new Writable[arrayValue.size()];
      int arrayValueIndex = 0;
      for (Object obj : arrayValue) {
        arrayValues[arrayValueIndex++] = avroToArrayWritable(obj, schema.getElementType());
      }
      // Hive 1.x will fail here, it requires values2 to be wrapped into another ArrayWritable
      return new ArrayWritable(Writable.class, arrayValues);
    case MAP:
      Map mapValue = (Map) value;
      Writable[] mapValues = new Writable[mapValue.size()];
      int mapValueIndex = 0;
      for (Object entry : mapValue.entrySet()) {
        Map.Entry mapEntry = (Map.Entry) entry;
        Writable[] nestedMapValues = new Writable[2];
        nestedMapValues[0] = new Text(mapEntry.getKey().toString());
        nestedMapValues[1] = avroToArrayWritable(mapEntry.getValue(), schema.getValueType());
        mapValues[mapValueIndex++] = new ArrayWritable(Writable.class, nestedMapValues);
      }
      // Hive 1.x will fail here, it requires values3 to be wrapped into another ArrayWritable
      return new ArrayWritable(Writable.class, mapValues);
    case UNION:
      List<Schema> types = schema.getTypes();
      if (types.size() != 2) {
        throw new IllegalArgumentException("Only support union with 2 fields");
      }
      Schema s1 = types.get(0);
      Schema s2 = types.get(1);
      if (s1.getType() == Schema.Type.NULL) {
        return avroToArrayWritable(value, s2);
      } else if (s2.getType() == Schema.Type.NULL) {
        return avroToArrayWritable(value, s1);
      } else {
        throw new IllegalArgumentException("Only support union with null");
      }
    case FIXED:
      if (schema.getLogicalType() != null && schema.getLogicalType().getName().equals("decimal")) {
        LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) LogicalTypes.fromSchema(schema);
        HiveDecimalWritable writable = new HiveDecimalWritable(((GenericFixed) value).bytes(),
            decimal.getScale());
        return HiveDecimalWritable.enforcePrecisionScale(writable,
            decimal.getPrecision(),
            decimal.getScale());
      }
      return new BytesWritable(((GenericFixed) value).bytes());
    default:
      return null;
  }
}