Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: TestOrcReaderMemoryUsage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Write a file that contains a number of rows with 1 BIGINT column, and some rows have null values.
 */
private static TempFile createSingleColumnFileWithNullValues(int rows)
        throws IOException, SerDeException
{
    Serializer serde = new OrcSerde();
    TempFile tempFile = new TempFile();
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(tempFile.getFile(), ORC_12, CompressionKind.NONE, BIGINT);
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", BIGINT);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);

    for (int i = 0; i < rows; i++) {
        if (i % 10 == 0) {
            objectInspector.setStructFieldData(row, field, null);
        }
        else {
            objectInspector.setStructFieldData(row, field, (long) i);
        }

        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }

    writer.close(false);
    return tempFile;
}
 
Example 2
Source Project: presto   Source File: TestOrcReaderMemoryUsage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Write a file that contains a number of rows with 1 VARCHAR column, and all values are not null.
 */
private static TempFile createSingleColumnVarcharFile(int count, int length)
        throws Exception
{
    Serializer serde = new OrcSerde();
    TempFile tempFile = new TempFile();
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(tempFile.getFile(), ORC_12, CompressionKind.NONE, VARCHAR);
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", VARCHAR);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);

    for (int i = 0; i < count; i++) {
        objectInspector.setStructFieldData(row, field, Strings.repeat("0", length));
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }

    writer.close(false);
    return tempFile;
}
 
Example 3
Source Project: presto   Source File: OrcTester.java    License: Apache License 2.0 6 votes vote down vote up
public static DataSize writeOrcFileColumnHive(File outputFile, RecordWriter recordWriter, Type type, Iterator<?> values)
        throws Exception
{
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", type);
    Object row = objectInspector.create();

    List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());
    Serializer serializer = new OrcSerde();

    while (values.hasNext()) {
        Object value = values.next();
        value = preprocessWriteValueHive(type, value);
        objectInspector.setStructFieldData(row, fields.get(0), value);

        Writable record = serializer.serialize(row, objectInspector);
        recordWriter.write(record);
    }

    recordWriter.close(false);
    return succinctBytes(outputFile.length());
}
 
Example 4
Source Project: presto   Source File: TestOrcReaderPositions.java    License: Apache License 2.0 6 votes vote down vote up
private static void createMultiStripeFile(File file)
        throws IOException, ReflectiveOperationException, SerDeException
{
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(file, ORC_12, CompressionKind.NONE, BIGINT);

    Serializer serde = new OrcSerde();
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", BIGINT);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);

    for (int i = 0; i < 300; i += 3) {
        if ((i > 0) && (i % 60 == 0)) {
            flushWriter(writer);
        }

        objectInspector.setStructFieldData(row, field, (long) i);
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }

    writer.close(false);
}
 
Example 5
Source Project: presto   Source File: TestOrcReaderPositions.java    License: Apache License 2.0 6 votes vote down vote up
private static void createSequentialFile(File file, int count)
        throws IOException, SerDeException
{
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(file, ORC_12, CompressionKind.NONE, BIGINT);

    Serializer serde = new OrcSerde();
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", BIGINT);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);

    for (int i = 0; i < count; i++) {
        objectInspector.setStructFieldData(row, field, (long) i);
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }

    writer.close(false);
}
 
Example 6
public StructConverter(ObjectInspector inputOI, SettableStructObjectInspector outputOI) {
  if (inputOI instanceof StructObjectInspector) {
    this.inputOI = (StructObjectInspector) inputOI;
    this.outputOI = outputOI;
    inputFields = this.inputOI.getAllStructFieldRefs();
    outputFields = outputOI.getAllStructFieldRefs();

    // If the output has some extra fields, set them to NULL.
    int minFields = Math.min(inputFields.size(), outputFields.size());
    fieldConverters = new ArrayList<Converter>(minFields);
    for (int f = 0; f < minFields; f++) {
      fieldConverters.add(getConverter(inputFields.get(f).getFieldObjectInspector(),
          outputFields.get(f).getFieldObjectInspector()));
    }
  } else if (!(inputOI instanceof VoidObjectInspector)) {
    throw new UnsupportedOperationException(
        "Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName()
            + "not supported yet.");
  }
}
 
Example 7
private static void createMultiStripeFile(File file)
        throws IOException, ReflectiveOperationException, SerDeException
{
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(file, ORC_12, OrcTester.Compression.NONE, javaLongObjectInspector);

    @SuppressWarnings("deprecation") Serializer serde = new OrcSerde();
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", javaLongObjectInspector);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);

    for (int i = 0; i < 300; i += 3) {
        if ((i > 0) && (i % 60 == 0)) {
            flushWriter(writer);
        }

        objectInspector.setStructFieldData(row, field, (long) i);
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }

    writer.close(false);
}
 
Example 8
private static void createSequentialFile(File file, int count)
        throws IOException, ReflectiveOperationException, SerDeException
{
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(file, ORC_12, OrcTester.Compression.NONE, javaLongObjectInspector);

    @SuppressWarnings("deprecation") Serializer serde = new OrcSerde();
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", javaLongObjectInspector);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);

    for (int i = 0; i < count; i++) {
        objectInspector.setStructFieldData(row, field, (long) i);
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }

    writer.close(false);
}
 
Example 9
Source Project: presto   Source File: TestOrcReaderMemoryUsage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write a file that contains a given number of maps where each row has 10 entries in total
 * and some entries have null keys/values.
 */
private static TempFile createSingleColumnMapFileWithNullValues(Type mapType, int rows)
        throws IOException, SerDeException
{
    Serializer serde = new OrcSerde();
    TempFile tempFile = new TempFile();
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(tempFile.getFile(), ORC_12, CompressionKind.NONE, mapType);
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", mapType);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);

    for (int i = 1; i <= rows; i++) {
        HashMap<Long, Long> map = new HashMap<>();

        for (int j = 1; j <= 8; j++) {
            Long value = (long) j;
            map.put(value, value);
        }

        // Add null values so that the StreamReader nullVectors are not empty.
        map.put(null, 0L);
        map.put(0L, null);

        objectInspector.setStructFieldData(row, field, map);
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }
    writer.close(false);
    return tempFile;
}
 
Example 10
Source Project: presto   Source File: TestOrcReaderPositions.java    License: Apache License 2.0 5 votes vote down vote up
private static void createGrowingSequentialFile(File file, int count, int step, int initialLength)
        throws IOException, SerDeException
{
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(file, ORC_12, CompressionKind.NONE, VARCHAR);

    Serializer serde = new OrcSerde();
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", VARCHAR);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);

    StringBuilder builder = new StringBuilder();
    for (int i = 0; i < initialLength; i++) {
        builder.append("0");
    }
    String seedString = builder.toString();

    // gradually grow the length of a cell
    int previousLength = initialLength;
    for (int i = 0; i < count; i++) {
        if ((i / step + 1) * initialLength > previousLength) {
            previousLength = (i / step + 1) * initialLength;
            builder.append(seedString);
        }
        objectInspector.setStructFieldData(row, field, builder.toString());
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }

    writer.close(false);
}
 
Example 11
Source Project: presto   Source File: RcFileTester.java    License: Apache License 2.0 5 votes vote down vote up
private static DataSize writeRcFileColumnOld(File outputFile, Format format, Compression compression, Type type, Iterator<?> values)
        throws Exception
{
    ObjectInspector columnObjectInspector = getJavaObjectInspector(type);
    RecordWriter recordWriter = createRcFileWriterOld(outputFile, compression, columnObjectInspector);

    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", columnObjectInspector);
    Object row = objectInspector.create();

    List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());
    Serializer serializer = format.createSerializer();

    Properties tableProperties = new Properties();
    tableProperties.setProperty("columns", "test");
    tableProperties.setProperty("columns.types", objectInspector.getTypeName());
    serializer.initialize(new JobConf(false), tableProperties);

    while (values.hasNext()) {
        Object value = values.next();
        value = preprocessWriteValueOld(type, value);
        objectInspector.setStructFieldData(row, fields.get(0), value);

        Writable record = serializer.serialize(row, objectInspector);
        recordWriter.write(record);
    }

    recordWriter.close(false);
    return DataSize.ofBytes(outputFile.length()).succinct();
}
 
Example 12
Source Project: presto   Source File: ParquetTester.java    License: Apache License 2.0 5 votes vote down vote up
private static void writeParquetColumn(
        JobConf jobConf,
        File outputFile,
        CompressionCodecName compressionCodecName,
        Properties tableProperties,
        SettableStructObjectInspector objectInspector,
        Iterator<?>[] valuesByField,
        Optional<MessageType> parquetSchema,
        boolean singleLevelArray)
        throws Exception
{
    RecordWriter recordWriter = new TestMapredParquetOutputFormat(parquetSchema, singleLevelArray)
            .getHiveRecordWriter(
                    jobConf,
                    new Path(outputFile.toURI()),
                    Text.class,
                    compressionCodecName != UNCOMPRESSED,
                    tableProperties,
                    () -> {});
    Object row = objectInspector.create();
    List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());
    while (stream(valuesByField).allMatch(Iterator::hasNext)) {
        for (int field = 0; field < fields.size(); field++) {
            Object value = valuesByField[field].next();
            objectInspector.setStructFieldData(row, fields.get(field), value);
        }
        ParquetHiveSerDe serde = new ParquetHiveSerDe();
        serde.initialize(jobConf, tableProperties, null);
        Writable record = serde.serialize(row, objectInspector);
        recordWriter.write(record);
    }
    recordWriter.close(false);
}
 
Example 13
Source Project: transport   Source File: HiveStruct.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public void setField(int index, StdData value) {
  if (_structObjectInspector instanceof SettableStructObjectInspector) {
    StructField field = _structObjectInspector.getAllStructFieldRefs().get(index);
    ((SettableStructObjectInspector) _structObjectInspector).setStructFieldData(_object,
        field, ((HiveData) value).getUnderlyingDataForObjectInspector(field.getFieldObjectInspector())
    );
    _isObjectModified = true;
  } else {
    throw new RuntimeException("Attempt to modify an immutable Hive object of type: "
        + _structObjectInspector.getClass());
  }
}
 
Example 14
Source Project: transport   Source File: HiveStruct.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public void setField(String name, StdData value) {
  if (_structObjectInspector instanceof SettableStructObjectInspector) {
    StructField field = _structObjectInspector.getStructFieldRef(name);
    ((SettableStructObjectInspector) _structObjectInspector).setStructFieldData(_object,
        field, ((HiveData) value).getUnderlyingDataForObjectInspector(field.getFieldObjectInspector()));
    _isObjectModified = true;
  } else {
    throw new RuntimeException("Attempt to modify an immutable Hive object of type: "
        + _structObjectInspector.getClass());
  }
}
 
Example 15
/**
 * Returns a converter that converts objects from one OI to another OI. The
 * returned (converted) object does not belong to the converter. Hence once convertor can be used
 * multiple times within one eval invocation.
 */
public Converter getConverter(ObjectInspector inputOI, ObjectInspector outputOI) {
  // If the inputOI is the same as the outputOI, just return an
  // IdentityConverter.
  if (inputOI.equals(outputOI)) {
    return new ObjectInspectorConverters.IdentityConverter();
  }
  Converter c = getConverterFromCache(inputOI, outputOI);
  if (c != null) {
    return c;
  }
  switch (outputOI.getCategory()) {
    case PRIMITIVE:
      return getConverter((PrimitiveObjectInspector) inputOI, (PrimitiveObjectInspector) outputOI);
    case STRUCT:
      c = new StructConverter(inputOI, (SettableStructObjectInspector) outputOI);
      break;
    case LIST:
      c = new ListConverter(inputOI, (SettableListObjectInspector) outputOI);
      break;
    case MAP:
      c = new MapConverter(inputOI, (SettableMapObjectInspector) outputOI);
      break;
    default:
      throw new UnsupportedOperationException(
          "Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName()
              + " not supported yet.");
  }
  cacheConverter(inputOI, outputOI, c);
  return c;
}
 
Example 16
Source Project: localization_nifi   Source File: NiFiOrcUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create an object of OrcStruct given a TypeInfo and a list of objects
 *
 * @param typeInfo The TypeInfo object representing the ORC record schema
 * @param objs     ORC objects/Writables
 * @return an OrcStruct containing the specified objects for the specified schema
 */
public static OrcStruct createOrcStruct(TypeInfo typeInfo, Object... objs) {
    SettableStructObjectInspector oi = (SettableStructObjectInspector) OrcStruct
            .createObjectInspector(typeInfo);
    List<StructField> fields = (List<StructField>) oi.getAllStructFieldRefs();
    OrcStruct result = (OrcStruct) oi.create();
    result.setNumFields(fields.size());
    for (int i = 0; i < fields.size(); i++) {
        oi.setStructFieldData(result, fields.get(i), objs[i]);
    }
    return result;
}
 
Example 17
Source Project: nifi   Source File: NiFiOrcUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create an object of OrcStruct given a TypeInfo and a list of objects
 *
 * @param typeInfo The TypeInfo object representing the ORC record schema
 * @param objs     ORC objects/Writables
 * @return an OrcStruct containing the specified objects for the specified schema
 */
@SuppressWarnings("unchecked")
public static OrcStruct createOrcStruct(TypeInfo typeInfo, Object... objs) {
    SettableStructObjectInspector oi = (SettableStructObjectInspector) OrcStruct
            .createObjectInspector(typeInfo);
    List<StructField> fields = (List<StructField>) oi.getAllStructFieldRefs();
    OrcStruct result = (OrcStruct) oi.create();
    result.setNumFields(fields.size());
    for (int i = 0; i < fields.size(); i++) {
        oi.setStructFieldData(result, fields.get(i), objs[i]);
    }
    return result;
}
 
Example 18
Source Project: nifi   Source File: NiFiOrcUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create an object of OrcStruct given a TypeInfo and a list of objects
 *
 * @param typeInfo The TypeInfo object representing the ORC record schema
 * @param objs     ORC objects/Writables
 * @return an OrcStruct containing the specified objects for the specified schema
 */
public static OrcStruct createOrcStruct(TypeInfo typeInfo, Object... objs) {
    SettableStructObjectInspector oi = (SettableStructObjectInspector) OrcStruct
            .createObjectInspector(typeInfo);
    List<StructField> fields = (List<StructField>) oi.getAllStructFieldRefs();
    OrcStruct result = (OrcStruct) oi.create();
    result.setNumFields(fields.size());
    for (int i = 0; i < fields.size(); i++) {
        oi.setStructFieldData(result, fields.get(i), objs[i]);
    }
    return result;
}
 
Example 19
Source Project: presto   Source File: OrcTester.java    License: Apache License 2.0 4 votes vote down vote up
static SettableStructObjectInspector createSettableStructObjectInspector(String name, Type type)
{
    return getStandardStructObjectInspector(ImmutableList.of(name), ImmutableList.of(getJavaObjectInspector(type)));
}
 
Example 20
Source Project: presto   Source File: RcFileTester.java    License: Apache License 2.0 4 votes vote down vote up
private static SettableStructObjectInspector createSettableStructObjectInspector(String name, ObjectInspector objectInspector)
{
    return getStandardStructObjectInspector(ImmutableList.of(name), ImmutableList.of(objectInspector));
}
 
Example 21
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static FieldSetter createFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type type)
{
    if (type.equals(BooleanType.BOOLEAN)) {
        return new BooleanFieldSetter(rowInspector, row, field);
    }

    if (type.equals(BigintType.BIGINT)) {
        return new BigintFieldBuilder(rowInspector, row, field);
    }

    if (type.equals(IntegerType.INTEGER)) {
        return new IntFieldSetter(rowInspector, row, field);
    }

    if (type.equals(SmallintType.SMALLINT)) {
        return new SmallintFieldSetter(rowInspector, row, field);
    }

    if (type.equals(TinyintType.TINYINT)) {
        return new TinyintFieldSetter(rowInspector, row, field);
    }

    if (type.equals(RealType.REAL)) {
        return new FloatFieldSetter(rowInspector, row, field);
    }

    if (type.equals(DoubleType.DOUBLE)) {
        return new DoubleFieldSetter(rowInspector, row, field);
    }

    if (type instanceof VarcharType) {
        return new VarcharFieldSetter(rowInspector, row, field, type);
    }

    if (type instanceof CharType) {
        return new CharFieldSetter(rowInspector, row, field, type);
    }

    if (type.equals(VarbinaryType.VARBINARY)) {
        return new BinaryFieldSetter(rowInspector, row, field);
    }

    if (type.equals(DateType.DATE)) {
        return new DateFieldSetter(rowInspector, row, field);
    }

    if (type.equals(TimestampType.TIMESTAMP)) {
        return new TimestampFieldSetter(rowInspector, row, field);
    }

    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return new DecimalFieldSetter(rowInspector, row, field, decimalType);
    }

    if (isArrayType(type)) {
        return new ArrayFieldSetter(rowInspector, row, field, type.getTypeParameters().get(0));
    }

    if (isMapType(type)) {
        return new MapFieldSetter(rowInspector, row, field, type.getTypeParameters().get(0), type.getTypeParameters().get(1));
    }

    if (isRowType(type)) {
        return new RowFieldSetter(rowInspector, row, field, type.getTypeParameters());
    }

    throw new IllegalArgumentException("unsupported type: " + type);
}
 
Example 22
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
protected FieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
    this.rowInspector = requireNonNull(rowInspector, "rowInspector is null");
    this.row = requireNonNull(row, "row is null");
    this.field = requireNonNull(field, "field is null");
}
 
Example 23
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public BooleanFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
    super(rowInspector, row, field);
}
 
Example 24
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public BigintFieldBuilder(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
    super(rowInspector, row, field);
}
 
Example 25
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public IntFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
    super(rowInspector, row, field);
}
 
Example 26
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public SmallintFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
    super(rowInspector, row, field);
}
 
Example 27
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public TinyintFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
    super(rowInspector, row, field);
}
 
Example 28
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public DoubleFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
    super(rowInspector, row, field);
}
 
Example 29
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public FloatFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
    super(rowInspector, row, field);
}
 
Example 30
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public VarcharFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type type)
{
    super(rowInspector, row, field);
    this.type = type;
}