org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector Java Examples

/**
 *
 * Initialize HiveUDF and create object inspectors. It requires that the argument length is = 1 and that the ObjectInspector of arguments[0] is of type org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector
 *
 * @param arguments array of length 1 containing one WritableBinaryObjectInspector
 *
 * @return ObjectInspector that is able to parse the result of the evaluate method of the UDF (List Object Inspector for Strings)
 *
 * @throws org.apache.hadoop.hive.ql.exec.UDFArgumentException in case the first argument is not of WritableBinaryObjectInspector
 * @throws org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException in case the number of arguments is != 1
 *
 */
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
	if (arguments==null) {
      		throw new UDFArgumentLengthException("namecoinExtractField only takes one argument: Binary ");
	}
  	if (arguments.length != 1) {
      		throw new UDFArgumentLengthException("namecoinExtractField only takes one argument: Binary ");
	}
	if (!(arguments[0] instanceof BinaryObjectInspector)) { 
		throw new UDFArgumentException("first argument must be a Binary containing a Namecoin script");
	}
	// these are only used for bitcointransaction structs exported to other formats, such as ORC
	this.wboi = (BinaryObjectInspector)arguments[0];
	// the UDF returns the hash value of a BitcoinTransaction as byte array
	return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
 

/**
 * Tests that after copying a lazy binary object, calling materialize on the original and the
 * copy doesn't advance the tree reader twice
 * @throws Exception
 */
@Test
public void TestCopyBinary() throws Exception {
  ReaderWriterProfiler.setProfilerOptions(null);
  OrcLazyBinary lazyBinary = new OrcLazyBinary(new LazyBinaryTreeReader(0, 0) {
    int nextCalls = 0;

    @Override
    public Object next(Object previous) throws IOException {
      if (nextCalls == 0) {
        nextCalls++;
        return new BytesWritable("a".getBytes());
      }

      throw new IOException("next should only be called once");
    }

    @Override
    protected boolean seekToRow(long currentRow) throws IOException {
      return true;
    }
  });

  BinaryObjectInspector binaryOI = (BinaryObjectInspector)
      OrcLazyObjectInspectorUtils.createLazyObjectInspector(TypeInfoFactory.binaryTypeInfo);

  OrcLazyBinary lazyBinary2 = (OrcLazyBinary) binaryOI.copyObject(lazyBinary);

  Assert.assertEquals("a", new String(((BytesWritable) lazyBinary.materialize()).getBytes()));
  Assert.assertEquals("a", new String(((BytesWritable) lazyBinary2.materialize()).getBytes()));
}
 

@Override
void write(Object obj) throws IOException {
  long rawDataSize = 0;
  if (obj != null) {
    BytesWritable val =
        ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
    stream.write(val.getBytes(), 0, val.getLength());
    length.write(val.getLength());

    // Raw data size is the length of the BytesWritable, i.e. the number of bytes
    rawDataSize = val.getLength();
  }
  super.write(obj, rawDataSize);
}
 

@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        BytesWritable val =
                ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
        stream.write(val.getBytes(), 0, val.getLength());
        length.write(val.getLength());
        indexStatistics.updateBinary(val);
        if (createBloomFilter) {
            bloomFilter.addBytes(val.getBytes(), val.getLength());
        }
    }
}
 

@Nonnull
public static BinaryObjectInspector asBinaryOI(@Nonnull final ObjectInspector argOI)
        throws UDFArgumentException {
    if (!BINARY_TYPE_NAME.equals(argOI.getTypeName())) {
        throw new UDFArgumentException("Argument type must be Binary: " + argOI.getTypeName());
    }
    return (BinaryObjectInspector) argOI;
}
 

@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        BytesWritable val =
                ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
        stream.write(val.getBytes(), 0, val.getLength());
        length.write(val.getLength());
        indexStatistics.updateBinary(val);
        if (createBloomFilter) {
            bloomFilter.addBytes(val.getBytes(), val.getLength());
        }
    }
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final byte[] value = ((BinaryObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
  checkSizeLimit(value.length);
  ((VarBinaryVector) outputVV).setSafe(outputIndex, value, 0, value.length);
}
 

public static ByteBuffer getByteBuffer(Object data, ObjectInspector objectInspector) {
  BytesWritable bw = ((BinaryObjectInspector) objectInspector).getPrimitiveWritableObject(data);
  byte[] result = new byte[bw.getLength()];
  System.arraycopy(bw.getBytes(), 0, result, 0, bw.getLength());
  return ByteBuffer.wrap(result);
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final byte[] value = ((BinaryObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
  checkSizeLimit(value.length);
  ((VarBinaryVector) outputVV).setSafe(outputIndex, value, 0, value.length);
}
 

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
  if (arguments[0] == null || arguments[0].get() == null) {
    return null;
  }

  Object input = arguments[0].get();
  switch(inputType) {
    case BOOLEAN:
      return ((BooleanObjectInspector)argumentOI).get(input) ? Boolean.TRUE : Boolean.FALSE;
    case BYTE:
      return new Byte(((ByteObjectInspector)argumentOI).get(input));
    case SHORT:
      return new Short(((ShortObjectInspector)argumentOI).get(input));
    case INT:
      return new Integer(((IntObjectInspector)argumentOI).get(input));
    case LONG:
      return new Long(((LongObjectInspector)argumentOI).get(input));
    case FLOAT:
      return new Float(((FloatObjectInspector)argumentOI).get(input));
    case DOUBLE:
      return new Double(((DoubleObjectInspector)argumentOI).get(input));
    case STRING:
      return PrimitiveObjectInspectorUtils.getString(input, (StringObjectInspector)argumentOI);
    case BINARY:
      return PrimitiveObjectInspectorUtils.getBinary(input, (BinaryObjectInspector) argumentOI).getBytes();
    case VARCHAR:
      if (outputType == PrimitiveCategory.CHAR) {
        HiveVarchar hiveVarchar = PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector) argumentOI);
        return new HiveChar(hiveVarchar.getValue(), HiveChar.MAX_CHAR_LENGTH);
      } else {
        return PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector)argumentOI);
      }
    case CHAR:
      return PrimitiveObjectInspectorUtils.getHiveChar(input, (HiveCharObjectInspector) argumentOI);
    case DATE:
      return PrimitiveObjectInspectorUtils.getDate(input, (DateObjectInspector) argumentOI);
    case TIMESTAMP:
      return PrimitiveObjectInspectorUtils.getTimestamp(input, (TimestampObjectInspector) argumentOI);
    case DECIMAL:
      // return type is a HiveVarchar
      HiveDecimal decimalValue =
          PrimitiveObjectInspectorUtils.getHiveDecimal(input, (HiveDecimalObjectInspector) argumentOI);
      return new HiveVarchar(decimalValue.toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
  }

  throw new UnsupportedOperationException(String.format("Unexpected input type '%s' in Test UDF", inputType));
}
 

private static void serializePrimitive(Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector)
{
    requireNonNull(builder, "parent builder is null");

    if (object == null) {
        builder.appendNull();
        return;
    }

    switch (inspector.getPrimitiveCategory()) {
        case BOOLEAN:
            BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
            return;
        case BYTE:
            TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
            return;
        case SHORT:
            SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
            return;
        case INT:
            IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
            return;
        case LONG:
            BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
            return;
        case FLOAT:
            RealType.REAL.writeLong(builder, floatToRawIntBits(((FloatObjectInspector) inspector).get(object)));
            return;
        case DOUBLE:
            DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
            return;
        case STRING:
            type.writeSlice(builder, Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
            return;
        case VARCHAR:
            type.writeSlice(builder, Slices.utf8Slice(((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(object).getValue()));
            return;
        case CHAR:
            CharType charType = (CharType) type;
            HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
            type.writeSlice(builder, truncateToLengthAndTrimSpaces(Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
            return;
        case DATE:
            DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
            return;
        case TIMESTAMP:
            TimestampType.TIMESTAMP.writeLong(builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
            return;
        case BINARY:
            VARBINARY.writeSlice(builder, Slices.wrappedBuffer(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
            return;
        case DECIMAL:
            DecimalType decimalType = (DecimalType) type;
            HiveDecimalWritable hiveDecimal = ((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
            if (decimalType.isShort()) {
                decimalType.writeLong(builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
            }
            else {
                decimalType.writeSlice(builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
            }
            return;
    }
    throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
}
 

@Override
public Object convert(ObjectInspector objectInspector, Object o, TypeInfo odpsTypeInfo) {
  BinaryObjectInspector binaryObjectInspector = (BinaryObjectInspector) objectInspector;
  return new Binary(binaryObjectInspector.getPrimitiveJavaObject(o));
}
 

private void resolvePrimitive(Object o, PrimitiveObjectInspector oi) throws IOException {

        if (!firstColumn) {
            builder.append(delimiter);
        }

        if (o == null) {
            builder.append(nullChar);
        } else {
            switch (oi.getPrimitiveCategory()) {
                case BOOLEAN:
                    builder.append(((BooleanObjectInspector) oi).get(o));
                    break;
                case SHORT:
                    builder.append(((ShortObjectInspector) oi).get(o));
                    break;
                case INT:
                    builder.append(((IntObjectInspector) oi).get(o));
                    break;
                case LONG:
                    builder.append(((LongObjectInspector) oi).get(o));
                    break;
                case FLOAT:
                    builder.append(((FloatObjectInspector) oi).get(o));
                    break;
                case DOUBLE:
                    builder.append(((DoubleObjectInspector) oi).get(o));
                    break;
                case DECIMAL:
                    builder.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o).bigDecimalValue());
                    break;
                case STRING:
                    builder.append(((StringObjectInspector) oi).getPrimitiveJavaObject(o));
                    break;
                case BINARY:
                    byte[] bytes = ((BinaryObjectInspector) oi).getPrimitiveJavaObject(o);
                    Utilities.byteArrayToOctalString(bytes, builder);
                    break;
                case TIMESTAMP:
                    builder.append(((TimestampObjectInspector) oi).getPrimitiveJavaObject(o));
                    break;
                case BYTE:  /* TINYINT */
                    builder.append(Short.valueOf(((ByteObjectInspector) oi).get(o)));
                    break;
                default:
                    throw new UnsupportedTypeException(oi.getTypeName()
                            + " conversion is not supported by HiveColumnarSerdeResolver");
            }
        }
        firstColumn = false;
    }
 

public HiveBinaryType(BinaryObjectInspector binaryObjectInspector) {
  _binaryObjectInspector = binaryObjectInspector;
}
 

public Text convert(Object input) {
  if (input == null) {
    return null;
  }
  Text t = new Text();

  switch (inputOI.getPrimitiveCategory()) {
    case VOID:
      return null;
    case BOOLEAN:
      t.set(((BooleanObjectInspector) inputOI).get(input) ? trueBytes
          : falseBytes);
      return t;
    case BYTE:
      out.reset();
      LazyInteger.writeUTF8NoException(out, ((ByteObjectInspector) inputOI).get(input));
      t.set(out.getData(), 0, out.getLength());
      return t;
    case SHORT:
      out.reset();
      LazyInteger.writeUTF8NoException(out, ((ShortObjectInspector) inputOI).get(input));
      t.set(out.getData(), 0, out.getLength());
      return t;
    case INT:
      out.reset();
      LazyInteger.writeUTF8NoException(out, ((IntObjectInspector) inputOI).get(input));
      t.set(out.getData(), 0, out.getLength());
      return t;
    case LONG:
      out.reset();
      LazyLong.writeUTF8NoException(out, ((LongObjectInspector) inputOI).get(input));
      t.set(out.getData(), 0, out.getLength());
      return t;
    case FLOAT:
      t.set(String.valueOf(((FloatObjectInspector) inputOI).get(input)));
      return t;
    case DOUBLE:
      t.set(String.valueOf(((DoubleObjectInspector) inputOI).get(input)));
      return t;
    case STRING:
      if (inputOI.preferWritable()) {
        t.set(((StringObjectInspector) inputOI).getPrimitiveWritableObject(input));
      } else {
        t.set(((StringObjectInspector) inputOI).getPrimitiveJavaObject(input));
      }
      return t;
    case CHAR:
      // when converting from char, the value should be stripped of any trailing spaces.
      if (inputOI.preferWritable()) {
        // char text value is already stripped of trailing space
        t.set(((HiveCharObjectInspector) inputOI).getPrimitiveWritableObject(input)
            .getStrippedValue());
      } else {
        t.set(((HiveCharObjectInspector) inputOI).getPrimitiveJavaObject(input).getStrippedValue());
      }
      return t;
    case VARCHAR:
      if (inputOI.preferWritable()) {
        t.set(((HiveVarcharObjectInspector) inputOI).getPrimitiveWritableObject(input)
            .toString());
      } else {
        t.set(((HiveVarcharObjectInspector) inputOI).getPrimitiveJavaObject(input).toString());
      }
      return t;
    case DATE:
      t.set(((DateObjectInspector) inputOI).getPrimitiveWritableObject(input).toString());
      return t;
    case TIMESTAMP:
      t.set(((TimestampObjectInspector) inputOI)
          .getPrimitiveWritableObject(input).toString());
      return t;
    case BINARY:
      BinaryObjectInspector binaryOI = (BinaryObjectInspector) inputOI;
      if (binaryOI.preferWritable()) {
        BytesWritable bytes = binaryOI.getPrimitiveWritableObject(input);
        t.set(bytes.getBytes(), 0, bytes.getLength());
      } else {
        t.set(binaryOI.getPrimitiveJavaObject(input));
      }
      return t;
    case DECIMAL:
      t.set(((HiveDecimalObjectInspector) inputOI).getPrimitiveWritableObject(input).toString());
      return t;
    default:
      throw new RuntimeException("Hive 2 Internal error: type = " + inputOI.getTypeName());
  }
}
 

public HiveBinary(Object object, BinaryObjectInspector binaryObjectInspector, StdFactory stdFactory) {
  super(stdFactory);
  _object = object;
  _binaryObjectInspector = binaryObjectInspector;
}
 

@Override
protected boolean isBinaryType(ObjectInspector dataType) {
  return dataType instanceof BinaryObjectInspector;
}
 

/**
 * It writes the primitive value to the Parquet RecordConsumer.
 *
 * @param value The object that contains the primitive value.
 * @param inspector The object inspector used to get the correct value type.
 */
private void writePrimitive(Object value, PrimitiveObjectInspector inspector)
{
    if (value == null) {
        return;
    }

    switch (inspector.getPrimitiveCategory()) {
        case VOID:
            return;
        case DOUBLE:
            recordConsumer.addDouble(((DoubleObjectInspector) inspector).get(value));
            break;
        case BOOLEAN:
            recordConsumer.addBoolean(((BooleanObjectInspector) inspector).get(value));
            break;
        case FLOAT:
            recordConsumer.addFloat(((FloatObjectInspector) inspector).get(value));
            break;
        case BYTE:
            recordConsumer.addInteger(((ByteObjectInspector) inspector).get(value));
            break;
        case INT:
            recordConsumer.addInteger(((IntObjectInspector) inspector).get(value));
            break;
        case LONG:
            recordConsumer.addLong(((LongObjectInspector) inspector).get(value));
            break;
        case SHORT:
            recordConsumer.addInteger(((ShortObjectInspector) inspector).get(value));
            break;
        case STRING:
            String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromString(v));
            break;
        case CHAR:
            String vChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(value).getStrippedValue();
            recordConsumer.addBinary(Binary.fromString(vChar));
            break;
        case VARCHAR:
            String vVarchar = ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(value).getValue();
            recordConsumer.addBinary(Binary.fromString(vVarchar));
            break;
        case BINARY:
            byte[] vBinary = ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromByteArray(vBinary));
            break;
        case TIMESTAMP:
            Timestamp ts = ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(NanoTimeUtils.getNanoTime(ts, false).toBinary());
            break;
        case DECIMAL:
            HiveDecimal vDecimal = ((HiveDecimal) inspector.getPrimitiveJavaObject(value));
            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
            recordConsumer.addBinary(decimalToBinary(vDecimal, decTypeInfo));
            break;
        case DATE:
            Date vDate = ((DateObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addInteger(DateWritable.dateToDays(vDate));
            break;
        default:
            throw new IllegalArgumentException("Unsupported primitive data type: " + inspector.getPrimitiveCategory());
    }
}