org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NamecoinExtractFieldUDF.java    From hadoopcryptoledger with Apache License 2.0 6 votes vote down vote up
/**
 *
 * Initialize HiveUDF and create object inspectors. It requires that the argument length is = 1 and that the ObjectInspector of arguments[0] is of type org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector
 *
 * @param arguments array of length 1 containing one WritableBinaryObjectInspector
 *
 * @return ObjectInspector that is able to parse the result of the evaluate method of the UDF (List Object Inspector for Strings)
 *
 * @throws org.apache.hadoop.hive.ql.exec.UDFArgumentException in case the first argument is not of WritableBinaryObjectInspector
 * @throws org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException in case the number of arguments is != 1
 *
 */
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
	if (arguments==null) {
      		throw new UDFArgumentLengthException("namecoinExtractField only takes one argument: Binary ");
	}
  	if (arguments.length != 1) {
      		throw new UDFArgumentLengthException("namecoinExtractField only takes one argument: Binary ");
	}
	if (!(arguments[0] instanceof BinaryObjectInspector)) { 
		throw new UDFArgumentException("first argument must be a Binary containing a Namecoin script");
	}
	// these are only used for bitcointransaction structs exported to other formats, such as ORC
	this.wboi = (BinaryObjectInspector)arguments[0];
	// the UDF returns the hash value of a BitcoinTransaction as byte array
	return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
 
Example #2
Source File: TestObjectInspector.java    From hive-dwrf with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that after copying a lazy binary object, calling materialize on the original and the
 * copy doesn't advance the tree reader twice
 * @throws Exception
 */
@Test
public void TestCopyBinary() throws Exception {
  ReaderWriterProfiler.setProfilerOptions(null);
  OrcLazyBinary lazyBinary = new OrcLazyBinary(new LazyBinaryTreeReader(0, 0) {
    int nextCalls = 0;

    @Override
    public Object next(Object previous) throws IOException {
      if (nextCalls == 0) {
        nextCalls++;
        return new BytesWritable("a".getBytes());
      }

      throw new IOException("next should only be called once");
    }

    @Override
    protected boolean seekToRow(long currentRow) throws IOException {
      return true;
    }
  });

  BinaryObjectInspector binaryOI = (BinaryObjectInspector)
      OrcLazyObjectInspectorUtils.createLazyObjectInspector(TypeInfoFactory.binaryTypeInfo);

  OrcLazyBinary lazyBinary2 = (OrcLazyBinary) binaryOI.copyObject(lazyBinary);

  Assert.assertEquals("a", new String(((BytesWritable) lazyBinary.materialize()).getBytes()));
  Assert.assertEquals("a", new String(((BytesWritable) lazyBinary2.materialize()).getBytes()));
}
 
Example #3
Source File: WriterImpl.java    From hive-dwrf with Apache License 2.0 5 votes vote down vote up
@Override
void write(Object obj) throws IOException {
  long rawDataSize = 0;
  if (obj != null) {
    BytesWritable val =
        ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
    stream.write(val.getBytes(), 0, val.getLength());
    length.write(val.getLength());

    // Raw data size is the length of the BytesWritable, i.e. the number of bytes
    rawDataSize = val.getLength();
  }
  super.write(obj, rawDataSize);
}
 
Example #4
Source File: OrcFlowFileWriter.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        BytesWritable val =
                ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
        stream.write(val.getBytes(), 0, val.getLength());
        length.write(val.getLength());
        indexStatistics.updateBinary(val);
        if (createBloomFilter) {
            bloomFilter.addBytes(val.getBytes(), val.getLength());
        }
    }
}
 
Example #5
Source File: HiveUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Nonnull
public static BinaryObjectInspector asBinaryOI(@Nonnull final ObjectInspector argOI)
        throws UDFArgumentException {
    if (!BINARY_TYPE_NAME.equals(argOI.getTypeName())) {
        throw new UDFArgumentException("Argument type must be Binary: " + argOI.getTypeName());
    }
    return (BinaryObjectInspector) argOI;
}
 
Example #6
Source File: OrcFlowFileWriter.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        BytesWritable val =
                ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
        stream.write(val.getBytes(), 0, val.getLength());
        length.write(val.getLength());
        indexStatistics.updateBinary(val);
        if (createBloomFilter) {
            bloomFilter.addBytes(val.getBytes(), val.getLength());
        }
    }
}
 
Example #7
Source File: HiveFieldConverter.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final byte[] value = ((BinaryObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
  checkSizeLimit(value.length);
  ((VarBinaryVector) outputVV).setSafe(outputIndex, value, 0, value.length);
}
 
Example #8
Source File: DynamoDBDataParser.java    From emr-dynamodb-connector with Apache License 2.0 4 votes vote down vote up
public static ByteBuffer getByteBuffer(Object data, ObjectInspector objectInspector) {
  BytesWritable bw = ((BinaryObjectInspector) objectInspector).getPrimitiveWritableObject(data);
  byte[] result = new byte[bw.getLength()];
  System.arraycopy(bw.getBytes(), 0, result, 0, bw.getLength());
  return ByteBuffer.wrap(result);
}
 
Example #9
Source File: HiveFieldConverter.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final byte[] value = ((BinaryObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
  checkSizeLimit(value.length);
  ((VarBinaryVector) outputVV).setSafe(outputIndex, value, 0, value.length);
}
 
Example #10
Source File: HiveTestUDFImpls.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
  if (arguments[0] == null || arguments[0].get() == null) {
    return null;
  }

  Object input = arguments[0].get();
  switch(inputType) {
    case BOOLEAN:
      return ((BooleanObjectInspector)argumentOI).get(input) ? Boolean.TRUE : Boolean.FALSE;
    case BYTE:
      return new Byte(((ByteObjectInspector)argumentOI).get(input));
    case SHORT:
      return new Short(((ShortObjectInspector)argumentOI).get(input));
    case INT:
      return new Integer(((IntObjectInspector)argumentOI).get(input));
    case LONG:
      return new Long(((LongObjectInspector)argumentOI).get(input));
    case FLOAT:
      return new Float(((FloatObjectInspector)argumentOI).get(input));
    case DOUBLE:
      return new Double(((DoubleObjectInspector)argumentOI).get(input));
    case STRING:
      return PrimitiveObjectInspectorUtils.getString(input, (StringObjectInspector)argumentOI);
    case BINARY:
      return PrimitiveObjectInspectorUtils.getBinary(input, (BinaryObjectInspector) argumentOI).getBytes();
    case VARCHAR:
      if (outputType == PrimitiveCategory.CHAR) {
        HiveVarchar hiveVarchar = PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector) argumentOI);
        return new HiveChar(hiveVarchar.getValue(), HiveChar.MAX_CHAR_LENGTH);
      } else {
        return PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector)argumentOI);
      }
    case CHAR:
      return PrimitiveObjectInspectorUtils.getHiveChar(input, (HiveCharObjectInspector) argumentOI);
    case DATE:
      return PrimitiveObjectInspectorUtils.getDate(input, (DateObjectInspector) argumentOI);
    case TIMESTAMP:
      return PrimitiveObjectInspectorUtils.getTimestamp(input, (TimestampObjectInspector) argumentOI);
    case DECIMAL:
      // return type is a HiveVarchar
      HiveDecimal decimalValue =
          PrimitiveObjectInspectorUtils.getHiveDecimal(input, (HiveDecimalObjectInspector) argumentOI);
      return new HiveVarchar(decimalValue.toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
  }

  throw new UnsupportedOperationException(String.format("Unexpected input type '%s' in Test UDF", inputType));
}
 
Example #11
Source File: SerDeUtils.java    From presto with Apache License 2.0 4 votes vote down vote up
private static void serializePrimitive(Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector)
{
    requireNonNull(builder, "parent builder is null");

    if (object == null) {
        builder.appendNull();
        return;
    }

    switch (inspector.getPrimitiveCategory()) {
        case BOOLEAN:
            BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
            return;
        case BYTE:
            TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
            return;
        case SHORT:
            SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
            return;
        case INT:
            IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
            return;
        case LONG:
            BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
            return;
        case FLOAT:
            RealType.REAL.writeLong(builder, floatToRawIntBits(((FloatObjectInspector) inspector).get(object)));
            return;
        case DOUBLE:
            DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
            return;
        case STRING:
            type.writeSlice(builder, Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
            return;
        case VARCHAR:
            type.writeSlice(builder, Slices.utf8Slice(((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(object).getValue()));
            return;
        case CHAR:
            CharType charType = (CharType) type;
            HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
            type.writeSlice(builder, truncateToLengthAndTrimSpaces(Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
            return;
        case DATE:
            DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
            return;
        case TIMESTAMP:
            TimestampType.TIMESTAMP.writeLong(builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
            return;
        case BINARY:
            VARBINARY.writeSlice(builder, Slices.wrappedBuffer(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
            return;
        case DECIMAL:
            DecimalType decimalType = (DecimalType) type;
            HiveDecimalWritable hiveDecimal = ((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
            if (decimalType.isShort()) {
                decimalType.writeLong(builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
            }
            else {
                decimalType.writeSlice(builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
            }
            return;
    }
    throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
}
 
Example #12
Source File: HiveBinaryObjectConverter.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 4 votes vote down vote up
@Override
public Object convert(ObjectInspector objectInspector, Object o, TypeInfo odpsTypeInfo) {
  BinaryObjectInspector binaryObjectInspector = (BinaryObjectInspector) objectInspector;
  return new Binary(binaryObjectInspector.getPrimitiveJavaObject(o));
}
 
Example #13
Source File: HiveColumnarSerdeResolver.java    From pxf with Apache License 2.0 4 votes vote down vote up
private void resolvePrimitive(Object o, PrimitiveObjectInspector oi) throws IOException {

        if (!firstColumn) {
            builder.append(delimiter);
        }

        if (o == null) {
            builder.append(nullChar);
        } else {
            switch (oi.getPrimitiveCategory()) {
                case BOOLEAN:
                    builder.append(((BooleanObjectInspector) oi).get(o));
                    break;
                case SHORT:
                    builder.append(((ShortObjectInspector) oi).get(o));
                    break;
                case INT:
                    builder.append(((IntObjectInspector) oi).get(o));
                    break;
                case LONG:
                    builder.append(((LongObjectInspector) oi).get(o));
                    break;
                case FLOAT:
                    builder.append(((FloatObjectInspector) oi).get(o));
                    break;
                case DOUBLE:
                    builder.append(((DoubleObjectInspector) oi).get(o));
                    break;
                case DECIMAL:
                    builder.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o).bigDecimalValue());
                    break;
                case STRING:
                    builder.append(((StringObjectInspector) oi).getPrimitiveJavaObject(o));
                    break;
                case BINARY:
                    byte[] bytes = ((BinaryObjectInspector) oi).getPrimitiveJavaObject(o);
                    Utilities.byteArrayToOctalString(bytes, builder);
                    break;
                case TIMESTAMP:
                    builder.append(((TimestampObjectInspector) oi).getPrimitiveJavaObject(o));
                    break;
                case BYTE:  /* TINYINT */
                    builder.append(Short.valueOf(((ByteObjectInspector) oi).get(o)));
                    break;
                default:
                    throw new UnsupportedTypeException(oi.getTypeName()
                            + " conversion is not supported by HiveColumnarSerdeResolver");
            }
        }
        firstColumn = false;
    }
 
Example #14
Source File: HiveBinaryType.java    From transport with BSD 2-Clause "Simplified" License 4 votes vote down vote up
public HiveBinaryType(BinaryObjectInspector binaryObjectInspector) {
  _binaryObjectInspector = binaryObjectInspector;
}
 
Example #15
Source File: CacheablePrimitiveObjectInspectorConverter.java    From transport with BSD 2-Clause "Simplified" License 4 votes vote down vote up
public Text convert(Object input) {
  if (input == null) {
    return null;
  }
  Text t = new Text();

  switch (inputOI.getPrimitiveCategory()) {
    case VOID:
      return null;
    case BOOLEAN:
      t.set(((BooleanObjectInspector) inputOI).get(input) ? trueBytes
          : falseBytes);
      return t;
    case BYTE:
      out.reset();
      LazyInteger.writeUTF8NoException(out, ((ByteObjectInspector) inputOI).get(input));
      t.set(out.getData(), 0, out.getLength());
      return t;
    case SHORT:
      out.reset();
      LazyInteger.writeUTF8NoException(out, ((ShortObjectInspector) inputOI).get(input));
      t.set(out.getData(), 0, out.getLength());
      return t;
    case INT:
      out.reset();
      LazyInteger.writeUTF8NoException(out, ((IntObjectInspector) inputOI).get(input));
      t.set(out.getData(), 0, out.getLength());
      return t;
    case LONG:
      out.reset();
      LazyLong.writeUTF8NoException(out, ((LongObjectInspector) inputOI).get(input));
      t.set(out.getData(), 0, out.getLength());
      return t;
    case FLOAT:
      t.set(String.valueOf(((FloatObjectInspector) inputOI).get(input)));
      return t;
    case DOUBLE:
      t.set(String.valueOf(((DoubleObjectInspector) inputOI).get(input)));
      return t;
    case STRING:
      if (inputOI.preferWritable()) {
        t.set(((StringObjectInspector) inputOI).getPrimitiveWritableObject(input));
      } else {
        t.set(((StringObjectInspector) inputOI).getPrimitiveJavaObject(input));
      }
      return t;
    case CHAR:
      // when converting from char, the value should be stripped of any trailing spaces.
      if (inputOI.preferWritable()) {
        // char text value is already stripped of trailing space
        t.set(((HiveCharObjectInspector) inputOI).getPrimitiveWritableObject(input)
            .getStrippedValue());
      } else {
        t.set(((HiveCharObjectInspector) inputOI).getPrimitiveJavaObject(input).getStrippedValue());
      }
      return t;
    case VARCHAR:
      if (inputOI.preferWritable()) {
        t.set(((HiveVarcharObjectInspector) inputOI).getPrimitiveWritableObject(input)
            .toString());
      } else {
        t.set(((HiveVarcharObjectInspector) inputOI).getPrimitiveJavaObject(input).toString());
      }
      return t;
    case DATE:
      t.set(((DateObjectInspector) inputOI).getPrimitiveWritableObject(input).toString());
      return t;
    case TIMESTAMP:
      t.set(((TimestampObjectInspector) inputOI)
          .getPrimitiveWritableObject(input).toString());
      return t;
    case BINARY:
      BinaryObjectInspector binaryOI = (BinaryObjectInspector) inputOI;
      if (binaryOI.preferWritable()) {
        BytesWritable bytes = binaryOI.getPrimitiveWritableObject(input);
        t.set(bytes.getBytes(), 0, bytes.getLength());
      } else {
        t.set(binaryOI.getPrimitiveJavaObject(input));
      }
      return t;
    case DECIMAL:
      t.set(((HiveDecimalObjectInspector) inputOI).getPrimitiveWritableObject(input).toString());
      return t;
    default:
      throw new RuntimeException("Hive 2 Internal error: type = " + inputOI.getTypeName());
  }
}
 
Example #16
Source File: HiveBinary.java    From transport with BSD 2-Clause "Simplified" License 4 votes vote down vote up
public HiveBinary(Object object, BinaryObjectInspector binaryObjectInspector, StdFactory stdFactory) {
  super(stdFactory);
  _object = object;
  _binaryObjectInspector = binaryObjectInspector;
}
 
Example #17
Source File: HiveTypeSystem.java    From transport with BSD 2-Clause "Simplified" License 4 votes vote down vote up
@Override
protected boolean isBinaryType(ObjectInspector dataType) {
  return dataType instanceof BinaryObjectInspector;
}
 
Example #18
Source File: TestDataWritableWriter.java    From presto with Apache License 2.0 4 votes vote down vote up
/**
 * It writes the primitive value to the Parquet RecordConsumer.
 *
 * @param value The object that contains the primitive value.
 * @param inspector The object inspector used to get the correct value type.
 */
private void writePrimitive(Object value, PrimitiveObjectInspector inspector)
{
    if (value == null) {
        return;
    }

    switch (inspector.getPrimitiveCategory()) {
        case VOID:
            return;
        case DOUBLE:
            recordConsumer.addDouble(((DoubleObjectInspector) inspector).get(value));
            break;
        case BOOLEAN:
            recordConsumer.addBoolean(((BooleanObjectInspector) inspector).get(value));
            break;
        case FLOAT:
            recordConsumer.addFloat(((FloatObjectInspector) inspector).get(value));
            break;
        case BYTE:
            recordConsumer.addInteger(((ByteObjectInspector) inspector).get(value));
            break;
        case INT:
            recordConsumer.addInteger(((IntObjectInspector) inspector).get(value));
            break;
        case LONG:
            recordConsumer.addLong(((LongObjectInspector) inspector).get(value));
            break;
        case SHORT:
            recordConsumer.addInteger(((ShortObjectInspector) inspector).get(value));
            break;
        case STRING:
            String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromString(v));
            break;
        case CHAR:
            String vChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(value).getStrippedValue();
            recordConsumer.addBinary(Binary.fromString(vChar));
            break;
        case VARCHAR:
            String vVarchar = ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(value).getValue();
            recordConsumer.addBinary(Binary.fromString(vVarchar));
            break;
        case BINARY:
            byte[] vBinary = ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromByteArray(vBinary));
            break;
        case TIMESTAMP:
            Timestamp ts = ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(NanoTimeUtils.getNanoTime(ts, false).toBinary());
            break;
        case DECIMAL:
            HiveDecimal vDecimal = ((HiveDecimal) inspector.getPrimitiveJavaObject(value));
            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
            recordConsumer.addBinary(decimalToBinary(vDecimal, decTypeInfo));
            break;
        case DATE:
            Date vDate = ((DateObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addInteger(DateWritable.dateToDays(vDate));
            break;
        default:
            throw new IllegalArgumentException("Unsupported primitive data type: " + inspector.getPrimitiveCategory());
    }
}