Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: indexr   Source File: IndexRSerde.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Writable serialize(Object obj, ObjectInspector objectInspector) throws SerDeException {
    if (!objectInspector.getCategory().equals(ObjectInspector.Category.STRUCT)) {
        throw new SerDeException("Cannot serialize " + objectInspector.getCategory() + ". Can only serialize a struct");
    }

    StructObjectInspector inspector = (StructObjectInspector) objectInspector;
    List<? extends StructField> fields = inspector.getAllStructFieldRefs();
    Writable[] arr = new Writable[fields.size()];
    for (int i = 0; i < fields.size(); i++) {
        StructField field = fields.get(i);
        Object subObj = inspector.getStructFieldData(obj, field);
        ObjectInspector subInspector = field.getFieldObjectInspector();
        arr[i] = createPrimitive(subObj, (PrimitiveObjectInspector) subInspector);
    }
    serdeSize = arr.length;
    return new ArrayWritable(Writable.class, arr);
}
 
Example 2
@Test
public void TestIntWrite() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       // {"properties":{"num":7}}
       addWritable(stuff, 7);
	Writable jsw = jserde.serialize(stuff, rowOI);
	JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
	jn = jn.findValue("properties");
	jn = jn.findValue("num");
	Assert.assertEquals(7, jn.getIntValue());
}
 
Example 3
Source Project: incubator-hivemall   Source File: GeneralLearnerBaseUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
protected StructObjectInspector getReturnOI(@Nonnull ObjectInspector featureOutputOI) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("feature");
    fieldOIs.add(featureOutputOI);
    fieldNames.add("weight");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    if (useCovariance()) {
        fieldNames.add("covar");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 4
@Nonnull
protected StructObjectInspector getOutputOI(@Nonnull FMHyperParameters params) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("feature");
    if (params.parseFeatureAsInt) {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    } else {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    }
    fieldNames.add("W_i");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("V_if");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableFloatObjectInspector));

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 5
@Override
protected StructObjectInspector getReturnOI(ObjectInspector featureRawOI) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("h");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("w0");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("w1");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("w2");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("hk");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("w3");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 6
@Test
public void TestNullGeom() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);

       //value.set("{\"attributes\":{},\"geometry\":null}");
	stuff.set(0, null);
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("shape", row, rowOI);
	Assert.assertNull(fieldData);
}
 
Example 7
Source Project: incubator-hivemall   Source File: LDAPredictUDAF.java    License: Apache License 2.0 6 votes vote down vote up
private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("wcList");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector));

    fieldNames.add("lambdaMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaFloatObjectInspector)));

    fieldNames.add("topics");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    fieldNames.add("alpha");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    fieldNames.add("delta");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 8
Source Project: HiveJdbcStorageHandler   Source File: JdbcSerDe.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method takes an object representing a row of data from Hive, and uses
 * the ObjectInspector to get the data for each column and serialize.
 */
@Override
public DbRecordWritable serialize(Object row, ObjectInspector inspector) throws SerDeException {
    final StructObjectInspector structInspector = (StructObjectInspector) inspector;
    final List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
    if(fields.size() != fieldCount) {
        throw new SerDeException(String.format("Required %d columns, received %d.", fieldCount, fields.size()));
    }

    cachedWritable.clear();

    for(int i = 0; i < fieldCount; i++) {
        StructField structField = fields.get(i);
        if(structField != null) {
            Object field = structInspector.getStructFieldData(row, structField);
            ObjectInspector fieldOI = structField.getFieldObjectInspector();
            Object javaObject = HiveJdbcBridgeUtils.deparseObject(field, fieldOI);
            cachedWritable.set(i, javaObject);
        }
    }

    return cachedWritable;
}
 
Example 9
Source Project: searchanalytics-bigdata   Source File: JSONSerDe.java    License: MIT License 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses the
 * ObjectInspector to extract the column data.
 *
 * @param obj
 *            - Hive object to deparse
 * @param oi
 *            - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(final Object obj, final ObjectInspector oi) {
	switch (oi.getCategory()) {
	case PRIMITIVE:
		return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
	case LIST:
		return deparseList(obj, (ListObjectInspector) oi);
	case MAP:
		return deparseMap(obj, (MapObjectInspector) oi);
	case STRUCT:
		return deparseStruct(obj, (StructObjectInspector) oi, false);
	case UNION:
		// Unsupported by JSON
	default:
		return null;
	}
}
 
Example 10
Source Project: hugegraph-loader   Source File: OrcFileLineFetcher.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void openReader(Readable readable) {
    Path path = new Path(this.source().path());
    try {
        this.reader = OrcFile.createReader(path, OrcFile.readerOptions(
                                                         this.conf));
        this.recordReader = this.reader.rows();
        this.inspector = (StructObjectInspector) this.reader
                                                     .getObjectInspector();
        this.row = null;
    } catch (IOException e) {
        throw new LoadException("Failed to open orc reader for '%s'",
                                e, readable);
    }
    this.resetOffset();
}
 
Example 11
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.STRING);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 12
Source Project: incubator-hivemall   Source File: OnehotEncodingUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) throws UDFArgumentException {
    Preconditions.checkNotNull(inputOIs);

    final int numOIs = inputOIs.length;
    final List<String> fieldNames = new ArrayList<String>(numOIs);
    final List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(numOIs);
    for (int i = 0; i < numOIs; i++) {
        fieldNames.add("f" + String.valueOf(i));
        ObjectInspector elemOI = ObjectInspectorUtils.getStandardObjectInspector(
            inputOIs[i], ObjectInspectorCopyOption.WRITABLE);
        ListObjectInspector listOI =
                ObjectInspectorFactory.getStandardListObjectInspector(elemOI);
        fieldOIs.add(listOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 13
@Test
public void TestPointOnly() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
	stuff.clear();
       addWritable(stuff, new Point(7.0, 4.0));
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
 
Example 14
Source Project: incubator-hivemall   Source File: AmplifierUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (!(argOIs.length >= 2)) {
        throw new UDFArgumentException("_FUNC_(int xtimes, *) takes at least two arguments");
    }
    this.xtimes = HiveUtils.getAsConstInt(argOIs[0]);
    if (!(xtimes >= 1)) {
        throw new UDFArgumentException("Illegal xtimes value: " + xtimes);
    }
    this.forwardObjs = new Object[argOIs.length - 1];

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    for (int i = 1; i < argOIs.length; i++) {
        fieldNames.add("c" + i);
        fieldOIs.add(argOIs[i]);
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 15
Source Project: bigdata-tutorial   Source File: JSONCDHSerDe.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses
 * the ObjectInspector to extract the column data.
 *
 * @param obj - Hive object to deparse
 * @param oi  - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(Object obj, ObjectInspector oi) {
	switch (oi.getCategory()) {
		case LIST:
			return deparseList(obj, (ListObjectInspector) oi);
		case MAP:
			return deparseMap(obj, (MapObjectInspector) oi);
		case PRIMITIVE:
			return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
		case STRUCT:
			return deparseStruct(obj, (StructObjectInspector) oi, false);
		case UNION:
			// Unsupported by JSON
		default:
			return null;
	}
}
 
Example 16
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        showHelp(String.format(
            "%s takes 2 or 3 arguments: array<string> x, double y [, CONSTANT string options]: %s",
            getClass().getSimpleName(), Arrays.toString(argOIs)));
    }
    this._xOI = HiveUtils.asListOI(argOIs, 0);
    HiveUtils.validateFeatureOI(_xOI.getListElementObjectInspector());
    this._yOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);

    this._params = newHyperParameters();
    processOptions(argOIs);

    this._model = null;
    this._t = 0L;
    this._numValidations = 0L;

    if (LOG.isInfoEnabled()) {
        LOG.info(_params);
    }

    return getOutputOI(_params);
}
 
Example 17
@Test
public void TestDateParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new GeoJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"properties\":{\"when\":\"2020-02-20\"}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2020-02-20",
						((DateWritable)fieldData).get().toString());
       value.set("{\"properties\":{\"when\":\"2017-05-05\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2017-05-05",
						((DateWritable)fieldData).get().toString());
}
 
Example 18
Source Project: dremio-oss   Source File: HiveAbstractReader.java    License: Apache License 2.0 6 votes vote down vote up
public HiveAbstractReader(final HiveTableXattr tableAttr, final SplitAndPartitionInfo split,
                          final List<SchemaPath> projectedColumns, final OperatorContext context, final JobConf jobConf,
                          final AbstractSerDe tableSerDe, final StructObjectInspector tableOI, final AbstractSerDe partitionSerDe,
                          final StructObjectInspector partitionOI, final ScanFilter filter,
                          final Collection<List<String>> referencedTables, final UserGroupInformation readerUgi) {
  super(context, projectedColumns);
  this.tableAttr = tableAttr;
  this.split = split;
  this.jobConf = jobConf;
  this.tableSerDe = tableSerDe;
  this.tableOI = tableOI;
  this.partitionSerDe = partitionSerDe == null ? tableSerDe : partitionSerDe;
  this.partitionOI = partitionOI == null ? tableOI : partitionOI;
  this.filter = filter;
  this.referencedTables = referencedTables;
  this.readerUgi = readerUgi;
}
 
Example 19
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
    Assert.assertNotNull(resultInspector);
    Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
    StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
    List<?> fields = structResultInspector.getAllStructFieldRefs();
    Assert.assertEquals(fields.size(), 3);

    ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
    Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
    Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

    ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
    Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
    Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.LONG);

    ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
    Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
    Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 20
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
	if (arguments==null) {
     		throw new UDFArgumentLengthException("ethereumGetSendAddress only takes two arguments: Struct<EthereumTransction>, INT chainId ");
	}
	if (arguments.length != 2) {
     		throw new UDFArgumentLengthException("ethereumGetSendAddress only takes two arguments: Struct<EthereumTransction>, INT chainId ");
	}
	if (!(arguments[0] instanceof StructObjectInspector)) { 
	throw new UDFArgumentException("first argument must be a Struct containing a EthereumTransction");
	}
	if (!(arguments[1] instanceof IntObjectInspector)) {
		throw new UDFArgumentException("second argument must be an int with the chainId");
	}
	this.ethereumUDFUtil=new EthereumUDFUtil((StructObjectInspector) arguments[0]);
	return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
}
 
Example 21
@Test
public void TestIntPoint() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num,shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "bigint,binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{\"num\":7},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, 7L);
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("num", row, rowOI);
	Assert.assertEquals(7, ((LongWritable)fieldData).get());

       //value.set("{\"attributes\":{\"num\":4},\"geometry\":{\"x\":7.0,\"y\":2.0}}");
	stuff.clear();
       addWritable(stuff, 4L);
       addWritable(stuff, new Point(7.0, 2.0));
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("num", row, rowOI);
	Assert.assertEquals(4, ((LongWritable)fieldData).get());
	fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(7.0, 2.0), (BytesWritable)fieldData);
}
 
Example 22
/**
* Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionOutputs 
*
*/

private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) {
int listLength=loi.getListLength(listOfOutputsObject);
List<BitcoinTransactionOutput> result=new ArrayList<>(listLength);
StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
	for (int i=0;i<listLength;i++) {
		Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i);
		StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value");
		StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength");
		StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript");
		if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) {
			LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i);
			return new ArrayList<>();
		}
		HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF));	
		byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF));
		byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF));
		BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript);
		result.add(currentBitcoinTransactionOutput);
	}
return result;
}
 
Example 23
/**
*
* Initialize HiveUDF and create object inspectors. It requires that the argument length is = 1 and that the ObjectInspector of arguments[0] is of type StructObjectInspector
*
* @param arguments array of length 1 containing one StructObjectInspector
*
* @return ObjectInspector that is able to parse the result of the evaluate method of the UDF (BinaryWritable)
*
* @throws org.apache.hadoop.hive.ql.exec.UDFArgumentException in case the first argument is not of StructObjectInspector
* @throws org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException in case the number of arguments is != 1
*
*/

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
	if (arguments==null) {
      		throw new UDFArgumentLengthException("bitcoinTransactionHash only takes one argument: Struct<BitcoinTransaction> ");
	}
  	if (arguments.length != 1) {
      		throw new UDFArgumentLengthException("bitcoinTransactionHash only takes one argument: Struct<BitcoinTransaction> ");
	}
	if (!(arguments[0] instanceof StructObjectInspector)) { 
		throw new UDFArgumentException("first argument must be a Struct containing a BitcoinTransaction");
	}
	this.soi = (StructObjectInspector)arguments[0];
	// these are only used for bitcointransaction structs exported to other formats, such as ORC
	this.wboi = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
	this.wioi = PrimitiveObjectInspectorFactory.writableIntObjectInspector;
	this.wloi = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
	this.wbyoi = PrimitiveObjectInspectorFactory.writableByteObjectInspector;
	this.hdoi = PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector;
	// the UDF returns the hash value of a BitcoinTransaction as byte array
	return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
}
 
Example 24
@Test
public void TestPointWrite() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       // {"properties":{},"geometry":{"type":"Point","coordinates":[15.0,5.0]}}
       addWritable(stuff, new Point(15.0, 5.0));
	Writable jsw = jserde.serialize(stuff, rowOI);
       String rslt = ((Text)jsw).toString();
	JsonNode jn = new ObjectMapper().readTree(rslt);
	jn = jn.findValue("geometry");
	Assert.assertNotNull(jn.findValue("type"));
	Assert.assertNotNull(jn.findValue("coordinates"));
}
 
Example 25
Source Project: presto   Source File: HiveUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static StructObjectInspector getTableObjectInspector(Deserializer deserializer)
{
    try {
        ObjectInspector inspector = deserializer.getObjectInspector();
        checkArgument(inspector.getCategory() == Category.STRUCT, "expected STRUCT: %s", inspector.getCategory());
        return (StructObjectInspector) inspector;
    }
    catch (SerDeException e) {
        throw new RuntimeException(e);
    }
}
 
Example 26
Source Project: azure-tables-hadoop   Source File: TestAzureEntitySerDe.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("serial")
public void testPropertyNotFound(boolean requireFieldExists) throws Exception {
	WritableEntity entity = new WritableEntity();
	entity.setProperties(new LinkedHashMap<String, EntityProperty>() {{
		put("a", new EntityProperty(7));
		put("b", new EntityProperty("hello"));
	}});
	AzureEntitySerDe serDe = new AzureEntitySerDe();
	Properties tbl = new Properties();
	tbl.put(LIST_COLUMNS, "a,b,c");
	tbl.put(LIST_COLUMN_TYPES, "int,string,int");
	Configuration conf = new Configuration();
	if (requireFieldExists) {
		conf.setBoolean(Keys.REQUIRE_FIELD_EXISTS.getKey(), true);
	}
	serDe.initialize(conf, tbl);
	StructObjectInspector inspector = (StructObjectInspector)serDe.getObjectInspector();
	assertEquals(7, inspector.getStructFieldData(entity,
			inspector.getStructFieldRef("a")));
	try {
		Object returned = inspector.getStructFieldData(entity,
				inspector.getStructFieldRef("c"));
		if (requireFieldExists) {
			fail("Should've thrown here.");
		} else {
			assertNull(returned);
		}
	} catch (IllegalArgumentException ex) {
		if (requireFieldExists) {
			assertEquals("No property found with name c. Properties found: a,b",
					ex.getMessage());
		} else {
			throw ex;
		}
	}
}
 
Example 27
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    final int numArgs = argOIs.length;
    if (numArgs != 2 && numArgs != 3) {
        throw new UDFArgumentException(
            "_FUNC_ takes 2 or 3 arguments: List<String|Int|BitInt> features, {Int|String} label [, constant String options]");
    }

    return super.initialize(argOIs);
}
 
Example 28
Source Project: incubator-hivemall   Source File: RecallUDAF.java    License: Apache License 2.0 5 votes vote down vote up
private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("sum");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("count");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 29
Source Project: nifi   Source File: OrcFlowFileWriter.java    License: Apache License 2.0 5 votes vote down vote up
private String getColumnNamesFromInspector(ObjectInspector inspector) {
    List<String> fieldNames = Lists.newArrayList();
    Joiner joiner = Joiner.on(",");
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector soi = (StructObjectInspector) inspector;
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        fieldNames.addAll(fields.stream().map((Function<StructField, String>) StructField::getFieldName).collect(Collectors.toList()));
    }
    return joiner.join(fieldNames);
}
 
Example 30
Source Project: incubator-hivemall   Source File: HitRateUDAF.java    License: Apache License 2.0 5 votes vote down vote up
private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("sum");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("count");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}