org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: indexr   Author: shunfei   File: IndexRSerde.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Writable serialize(Object obj, ObjectInspector objectInspector) throws SerDeException {
    if (!objectInspector.getCategory().equals(ObjectInspector.Category.STRUCT)) {
        throw new SerDeException("Cannot serialize " + objectInspector.getCategory() + ". Can only serialize a struct");
    }

    StructObjectInspector inspector = (StructObjectInspector) objectInspector;
    List<? extends StructField> fields = inspector.getAllStructFieldRefs();
    Writable[] arr = new Writable[fields.size()];
    for (int i = 0; i < fields.size(); i++) {
        StructField field = fields.get(i);
        Object subObj = inspector.getStructFieldData(obj, field);
        ObjectInspector subInspector = field.getFieldObjectInspector();
        arr[i] = createPrimitive(subObj, (PrimitiveObjectInspector) subInspector);
    }
    serdeSize = arr.length;
    return new ArrayWritable(Writable.class, arr);
}
 
Example #2
Source Project: spatial-framework-for-hadoop   Author: Esri   File: TestGeoJsonSerDe.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void TestIntWrite() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       // {"properties":{"num":7}}
       addWritable(stuff, 7);
	Writable jsw = jserde.serialize(stuff, rowOI);
	JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
	jn = jn.findValue("properties");
	jn = jn.findValue("num");
	Assert.assertEquals(7, jn.getIntValue());
}
 
Example #3
Source Project: incubator-hivemall   Author: apache   File: GeneralLearnerBaseUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
protected StructObjectInspector getReturnOI(@Nonnull ObjectInspector featureOutputOI) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("feature");
    fieldOIs.add(featureOutputOI);
    fieldNames.add("weight");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    if (useCovariance()) {
        fieldNames.add("covar");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #4
Source Project: incubator-hivemall   Author: apache   File: FactorizationMachineUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
protected StructObjectInspector getOutputOI(@Nonnull FMHyperParameters params) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("feature");
    if (params.parseFeatureAsInt) {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    } else {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    }
    fieldNames.add("W_i");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("V_if");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableFloatObjectInspector));

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #5
Source Project: incubator-hivemall   Author: apache   File: KernelExpansionPassiveAggressiveUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected StructObjectInspector getReturnOI(ObjectInspector featureRawOI) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("h");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("w0");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("w1");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("w2");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("hk");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("w3");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #6
Source Project: spatial-framework-for-hadoop   Author: Esri   File: TestEsriJsonSerDe.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void TestNullGeom() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);

       //value.set("{\"attributes\":{},\"geometry\":null}");
	stuff.set(0, null);
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("shape", row, rowOI);
	Assert.assertNull(fieldData);
}
 
Example #7
Source Project: incubator-hivemall   Author: apache   File: LDAPredictUDAF.java    License: Apache License 2.0 6 votes vote down vote up
private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("wcList");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector));

    fieldNames.add("lambdaMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaFloatObjectInspector)));

    fieldNames.add("topics");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    fieldNames.add("alpha");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    fieldNames.add("delta");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #8
Source Project: HiveJdbcStorageHandler   Author: myui   File: JdbcSerDe.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method takes an object representing a row of data from Hive, and uses
 * the ObjectInspector to get the data for each column and serialize.
 */
@Override
public DbRecordWritable serialize(Object row, ObjectInspector inspector) throws SerDeException {
    final StructObjectInspector structInspector = (StructObjectInspector) inspector;
    final List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
    if(fields.size() != fieldCount) {
        throw new SerDeException(String.format("Required %d columns, received %d.", fieldCount, fields.size()));
    }

    cachedWritable.clear();

    for(int i = 0; i < fieldCount; i++) {
        StructField structField = fields.get(i);
        if(structField != null) {
            Object field = structInspector.getStructFieldData(row, structField);
            ObjectInspector fieldOI = structField.getFieldObjectInspector();
            Object javaObject = HiveJdbcBridgeUtils.deparseObject(field, fieldOI);
            cachedWritable.set(i, javaObject);
        }
    }

    return cachedWritable;
}
 
Example #9
Source Project: searchanalytics-bigdata   Author: jaibeermalik   File: JSONSerDe.java    License: MIT License 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses the
 * ObjectInspector to extract the column data.
 *
 * @param obj
 *            - Hive object to deparse
 * @param oi
 *            - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(final Object obj, final ObjectInspector oi) {
	switch (oi.getCategory()) {
	case PRIMITIVE:
		return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
	case LIST:
		return deparseList(obj, (ListObjectInspector) oi);
	case MAP:
		return deparseMap(obj, (MapObjectInspector) oi);
	case STRUCT:
		return deparseStruct(obj, (StructObjectInspector) oi, false);
	case UNION:
		// Unsupported by JSON
	default:
		return null;
	}
}
 
Example #10
Source Project: hugegraph-loader   Author: hugegraph   File: OrcFileLineFetcher.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void openReader(Readable readable) {
    Path path = new Path(this.source().path());
    try {
        this.reader = OrcFile.createReader(path, OrcFile.readerOptions(
                                                         this.conf));
        this.recordReader = this.reader.rows();
        this.inspector = (StructObjectInspector) this.reader
                                                     .getObjectInspector();
        this.row = null;
    } catch (IOException e) {
        throw new LoadException("Failed to open orc reader for '%s'",
                                e, readable);
    }
    this.resetOffset();
}
 
Example #11
Source Project: incubator-datasketches-hive   Author: apache   File: DataToSketchUDAFTest.java    License: Apache License 2.0 6 votes vote down vote up
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.STRING);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example #12
Source Project: incubator-hivemall   Author: apache   File: OnehotEncodingUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) throws UDFArgumentException {
    Preconditions.checkNotNull(inputOIs);

    final int numOIs = inputOIs.length;
    final List<String> fieldNames = new ArrayList<String>(numOIs);
    final List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(numOIs);
    for (int i = 0; i < numOIs; i++) {
        fieldNames.add("f" + String.valueOf(i));
        ObjectInspector elemOI = ObjectInspectorUtils.getStandardObjectInspector(
            inputOIs[i], ObjectInspectorCopyOption.WRITABLE);
        ListObjectInspector listOI =
                ObjectInspectorFactory.getStandardListObjectInspector(elemOI);
        fieldOIs.add(listOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #13
Source Project: spatial-framework-for-hadoop   Author: Esri   File: TestEsriJsonSerDe.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void TestPointOnly() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
	stuff.clear();
       addWritable(stuff, new Point(7.0, 4.0));
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
 
Example #14
Source Project: incubator-hivemall   Author: apache   File: AmplifierUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (!(argOIs.length >= 2)) {
        throw new UDFArgumentException("_FUNC_(int xtimes, *) takes at least two arguments");
    }
    this.xtimes = HiveUtils.getAsConstInt(argOIs[0]);
    if (!(xtimes >= 1)) {
        throw new UDFArgumentException("Illegal xtimes value: " + xtimes);
    }
    this.forwardObjs = new Object[argOIs.length - 1];

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    for (int i = 1; i < argOIs.length; i++) {
        fieldNames.add("c" + i);
        fieldOIs.add(argOIs[i]);
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #15
Source Project: bigdata-tutorial   Author: micmiu   File: JSONCDHSerDe.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses
 * the ObjectInspector to extract the column data.
 *
 * @param obj - Hive object to deparse
 * @param oi  - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(Object obj, ObjectInspector oi) {
	switch (oi.getCategory()) {
		case LIST:
			return deparseList(obj, (ListObjectInspector) oi);
		case MAP:
			return deparseMap(obj, (MapObjectInspector) oi);
		case PRIMITIVE:
			return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
		case STRUCT:
			return deparseStruct(obj, (StructObjectInspector) oi, false);
		case UNION:
			// Unsupported by JSON
		default:
			return null;
	}
}
 
Example #16
Source Project: incubator-hivemall   Author: apache   File: FactorizationMachineUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        showHelp(String.format(
            "%s takes 2 or 3 arguments: array<string> x, double y [, CONSTANT string options]: %s",
            getClass().getSimpleName(), Arrays.toString(argOIs)));
    }
    this._xOI = HiveUtils.asListOI(argOIs, 0);
    HiveUtils.validateFeatureOI(_xOI.getListElementObjectInspector());
    this._yOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);

    this._params = newHyperParameters();
    processOptions(argOIs);

    this._model = null;
    this._t = 0L;
    this._numValidations = 0L;

    if (LOG.isInfoEnabled()) {
        LOG.info(_params);
    }

    return getOutputOI(_params);
}
 
Example #17
Source Project: spatial-framework-for-hadoop   Author: Esri   File: TestGeoJsonSerDe.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void TestDateParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new GeoJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"properties\":{\"when\":\"2020-02-20\"}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2020-02-20",
						((DateWritable)fieldData).get().toString());
       value.set("{\"properties\":{\"when\":\"2017-05-05\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2017-05-05",
						((DateWritable)fieldData).get().toString());
}
 
Example #18
Source Project: dremio-oss   Author: dremio   File: HiveAbstractReader.java    License: Apache License 2.0 6 votes vote down vote up
public HiveAbstractReader(final HiveTableXattr tableAttr, final SplitAndPartitionInfo split,
                          final List<SchemaPath> projectedColumns, final OperatorContext context, final JobConf jobConf,
                          final AbstractSerDe tableSerDe, final StructObjectInspector tableOI, final AbstractSerDe partitionSerDe,
                          final StructObjectInspector partitionOI, final ScanFilter filter,
                          final Collection<List<String>> referencedTables, final UserGroupInformation readerUgi) {
  super(context, projectedColumns);
  this.tableAttr = tableAttr;
  this.split = split;
  this.jobConf = jobConf;
  this.tableSerDe = tableSerDe;
  this.tableOI = tableOI;
  this.partitionSerDe = partitionSerDe == null ? tableSerDe : partitionSerDe;
  this.partitionOI = partitionOI == null ? tableOI : partitionOI;
  this.filter = filter;
  this.referencedTables = referencedTables;
  this.readerUgi = readerUgi;
}
 
Example #19
Source Project: incubator-datasketches-hive   Author: apache   File: DataToSketchUDAFTest.java    License: Apache License 2.0 6 votes vote down vote up
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
    Assert.assertNotNull(resultInspector);
    Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
    StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
    List<?> fields = structResultInspector.getAllStructFieldRefs();
    Assert.assertEquals(fields.size(), 3);

    ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
    Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
    Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

    ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
    Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
    Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.LONG);

    ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
    Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
    Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example #20
Source Project: hadoopcryptoledger   Author: ZuInnoTe   File: EthereumGetSendAddressUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
	if (arguments==null) {
     		throw new UDFArgumentLengthException("ethereumGetSendAddress only takes two arguments: Struct<EthereumTransction>, INT chainId ");
	}
	if (arguments.length != 2) {
     		throw new UDFArgumentLengthException("ethereumGetSendAddress only takes two arguments: Struct<EthereumTransction>, INT chainId ");
	}
	if (!(arguments[0] instanceof StructObjectInspector)) { 
	throw new UDFArgumentException("first argument must be a Struct containing a EthereumTransction");
	}
	if (!(arguments[1] instanceof IntObjectInspector)) {
		throw new UDFArgumentException("second argument must be an int with the chainId");
	}
	this.ethereumUDFUtil=new EthereumUDFUtil((StructObjectInspector) arguments[0]);
	return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
}
 
Example #21
Source Project: spatial-framework-for-hadoop   Author: Esri   File: TestEsriJsonSerDe.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void TestIntPoint() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num,shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "bigint,binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{\"num\":7},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, 7L);
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("num", row, rowOI);
	Assert.assertEquals(7, ((LongWritable)fieldData).get());

       //value.set("{\"attributes\":{\"num\":4},\"geometry\":{\"x\":7.0,\"y\":2.0}}");
	stuff.clear();
       addWritable(stuff, 4L);
       addWritable(stuff, new Point(7.0, 2.0));
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("num", row, rowOI);
	Assert.assertEquals(4, ((LongWritable)fieldData).get());
	fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(7.0, 2.0), (BytesWritable)fieldData);
}
 
Example #22
Source Project: hadoopcryptoledger   Author: ZuInnoTe   File: BitcoinTransactionHashUDF.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionOutputs 
*
*/

private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) {
int listLength=loi.getListLength(listOfOutputsObject);
List<BitcoinTransactionOutput> result=new ArrayList<>(listLength);
StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
	for (int i=0;i<listLength;i++) {
		Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i);
		StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value");
		StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength");
		StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript");
		if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) {
			LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i);
			return new ArrayList<>();
		}
		HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF));	
		byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF));
		byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF));
		BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript);
		result.add(currentBitcoinTransactionOutput);
	}
return result;
}
 
Example #23
Source Project: hadoopcryptoledger   Author: ZuInnoTe   File: BitcoinTransactionHashSegwitUDF.java    License: Apache License 2.0 6 votes vote down vote up
/**
*
* Initialize HiveUDF and create object inspectors. It requires that the argument length is = 1 and that the ObjectInspector of arguments[0] is of type StructObjectInspector
*
* @param arguments array of length 1 containing one StructObjectInspector
*
* @return ObjectInspector that is able to parse the result of the evaluate method of the UDF (BinaryWritable)
*
* @throws org.apache.hadoop.hive.ql.exec.UDFArgumentException in case the first argument is not of StructObjectInspector
* @throws org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException in case the number of arguments is != 1
*
*/

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
	if (arguments==null) {
      		throw new UDFArgumentLengthException("bitcoinTransactionHash only takes one argument: Struct<BitcoinTransaction> ");
	}
  	if (arguments.length != 1) {
      		throw new UDFArgumentLengthException("bitcoinTransactionHash only takes one argument: Struct<BitcoinTransaction> ");
	}
	if (!(arguments[0] instanceof StructObjectInspector)) { 
		throw new UDFArgumentException("first argument must be a Struct containing a BitcoinTransaction");
	}
	this.soi = (StructObjectInspector)arguments[0];
	// these are only used for bitcointransaction structs exported to other formats, such as ORC
	this.wboi = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
	this.wioi = PrimitiveObjectInspectorFactory.writableIntObjectInspector;
	this.wloi = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
	this.wbyoi = PrimitiveObjectInspectorFactory.writableByteObjectInspector;
	this.hdoi = PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector;
	// the UDF returns the hash value of a BitcoinTransaction as byte array
	return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
}
 
Example #24
Source Project: spatial-framework-for-hadoop   Author: Esri   File: TestGeoJsonSerDe.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void TestPointWrite() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       // {"properties":{},"geometry":{"type":"Point","coordinates":[15.0,5.0]}}
       addWritable(stuff, new Point(15.0, 5.0));
	Writable jsw = jserde.serialize(stuff, rowOI);
       String rslt = ((Text)jsw).toString();
	JsonNode jn = new ObjectMapper().readTree(rslt);
	jn = jn.findValue("geometry");
	Assert.assertNotNull(jn.findValue("type"));
	Assert.assertNotNull(jn.findValue("coordinates"));
}
 
Example #25
Source Project: presto   Author: prestosql   File: HiveUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static StructObjectInspector getTableObjectInspector(Deserializer deserializer)
{
    try {
        ObjectInspector inspector = deserializer.getObjectInspector();
        checkArgument(inspector.getCategory() == Category.STRUCT, "expected STRUCT: %s", inspector.getCategory());
        return (StructObjectInspector) inspector;
    }
    catch (SerDeException e) {
        throw new RuntimeException(e);
    }
}
 
Example #26
Source Project: azure-tables-hadoop   Author: mooso   File: TestAzureEntitySerDe.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("serial")
public void testPropertyNotFound(boolean requireFieldExists) throws Exception {
	WritableEntity entity = new WritableEntity();
	entity.setProperties(new LinkedHashMap<String, EntityProperty>() {{
		put("a", new EntityProperty(7));
		put("b", new EntityProperty("hello"));
	}});
	AzureEntitySerDe serDe = new AzureEntitySerDe();
	Properties tbl = new Properties();
	tbl.put(LIST_COLUMNS, "a,b,c");
	tbl.put(LIST_COLUMN_TYPES, "int,string,int");
	Configuration conf = new Configuration();
	if (requireFieldExists) {
		conf.setBoolean(Keys.REQUIRE_FIELD_EXISTS.getKey(), true);
	}
	serDe.initialize(conf, tbl);
	StructObjectInspector inspector = (StructObjectInspector)serDe.getObjectInspector();
	assertEquals(7, inspector.getStructFieldData(entity,
			inspector.getStructFieldRef("a")));
	try {
		Object returned = inspector.getStructFieldData(entity,
				inspector.getStructFieldRef("c"));
		if (requireFieldExists) {
			fail("Should've thrown here.");
		} else {
			assertNull(returned);
		}
	} catch (IllegalArgumentException ex) {
		if (requireFieldExists) {
			assertEquals("No property found with name c. Properties found: a,b",
					ex.getMessage());
		} else {
			throw ex;
		}
	}
}
 
Example #27
Source Project: incubator-hivemall   Author: apache   File: MulticlassAROWClassifierUDTF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    final int numArgs = argOIs.length;
    if (numArgs != 2 && numArgs != 3) {
        throw new UDFArgumentException(
            "_FUNC_ takes 2 or 3 arguments: List<String|Int|BitInt> features, {Int|String} label [, constant String options]");
    }

    return super.initialize(argOIs);
}
 
Example #28
Source Project: incubator-hivemall   Author: apache   File: RecallUDAF.java    License: Apache License 2.0 5 votes vote down vote up
private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("sum");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("count");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #29
Source Project: nifi   Author: apache   File: OrcFlowFileWriter.java    License: Apache License 2.0 5 votes vote down vote up
private String getColumnNamesFromInspector(ObjectInspector inspector) {
    List<String> fieldNames = Lists.newArrayList();
    Joiner joiner = Joiner.on(",");
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector soi = (StructObjectInspector) inspector;
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        fieldNames.addAll(fields.stream().map((Function<StructField, String>) StructField::getFieldName).collect(Collectors.toList()));
    }
    return joiner.join(fieldNames);
}
 
Example #30
Source Project: incubator-hivemall   Author: apache   File: HitRateUDAF.java    License: Apache License 2.0 5 votes vote down vote up
private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("sum");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("count");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}