Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector#getStructFieldRef()
The following examples show how to use
org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector#getStructFieldRef() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7},\"geometry\":null}");
value.set("{\"attributes\":{\"num\":7}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("num");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
value.set("{\"attributes\":{\"num\":9}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("num");
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
Example 2
| Source File: BitcoinTransactionHashUDF.java From hadoopcryptoledger with Apache License 2.0 | 6 votes |
|
/**
* Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionOutputs
*
*/
private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) {
int listLength=loi.getListLength(listOfOutputsObject);
List<BitcoinTransactionOutput> result=new ArrayList<>(listLength);
StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
for (int i=0;i<listLength;i++) {
Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i);
StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value");
StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength");
StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript");
if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) {
LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i);
return new ArrayList<>();
}
HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF));
byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF));
byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF));
BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript);
result.add(currentBitcoinTransactionOutput);
}
return result;
}
Example 3
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestEpochParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{\"when\":147147147147}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
((DateWritable)fieldData).get().toString());
value.set("{\"properties\":{\"when\":142857142857}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
((DateWritable)fieldData).get().toString());
}
Example 4
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestEpochParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{\"when\":147147147147}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
((DateWritable)fieldData).get().toString());
value.set("{\"attributes\":{\"when\":142857142857}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
((DateWritable)fieldData).get().toString());
}
Example 5
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestDateParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{\"when\":\"2020-02-20\"}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2020-02-20",
((DateWritable)fieldData).get().toString());
value.set("{\"attributes\":{\"when\":\"2017-05-05\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2017-05-05",
((DateWritable)fieldData).get().toString());
}
Example 6
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"coordinates\":[15.0,5.0]}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("shape");
Object fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"type\":\"Point\",\"coordinates\":[7.0,4.0]}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("shape");
fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
Example 7
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{\"num\":7}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("num");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
value.set("{\"properties\":{\"num\":9}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("num");
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
Example 8
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("shape");
Object fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("shape");
fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
Example 9
| Source File: BitcoinTransactionHashUDF.java From hadoopcryptoledger with Apache License 2.0 | 5 votes |
|
/**
* Read list of Bitcoin transaction inputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfInputsObject object containing the list of inputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionInputs
*
*/
private List<BitcoinTransactionInput> readListOfInputsFromTable(ListObjectInspector loi, Object listOfInputsObject) {
int listLength=loi.getListLength(listOfInputsObject);
List<BitcoinTransactionInput> result = new ArrayList<>(listLength);
StructObjectInspector listOfInputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
for (int i=0;i<listLength;i++) {
Object currentlistofinputsObject = loi.getListElement(listOfInputsObject,i);
StructField prevtransactionhashSF = listOfInputsElementObjectInspector.getStructFieldRef("prevtransactionhash");
StructField previoustxoutindexSF = listOfInputsElementObjectInspector.getStructFieldRef("previoustxoutindex");
StructField txinscriptlengthSF = listOfInputsElementObjectInspector.getStructFieldRef("txinscriptlength");
StructField txinscriptSF = listOfInputsElementObjectInspector.getStructFieldRef("txinscript");
StructField seqnoSF = listOfInputsElementObjectInspector.getStructFieldRef("seqno");
boolean prevFieldsNull = (prevtransactionhashSF==null) || (previoustxoutindexSF==null);
boolean inFieldsNull = (txinscriptlengthSF==null) || (txinscriptSF==null);
boolean otherAttribNull = seqnoSF==null;
if (prevFieldsNull || inFieldsNull || otherAttribNull) {
LOG.warn("Invalid BitcoinTransactionInput detected at position "+i);
return new ArrayList<>();
}
byte[] currentPrevTransactionHash = wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,prevtransactionhashSF));
long currentPreviousTxOutIndex = wloi.get(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,previoustxoutindexSF));
byte[] currentTxInScriptLength= wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,txinscriptlengthSF));
byte[] currentTxInScript= wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,txinscriptSF));
long currentSeqNo = wloi.get(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,seqnoSF));
BitcoinTransactionInput currentBitcoinTransactionInput = new BitcoinTransactionInput(currentPrevTransactionHash,currentPreviousTxOutIndex,currentTxInScriptLength,currentTxInScript,currentSeqNo);
result.add(currentBitcoinTransactionInput);
}
return result;
}
Example 10
| Source File: UDAFToOrderedMap.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
super.init(mode, argOIs);
// initialize input
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
this.inputValueOI = argOIs[1];
this.sizeOI = HiveUtils.asIntegerOI(argOIs[2]);
} else {// from partial aggregation
StructObjectInspector soi = (StructObjectInspector) argOIs[0];
this.internalMergeOI = soi;
this.partialMapField = soi.getStructFieldRef("partialMap");
// re-extract input key/value OIs
MapObjectInspector partialMapOI =
(MapObjectInspector) partialMapField.getFieldObjectInspector();
this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(
partialMapOI.getMapKeyObjectInspector());
this.inputValueOI = partialMapOI.getMapValueObjectInspector();
this.partialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
this.sizeField = soi.getStructFieldRef("size");
this.sizeOI = (PrimitiveObjectInspector) sizeField.getFieldObjectInspector();
}
// initialize output
final ObjectInspector outputOI;
if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
outputOI = internalMergeOI(inputKeyOI, inputValueOI);
} else {// terminate
outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
}
return outputOI;
}
Example 11
| Source File: ArrayAvgGenericUDAF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
assert (parameters.length == 1);
super.init(mode, parameters);
// initialize input
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
this.inputListOI = (ListObjectInspector) parameters[0];
this.inputListElemOI =
HiveUtils.asDoubleCompatibleOI(inputListOI.getListElementObjectInspector());
} else {// from partial aggregation
StructObjectInspector soi = (StructObjectInspector) parameters[0];
this.internalMergeOI = soi;
this.sizeField = soi.getStructFieldRef("size");
this.sumField = soi.getStructFieldRef("sum");
this.countField = soi.getStructFieldRef("count");
this.sizeOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector;
this.sumOI = ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
this.countOI = ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableLongObjectInspector);
}
// initialize output
final ObjectInspector outputOI;
if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
outputOI = internalMergeOI();
} else {// terminate
outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
}
return outputOI;
}
Example 12
| Source File: BitcoinTransactionHashSegwitUDF.java From hadoopcryptoledger with Apache License 2.0 | 5 votes |
|
/**
* Read list of Bitcoin ScriptWitness items from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfScriptWitnessItemObject object containing the list of scriptwitnessitems of a Bitcoin Transaction
*
* @return a list of BitcoinScriptWitnessItem
*
*/
private List<BitcoinScriptWitnessItem> readListOfBitcoinScriptWitnessFromTable(ListObjectInspector loi, Object listOfScriptWitnessItemObject) {
int listLength=loi.getListLength(listOfScriptWitnessItemObject);
List<BitcoinScriptWitnessItem> result = new ArrayList<>(listLength);
StructObjectInspector listOfScriptwitnessItemElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
for (int i=0;i<listLength;i++) {
Object currentlistofscriptwitnessitemObject = loi.getListElement(listOfScriptWitnessItemObject,i);
StructField stackitemcounterSF = listOfScriptwitnessItemElementObjectInspector.getStructFieldRef("stackitemcounter");
StructField scriptwitnesslistSF = listOfScriptwitnessItemElementObjectInspector.getStructFieldRef("scriptwitnesslist");
boolean scriptwitnessitemNull = (stackitemcounterSF==null) || (scriptwitnesslistSF==null) ;
if (scriptwitnessitemNull) {
LOG.warn("Invalid BitcoinScriptWitnessItem detected at position "+i);
return new ArrayList<>();
}
byte[] stackItemCounter = wboi.getPrimitiveJavaObject(listOfScriptwitnessItemElementObjectInspector.getStructFieldData(currentlistofscriptwitnessitemObject,stackitemcounterSF));
Object listofscriptwitnessObject = soi.getStructFieldData(currentlistofscriptwitnessitemObject,scriptwitnesslistSF);
ListObjectInspector loiScriptWitness=(ListObjectInspector)scriptwitnesslistSF.getFieldObjectInspector();
StructObjectInspector listOfScriptwitnessElementObjectInspector = (StructObjectInspector)loiScriptWitness.getListElementObjectInspector();
int listWitnessLength = loiScriptWitness.getListLength(listofscriptwitnessObject);
List<BitcoinScriptWitness> currentScriptWitnessList = new ArrayList<>(listWitnessLength);
for (int j=0;j<listWitnessLength;j++) {
Object currentlistofscriptwitnessObject = loi.getListElement(listofscriptwitnessObject,j);
StructField witnessscriptlengthSF = listOfScriptwitnessElementObjectInspector.getStructFieldRef("witnessscriptlength");
StructField witnessscriptSF = listOfScriptwitnessElementObjectInspector.getStructFieldRef("witnessscript");
boolean scriptwitnessNull = (witnessscriptlengthSF==null) || (witnessscriptSF==null);
if (scriptwitnessNull) {
LOG.warn("Invalid BitcoinScriptWitness detected at position "+j+ "for BitcoinScriptWitnessItem "+i);
return new ArrayList<>();
}
byte[] scriptWitnessLength = wboi.getPrimitiveJavaObject(listOfScriptwitnessElementObjectInspector.getStructFieldData(currentlistofscriptwitnessObject,witnessscriptlengthSF));
byte[] scriptWitness = wboi.getPrimitiveJavaObject(listOfScriptwitnessElementObjectInspector.getStructFieldData(currentlistofscriptwitnessObject,witnessscriptSF));
currentScriptWitnessList.add(new BitcoinScriptWitness(scriptWitnessLength,scriptWitness));
}
BitcoinScriptWitnessItem currentBitcoinScriptWitnessItem = new BitcoinScriptWitnessItem(stackItemCounter,currentScriptWitnessList);
result.add(currentBitcoinScriptWitnessItem);
}
return result;
}
Example 13
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 5 votes |
|
@Test
public void TestTimeParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "timestamp");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{\"when\":\"2020-02-20\"}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(
new java.text.SimpleDateFormat("yyyy-MM-dd").parse("2020-02-20").getTime(),
((TimestampWritable)fieldData).getTimestamp().getTime());
value.set("{\"attributes\":{\"when\":\"2017-05-05 05:05\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(
new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm").parse("2017-05-05 05:05").getTime(),
((TimestampWritable)fieldData).getTimestamp().getTime());
value.set("{\"attributes\":{\"when\":\"2017-08-09 10:11:12\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(
new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2017-08-09 10:11:12").getTime(),
((TimestampWritable)fieldData).getTimestamp().getTime());
value.set("{\"attributes\":{\"when\":\"2017-06-05 04:03:02.123456789\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(
new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse("2017-06-05 04:03:02.123").getTime(),
((TimestampWritable)fieldData).getTimestamp().getTime()); // ns parsed but not checked
}
Example 14
| Source File: BuildBinsUDAF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] OIs) throws HiveException {
super.init(mode, OIs);
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
weightOI = HiveUtils.asDoubleCompatibleOI(OIs[0]);
// set const values
nBins = HiveUtils.getConstInt(OIs[1]);
if (OIs.length == 3) {
autoShrink = HiveUtils.getConstBoolean(OIs[2]);
}
// check value of `num_of_bins`
if (nBins < 2) {
throw new UDFArgumentException(
"Only greater than or equal to 2 is accepted but " + nBins
+ " was passed as `num_of_bins`.");
}
quantiles = getQuantiles();
} else {
structOI = (StructObjectInspector) OIs[0];
autoShrinkField = structOI.getStructFieldRef("autoShrink");
histogramField = structOI.getStructFieldRef("histogram");
quantilesField = structOI.getStructFieldRef("quantiles");
autoShrinkOI =
(WritableBooleanObjectInspector) autoShrinkField.getFieldObjectInspector();
histogramOI =
(StandardListObjectInspector) histogramField.getFieldObjectInspector();
quantilesOI =
(StandardListObjectInspector) quantilesField.getFieldObjectInspector();
histogramElOI =
(WritableDoubleObjectInspector) histogramOI.getListElementObjectInspector();
quantileOI =
(WritableDoubleObjectInspector) quantilesOI.getListElementObjectInspector();
}
if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
return ObjectInspectorFactory.getStandardStructObjectInspector(
Arrays.asList("autoShrink", "histogram", "quantiles"), fieldOIs);
} else {
return ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
}
}
Example 15
| Source File: MulticlassOnlineClassifierUDTF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
private long loadPredictionModel(Map<Object, PredictionModel> label2model, File file,
PrimitiveObjectInspector labelOI, PrimitiveObjectInspector featureOI,
WritableFloatObjectInspector weightOI) throws IOException, SerDeException {
long count = 0L;
if (!file.exists()) {
return count;
}
if (!file.getName().endsWith(".crc")) {
if (file.isDirectory()) {
for (File f : file.listFiles()) {
count += loadPredictionModel(label2model, f, labelOI, featureOI, weightOI);
}
} else {
LazySimpleSerDe serde = HiveUtils.getLineSerde(labelOI, featureOI, weightOI);
StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector();
StructField c1ref = lineOI.getStructFieldRef("c1");
StructField c2ref = lineOI.getStructFieldRef("c2");
StructField c3ref = lineOI.getStructFieldRef("c3");
PrimitiveObjectInspector c1refOI =
(PrimitiveObjectInspector) c1ref.getFieldObjectInspector();
PrimitiveObjectInspector c2refOI =
(PrimitiveObjectInspector) c2ref.getFieldObjectInspector();
FloatObjectInspector c3refOI =
(FloatObjectInspector) c3ref.getFieldObjectInspector();
BufferedReader reader = null;
try {
reader = HadoopUtils.getBufferedReader(file);
String line;
while ((line = reader.readLine()) != null) {
count++;
Text lineText = new Text(line);
Object lineObj = serde.deserialize(lineText);
List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj);
Object f0 = fields.get(0);
Object f1 = fields.get(1);
Object f2 = fields.get(2);
if (f0 == null || f1 == null || f2 == null) {
continue; // avoid the case that key or value is null
}
Object label = c1refOI.getPrimitiveWritableObject(c1refOI.copyObject(f0));
PredictionModel model = label2model.get(label);
if (model == null) {
model = createModel();
label2model.put(label, model);
}
Object k = c2refOI.getPrimitiveWritableObject(c2refOI.copyObject(f1));
float v = c3refOI.get(f2);
model.set(k, new WeightValue(v, false));
}
} finally {
IOUtils.closeQuietly(reader);
}
}
}
return count;
}
Example 16
| Source File: UDAFToOrderedList.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
super.init(mode, argOIs);
// initialize input
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
// this flag will be used in `processOptions` and `iterate` (= when Mode.PARTIAL1 or Mode.COMPLETE)
this.sortByKey = (argOIs.length == 2 && !HiveUtils.isConstString(argOIs[1]))
|| (argOIs.length == 3 && HiveUtils.isConstString(argOIs[2]));
if (sortByKey) {
this.valueOI = argOIs[0];
this.keyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[1]);
} else {
// sort values by value itself
this.valueOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
this.keyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
}
processOptions(argOIs);
} else {// from partial aggregation
StructObjectInspector soi = (StructObjectInspector) argOIs[0];
this.internalMergeOI = soi;
// re-extract input value OI
this.valueListField = soi.getStructFieldRef("valueList");
StandardListObjectInspector valueListOI =
(StandardListObjectInspector) valueListField.getFieldObjectInspector();
this.valueOI = valueListOI.getListElementObjectInspector();
this.valueListOI = ObjectInspectorFactory.getStandardListObjectInspector(valueOI);
// re-extract input key OI
this.keyListField = soi.getStructFieldRef("keyList");
StandardListObjectInspector keyListOI =
(StandardListObjectInspector) keyListField.getFieldObjectInspector();
this.keyOI = HiveUtils.asPrimitiveObjectInspector(
keyListOI.getListElementObjectInspector());
this.keyListOI = ObjectInspectorFactory.getStandardListObjectInspector(keyOI);
this.sizeField = soi.getStructFieldRef("size");
this.reverseOrderField = soi.getStructFieldRef("reverseOrder");
List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
this.outKVField = HiveUtils.getStructFieldRef("outKV", fieldRefs);
if (outKVField != null) {
this.outKV = true;
}
this.outVKField = HiveUtils.getStructFieldRef("outVK", fieldRefs);
if (outVKField != null) {
this.outVK = true;
}
}
// initialize output
final ObjectInspector outputOI;
if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
outputOI = internalMergeOI(valueOI, keyOI, outKV, outVK);
} else {// terminate
if (outKV) {
outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
ObjectInspectorUtils.getStandardObjectInspector(keyOI),
ObjectInspectorUtils.getStandardObjectInspector(valueOI));
} else if (outVK) {
outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
ObjectInspectorUtils.getStandardObjectInspector(valueOI),
ObjectInspectorUtils.getStandardObjectInspector(keyOI));
} else {
outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorUtils.getStandardObjectInspector(valueOI));
}
}
return outputOI;
}
Example 17
| Source File: DistributedCacheLookupUDF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
private static void loadValues(Object2ObjectMap<Object, Object> map, File file,
PrimitiveObjectInspector keyOI, PrimitiveObjectInspector valueOI)
throws IOException, SerDeException {
if (!file.exists()) {
return;
}
if (!file.getName().endsWith(".crc")) {
if (file.isDirectory()) {
for (File f : file.listFiles()) {
loadValues(map, f, keyOI, valueOI);
}
} else {
LazySimpleSerDe serde = HiveUtils.getKeyValueLineSerde(keyOI, valueOI);
StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector();
StructField keyRef = lineOI.getStructFieldRef("key");
StructField valueRef = lineOI.getStructFieldRef("value");
PrimitiveObjectInspector keyRefOI =
(PrimitiveObjectInspector) keyRef.getFieldObjectInspector();
PrimitiveObjectInspector valueRefOI =
(PrimitiveObjectInspector) valueRef.getFieldObjectInspector();
BufferedReader reader = null;
try {
reader = HadoopUtils.getBufferedReader(file);
String line;
while ((line = reader.readLine()) != null) {
Text lineText = new Text(line);
Object lineObj = serde.deserialize(lineText);
List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj);
Object f0 = fields.get(0);
Object f1 = fields.get(1);
Object k = keyRefOI.getPrimitiveJavaObject(f0);
Object v = valueRefOI.getPrimitiveWritableObject(valueRefOI.copyObject(f1));
map.put(k, v);
}
} finally {
IOUtils.closeQuietly(reader);
}
}
}
}
Example 18
| Source File: PLSAPredictUDAF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
assert (parameters.length == 1 || parameters.length == 4 || parameters.length == 5);
super.init(mode, parameters);
// initialize input
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
processOptions(parameters);
this.wordOI = HiveUtils.asStringOI(parameters[0]);
this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]);
this.labelOI = HiveUtils.asIntegerOI(parameters[2]);
this.probOI = HiveUtils.asDoubleCompatibleOI(parameters[3]);
} else {// from partial aggregation
StructObjectInspector soi = (StructObjectInspector) parameters[0];
this.internalMergeOI = soi;
this.wcListField = soi.getStructFieldRef("wcList");
this.probMapField = soi.getStructFieldRef("probMap");
this.topicsOptionField = soi.getStructFieldRef("topics");
this.alphaOptionField = soi.getStructFieldRef("alpha");
this.deltaOptionField = soi.getStructFieldRef("delta");
this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI);
this.probMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
this.probMapValueElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
this.probMapValueOI =
ObjectInspectorFactory.getStandardListObjectInspector(probMapValueElemOI);
this.probMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(probMapKeyOI,
probMapValueOI);
}
// initialize output
final ObjectInspector outputOI;
if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
outputOI = internalMergeOI();
} else {
final ArrayList<String> fieldNames = new ArrayList<String>();
final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
fieldNames.add("label");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
fieldNames.add("probability");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs));
}
return outputOI;
}
Example 19
| Source File: LDAPredictUDAF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
assert (parameters.length == 1 || parameters.length == 4 || parameters.length == 5);
super.init(mode, parameters);
// initialize input
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
processOptions(parameters);
this.wordOI = HiveUtils.asStringOI(parameters[0]);
this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]);
this.labelOI = HiveUtils.asIntegerOI(parameters[2]);
this.lambdaOI = HiveUtils.asDoubleCompatibleOI(parameters[3]);
} else {// from partial aggregation
StructObjectInspector soi = (StructObjectInspector) parameters[0];
this.internalMergeOI = soi;
this.wcListField = soi.getStructFieldRef("wcList");
this.lambdaMapField = soi.getStructFieldRef("lambdaMap");
this.topicsOptionField = soi.getStructFieldRef("topics");
this.alphaOptionField = soi.getStructFieldRef("alpha");
this.deltaOptionField = soi.getStructFieldRef("delta");
this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI);
this.lambdaMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
this.lambdaMapValueElemOI =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
this.lambdaMapValueOI =
ObjectInspectorFactory.getStandardListObjectInspector(lambdaMapValueElemOI);
this.lambdaMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
lambdaMapKeyOI, lambdaMapValueOI);
}
// initialize output
final ObjectInspector outputOI;
if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
outputOI = internalMergeOI();
} else {
final ArrayList<String> fieldNames = new ArrayList<String>();
final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
fieldNames.add("label");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
fieldNames.add("probability");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs));
}
return outputOI;
}
Example 20
| Source File: HiveJsonStructReader.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
private StructField getStructField(StructObjectInspector oi, String name) {
return oi.getStructFieldRef(name);
}
