Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory#getPrimitiveJavaObjectInspector()
The following examples show how to use
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory#getPrimitiveJavaObjectInspector() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TsFileSerDe.java From incubator-iotdb with Apache License 2.0 | 6 votes |
|
private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws TsFileSerDeException {
if(!supportedCategories(ti)) {
throw new TsFileSerDeException("Don't yet support this type: " + ti);
}
ObjectInspector result;
switch(ti.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
result = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti);
break;
// these types is not supported in TsFile
case LIST:
case MAP:
case STRUCT:
case UNION:
throw new TsFileSerDeException("The type is not supported in TsFile: " + ti);
default:
throw new TsFileSerDeException("No Hive categories matched: " + ti);
}
return result;
}
Example 2
| Source File: HiveTestUDFImpls.java From dremio-oss with Apache License 2.0 | 6 votes |
|
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length != 1) {
throw new UDFArgumentLengthException(String.format("%s needs 1 argument, got %d", udfName, arguments.length));
}
if (arguments[0].getCategory() != Category.PRIMITIVE ||
((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() != inputType) {
String actual = arguments[0].getCategory() + (arguments[0].getCategory() == Category.PRIMITIVE ?
"[" + ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() + "]" : "");
throw new UDFArgumentException(
String.format("%s only takes primitive type %s, got %s", udfName, inputType, actual));
}
argumentOI = arguments[0];
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(outputType);
}
Example 3
| Source File: Geoloc.java From hiped2 with Apache License 2.0 | 6 votes |
|
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length != 2) {
throw new UDFArgumentLengthException(
"The function COUNTRY(ip, geolocfile) takes exactly 2 arguments.");
}
converters = new ObjectInspectorConverters.Converter[arguments.length];
for (int i = 0; i < arguments.length; i++) {
converters[i] = ObjectInspectorConverters.getConverter(arguments[i],
PrimitiveObjectInspectorFactory.javaStringObjectInspector);
}
return PrimitiveObjectInspectorFactory
.getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
}
Example 4
| Source File: BlurObjectInspectorGenerator.java From incubator-retired-blur with Apache License 2.0 | 6 votes |
|
private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeException {
switch (ti.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti);
case STRUCT:
StructTypeInfo sti = (StructTypeInfo) ti;
List<ObjectInspector> ois = new ArrayList<ObjectInspector>(sti.getAllStructFieldTypeInfos().size());
for (TypeInfo typeInfo : sti.getAllStructFieldTypeInfos()) {
ois.add(createObjectInspectorWorker(typeInfo));
}
return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois);
case LIST:
ListTypeInfo lti = (ListTypeInfo) ti;
TypeInfo listElementTypeInfo = lti.getListElementTypeInfo();
return ObjectInspectorFactory.getStandardListObjectInspector(createObjectInspectorWorker(listElementTypeInfo));
default:
throw new SerDeException("No Hive categories matched for [" + ti + "]");
}
}
Example 5
| Source File: HiveInspectors.java From flink with Apache License 2.0 | 5 votes |
|
private static ObjectInspector getObjectInspector(TypeInfo type) {
switch (type.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
case LIST:
ListTypeInfo listType = (ListTypeInfo) type;
return ObjectInspectorFactory.getStandardListObjectInspector(
getObjectInspector(listType.getListElementTypeInfo()));
case MAP:
MapTypeInfo mapType = (MapTypeInfo) type;
return ObjectInspectorFactory.getStandardMapObjectInspector(
getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
case STRUCT:
StructTypeInfo structType = (StructTypeInfo) type;
List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
for (TypeInfo fieldType : fieldTypes) {
fieldInspectors.add(getObjectInspector(fieldType));
}
return ObjectInspectorFactory.getStandardStructObjectInspector(
structType.getAllStructFieldNames(), fieldInspectors);
default:
throw new CatalogException("Unsupported Hive type category " + type.getCategory());
}
}
Example 6
| Source File: MajorityVoteUDAF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@Override
public ObjectInspector init(@Nonnull Mode mode, @Nonnull ObjectInspector[] argOIs)
throws HiveException {
assert (argOIs.length == 1);
super.init(mode, argOIs);
// initialize input
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
this.keyInputOI = asPrimitiveObjectInspector(argOIs[0]);
} else {// from partial aggregation
this.partialOI = (StandardMapObjectInspector) argOIs[0];
this.keyInputOI = asPrimitiveObjectInspector(partialOI.getMapKeyObjectInspector());
this.counterInputOI = asLongOI(partialOI.getMapValueObjectInspector());
}
this.keyOutputOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
keyInputOI.getTypeInfo());
// initialize output
final ObjectInspector outputOI;
if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(keyOutputOI,
PrimitiveObjectInspectorFactory.javaLongObjectInspector);
} else {// terminate
outputOI = keyOutputOI;
}
return outputOI;
}
Example 7
| Source File: FMeasureUDAFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
private void binarySetUp(Object actual, Object predicted, double beta, String average)
throws Exception {
fmeasure = new FMeasureUDAF();
inputOIs = new ObjectInspector[3];
String actualClassName = actual.getClass().getName();
if (actualClassName.equals("java.lang.Integer")) {
inputOIs[0] = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.INT);
} else if (actualClassName.equals("java.lang.Boolean")) {
inputOIs[0] = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN);
} else if ((actualClassName.equals("java.lang.String"))) {
inputOIs[0] = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.STRING);
}
String predicatedClassName = predicted.getClass().getName();
if (predicatedClassName.equals("java.lang.Integer")) {
inputOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.INT);
} else if (predicatedClassName.equals("java.lang.Boolean")) {
inputOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN);
} else if ((predicatedClassName.equals("java.lang.String"))) {
inputOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.STRING);
}
inputOIs[2] = ObjectInspectorUtils.getConstantObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector,
"-beta " + beta + " -average " + average);
evaluator =
fmeasure.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));
agg = (FMeasureUDAF.FMeasureAggregationBuffer) evaluator.getNewAggregationBuffer();
}
Example 8
| Source File: PLSAPredictUDAFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@SuppressWarnings("unchecked")
@Test
public void testTerminateWithSameTopicProbability() throws Exception {
udaf = new PLSAPredictUDAF();
inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.STRING),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.INT),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
ObjectInspectorUtils.getConstantObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-topics 2")};
evaluator = udaf.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));
agg = (PLSAPredictUDAF.PLSAPredictAggregationBuffer) evaluator.getNewAggregationBuffer();
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
// Assume that all words in a document are NOT in vocabulary that composes a LDA model.
// Hence, the document should be assigned to topic #1 (#2) with probability 0.5 (0.5).
for (int i = 0; i < words.length; i++) {
String word = words[i];
evaluator.iterate(agg, new Object[] {word, 0.f, labels[i], probs[i]});
}
// Probability for each of the two topics should be same.
List<Object[]> result = (List<Object[]>) evaluator.terminate(agg);
Assert.assertEquals(result.size(), 2);
Assert.assertEquals(result.get(0)[1], result.get(1)[1]);
}
Example 9
| Source File: LDAPredictUDAFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@SuppressWarnings("unchecked")
@Test
public void testTerminateWithSameTopicProbability() throws Exception {
udaf = new LDAPredictUDAF();
inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.STRING),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.INT),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
ObjectInspectorUtils.getConstantObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-topics 2")};
evaluator = udaf.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));
agg = (LDAPredictUDAF.OnlineLDAPredictAggregationBuffer) evaluator.getNewAggregationBuffer();
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
// Assume that all words in a document are NOT in vocabulary that composes a LDA model.
// Hence, the document should be assigned to topic #1 (#2) with probability 0.5 (0.5).
for (int i = 0; i < 18; i++) {
evaluator.iterate(agg, new Object[] {words[i], 0.f, labels[i], lambdas[i]});
}
// Probability for each of the two topics should be same.
List<Object[]> result = (List<Object[]>) evaluator.terminate(agg);
Assert.assertEquals(result.size(), 2);
Assert.assertEquals(result.get(0)[1], result.get(1)[1]);
}
Example 10
| Source File: HiveInspectors.java From flink with Apache License 2.0 | 5 votes |
|
private static ObjectInspector getObjectInspector(TypeInfo type) {
switch (type.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
case LIST:
ListTypeInfo listType = (ListTypeInfo) type;
return ObjectInspectorFactory.getStandardListObjectInspector(
getObjectInspector(listType.getListElementTypeInfo()));
case MAP:
MapTypeInfo mapType = (MapTypeInfo) type;
return ObjectInspectorFactory.getStandardMapObjectInspector(
getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
case STRUCT:
StructTypeInfo structType = (StructTypeInfo) type;
List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
for (TypeInfo fieldType : fieldTypes) {
fieldInspectors.add(getObjectInspector(fieldType));
}
return ObjectInspectorFactory.getStandardStructObjectInspector(
structType.getAllStructFieldNames(), fieldInspectors);
default:
throw new CatalogException("Unsupported Hive type category " + type.getCategory());
}
}
Example 11
| Source File: XmlObjectInspectorFactory.java From Hive-XML-SerDe with Apache License 2.0 | 5 votes |
|
/**
* Returns the standard java object inspector
*
* @param typeInfo
* the type info
* @param xmlProcessor
* the XML processor
* @return the standard java object inspector
*/
public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo(TypeInfo typeInfo, XmlProcessor xmlProcessor) {
switch (typeInfo.getCategory()) {
case PRIMITIVE: {
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
}
case LIST: {
ObjectInspector listElementObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo).getListElementTypeInfo(),
xmlProcessor);
return new XmlListObjectInspector(listElementObjectInspector, xmlProcessor);
}
case MAP: {
MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
ObjectInspector mapKeyObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo(),
xmlProcessor);
ObjectInspector mapValueObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo(),
xmlProcessor);
return new XmlMapObjectInspector(mapKeyObjectInspector, mapValueObjectInspector, xmlProcessor);
}
case STRUCT: {
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
List<String> structFieldNames = structTypeInfo.getAllStructFieldNames();
List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
for (int fieldIndex = 0; fieldIndex < fieldTypeInfos.size(); ++fieldIndex) {
structFieldObjectInspectors.add(getStandardJavaObjectInspectorFromTypeInfo(fieldTypeInfos.get(fieldIndex), xmlProcessor));
}
return getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors, xmlProcessor);
}
default: {
throw new IllegalStateException();
}
}
}
Example 12
| Source File: AUCUDAFTest.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
@Before
public void setUp() throws Exception {
auc = new AUCUDAF();
inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.DOUBLE),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.INT)};
evaluator = auc.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));
ArrayList<String> fieldNames = new ArrayList<String>();
ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
fieldNames.add("indexScore");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
fieldNames.add("area");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
fieldNames.add("fp");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
fieldNames.add("tp");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
fieldNames.add("fpPrev");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
fieldNames.add("tpPrev");
fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
MapObjectInspector areaPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
fieldNames.add("areaPartialMap");
fieldOIs.add(areaPartialMapOI);
MapObjectInspector fpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
PrimitiveObjectInspectorFactory.writableLongObjectInspector);
fieldNames.add("fpPartialMap");
fieldOIs.add(fpPartialMapOI);
MapObjectInspector tpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
PrimitiveObjectInspectorFactory.writableLongObjectInspector);
fieldNames.add("tpPartialMap");
fieldOIs.add(tpPartialMapOI);
MapObjectInspector fpPrevPartialMapOI =
ObjectInspectorFactory.getStandardMapObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
PrimitiveObjectInspectorFactory.writableLongObjectInspector);
fieldNames.add("fpPrevPartialMap");
fieldOIs.add(fpPrevPartialMapOI);
MapObjectInspector tpPrevPartialMapOI =
ObjectInspectorFactory.getStandardMapObjectInspector(
PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
PrimitiveObjectInspectorFactory.writableLongObjectInspector);
fieldNames.add("tpPrevPartialMap");
fieldOIs.add(tpPrevPartialMapOI);
partialOI = new ObjectInspector[2];
partialOI[0] =
ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
agg = (AUCUDAF.ClassificationAUCAggregationBuffer) evaluator.getNewAggregationBuffer();
}
Example 13
| Source File: PLSAPredictUDAFTest.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
@Test
public void testMerge() throws Exception {
udaf = new PLSAPredictUDAF();
inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.STRING),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.INT),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
ObjectInspectorUtils.getConstantObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-topics 2")};
evaluator = udaf.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));
agg = (PLSAPredictUDAF.PLSAPredictAggregationBuffer) evaluator.getNewAggregationBuffer();
final Map<String, Float> doc = new HashMap<String, Float>();
doc.put("apples", 1.f);
doc.put("avocados", 1.f);
doc.put("colds", 1.f);
doc.put("flu", 1.f);
doc.put("like", 2.f);
doc.put("oranges", 1.f);
Object[] partials = new Object[3];
// bin #1
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
for (int i = 0; i < 6; i++) {
evaluator.iterate(agg, new Object[] {words[i], doc.get(words[i]), labels[i], probs[i]});
}
partials[0] = evaluator.terminatePartial(agg);
// bin #2
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
for (int i = 6; i < 12; i++) {
evaluator.iterate(agg, new Object[] {words[i], doc.get(words[i]), labels[i], probs[i]});
}
partials[1] = evaluator.terminatePartial(agg);
// bin #3
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
for (int i = 12; i < 18; i++) {
evaluator.iterate(agg, new Object[] {words[i], doc.get(words[i]), labels[i], probs[i]});
}
partials[2] = evaluator.terminatePartial(agg);
// merge in a different order
final int[][] orders = new int[][] {{0, 1, 2}, {1, 0, 2}, {1, 2, 0}, {2, 1, 0}};
for (int i = 0; i < orders.length; i++) {
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL2, partialOI);
evaluator.reset(agg);
evaluator.merge(agg, partials[orders[i][0]]);
evaluator.merge(agg, partials[orders[i][1]]);
evaluator.merge(agg, partials[orders[i][2]]);
float[] distr = agg.get();
Assert.assertTrue(distr[0] < distr[1]);
}
}
Example 14
| Source File: LDAPredictUDAFTest.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
@Test
public void testMerge() throws Exception {
udaf = new LDAPredictUDAF();
inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.STRING),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.INT),
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
ObjectInspectorUtils.getConstantObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-topics 2")};
evaluator = udaf.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));
agg = (LDAPredictUDAF.OnlineLDAPredictAggregationBuffer) evaluator.getNewAggregationBuffer();
final Map<String, Float> doc = new HashMap<String, Float>();
doc.put("apples", 1.f);
doc.put("avocados", 1.f);
doc.put("colds", 1.f);
doc.put("flu", 1.f);
doc.put("like", 2.f);
doc.put("oranges", 1.f);
Object[] partials = new Object[3];
// bin #1
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
for (int i = 0; i < 6; i++) {
evaluator.iterate(agg,
new Object[] {words[i], doc.get(words[i]), labels[i], lambdas[i]});
}
partials[0] = evaluator.terminatePartial(agg);
// bin #2
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
for (int i = 6; i < 12; i++) {
evaluator.iterate(agg,
new Object[] {words[i], doc.get(words[i]), labels[i], lambdas[i]});
}
partials[1] = evaluator.terminatePartial(agg);
// bin #3
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
for (int i = 12; i < 18; i++) {
evaluator.iterate(agg,
new Object[] {words[i], doc.get(words[i]), labels[i], lambdas[i]});
}
partials[2] = evaluator.terminatePartial(agg);
// merge in a different order
final int[][] orders = new int[][] {{0, 1, 2}, {1, 0, 2}, {1, 2, 0}, {2, 1, 0}};
for (int i = 0; i < orders.length; i++) {
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL2, partialOI);
evaluator.reset(agg);
evaluator.merge(agg, partials[orders[i][0]]);
evaluator.merge(agg, partials[orders[i][1]]);
evaluator.merge(agg, partials[orders[i][2]]);
float[] distr = agg.get();
Assert.assertTrue(distr[0] < distr[1]);
}
}
Example 15
| Source File: CobolNumberField.java From Cobol-to-Hive with Apache License 2.0 | 4 votes |
|
@Override
public Object deserialize(byte[] rowBytes) throws CobolSerdeException {
byte[] temp = transcodeField(super.getBytes(rowBytes));
String s1 = new String(temp);
if (this.compType > 0) {
if (this.compType == 3) {
s1 = unpackData(super.getBytes(rowBytes), this.decimalLocation);
}else if(this.compType == 4){
s1 = getBinary(super.getBytes(rowBytes), this.decimalLocation);
}
} //} else if (this.decimalLocation > 0) {
else {
//Now calling unpackSign on all numeric fields for which compType resolves to 0.
//
//The function will check to see if the least significant byte has been overpunched with a sign and
//return a negative number if a negative sign is found.
s1 = unpackSign(super.getBytes(rowBytes), this.decimalLocation);
}
// else if (this.decimalLocation > 0) {
// s1 = s1.substring(0, this.length * this.divideFactor
// - this.decimalLocation)
// + "."
// + s1.substring(this.length * this.divideFactor
// - this.decimalLocation);
// }
// System.out.println(name + "\t - " + s1 + "\t:" + offset + "\t@"
// + length);
try {
switch (((PrimitiveTypeInfo) this.typeInfo).getPrimitiveCategory()) {
case LONG:
return Long.parseLong(s1.trim());
case SHORT:
return Short.parseShort(s1.trim());
case INT:
return Integer.parseInt(s1.trim());
case BYTE:
return Byte.parseByte(s1.trim());
case FLOAT:
return Float.parseFloat(s1.trim());
case DOUBLE:
return Double.parseDouble(s1.trim());
case DECIMAL:
BigDecimal bd = new BigDecimal(s1);
HiveDecimal dec = HiveDecimal.create(bd);
JavaHiveDecimalObjectInspector oi = (JavaHiveDecimalObjectInspector) PrimitiveObjectInspectorFactory
.getPrimitiveJavaObjectInspector((DecimalTypeInfo) this.typeInfo);
return oi.set(null, dec);
}
} catch (Exception e) {
return null; // if cannot be converted make it null
}
return null;
}
