Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardStructObjectInspector()
The following examples show how to use
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardStructObjectInspector() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GeneralLearnerBaseUDTF.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Nonnull protected StructObjectInspector getReturnOI(@Nonnull ObjectInspector featureOutputOI) { ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("feature"); fieldOIs.add(featureOutputOI); fieldNames.add("weight"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector); if (useCovariance()) { fieldNames.add("covar"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector); } return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 2
Source File: FMeasureUDAF.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Nonnull private static StructObjectInspector internalMergeOI() { List<String> fieldNames = new ArrayList<>(); List<ObjectInspector> fieldOIs = new ArrayList<>(); fieldNames.add("tp"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("totalActual"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("totalPredicted"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("beta"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); fieldNames.add("average"); fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 3
Source File: UnionDoubleSummaryWithModeSketchUDAF.java From incubator-datasketches-hive with Apache License 2.0 | 6 votes |
@Override public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException { final ObjectInspector resultInspector = super.init(mode, inspectors); if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) { // input is original data if (inspectors.length > 2) { summaryModeInspector_ = (PrimitiveObjectInspector) inspectors[2]; } } if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) { // intermediate results need to include the nominal number of entries and the summary mode return ObjectInspectorFactory.getStandardStructObjectInspector( Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SUMMARY_MODE_FIELD, SKETCH_FIELD), Arrays.asList( PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT), PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.STRING), PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY) ) ); } return resultInspector; }
Example 4
Source File: UnionSketchUDAF.java From incubator-datasketches-hive with Apache License 2.0 | 5 votes |
@Override public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException { super.init(mode, inspectors); if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) { // input is original data sketchInspector_ = (PrimitiveObjectInspector) inspectors[0]; if (inspectors.length > 1) { nominalNumEntriesInspector_ = (PrimitiveObjectInspector) inspectors[1]; } } else { // input for PARTIAL2 and FINAL is the output from PARTIAL1 intermediateInspector_ = (StructObjectInspector) inspectors[0]; } if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) { // intermediate results need to include the the nominal number of entries return ObjectInspectorFactory.getStandardStructObjectInspector( Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SKETCH_FIELD), Arrays.asList( PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT), PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY) ) ); } // final results include just the sketch return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY); }
Example 5
Source File: TestRecordReaderImpl.java From hive-dwrf with Apache License 2.0 | 5 votes |
public static StructObjectInspector getObjectInspectorFor(final ImmutableList<String> names, final ImmutableList<String> columnTypeNames) { List<ObjectInspector> inspectors = new ArrayList<>(); for (int i = 0; i < columnTypeNames.size(); i++) { inspectors.add(createJavaObjectInspectorFromFieldSchema(columnTypeNames.get(i))); } return ObjectInspectorFactory.getStandardStructObjectInspector( names, inspectors); }
Example 6
Source File: TsFileSerDe.java From incubator-iotdb with Apache License 2.0 | 5 votes |
private ObjectInspector createObjectInspector() throws TsFileSerDeException { List<ObjectInspector> columnOIs = new ArrayList<>(columnNames.size()); // At this point we've verified the types are correct. for(int i = 0; i < columnNames.size(); i++) { columnOIs.add(i, createObjectInspectorWorker(columnTypes.get(i))); } return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs); }
Example 7
Source File: ArrayAvgGenericUDAF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
private static StructObjectInspector internalMergeOI() { ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("size"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); fieldNames.add("sum"); fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)); fieldNames.add("count"); fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableLongObjectInspector)); return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 8
Source File: RandomForestEnsembleUDAF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Override public ObjectInspector init(@Nonnull Mode mode, @Nonnull ObjectInspector[] argOIs) throws HiveException { super.init(mode, argOIs); // initialize input if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data this.yhatOI = HiveUtils.asIntegerOI(argOIs, 0); } else {// from partial aggregation this.internalMergeOI = (StandardMapObjectInspector) argOIs[0]; this.keyOI = HiveUtils.asIntOI(internalMergeOI.getMapKeyObjectInspector()); this.valueOI = HiveUtils.asIntOI(internalMergeOI.getMapValueObjectInspector()); } // initialize output final ObjectInspector outputOI; if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial outputOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaIntObjectInspector); } else {// terminate List<String> fieldNames = new ArrayList<>(3); List<ObjectInspector> fieldOIs = new ArrayList<>(3); fieldNames.add("label"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); fieldNames.add("probability"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); fieldNames.add("probabilities"); fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)); outputOI = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); } return outputOI; }
Example 9
Source File: LDAPredictUDAFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Before public void setUp() throws Exception { ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("wcList"); fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.javaStringObjectInspector)); fieldNames.add("lambdaMap"); fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaStringObjectInspector, ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.javaFloatObjectInspector))); fieldNames.add("topics"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); fieldNames.add("alpha"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector); fieldNames.add("delta"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); partialOI = new ObjectInspector[4]; partialOI[0] = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); words = new String[] {"fruits", "vegetables", "healthy", "flu", "apples", "oranges", "like", "avocados", "colds", "colds", "avocados", "oranges", "like", "apples", "flu", "healthy", "vegetables", "fruits"}; labels = new int[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1}; lambdas = new float[] {0.3339331f, 0.3324783f, 0.33209667f, 3.2804057E-4f, 3.0303953E-4f, 2.4860457E-4f, 2.41481E-4f, 2.3554532E-4f, 1.352576E-4f, 0.1660153f, 0.16596903f, 0.1659654f, 0.1659627f, 0.16593699f, 0.1659259f, 0.0017611005f, 0.0015791848f, 8.84464E-4f}; }
Example 10
Source File: UnionSketchUDAF.java From incubator-datasketches-hive with Apache License 2.0 | 5 votes |
/** * Receives the passed in argument object inspectors and returns the desired * return type's object inspector to inform hive of return type of UDAF. * * @param mode * Mode (i.e. PARTIAL 1, COMPLETE...) for determining input and output * object inspector type. * @param parameters * List of object inspectors for input arguments. * @return The object inspector type indicates the UDAF return type (i.e. * returned type of terminate(...)). */ @Override public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException { super.init(mode, parameters); if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) { inputInspector_ = (PrimitiveObjectInspector) parameters[0]; if (parameters.length > 1) { lgKInspector_ = (PrimitiveObjectInspector) parameters[1]; } if (parameters.length > 2) { seedInspector_ = (PrimitiveObjectInspector) parameters[2]; } } else { // mode = partial2 || final intermediateInspector_ = (StandardStructObjectInspector) parameters[0]; } if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) { // intermediate results need to include the lgK and the target HLL type return ObjectInspectorFactory.getStandardStructObjectInspector( Arrays.asList(LG_K_FIELD, SEED_FIELD, SKETCH_FIELD), Arrays.asList( PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT), PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.LONG), PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY) ) ); } // final results include just the sketch return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY); }
Example 11
Source File: DynamoDBSerDeTest.java From emr-dynamodb-connector with Apache License 2.0 | 5 votes |
private Map<String, AttributeValue> getSerializedItem(List<String> attributeNames, List<ObjectInspector> colOIs, Map<String, String> typeMapping, List<Object> rowData, boolean nullSerialization) throws SerDeException { List<String> colTypes = Lists.newArrayList(); List<String> colMappings = Lists.newArrayList(); for (int i = 0; i < attributeNames.size(); i++) { String attributeName = attributeNames.get(i); colTypes.add(colOIs.get(i).getTypeName()); if (!HiveDynamoDBTypeFactory.isHiveDynamoDBItemMapType(colOIs.get(i)) || typeMapping.containsKey(attributeName)) { colMappings.add(attributeName + ":" + attributeName); } } List<String> typeMapList = Lists.newArrayList(); for (Map.Entry<String, String> colType : typeMapping.entrySet()) { typeMapList.add(colType.getKey() + ":" + colType.getValue()); } Properties props = new Properties(); props.setProperty(serdeConstants.LIST_COLUMNS, StringUtils.join(attributeNames, ",")); props.setProperty(serdeConstants.LIST_COLUMN_TYPES, StringUtils.join(colTypes, ",")); props.setProperty(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, StringUtils.join(colMappings, ",")); props.setProperty(DynamoDBConstants.DYNAMODB_TYPE_MAPPING, StringUtils.join(typeMapList, ",")); props.setProperty(DynamoDBConstants.DYNAMODB_NULL_SERIALIZATION, Boolean.toString(nullSerialization)); DynamoDBSerDe serde = new DynamoDBSerDe(); serde.initialize(null, props); StructObjectInspector rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(attributeNames, colOIs); DynamoDBItemWritable item = (DynamoDBItemWritable) serde.serialize(rowData, rowOI); return item.getItem(); }
Example 12
Source File: BinarizeLabelUDTF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Override public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { if (argOIs.length < 3) { throw new UDFArgumentException("binarize_label(int/long positive, " + "int/long negative, *) takes at least three arguments"); } this.positiveOI = HiveUtils.asIntCompatibleOI(argOIs[0]); this.negativeOI = HiveUtils.asIntCompatibleOI(argOIs[1]); this.positiveObjs = new Object[argOIs.length - 1]; positiveObjs[positiveObjs.length - 1] = 1; this.negativeObjs = new Object[argOIs.length - 1]; negativeObjs[negativeObjs.length - 1] = 0; ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); for (int i = 2; i < argOIs.length; i++) { fieldNames.add("c" + (i - 2)); // Use negative label ObjectInspector here. OIs for positive // label and negative labels must be same. fieldOIs.add(argOIs[i]); } fieldNames.add("c" + (argOIs.length - 2)); fieldOIs.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 13
Source File: HiveWriterFactory.java From flink with Apache License 2.0 | 5 votes |
private void checkInitialize() throws Exception { if (initialized) { return; } JobConf jobConf = confWrapper.conf(); Object serdeLib = Class.forName(serDeInfo.getSerializationLib()).newInstance(); Preconditions.checkArgument(serdeLib instanceof Serializer && serdeLib instanceof Deserializer, "Expect a SerDe lib implementing both Serializer and Deserializer, but actually got " + serdeLib.getClass().getName()); this.recordSerDe = (Serializer) serdeLib; ReflectionUtils.setConf(recordSerDe, jobConf); // TODO: support partition properties, for now assume they're same as table properties SerDeUtils.initializeSerDe((Deserializer) recordSerDe, jobConf, tableProperties, null); this.formatFields = allColumns.length - partitionColumns.length; this.hiveConversions = new HiveObjectConversion[formatFields]; this.converters = new DataFormatConverter[formatFields]; List<ObjectInspector> objectInspectors = new ArrayList<>(hiveConversions.length); for (int i = 0; i < formatFields; i++) { DataType type = allTypes[i]; ObjectInspector objectInspector = HiveInspectors.getObjectInspector(type); objectInspectors.add(objectInspector); hiveConversions[i] = HiveInspectors.getConversion( objectInspector, type.getLogicalType(), hiveShim); converters[i] = DataFormatConverters.getConverterForDataType(type); } this.formatInspector = ObjectInspectorFactory.getStandardStructObjectInspector( Arrays.asList(allColumns).subList(0, formatFields), objectInspectors); this.initialized = true; }
Example 14
Source File: DIMSUMMapperUDTF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Override public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { if (argOIs.length != 2 && argOIs.length != 3) { throw new UDFArgumentException(getClass().getSimpleName() + " takes 2 or 3 arguments: array<string> x, map<long, double> colNorms " + "[, CONSTANT STRING options]: " + Arrays.toString(argOIs)); } this.rowOI = HiveUtils.asListOI(argOIs[0]); HiveUtils.validateFeatureOI(rowOI.getListElementObjectInspector()); this.colNormsOI = HiveUtils.asMapOI(argOIs[1]); processOptions(argOIs); this.rnd = RandomNumberGeneratorFactory.createPRNG(1001); this.colNorms = null; this.colProbs = null; ArrayList<String> fieldNames = new ArrayList<String>(); fieldNames.add("j"); fieldNames.add("k"); fieldNames.add("b_jk"); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); if (parseFeatureAsInt) { fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); } else { fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector); fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector); } fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 15
Source File: XGBoostTrainUDTF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Override public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs) throws UDFArgumentException { if (argOIs.length != 2 && argOIs.length != 3) { showHelp("Invalid argment length=" + argOIs.length); } processOptions(argOIs); ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0); ObjectInspector elemOI = listOI.getListElementObjectInspector(); this.featureListOI = listOI; if (HiveUtils.isNumberOI(elemOI)) { this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI); this.denseInput = true; this.matrixBuilder = new DenseDMatrixBuilder(8192); } else if (HiveUtils.isStringOI(elemOI)) { this.featureElemOI = HiveUtils.asStringOI(elemOI); this.denseInput = false; this.matrixBuilder = new SparseDMatrixBuilder(8192); } else { throw new UDFArgumentException( "train_xgboost takes array<double> or array<string> for the first argument: " + listOI.getTypeName()); } this.targetOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1); this.labels = new FloatArrayList(1024); final List<String> fieldNames = new ArrayList<>(2); final List<ObjectInspector> fieldOIs = new ArrayList<>(2); fieldNames.add("model_id"); fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); fieldNames.add("model"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector); return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 16
Source File: HiveGenericUDTFTest.java From flink with Apache License 2.0 | 4 votes |
@Override public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { return ObjectInspectorFactory.getStandardStructObjectInspector( Collections.singletonList("col1"), Collections.singletonList(PrimitiveObjectInspectorFactory.javaStringObjectInspector)); }
Example 17
Source File: SingularSpectrumTransformUDF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
@Override public ObjectInspector initialize(@Nonnull ObjectInspector[] argOIs) throws UDFArgumentException { if (argOIs.length < 1 || argOIs.length > 2) { throw new UDFArgumentException( "_FUNC_(double|array<double> x [, const string options]) takes 1 or 2 arguments: " + Arrays.toString(argOIs)); } this._params = new Parameters(); if (argOIs.length == 2) { String options = HiveUtils.getConstString(argOIs[1]); processOptions(options); } ObjectInspector argOI0 = argOIs[0]; PrimitiveObjectInspector xOI = HiveUtils.asDoubleCompatibleOI(argOI0); this._sst = new SingularSpectrumTransform(_params, xOI); this._scores = new double[1]; final Object[] result; final ArrayList<String> fieldNames = new ArrayList<String>(); final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("changepoint_score"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); if (_params.changepointThreshold != -1d) { fieldNames.add("is_changepoint"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); result = new Object[2]; this._isChangepoint = new BooleanWritable(false); result[1] = _isChangepoint; } else { result = new Object[1]; } this._changepointScore = new DoubleWritable(0.d); result[0] = _changepointScore; this._result = result; return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 18
Source File: ChangeFinderUDF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
@Override public ObjectInspector initialize(@Nonnull ObjectInspector[] argOIs) throws UDFArgumentException { if (argOIs.length < 1 || argOIs.length > 2) { throw new UDFArgumentException( "_FUNC_(double|array<double> x [, const string options]) takes 1 or 2 arguments: " + Arrays.toString(argOIs)); } this._params = new Parameters(); if (argOIs.length == 2) { String options = HiveUtils.getConstString(argOIs[1]); processOptions(options); } ObjectInspector argOI0 = argOIs[0]; if (HiveUtils.isListOI(argOI0)) { ListObjectInspector listOI = HiveUtils.asListOI(argOI0); this._changeFinder = new ChangeFinder2D(_params, listOI); } else if (HiveUtils.isNumberOI(argOI0)) { PrimitiveObjectInspector xOI = HiveUtils.asDoubleCompatibleOI(argOI0); this._changeFinder = new ChangeFinder1D(_params, xOI); } this._scores = new double[2]; final Object[] result; final ArrayList<String> fieldNames = new ArrayList<String>(); final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("outlier_score"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); fieldNames.add("changepoint_score"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); if (_params.outlierThreshold != -1d) { fieldNames.add("is_outlier"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); this._isOutlier = new BooleanWritable(false); if (_params.changepointThreshold != -1d) { fieldNames.add("is_changepoint"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); result = new Object[4]; this._isChangepoint = new BooleanWritable(false); result[3] = _isChangepoint; } else { result = new Object[3]; } result[2] = _isOutlier; } else { result = new Object[2]; } this._outlierScore = new DoubleWritable(0d); result[0] = _outlierScore; this._changepointScore = new DoubleWritable(0d); result[1] = _changepointScore; this._result = result; return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 19
Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0 | 4 votes |
@Test public void testPartial2() throws HiveException { Funnel udaf = new Funnel(); // Construct the object inspector for udaf evaluator ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column PrimitiveObjectInspectorFactory.javaLongObjectInspector, // timestamp_column ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1 ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1 }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); // Construct the struct object inspector List<String> fieldNames = new ArrayList<>(); fieldNames.add("action"); fieldNames.add("timestamp"); fieldNames.add("funnel"); List<ObjectInspector> fieldInspectors = new ArrayList<>(); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)); ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors); ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector}; ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, evaluatorInputObjectInspectorList); // Create the two structs to merge List<Object> parameter1 = new ArrayList<>(); parameter1.add(Arrays.asList("beta")); parameter1.add(Arrays.asList(300L)); parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null)); List<Object> parameter2 = new ArrayList<>(); parameter2.add(Arrays.asList("gamma", "alpha")); parameter2.add(Arrays.asList(400L, 200L)); parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null)); // Process the data AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.merge(agg, parameter1); udafEvaluator.merge(agg, parameter2); Object result = udafEvaluator.terminatePartial(agg); // Expected List<Object> expected = new ArrayList<>(); expected.add(Arrays.asList("beta", "gamma", "alpha")); expected.add(Arrays.asList(300L, 400L, 200L)); expected.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null)); Assert.assertEquals(expected, result); }
Example 20
Source File: BuildBinsUDAF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
@Override public ObjectInspector init(Mode mode, ObjectInspector[] OIs) throws HiveException { super.init(mode, OIs); if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) { weightOI = HiveUtils.asDoubleCompatibleOI(OIs[0]); // set const values nBins = HiveUtils.getConstInt(OIs[1]); if (OIs.length == 3) { autoShrink = HiveUtils.getConstBoolean(OIs[2]); } // check value of `num_of_bins` if (nBins < 2) { throw new UDFArgumentException( "Only greater than or equal to 2 is accepted but " + nBins + " was passed as `num_of_bins`."); } quantiles = getQuantiles(); } else { structOI = (StructObjectInspector) OIs[0]; autoShrinkField = structOI.getStructFieldRef("autoShrink"); histogramField = structOI.getStructFieldRef("histogram"); quantilesField = structOI.getStructFieldRef("quantiles"); autoShrinkOI = (WritableBooleanObjectInspector) autoShrinkField.getFieldObjectInspector(); histogramOI = (StandardListObjectInspector) histogramField.getFieldObjectInspector(); quantilesOI = (StandardListObjectInspector) quantilesField.getFieldObjectInspector(); histogramElOI = (WritableDoubleObjectInspector) histogramOI.getListElementObjectInspector(); quantileOI = (WritableDoubleObjectInspector) quantilesOI.getListElementObjectInspector(); } if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) { final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)); fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)); return ObjectInspectorFactory.getStandardStructObjectInspector( Arrays.asList("autoShrink", "histogram", "quantiles"), fieldOIs); } else { return ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); } }