Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardStructObjectInspector()

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardStructObjectInspector() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Nonnull
protected StructObjectInspector getReturnOI(@Nonnull ObjectInspector featureOutputOI) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("feature");
    fieldOIs.add(featureOutputOI);
    fieldNames.add("weight");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    if (useCovariance()) {
        fieldNames.add("covar");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 2
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
  final ObjectInspector resultInspector = super.init(mode, inspectors);
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    // input is original data
    if (inspectors.length > 2) {
      summaryModeInspector_ = (PrimitiveObjectInspector) inspectors[2];
    }
  }
  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the nominal number of entries and the summary mode
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SUMMARY_MODE_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.STRING),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  return resultInspector;
}
 
Example 3
Source Project: incubator-hivemall   File: FMeasureUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOI() {
    List<String> fieldNames = new ArrayList<>();
    List<ObjectInspector> fieldOIs = new ArrayList<>();

    fieldNames.add("tp");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("totalActual");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("totalPredicted");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("beta");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("average");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 4
@Override
public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        showHelp("Invalid argment length=" + argOIs.length);
    }
    processOptions(argOIs);

    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0);
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    this.featureListOI = listOI;
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseInput = true;
        this.matrixBuilder = new DenseDMatrixBuilder(8192);
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.featureElemOI = HiveUtils.asStringOI(elemOI);
        this.denseInput = false;
        this.matrixBuilder = new SparseDMatrixBuilder(8192);
    } else {
        throw new UDFArgumentException(
            "train_xgboost takes array<double> or array<string> for the first argument: "
                    + listOI.getTypeName());
    }
    this.targetOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);
    this.labels = new FloatArrayList(1024);

    final List<String> fieldNames = new ArrayList<>(2);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(2);
    fieldNames.add("model_id");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    fieldNames.add("model");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 5
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
  super.init(mode, inspectors);
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    // input is original data
    sketchInspector_ = (PrimitiveObjectInspector) inspectors[0];
    if (inspectors.length > 1) {
      nominalNumEntriesInspector_ = (PrimitiveObjectInspector) inspectors[1];
    }
  } else {
    // input for PARTIAL2 and FINAL is the output from PARTIAL1
    intermediateInspector_ = (StructObjectInspector) inspectors[0];
  }

  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the the nominal number of entries
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  // final results include just the sketch
  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 
Example 6
Source Project: hive-dwrf   File: TestRecordReaderImpl.java    License: Apache License 2.0 5 votes vote down vote up
public static StructObjectInspector getObjectInspectorFor(final ImmutableList<String> names,
                                                          final ImmutableList<String> columnTypeNames) {
  List<ObjectInspector> inspectors = new ArrayList<>();
  for (int i = 0; i < columnTypeNames.size(); i++) {
    inspectors.add(createJavaObjectInspectorFromFieldSchema(columnTypeNames.get(i)));
  }
  return ObjectInspectorFactory.getStandardStructObjectInspector(
      names,
      inspectors);
}
 
Example 7
Source Project: incubator-iotdb   File: TsFileSerDe.java    License: Apache License 2.0 5 votes vote down vote up
private ObjectInspector createObjectInspector() throws TsFileSerDeException {
  List<ObjectInspector> columnOIs = new ArrayList<>(columnNames.size());

  // At this point we've verified the types are correct.
  for(int i = 0; i < columnNames.size(); i++) {
    columnOIs.add(i, createObjectInspectorWorker(columnTypes.get(i)));
  }
  return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs);
}
 
Example 8
private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("size");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("sum");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
    fieldNames.add("count");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableLongObjectInspector));

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 9
private Map<String, AttributeValue> getSerializedItem(List<String> attributeNames, List<ObjectInspector> colOIs,
                                                      Map<String, String> typeMapping, List<Object> rowData,
                                                      boolean nullSerialization)
    throws SerDeException {
  List<String> colTypes = Lists.newArrayList();
  List<String> colMappings = Lists.newArrayList();
  for (int i = 0; i < attributeNames.size(); i++) {
    String attributeName = attributeNames.get(i);
    colTypes.add(colOIs.get(i).getTypeName());
    if (!HiveDynamoDBTypeFactory.isHiveDynamoDBItemMapType(colOIs.get(i)) ||
        typeMapping.containsKey(attributeName)) {
      colMappings.add(attributeName + ":" + attributeName);
    }
  }

  List<String> typeMapList = Lists.newArrayList();
  for (Map.Entry<String, String> colType : typeMapping.entrySet()) {
    typeMapList.add(colType.getKey() + ":" + colType.getValue());
  }

  Properties props = new Properties();
  props.setProperty(serdeConstants.LIST_COLUMNS, StringUtils.join(attributeNames, ","));
  props.setProperty(serdeConstants.LIST_COLUMN_TYPES, StringUtils.join(colTypes, ","));
  props.setProperty(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, StringUtils.join(colMappings, ","));
  props.setProperty(DynamoDBConstants.DYNAMODB_TYPE_MAPPING, StringUtils.join(typeMapList, ","));
  props.setProperty(DynamoDBConstants.DYNAMODB_NULL_SERIALIZATION, Boolean.toString(nullSerialization));

  DynamoDBSerDe serde = new DynamoDBSerDe();
  serde.initialize(null, props);

  StructObjectInspector rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(attributeNames, colOIs);
  DynamoDBItemWritable item = (DynamoDBItemWritable) serde.serialize(rowData, rowOI);
  return item.getItem();
}
 
Example 10
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length < 3) {
        throw new UDFArgumentException("binarize_label(int/long positive, "
                + "int/long negative, *) takes at least three arguments");
    }
    this.positiveOI = HiveUtils.asIntCompatibleOI(argOIs[0]);
    this.negativeOI = HiveUtils.asIntCompatibleOI(argOIs[1]);

    this.positiveObjs = new Object[argOIs.length - 1];
    positiveObjs[positiveObjs.length - 1] = 1;
    this.negativeObjs = new Object[argOIs.length - 1];
    negativeObjs[negativeObjs.length - 1] = 0;

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (int i = 2; i < argOIs.length; i++) {
        fieldNames.add("c" + (i - 2));
        // Use negative label ObjectInspector here. OIs for positive
        // label and negative labels must be same.
        fieldOIs.add(argOIs[i]);
    }
    fieldNames.add("c" + (argOIs.length - 2));
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 11
Source Project: flink   File: HiveWriterFactory.java    License: Apache License 2.0 5 votes vote down vote up
private void checkInitialize() throws Exception {
	if (initialized) {
		return;
	}

	JobConf jobConf = confWrapper.conf();
	Object serdeLib = Class.forName(serDeInfo.getSerializationLib()).newInstance();
	Preconditions.checkArgument(serdeLib instanceof Serializer && serdeLib instanceof Deserializer,
			"Expect a SerDe lib implementing both Serializer and Deserializer, but actually got "
					+ serdeLib.getClass().getName());
	this.recordSerDe = (Serializer) serdeLib;
	ReflectionUtils.setConf(recordSerDe, jobConf);

	// TODO: support partition properties, for now assume they're same as table properties
	SerDeUtils.initializeSerDe((Deserializer) recordSerDe, jobConf, tableProperties, null);

	this.formatFields = allColumns.length - partitionColumns.length;
	this.hiveConversions = new HiveObjectConversion[formatFields];
	this.converters = new DataFormatConverter[formatFields];
	List<ObjectInspector> objectInspectors = new ArrayList<>(hiveConversions.length);
	for (int i = 0; i < formatFields; i++) {
		DataType type = allTypes[i];
		ObjectInspector objectInspector = HiveInspectors.getObjectInspector(type);
		objectInspectors.add(objectInspector);
		hiveConversions[i] = HiveInspectors.getConversion(
				objectInspector, type.getLogicalType(), hiveShim);
		converters[i] = DataFormatConverters.getConverterForDataType(type);
	}

	this.formatInspector = ObjectInspectorFactory.getStandardStructObjectInspector(
			Arrays.asList(allColumns).subList(0, formatFields),
			objectInspectors);
	this.initialized = true;
}
 
Example 12
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        throw new UDFArgumentException(getClass().getSimpleName()
                + " takes 2 or 3 arguments: array<string> x, map<long, double> colNorms "
                + "[, CONSTANT STRING options]: " + Arrays.toString(argOIs));
    }

    this.rowOI = HiveUtils.asListOI(argOIs[0]);
    HiveUtils.validateFeatureOI(rowOI.getListElementObjectInspector());

    this.colNormsOI = HiveUtils.asMapOI(argOIs[1]);

    processOptions(argOIs);

    this.rnd = RandomNumberGeneratorFactory.createPRNG(1001);
    this.colNorms = null;
    this.colProbs = null;

    ArrayList<String> fieldNames = new ArrayList<String>();
    fieldNames.add("j");
    fieldNames.add("k");
    fieldNames.add("b_jk");

    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    if (parseFeatureAsInt) {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    } else {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    }
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 13
/**
 * Receives the passed in argument object inspectors and returns the desired
 * return type's object inspector to inform hive of return type of UDAF.
 *
 * @param mode
 *          Mode (i.e. PARTIAL 1, COMPLETE...) for determining input and output
 *          object inspector type.
 * @param parameters
 *          List of object inspectors for input arguments.
 * @return The object inspector type indicates the UDAF return type (i.e.
 *         returned type of terminate(...)).
 */
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);

  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    inputInspector_ = (PrimitiveObjectInspector) parameters[0];
    if (parameters.length > 1) {
      lgKInspector_ = (PrimitiveObjectInspector) parameters[1];
    }
    if (parameters.length > 2) {
      seedInspector_ = (PrimitiveObjectInspector) parameters[2];
    }
  } else {
    // mode = partial2 || final
    intermediateInspector_ = (StandardStructObjectInspector) parameters[0];
  }

  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the lgK and the target HLL type
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(LG_K_FIELD, SEED_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.LONG),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  // final results include just the sketch
  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 
Example 14
@Before
public void setUp() throws Exception {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("wcList");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector));

    fieldNames.add("lambdaMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaFloatObjectInspector)));

    fieldNames.add("topics");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    fieldNames.add("alpha");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    fieldNames.add("delta");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    partialOI = new ObjectInspector[4];
    partialOI[0] =
            ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);

    words = new String[] {"fruits", "vegetables", "healthy", "flu", "apples", "oranges", "like",
            "avocados", "colds", "colds", "avocados", "oranges", "like", "apples", "flu",
            "healthy", "vegetables", "fruits"};
    labels = new int[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1};
    lambdas = new float[] {0.3339331f, 0.3324783f, 0.33209667f, 3.2804057E-4f, 3.0303953E-4f,
            2.4860457E-4f, 2.41481E-4f, 2.3554532E-4f, 1.352576E-4f, 0.1660153f, 0.16596903f,
            0.1659654f, 0.1659627f, 0.16593699f, 0.1659259f, 0.0017611005f, 0.0015791848f,
            8.84464E-4f};
}
 
Example 15
@Override
public ObjectInspector init(@Nonnull Mode mode, @Nonnull ObjectInspector[] argOIs)
        throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.yhatOI = HiveUtils.asIntegerOI(argOIs, 0);
    } else {// from partial aggregation
        this.internalMergeOI = (StandardMapObjectInspector) argOIs[0];
        this.keyOI = HiveUtils.asIntOI(internalMergeOI.getMapKeyObjectInspector());
        this.valueOI = HiveUtils.asIntOI(internalMergeOI.getMapValueObjectInspector());
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.javaIntObjectInspector,
            PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    } else {// terminate
        List<String> fieldNames = new ArrayList<>(3);
        List<ObjectInspector> fieldOIs = new ArrayList<>(3);
        fieldNames.add("label");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldNames.add("probability");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
        fieldNames.add("probabilities");
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
        outputOI = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames,
            fieldOIs);
    }
    return outputOI;
}
 
Example 16
@Override
public ObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length < 1 || argOIs.length > 2) {
        throw new UDFArgumentException(
            "_FUNC_(double|array<double> x [, const string options]) takes 1 or 2 arguments: "
                    + Arrays.toString(argOIs));
    }

    this._params = new Parameters();
    if (argOIs.length == 2) {
        String options = HiveUtils.getConstString(argOIs[1]);
        processOptions(options);
    }

    ObjectInspector argOI0 = argOIs[0];
    PrimitiveObjectInspector xOI = HiveUtils.asDoubleCompatibleOI(argOI0);
    this._sst = new SingularSpectrumTransform(_params, xOI);

    this._scores = new double[1];

    final Object[] result;
    final ArrayList<String> fieldNames = new ArrayList<String>();
    final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("changepoint_score");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    if (_params.changepointThreshold != -1d) {
        fieldNames.add("is_changepoint");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
        result = new Object[2];
        this._isChangepoint = new BooleanWritable(false);
        result[1] = _isChangepoint;
    } else {
        result = new Object[1];
    }
    this._changepointScore = new DoubleWritable(0.d);
    result[0] = _changepointScore;
    this._result = result;

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 17
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] OIs) throws HiveException {
    super.init(mode, OIs);

    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
        weightOI = HiveUtils.asDoubleCompatibleOI(OIs[0]);

        // set const values
        nBins = HiveUtils.getConstInt(OIs[1]);
        if (OIs.length == 3) {
            autoShrink = HiveUtils.getConstBoolean(OIs[2]);
        }

        // check value of `num_of_bins`
        if (nBins < 2) {
            throw new UDFArgumentException(
                "Only greater than or equal to 2 is accepted but " + nBins
                        + " was passed as `num_of_bins`.");
        }

        quantiles = getQuantiles();
    } else {
        structOI = (StructObjectInspector) OIs[0];
        autoShrinkField = structOI.getStructFieldRef("autoShrink");
        histogramField = structOI.getStructFieldRef("histogram");
        quantilesField = structOI.getStructFieldRef("quantiles");
        autoShrinkOI =
                (WritableBooleanObjectInspector) autoShrinkField.getFieldObjectInspector();
        histogramOI =
                (StandardListObjectInspector) histogramField.getFieldObjectInspector();
        quantilesOI =
                (StandardListObjectInspector) quantilesField.getFieldObjectInspector();
        histogramElOI =
                (WritableDoubleObjectInspector) histogramOI.getListElementObjectInspector();
        quantileOI =
                (WritableDoubleObjectInspector) quantilesOI.getListElementObjectInspector();
    }

    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
        final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));

        return ObjectInspectorFactory.getStandardStructObjectInspector(
            Arrays.asList("autoShrink", "histogram", "quantiles"), fieldOIs);
    } else {
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    }
}
 
Example 18
Source Project: hive-funnel-udf   File: FunnelTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testPartial2() throws HiveException {
    Funnel udaf = new Funnel();

    // Construct the object inspector for udaf evaluator
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };
    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    // Construct the struct object inspector
    List<String> fieldNames = new ArrayList<>();
    fieldNames.add("action");
    fieldNames.add("timestamp");
    fieldNames.add("funnel");
    List<ObjectInspector> fieldInspectors = new ArrayList<>();
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors);
    ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector};

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, evaluatorInputObjectInspectorList);

    // Create the two structs to merge
    List<Object> parameter1 = new ArrayList<>();
    parameter1.add(Arrays.asList("beta"));
    parameter1.add(Arrays.asList(300L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    List<Object> parameter2 = new ArrayList<>();
    parameter2.add(Arrays.asList("gamma", "alpha"));
    parameter2.add(Arrays.asList(400L, 200L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, parameter1);
    udafEvaluator.merge(agg, parameter2);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected
    List<Object> expected = new ArrayList<>();
    expected.add(Arrays.asList("beta", "gamma", "alpha"));
    expected.add(Arrays.asList(300L, 400L, 200L));
    expected.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    Assert.assertEquals(expected, result);
}
 
Example 19
@Override
public ObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length < 1 || argOIs.length > 2) {
        throw new UDFArgumentException(
            "_FUNC_(double|array<double> x [, const string options]) takes 1 or 2 arguments: "
                    + Arrays.toString(argOIs));
    }

    this._params = new Parameters();
    if (argOIs.length == 2) {
        String options = HiveUtils.getConstString(argOIs[1]);
        processOptions(options);
    }

    ObjectInspector argOI0 = argOIs[0];
    if (HiveUtils.isListOI(argOI0)) {
        ListObjectInspector listOI = HiveUtils.asListOI(argOI0);
        this._changeFinder = new ChangeFinder2D(_params, listOI);
    } else if (HiveUtils.isNumberOI(argOI0)) {
        PrimitiveObjectInspector xOI = HiveUtils.asDoubleCompatibleOI(argOI0);
        this._changeFinder = new ChangeFinder1D(_params, xOI);
    }

    this._scores = new double[2];

    final Object[] result;
    final ArrayList<String> fieldNames = new ArrayList<String>();
    final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("outlier_score");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("changepoint_score");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    if (_params.outlierThreshold != -1d) {
        fieldNames.add("is_outlier");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
        this._isOutlier = new BooleanWritable(false);
        if (_params.changepointThreshold != -1d) {
            fieldNames.add("is_changepoint");
            fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
            result = new Object[4];
            this._isChangepoint = new BooleanWritable(false);
            result[3] = _isChangepoint;
        } else {
            result = new Object[3];
        }
        result[2] = _isOutlier;
    } else {
        result = new Object[2];
    }
    this._outlierScore = new DoubleWritable(0d);
    result[0] = _outlierScore;
    this._changepointScore = new DoubleWritable(0d);
    result[1] = _changepointScore;
    this._result = result;

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 20
Source Project: flink   File: HiveGenericUDTFTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
	return ObjectInspectorFactory.getStandardStructObjectInspector(
		Collections.singletonList("col1"),
		Collections.singletonList(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
}