Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static boolean isWritableType(TypeInfo typeInfo)
{
    switch (typeInfo.getCategory()) {
        case PRIMITIVE:
            PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
            return isWritablePrimitiveType(primitiveCategory);
        case MAP:
            MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
            return isWritableType(mapTypeInfo.getMapKeyTypeInfo()) && isWritableType(mapTypeInfo.getMapValueTypeInfo());
        case LIST:
            ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
            return isWritableType(listTypeInfo.getListElementTypeInfo());
        case STRUCT:
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            return structTypeInfo.getAllStructFieldTypeInfos().stream().allMatch(HiveWriteUtils::isWritableType);
    }
    return false;
}
 
Example 2
Source Project: presto   Source File: HiveWriteUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static boolean isWritablePrimitiveType(PrimitiveCategory primitiveCategory)
{
    switch (primitiveCategory) {
        case BOOLEAN:
        case LONG:
        case INT:
        case SHORT:
        case BYTE:
        case FLOAT:
        case DOUBLE:
        case STRING:
        case DATE:
        case TIMESTAMP:
        case BINARY:
        case DECIMAL:
        case VARCHAR:
        case CHAR:
            return true;
    }
    return false;
}
 
Example 3
public MDSMapObjectInspector( final MapTypeInfo typeInfo ){
  TypeInfo keyTypeInfo = typeInfo.getMapKeyTypeInfo();
  if( keyTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && ( (PrimitiveTypeInfo)keyTypeInfo ).getPrimitiveCategory() == PrimitiveCategory.STRING ){
    keyObjectInspector = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
  }
  else{
    throw new RuntimeException( "Map key type is string only." );
  }

  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getMapValueTypeInfo() ); 

  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getMapValueTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 
Example 4
private static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 2);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.LONG);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 5
Source Project: metacat   Source File: HiveTypeConverter.java    License: Apache License 2.0 6 votes vote down vote up
private static Type getPrimitiveType(final ObjectInspector fieldInspector) {
    final PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) fieldInspector)
        .getPrimitiveCategory();
    if (HiveTypeMapping.getHIVE_TO_CANONICAL().containsKey(primitiveCategory.name())) {
        return HiveTypeMapping.getHIVE_TO_CANONICAL().get(primitiveCategory.name());
    }
    switch (primitiveCategory) {
        case DECIMAL:
            final DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) ((PrimitiveObjectInspector) fieldInspector)
                .getTypeInfo();
            return DecimalType.createDecimalType(decimalTypeInfo.precision(), decimalTypeInfo.getScale());
        case CHAR:
            final int cLength = ((CharTypeInfo) ((PrimitiveObjectInspector)
                fieldInspector).getTypeInfo()).getLength();
            return CharType.createCharType(cLength);
        case VARCHAR:
            final int vLength = ((VarcharTypeInfo) ((PrimitiveObjectInspector) fieldInspector)
                .getTypeInfo()).getLength();
            return VarcharType.createVarcharType(vLength);
        default:
            return null;
    }
}
 
Example 6
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);
  inputObjectInspector = (PrimitiveObjectInspector) parameters[0];

  // Parameters:
  // In PARTIAL1 and COMPLETE mode, the parameters are original data.
  // In PARTIAL2 and FINAL mode, the parameters are partial aggregations.
  if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
    if (parameters.length > 1) {
      kObjectInspector = (PrimitiveObjectInspector) parameters[1];
    }
  }

  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 
Example 7
Source Project: spatial-framework-for-hadoop   Source File: ST_Bin.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] OIs)
		throws UDFArgumentException {
	
	if (OIs.length != 2) {
		throw new UDFArgumentException("Function takes exactly 2 arguments");
	}

	if (OIs[0].getCategory() != Category.PRIMITIVE) {
		throw new UDFArgumentException("Argument 0 must be a number - got: " + OIs[0].getCategory());
	}

	oiBinSize = (PrimitiveObjectInspector)OIs[0];
	if (!EnumSet.of(PrimitiveCategory.DECIMAL,PrimitiveCategory.DOUBLE,PrimitiveCategory.INT,PrimitiveCategory.LONG,PrimitiveCategory.SHORT, PrimitiveCategory.FLOAT).contains(oiBinSize.getPrimitiveCategory())) {
		throw new UDFArgumentException("Argument 0 must be a number - got: " + oiBinSize.getPrimitiveCategory());
	}

	geomHelper = HiveGeometryOIHelper.create(OIs[1], 1);
	binSizeIsConstant = ObjectInspectorUtils.isConstantObjectInspector(OIs[0]);

	return PrimitiveObjectInspectorFactory.javaLongObjectInspector;
}
 
Example 8
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);
  inputInspector_ = (PrimitiveObjectInspector) parameters[0];

  // Parameters:
  // In PARTIAL1 and COMPLETE mode, the parameters are original data.
  // In PARTIAL2 and FINAL mode, the parameters are partial aggregations.
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    if (parameters.length > 1) {
      kInspector_ = (PrimitiveObjectInspector) parameters[1];
    }
  }

  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 
Example 9
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
  final ObjectInspector resultInspector = super.init(mode, inspectors);
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    // input is original data
    if (inspectors.length > 2) {
      summaryModeInspector_ = (PrimitiveObjectInspector) inspectors[2];
    }
  }
  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the nominal number of entries and the summary mode
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SUMMARY_MODE_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.STRING),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  return resultInspector;
}
 
Example 10
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info) throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();

  if (inspectors.length < 1) {
    throw new UDFArgumentException("Expected at least 1 argument");
  }
  ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[0], 0, PrimitiveCategory.BINARY);

  // nominal number of entries
  if (inspectors.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[1], 1);
  }

  checkExtraArguments(inspectors);

  return createEvaluator();
}
 
Example 11
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info) throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();

  if (inspectors.length < 1) {
    throw new UDFArgumentException("Expected at least 1 argument");
  }
  if (inspectors.length > 3) {
    throw new UDFArgumentTypeException(inspectors.length - 1, "Expected no more than 3 arguments");
  }

  ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[0], 0, PrimitiveCategory.BINARY);

  // nominal number of entries
  if (inspectors.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[1], 1);
  }

  // number of double values per key
  if (inspectors.length > 2) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[2], 2);
  }

  return new UnionArrayOfDoublesSketchEvaluator();
}
 
Example 12
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
  final ObjectInspector resultInspector = super.init(mode, inspectors);
  if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
    // input is original data
    if (inspectors.length > 4) {
      summaryModeInspector_ = (PrimitiveObjectInspector) inspectors[4];
    }
  }
  if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
    // intermediate results need to include the nominal number of entries and the summary mode
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SUMMARY_MODE_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.STRING),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  return resultInspector;
}
 
Example 13
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info)
    throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();
  if (inspectors.length < 1) {
    throw new UDFArgumentException("Please specify at least 1 argument");
  }
  if (inspectors.length > 2) {
    throw new
    UDFArgumentTypeException(inspectors.length - 1, "Please specify no more than 2 arguments");
  }
  ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[0], 0,
      PrimitiveCategory.BINARY);
  if (inspectors.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[1], 1);
  }
  return new IntersectSketchUDAFEvaluator();
}
 
Example 14
/**
 * Perform argument count check and argument type checking, returns an
 * appropriate evaluator to perform based on input type (which should always
 * be BINARY sketch). Also check sketch size and seed params if they are passed in.
 *
 * @see org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver
 * #getEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo)
 *
 * @param info The parameter info to validate
 * @return The GenericUDAFEvaluator to use to compute the function.
 */
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info) throws SemanticException {
  final ObjectInspector[] parameters = info.getParameterObjectInspectors();

  if (parameters.length < 1) {
    throw new UDFArgumentException("Please specify at least 1 argument");
  }

  if (parameters.length > 3) {
    throw new UDFArgumentTypeException(parameters.length - 1, "Please specify no more than 3 arguments");
  }

  ObjectInspectorValidator.validateGivenPrimitiveCategory(parameters[0], 0, PrimitiveCategory.BINARY);

  if (parameters.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(parameters[1], 1);
  }

  if (parameters.length > 2) {
    ObjectInspectorValidator.validateIntegralParameter(parameters[2], 2);
  }
  return new UnionSketchUDAFEvaluator();
}
 
Example 15
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
    Assert.assertNotNull(resultInspector);
    Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
    StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
    List<?> fields = structResultInspector.getAllStructFieldRefs();
    Assert.assertEquals(fields.size(), 3);

    ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
    Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
    Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

    ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
    Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
    Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.LONG);

    ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
    Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
    Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 16
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 17
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.STRING);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 18
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.LONG);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 19
private static void checkResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  List<? extends StructField> fields = ((StructObjectInspector) resultInspector).getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 4);
  Assert.assertEquals(fields.get(0).getFieldObjectInspector().getCategory(), ObjectInspector.Category.PRIMITIVE);
  Assert.assertEquals(
    ((PrimitiveObjectInspector) fields.get(0).getFieldObjectInspector()).getPrimitiveCategory(),
    PrimitiveObjectInspector.PrimitiveCategory.STRING
  );
  Assert.assertEquals(fields.get(1).getFieldObjectInspector().getCategory(), ObjectInspector.Category.PRIMITIVE);
  Assert.assertEquals(
    ((PrimitiveObjectInspector) fields.get(1).getFieldObjectInspector()).getPrimitiveCategory(),
    PrimitiveObjectInspector.PrimitiveCategory.LONG
  );
  Assert.assertEquals(fields.get(2).getFieldObjectInspector().getCategory(), ObjectInspector.Category.PRIMITIVE);
  Assert.assertEquals(
    ((PrimitiveObjectInspector) fields.get(2).getFieldObjectInspector()).getPrimitiveCategory(),
    PrimitiveObjectInspector.PrimitiveCategory.LONG
  );
  Assert.assertEquals(fields.get(3).getFieldObjectInspector().getCategory(), ObjectInspector.Category.PRIMITIVE);
  Assert.assertEquals(
    ((PrimitiveObjectInspector) fields.get(3).getFieldObjectInspector()).getPrimitiveCategory(),
    PrimitiveObjectInspector.PrimitiveCategory.LONG
  );
}
 
Example 20
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.STRING);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 21
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.STRING);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 22
Source Project: presto   Source File: TestHiveFileFormats.java    License: Apache License 2.0 5 votes vote down vote up
private static List<TestColumn> getTestColumnsSupportedByParquet()
{
    // Write of complex hive data to Parquet is broken
    // TODO: empty arrays or maps with null keys don't seem to work
    // Parquet does not support DATE
    return TEST_COLUMNS.stream()
            .filter(column -> !ImmutableSet.of("t_null_array_int", "t_array_empty", "t_map_null_key", "t_map_null_key_complex_value", "t_map_null_key_complex_key_value")
                    .contains(column.getName()))
            .filter(column -> column.isPartitionKey() || (
                    !hasType(column.getObjectInspector(), PrimitiveCategory.DATE)) &&
                    !hasType(column.getObjectInspector(), PrimitiveCategory.SHORT) &&
                    !hasType(column.getObjectInspector(), PrimitiveCategory.BYTE))
            .collect(toList());
}
 
Example 23
Source Project: presto   Source File: TestHiveFileFormats.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean hasType(ObjectInspector objectInspector, PrimitiveCategory... types)
{
    if (objectInspector instanceof PrimitiveObjectInspector) {
        PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) objectInspector;
        PrimitiveCategory primitiveCategory = primitiveInspector.getPrimitiveCategory();
        for (PrimitiveCategory type : types) {
            if (primitiveCategory == type) {
                return true;
            }
        }
        return false;
    }
    if (objectInspector instanceof ListObjectInspector) {
        ListObjectInspector listInspector = (ListObjectInspector) objectInspector;
        return hasType(listInspector.getListElementObjectInspector(), types);
    }
    if (objectInspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) objectInspector;
        return hasType(mapInspector.getMapKeyObjectInspector(), types) ||
                hasType(mapInspector.getMapValueObjectInspector(), types);
    }
    if (objectInspector instanceof StructObjectInspector) {
        for (StructField field : ((StructObjectInspector) objectInspector).getAllStructFieldRefs()) {
            if (hasType(field.getFieldObjectInspector(), types)) {
                return true;
            }
        }
        return false;
    }
    throw new IllegalArgumentException("Unknown object inspector type " + objectInspector);
}
 
Example 24
Source Project: Hive-XML-SerDe   Source File: JavaXPathProcessorTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testSimpleAttribute() {
    final List<XmlQuery> queries = Arrays.asList(new XmlQuery[] {new XmlQuery("/root/@a", "field1")});
    String data = "<root a=\"value\"/>";
    JavaXmlProcessor xPathProcessor = new JavaXmlProcessor();
    xPathProcessor.initialize(new XmlProcessorContext() {

        @Override
        public List<XmlQuery> getXmlQueries() {
            return queries;
        }

        @Override
        public Map<String, XmlMapEntry> getXmlMapSpecification() {
            // TODO Auto-generated method stub
            return null;
        }

        @Override
        public Properties getProperties() {
            // TODO Auto-generated method stub
            return null;
        }

    });
    Map<String, NodeArray> out = xPathProcessor.parse(data);
    Node node = out.get("field1").get(0);
    assertTrue(node.getNodeType() == Node.ATTRIBUTE_NODE);
    assertEquals(xPathProcessor.getPrimitiveObjectValue(node, PrimitiveCategory.STRING), "value");
}
 
Example 25
@Override
public ObjectInspector initialize(ObjectInspector[] arguments)
		throws UDFArgumentException {
	
	if (arguments.length != 1) {
		throw new UDFArgumentLengthException("ST_GeomFromJson takes only one argument");
	}

	ObjectInspector argJsonOI = arguments[0];
	
	if (argJsonOI.getCategory() == Category.PRIMITIVE)
	{
		PrimitiveObjectInspector poi = (PrimitiveObjectInspector)argJsonOI;
		
		if (poi.getPrimitiveCategory() != PrimitiveCategory.STRING)
		{
			throw new UDFArgumentTypeException(0, "ST_GeomFromJson argument category must be either a string primitive or struct");
		}
	} else if (argJsonOI.getCategory() != Category.STRUCT) {
		
	} else {
		throw new UDFArgumentTypeException(0, "ST_GeomFromJson argument category must be either a string primitive or struct");
	}
	
	jsonOI = argJsonOI;

	return GeometryUtils.geometryTransportObjectInspector;
}
 
Example 26
Source Project: Hive-XML-SerDe   Source File: XmlMapObjectInspector.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @see org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector#getMapValueElement(java.lang.Object, java.lang.Object)
 */
@Override
@SuppressWarnings({"rawtypes"})
public Object getMapValueElement(Object data, Object key) {
    ObjectInspector mapKeyObjectInspector = getMapValueObjectInspector();
    Category mapKeyCategory = mapKeyObjectInspector.getCategory();
    if (mapKeyCategory == Category.PRIMITIVE) {
        PrimitiveObjectInspector primitiveMapKeyObjectInspector = (PrimitiveObjectInspector) mapKeyObjectInspector;
        PrimitiveCategory primitiveMapKeyCategory = primitiveMapKeyObjectInspector.getPrimitiveCategory();
        key = this.xmlProcessor.getPrimitiveObjectValue(key, primitiveMapKeyCategory);
    }
    Map map = getMap(data);
    return map == null ? null : map.get(key);
}
 
Example 27
/**
 * Perform argument count check and argument type checking, returns an
 * appropriate evaluator to perform based on input type (which should always
 * be BINARY sketch). Also check lgK and target HLL type parameters if they are passed in.
 *
 * @see org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver
 * #getEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo)
 *
 * @param info The parameter info to validate
 * @return The GenericUDAFEvaluator to use to compute the function.
 */
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info) throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();

  if (inspectors.length < 1) {
    throw new UDFArgumentException("Please specify at least 1 argument");
  }

  if (inspectors.length > 3) {
    throw new UDFArgumentTypeException(inspectors.length - 1, "Please specify no more than 3 arguments");
  }

  ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[0], 0, PrimitiveCategory.BINARY);

  // Validate second argument if present
  if (inspectors.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[1], 1);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[1])) {
      throw new UDFArgumentTypeException(1, "The second argument must be a constant");
    }
  }

  // Validate third argument if present
  if (inspectors.length > 2) {
    ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[2], 2, PrimitiveCategory.STRING);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[2])) {
      throw new UDFArgumentTypeException(2, "The third argument must be a constant");
    }
  }

  return new UnionSketchUDAFEvaluator();
}
 
Example 28
/**
 * Receives the passed in argument object inspectors and returns the desired
 * return type's object inspector to inform hive of return type of UDAF.
 *
 * @param mode
 *          Mode (i.e. PARTIAL 1, COMPLETE...) for determining input and output
 *          object inspector type.
 * @param parameters
 *          List of object inspectors for input arguments.
 * @return The object inspector type indicates the UDAF return type (i.e.
 *         returned type of terminate(...)).
 */
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);

  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    inputInspector_ = (PrimitiveObjectInspector) parameters[0];
    if (parameters.length > 1) {
      lgKInspector_ = (PrimitiveObjectInspector) parameters[1];
    }
    if (parameters.length > 2) {
      hllTypeInspector_ = (PrimitiveObjectInspector) parameters[2];
    }
  } else {
    // mode = partial2 || final
    intermediateInspector_ = (StandardStructObjectInspector) parameters[0];
  }

  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the lgK and the target HLL type
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(LG_K_FIELD, HLL_TYPE_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.STRING),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  // final results include just the sketch
  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 
Example 29
/**
 * Performs argument number and type validation. DataToSketch expects
 * to receive between one and three arguments.
 * <ul>
 * <li>The first (required) is the value to add to the sketch and must be a primitive.</li>
 *
 * <li>The second (optional) is the lgK from 4 to 21 (default 12).
 * This must be an integral value and must be constant.</li>
 *
 * <li>The third (optional) is the target HLL type and must be a string 'HLL_4',
 * 'HLL_6' or 'HLL_8' (default 'HLL_4').</li>
 * </ul>
 *
 * @see org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver
 * #getEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo)
 *
 * @param info Parameter info to validate
 * @return The GenericUDAFEvaluator that should be used to calculate the function.
 */
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info) throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();

  // Validate the correct number of parameters
  if (inspectors.length < 1) {
    throw new UDFArgumentException("Please specify at least 1 argument");
  }

  if (inspectors.length > 3) {
    throw new UDFArgumentException("Please specify no more than 3 arguments");
  }

  // Validate first parameter type
  ObjectInspectorValidator.validateCategoryPrimitive(inspectors[0], 0);

  // Validate second argument if present
  if (inspectors.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[1], 1);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[1])) {
      throw new UDFArgumentTypeException(1, "The second argument must be a constant");
    }
  }

  // Validate third argument if present
  if (inspectors.length > 2) {
    ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[2], 2, PrimitiveCategory.STRING);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[2])) {
      throw new UDFArgumentTypeException(2, "The third argument must be a constant");
    }
  }

  return new DataToSketchEvaluator();
}
 
Example 30
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);
  mode_ = mode;
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    // input is original data
    inputInspector_ = (PrimitiveObjectInspector) parameters[0];
    if (parameters.length > 1) {
      lgKInspector_ = (PrimitiveObjectInspector) parameters[1];
    }
    if (parameters.length > 2) {
      hllTypeInspector_ = (PrimitiveObjectInspector) parameters[2];
    }
  } else {
    // input for PARTIAL2 and FINAL is the output from PARTIAL1
    intermediateInspector_ = (StructObjectInspector) parameters[0];
  }

  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the lgK and the target HLL type
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(LG_K_FIELD, HLL_TYPE_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.STRING),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  // final results include just the sketch
  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}