Java Code Examples for org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils#getTypeInfoFromTypeString()

The following examples show how to use org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils#getTypeInfoFromTypeString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JsonSerdeUtilsTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTopLevelPrimitive() throws Exception {
    Double expected1 = Double.valueOf(3.3);
    Text json1 = new Text("3.3");
    TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("double");

    Object deserialized1 = JsonSerdeUtils.deserialize(json1, type1);
    Assert.assertEquals(expected1, deserialized1);
    Text serialized1 = JsonSerdeUtils.serialize(deserialized1,
        HCatRecordObjectInspectorFactory.getStandardObjectInspectorFromTypeInfo(type1));
    Assert.assertEquals(json1, serialized1);

    Boolean expected2 = Boolean.FALSE;
    Text json2 = new Text("false");

    Boolean deserialized2 = JsonSerdeUtils.deserialize(json2);
    Assert.assertEquals(expected2, deserialized2);
    Text serialized2 = JsonSerdeUtils.serialize(deserialized2,
        PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    Assert.assertEquals(json2, serialized2);
}
 
Example 2
Source File: JsonSerdeUtilsTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTopLevelArray() throws Exception {
    List<String> expected1 = Arrays.asList("Taro", "Tanaka");
    Text json1 = new Text("[\"Taro\",\"Tanaka\"]");
    TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("array<string>");

    List<Object> deserialized1 = JsonSerdeUtils.deserialize(json1, type1);
    assertRecordEquals(expected1, deserialized1);
    Text serialized1 = JsonSerdeUtils.serialize(deserialized1,
        HCatRecordObjectInspectorFactory.getStandardObjectInspectorFromTypeInfo(type1));
    Assert.assertEquals(json1, serialized1);

    List<Double> expected2 = Arrays.asList(1.1d, 2.2d, 3.3d);
    Text json2 = new Text("[1.1,2.2,3.3]");
    TypeInfo type2 = TypeInfoUtils.getTypeInfoFromTypeString("array<double>");

    List<Object> deserialized2 = JsonSerdeUtils.deserialize(json2, type2);
    assertRecordEquals(expected2, deserialized2);
    Text serialized2 = JsonSerdeUtils.serialize(deserialized2,
        HCatRecordObjectInspectorFactory.getStandardObjectInspectorFromTypeInfo(type2));
    Assert.assertEquals(json2, serialized2);
}
 
Example 3
Source File: HiveTypeConverter.java    From metacat with Apache License 2.0 6 votes vote down vote up
@Override
public Type toMetacatType(final String type) {
    // Hack to fix presto "varchar" type coming in with no length which is required by Hive.
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
        "varchar".equals(type.toLowerCase()) ? serdeConstants.STRING_TYPE_NAME : type);
    ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo);
    // The standard struct object inspector forces field names to lower case, however in Metacat we need to preserve
    // the original case of the struct fields so we wrap it with our wrapper to force the fieldNames to keep
    // their original case
    if (typeInfo.getCategory().equals(ObjectInspector.Category.STRUCT)) {
        final StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        final StandardStructObjectInspector objectInspector = (StandardStructObjectInspector) oi;
        oi = new HiveTypeConverter.SameCaseStandardStructObjectInspector(
            structTypeInfo.getAllStructFieldNames(), objectInspector);
    }
    return getCanonicalType(oi);
}
 
Example 4
Source File: HiveUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Nonnull
public static ObjectInspector getObjectInspector(@Nonnull final String typeString,
        final boolean preferWritable) {
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeString);
    if (preferWritable) {
        return TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo);
    } else {
        return TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo);
    }
}
 
Example 5
Source File: PutORCTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
private void verifyORCUsers(final Path orcUsers, final int numExpectedUsers, BiFunction<List<Object>, Integer, Void> assertFunction) throws IOException {
    Reader reader = OrcFile.createReader(orcUsers, OrcFile.readerOptions(testConf));
    RecordReader recordReader = reader.rows();

    TypeInfo typeInfo =
            TypeInfoUtils.getTypeInfoFromTypeString("struct<name:string,favorite_number:int,favorite_color:string,scale:double>");
    StructObjectInspector inspector = (StructObjectInspector)
            OrcStruct.createObjectInspector(typeInfo);

    int currUser = 0;
    Object nextRecord = null;
    while ((nextRecord = recordReader.next(nextRecord)) != null) {
        Assert.assertNotNull(nextRecord);
        Assert.assertTrue("Not an OrcStruct", nextRecord instanceof OrcStruct);
        List<Object> x = inspector.getStructFieldsDataAsList(nextRecord);

        if (assertFunction == null) {
            assertEquals("name" + currUser, x.get(0).toString());
            assertEquals(currUser, ((IntWritable) x.get(1)).get());
            assertEquals("blue" + currUser, x.get(2).toString());
            assertEquals(10.0 * currUser, ((DoubleWritable) x.get(3)).get(), Double.MIN_VALUE);
        } else {
            assertFunction.apply(x, currUser);
        }
        currUser++;
    }

    assertEquals(numExpectedUsers, currUser);
}
 
Example 6
Source File: CobolStringField.java    From Cobol-to-Hive with Apache License 2.0 5 votes vote down vote up
public CobolStringField(String debugInfo, int levelNo, String name,
		String picClause) {
	super();
	super.debugInfo = debugInfo;
	super.levelNo = levelNo;
	super.type = CobolFieldType.STRING;
	super.name = name;
	String fieldType = "string";
	if (picClause.contains("(")) {
		String[] s = picClause.split("\\(|\\)|\\.");
		if (s.length == 2) {
			try {
				super.length = Integer.parseInt(s[1]);
			} catch (NumberFormatException e) {
				throw e;
			}
		} else {
			throw new RuntimeException(
					"Alphanumeric Picture clause has more brackets:"
							+ this.debugInfo);
		}
	} else {
		if (picClause.trim().toLowerCase().matches("[x|a]+\\."))
			super.length = picClause.length() -1;
		else if (picClause.trim().toLowerCase().matches("[x|a]+"))
			super.length = picClause.length();
		else {
			throw new RuntimeException(
					"Alphanumeric Picture clause incorrect '"
							+ this.debugInfo);

		}
	}
	if (super.length < 65355) {
		fieldType = "varchar(" + this.length + ")";
	}
	super.typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fieldType);
	this.oi = TypeInfoUtils
			.getStandardJavaObjectInspectorFromTypeInfo(this.typeInfo);
}
 
Example 7
Source File: JsonSerdeUtilsTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testTopLevelMap() throws Exception {
    Map<String, Integer> expected1 = ImmutableMap.of("one", 1, "two", 2);
    Text json1 = new Text("{\"one\":1,\"two\":2}");
    TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("map<string,int>");

    Map<String, Integer> deserialized1 = JsonSerdeUtils.deserialize(json1, type1);
    Assert.assertEquals(expected1, deserialized1);
    Text serialized1 = JsonSerdeUtils.serialize(deserialized1,
        HCatRecordObjectInspectorFactory.getStandardObjectInspectorFromTypeInfo(type1));
    Assert.assertEquals(json1, serialized1);
}
 
Example 8
Source File: JsonSerdeUtilsTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testStructWithoutColumnNames() throws Exception {
    Text json1 = new Text("{ \"person\" : { \"name\" : \"makoto\" , \"age\" : 37 } }");
    TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("struct<name:string,age:int>");
    List<Object> expected1 = Arrays.<Object>asList("makoto", 37);

    List<Object> deserialized1 =
            JsonSerdeUtils.deserialize(json1, Arrays.asList("person"), Arrays.asList(type1));

    assertRecordEquals(expected1, deserialized1);
}
 
Example 9
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private static List<ColumnInfo> buildColumnInfo(final Table table, final InputFormat<?, ?> format, boolean
  includeComplexParquetCols) {
  final List<ColumnInfo> columnInfos = new ArrayList<>();
  for (FieldSchema hiveField : table.getSd().getCols()) {
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
    Field f = HiveSchemaConverter.getArrowFieldFromHiveType(hiveField.getName(), typeInfo, format, includeComplexParquetCols);
    if (f != null) {
      columnInfos.add(getColumnInfo(typeInfo));
    }
  }
  return columnInfos;
}
 
Example 10
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private static boolean isFieldTypeVarchar(FieldSchema hiveField) {
  final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
  if (typeInfo.getCategory() == Category.PRIMITIVE) {
    PrimitiveTypeInfo pTypeInfo = (PrimitiveTypeInfo) typeInfo;
    if (pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.VARCHAR ||
      pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.CHAR) {
      return true;
    }
  }
  return false;
}
 
Example 11
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private static List<ColumnInfo> buildColumnInfo(final Table table, final InputFormat<?, ?> format, final boolean includeComplexParquetCols) {
  final List<ColumnInfo> columnInfos = new ArrayList<>();
  for (FieldSchema hiveField : table.getSd().getCols()) {
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
    Field f = HiveSchemaConverter.getArrowFieldFromHiveType(hiveField.getName(), typeInfo, format, includeComplexParquetCols);
    if (f != null) {
      columnInfos.add(getColumnInfo(typeInfo));
    }
  }
  return columnInfos;
}
 
Example 12
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private static boolean isFieldTypeVarchar(FieldSchema hiveField) {
  final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
  if (typeInfo.getCategory() == Category.PRIMITIVE) {
    PrimitiveTypeInfo pTypeInfo = (PrimitiveTypeInfo) typeInfo;
    if (pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.VARCHAR ||
      pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.CHAR) {
      return true;
    }
  }
  return false;
}
 
Example 13
Source File: HiveSchemaConverter.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
/**
 * iterates over all fields of a table and checks if any field exceeds
 * maximum allowed nested level
 * @param table
 * @param maxNestedLevels
 */
public static void checkFieldNestedLevels(final Table table, int maxNestedLevels) {
  for (FieldSchema hiveField : table.getSd().getCols()) {
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
    int depth = findFieldDepth(typeInfo);
    if (depth > maxNestedLevels) {
      throw new ColumnNestedTooDeepException(hiveField.getName(), maxNestedLevels);
    }
  }
}
 
Example 14
Source File: TestRecordReaderImpl.java    From hive-dwrf with Apache License 2.0 4 votes vote down vote up
public static ObjectInspector createJavaObjectInspectorFromFieldSchema(String columnTypeString) {
  TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(columnTypeString);
  return TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo);
}
 
Example 15
Source File: TestNiFiOrcUtils.java    From nifi with Apache License 2.0 4 votes vote down vote up
public static TypeInfo buildNestedComplexOrcSchema() {
    return TypeInfoUtils.getTypeInfoFromTypeString("struct<myMapOfArray:map<string,array<double>>,myArrayOfMap:array<map<string,string>>>");
}
 
Example 16
Source File: TestNiFiOrcUtils.java    From nifi with Apache License 2.0 4 votes vote down vote up
public static TypeInfo buildComplexOrcSchema() {
    return TypeInfoUtils.getTypeInfoFromTypeString("struct<myInt:int,myMap:map<string,double>,myEnum:string,myLongOrFloat:uniontype<int>,myIntList:array<int>>");
}
 
Example 17
Source File: TestNiFiOrcUtils.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
public static TypeInfo buildComplexOrcSchema() {
    return TypeInfoUtils.getTypeInfoFromTypeString("struct<myInt:int,myMap:map<string,double>,myEnum:string,myLongOrFloat:uniontype<int>,myIntList:array<int>>");
}
 
Example 18
Source File: CobolFieldDecl.java    From Cobol-to-Hive with Apache License 2.0 4 votes vote down vote up
private void setFieldTypeInfo() {
	this.fieldTypeInfo = TypeInfoUtils
			.getTypeInfoFromTypeString(this.fieldType);
}
 
Example 19
Source File: TestNiFiOrcUtils.java    From nifi with Apache License 2.0 4 votes vote down vote up
public static TypeInfo buildNestedComplexOrcSchema() {
    return TypeInfoUtils.getTypeInfoFromTypeString("struct<myMapOfArray:map<string,array<double>>,myArrayOfMap:array<map<string,string>>>");
}
 
Example 20
Source File: FileLoadTest.java    From hugegraph-loader with Apache License 2.0 4 votes vote down vote up
@Test
public void testOrcCompressFile() {
    // TODO: add test for blob and uuid
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
            "struct<" +
            "name:string," +
            "p_boolean:boolean," +
            "p_byte:tinyint," +
            "p_int:int," +
            "p_long:bigint," +
            "p_float:float," +
            "p_double:double," +
            "p_string:string," +
            "p_date:date" +
            ">");

    Date now = Date.valueOf("2019-12-09");
    ioUtil.writeOrc("vertex_person.orc", typeInfo,
                    "marko", true, (byte) 1, 2, 3L,
                    4.0F, 5.0D, "marko", now);
    String[] args = new String[]{
            "-f", structPath("orc_compress_file/struct.json"),
            "-s", configPath("orc_compress_file/schema.groovy"),
            "-g", GRAPH,
            "-h", SERVER,
            "--batch-insert-threads", "2",
            "--test-mode", "true"
    };
    HugeGraphLoader.main(args);

    List<Vertex> vertices = CLIENT.graph().listVertices();
    Assert.assertEquals(1, vertices.size());

    Vertex vertex = vertices.get(0);
    Assert.assertEquals(true, vertex.property("p_boolean"));
    Assert.assertEquals(1, vertex.property("p_byte"));
    Assert.assertEquals(2, vertex.property("p_int"));
    Assert.assertEquals(3, vertex.property("p_long"));
    Assert.assertEquals(4.0D, vertex.property("p_float"));
    Assert.assertEquals(5.0D, vertex.property("p_double"));
    Assert.assertEquals("marko", vertex.property("p_string"));
    Assert.assertEquals(now.toEpochMilli(), vertex.property("p_date"));
}