Java Code Examples for org.apache.hadoop.hive.common.type.HiveVarchar

The following examples show how to use org.apache.hadoop.hive.common.type.HiveVarchar. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: HiveTypeUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypeInfo visit(VarCharType varCharType) {
	// Flink's StringType is defined as VARCHAR(Integer.MAX_VALUE)
	// We don't have more information in LogicalTypeRoot to distinguish StringType and a VARCHAR(Integer.MAX_VALUE) instance
	// Thus always treat VARCHAR(Integer.MAX_VALUE) as StringType
	if (varCharType.getLength() == Integer.MAX_VALUE) {
		return TypeInfoFactory.stringTypeInfo;
	}
	if (varCharType.getLength() > HiveVarchar.MAX_VARCHAR_LENGTH) {
		throw new CatalogException(
				String.format("HiveCatalog doesn't support varchar type with length of '%d'. " +
							"The maximum length is %d",
							varCharType.getLength(), HiveVarchar.MAX_VARCHAR_LENGTH));
	}
	return TypeInfoFactory.getVarcharTypeInfo(varCharType.getLength());
}
 
Example 2
Source Project: flink   Source File: HiveCatalogDataTypeTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDataTypes() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.TINYINT(),
		DataTypes.SMALLINT(),
		DataTypes.INT(),
		DataTypes.BIGINT(),
		DataTypes.FLOAT(),
		DataTypes.DOUBLE(),
		DataTypes.BOOLEAN(),
		DataTypes.STRING(),
		DataTypes.BYTES(),
		DataTypes.DATE(),
		DataTypes.TIMESTAMP(),
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH),
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH),
		DataTypes.DECIMAL(5, 3)
	};

	verifyDataTypes(types);
}
 
Example 3
@Override
public Object convert(Object input) {
  if (input == null) {
    return null;
  }
  // unfortunately we seem to get instances of varchar object inspectors without params
  // when an old-style UDF has an evaluate() method with varchar arguments.
  // If we disallow varchar in old-style UDFs and only allow GenericUDFs to be defined
  // with varchar arguments, then we might be able to enforce this properly.
  //if (typeParams == null) {
  //  throw new RuntimeException("varchar type used without type params");
  //}
  HiveVarcharWritable hc = new HiveVarcharWritable();
  switch (inputOI.getPrimitiveCategory()) {
    case BOOLEAN:
      return outputOI.set(hc,
          ((BooleanObjectInspector) inputOI).get(input)
              ? new HiveVarchar("TRUE", -1) : new HiveVarchar("FALSE", -1));
    default:
      return outputOI.set(hc, PrimitiveObjectInspectorUtils.getHiveVarchar(input, inputOI));
  }
}
 
Example 4
Source Project: pxf   Source File: HiveORCSearchArgumentBuilder.java    License: Apache License 2.0 6 votes vote down vote up
private static Object boxLiteral(Object literal) {
    if (literal instanceof String ||
            literal instanceof Long ||
            literal instanceof Double ||
            literal instanceof Date ||
            literal instanceof Timestamp ||
            literal instanceof HiveDecimal ||
            literal instanceof BigDecimal ||
            literal instanceof Boolean) {
        return literal;
    } else if (literal instanceof HiveChar ||
            literal instanceof HiveVarchar) {
        return StringUtils.stripEnd(literal.toString(), null);
    } else if (literal instanceof Byte ||
            literal instanceof Short ||
            literal instanceof Integer) {
        return ((Number) literal).longValue();
    } else if (literal instanceof Float) {
        // to avoid change in precision when upcasting float to double
        // we convert the literal to string and parse it as double. (HIVE-8460)
        return Double.parseDouble(literal.toString());
    } else {
        throw new IllegalArgumentException("Unknown type for literal " +
                literal);
    }
}
 
Example 5
private Object convertClobType(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  ClobRef cr = (ClobRef) val;
  String s = cr.isExternal() ? cr.toString() : cr.getData();

  if (hfsType == HCatFieldSchema.Type.STRING) {
    return s;
  } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
    VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
    HiveVarchar hvc = new HiveVarchar(s, vti.getLength());
    return hvc;
  } else if (hfsType == HCatFieldSchema.Type.CHAR) {
    CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
    HiveChar hc = new HiveChar(s, cti.getLength());
    return hc;
  }
  return null;
}
 
Example 6
public void testStringTypes() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "char(14)", Types.CHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.CHAR, 14, 0,
        new HiveChar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.VARCHAR, 14, 0,
        new HiveVarchar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
      "longvarchar", Types.LONGVARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
 
Example 7
public void testTableCreation() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--create-hcatalog-table");
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
    null, true, false);
}
 
Example 8
public void testTableCreationWithPartition() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
     new HiveVarchar("1", 20), "1", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
     new HiveVarchar("2", 20), "2", KeyType.STATIC_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--hive-partition-key");
  addlArgsArray.add("col1");
  addlArgsArray.add("--hive-partition-value");
  addlArgsArray.add("2");
  addlArgsArray.add("--create-hcatalog-table");
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
 
Example 9
public void testTableCreationWithMultipleStaticPartKeys() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("2", 20), "2", KeyType.STATIC_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--hcatalog-partition-keys");
  addlArgsArray.add("col0,col1");
  addlArgsArray.add("--hcatalog-partition-values");
  addlArgsArray.add("1,2");
  addlArgsArray.add("--create-hcatalog-table");
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
 
Example 10
public void testTableCreationWithStorageStanza() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("1", 20), "1", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("2", 20), "2", KeyType.STATIC_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--hive-partition-key");
  addlArgsArray.add("col1");
  addlArgsArray.add("--hive-partition-value");
  addlArgsArray.add("2");
  addlArgsArray.add("--create-hcatalog-table");
  addlArgsArray.add("--hcatalog-storage-stanza");
  addlArgsArray.add(HCatalogTestUtils.STORED_AS_TEXT);
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
 
Example 11
public void testCreateTableWithPreExistingTable() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("1", 20), "1", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--create-hcatalog-table");
  setExtraArgs(addlArgsArray);
  try {
    // Precreate table
    utils.createHCatTable(CreateMode.CREATE, TOTAL_RECORDS, table, cols);
    runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
      null, true, false);
    fail("HCatalog job with --create-hcatalog-table and pre-existing"
      + " table should fail");
  } catch (Exception e) {
    LOG.debug("Caught expected exception while running "
      + " create-hcatalog-table with pre-existing table test", e);
  }
}
 
Example 12
public void testStringTypes() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "char(14)", Types.CHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.CHAR, 14, 0,
        new HiveChar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.VARCHAR, 14, 0,
        new HiveVarchar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
      "longvarchar", Types.LONGVARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
 
Example 13
Source Project: hadoop-etl-udfs   Source File: HdfsSerDeImportService.java    License: MIT License 6 votes vote down vote up
private static Object getJavaObjectFromPrimitiveData(Object data, ObjectInspector objInsp) {
    assert(objInsp.getCategory() == Category.PRIMITIVE);
    if (data == null) {
        return null;
    }
    if (data instanceof BytesWritable && objInsp instanceof WritableHiveDecimalObjectInspector) {
        // BytesWritable cannot be directly cast to HiveDecimalWritable
        WritableHiveDecimalObjectInspector oi = (WritableHiveDecimalObjectInspector) objInsp;
        data = oi.create(((BytesWritable) data).getBytes(), oi.scale());
    }
    Object obj = ObjectInspectorUtils.copyToStandardJavaObject(data, objInsp);
    if (obj instanceof HiveDecimal) {
        obj = ((HiveDecimal) obj).bigDecimalValue();
    } else if (obj instanceof HiveVarchar || obj instanceof HiveChar) {
        obj = obj.toString();
    } else if (obj instanceof byte[]) {
        obj = Hex.encodeHexString((byte[]) obj);
    }
    return obj;
}
 
Example 14
Source Project: hadoop-etl-udfs   Source File: HdfsSerDeImportService.java    License: MIT License 6 votes vote down vote up
private static Object getJavaObjectFromFieldData(Object data, ObjectInspector objInsp) {
    if (data == null) {
        return null;
    }
    if (objInsp.getCategory() == Category.PRIMITIVE) {
        Object obj = ObjectInspectorUtils.copyToStandardJavaObject(data, objInsp);
        if (obj instanceof HiveDecimal) {
            obj = ((HiveDecimal) obj).bigDecimalValue();
        } else if (obj instanceof HiveVarchar || obj instanceof HiveChar) {
            obj = obj.toString();
        } else if (obj instanceof byte[]) {
            obj = Hex.encodeHexString((byte[]) obj);
        }
        return obj;
    } else if (objInsp.getCategory() == Category.LIST) {
        return getJsonArrayFromFieldData(data, objInsp, Json.createBuilderFactory(null)).build().toString();
    } else {
        return getJsonObjectFromFieldData(data, objInsp, Json.createBuilderFactory(null)).build().toString();
    }
}
 
Example 15
Source Project: flink   Source File: HiveTypeUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypeInfo visit(VarCharType varCharType) {
	// Flink's StringType is defined as VARCHAR(Integer.MAX_VALUE)
	// We don't have more information in LogicalTypeRoot to distinguish StringType and a VARCHAR(Integer.MAX_VALUE) instance
	// Thus always treat VARCHAR(Integer.MAX_VALUE) as StringType
	if (varCharType.getLength() == Integer.MAX_VALUE) {
		return TypeInfoFactory.stringTypeInfo;
	}
	// Flink and Hive have different length limit for VARCHAR. Promote it to STRING if it exceeds the limits of
	// Hive and we're told not to check precision. This can be useful when calling Hive UDF to process data.
	if (varCharType.getLength() > HiveVarchar.MAX_VARCHAR_LENGTH || varCharType.getLength() < 1) {
		if (checkPrecision) {
			throw new CatalogException(
					String.format("HiveCatalog doesn't support varchar type with length of '%d'. " +
									"The supported length is [%d, %d]",
							varCharType.getLength(), 1, HiveVarchar.MAX_VARCHAR_LENGTH));
		} else {
			return TypeInfoFactory.stringTypeInfo;
		}
	}
	return TypeInfoFactory.getVarcharTypeInfo(varCharType.getLength());
}
 
Example 16
Source Project: flink   Source File: HiveCatalogDataTypeTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDataTypes() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.TINYINT(),
		DataTypes.SMALLINT(),
		DataTypes.INT(),
		DataTypes.BIGINT(),
		DataTypes.FLOAT(),
		DataTypes.DOUBLE(),
		DataTypes.BOOLEAN(),
		DataTypes.STRING(),
		DataTypes.BYTES(),
		DataTypes.DATE(),
		DataTypes.TIMESTAMP(9),
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH),
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH),
		DataTypes.DECIMAL(5, 3)
	};

	verifyDataTypes(types);
}
 
Example 17
Source Project: presto   Source File: TestHiveFileFormats.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailForLongVarcharPartitionColumn()
        throws Exception
{
    TestColumn partitionColumn = new TestColumn("partition_column", getPrimitiveJavaObjectInspector(new VarcharTypeInfo(3)), "test", utf8Slice("tes"), true);
    TestColumn varcharColumn = new TestColumn("varchar_column", getPrimitiveJavaObjectInspector(new VarcharTypeInfo(3)), new HiveVarchar("tes", 3), utf8Slice("tes"));

    List<TestColumn> columns = ImmutableList.of(partitionColumn, varcharColumn);

    HiveErrorCode expectedErrorCode = HiveErrorCode.HIVE_INVALID_PARTITION_VALUE;
    String expectedMessage = "Invalid partition value 'test' for varchar(3) partition key: partition_column";

    assertThatFileFormat(RCTEXT)
            .withColumns(columns)
            .isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage)
            .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);

    assertThatFileFormat(RCBINARY)
            .withColumns(columns)
            .isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage)
            .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);

    assertThatFileFormat(ORC)
            .withColumns(columns)
            .isFailingForPageSource(new OrcPageSourceFactory(new OrcReaderOptions(), HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage);

    assertThatFileFormat(PARQUET)
            .withColumns(columns)
            .withSession(PARQUET_SESSION)
            .isFailingForPageSource(new ParquetPageSourceFactory(HDFS_ENVIRONMENT, STATS, new ParquetReaderConfig()), expectedErrorCode, expectedMessage);

    assertThatFileFormat(SEQUENCEFILE)
            .withColumns(columns)
            .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);

    assertThatFileFormat(TEXTFILE)
            .withColumns(columns)
            .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
}
 
Example 18
Source Project: presto   Source File: TestHiveBucketing.java    License: Apache License 2.0 5 votes vote down vote up
public static int getHiveBucketHashCode(BucketingVersion bucketingVersion, List<Entry<ObjectInspector, Object>> columnBindings)
{
    ObjectInspector[] objectInspectors = new ObjectInspector[columnBindings.size()];
    Object[] objects = new Object[columnBindings.size()];

    int i = 0;
    for (Entry<ObjectInspector, Object> entry : columnBindings) {
        objectInspectors[i] = entry.getKey();
        if (entry.getValue() != null && entry.getKey() instanceof JavaHiveVarcharObjectInspector) {
            JavaHiveVarcharObjectInspector varcharObjectInspector = (JavaHiveVarcharObjectInspector) entry.getKey();
            objects[i] = new HiveVarchar(((String) entry.getValue()), varcharObjectInspector.getMaxLength());
        }
        else {
            objects[i] = entry.getValue();
        }
        i++;
    }

    switch (bucketingVersion) {
        case BUCKETING_V1:
            @SuppressWarnings("deprecation")
            int hashCodeOld = ObjectInspectorUtils.getBucketHashCodeOld(objects, objectInspectors);
            return hashCodeOld;
        case BUCKETING_V2:
            return ObjectInspectorUtils.getBucketHashCode(objects, objectInspectors);
        default:
            throw new IllegalArgumentException("Unsupported bucketing version: " + bucketingVersion);
    }
}
 
Example 19
Source Project: flink   Source File: HiveInspectors.java    License: Apache License 2.0 5 votes vote down vote up
private static ConstantObjectInspector getPrimitiveJavaConstantObjectInspector(PrimitiveTypeInfo typeInfo, Object value) {
	switch (typeInfo.getPrimitiveCategory()) {
		case BOOLEAN:
			return new JavaConstantBooleanObjectInspector((Boolean) value);
		case BYTE:
			return new JavaConstantByteObjectInspector((Byte) value);
		case SHORT:
			return new JavaConstantShortObjectInspector((Short) value);
		case INT:
			return new JavaConstantIntObjectInspector((Integer) value);
		case LONG:
			return new JavaConstantLongObjectInspector((Long) value);
		case FLOAT:
			return new JavaConstantFloatObjectInspector((Float) value);
		case DOUBLE:
			return new JavaConstantDoubleObjectInspector((Double) value);
		case STRING:
			return new JavaConstantStringObjectInspector((String) value);
		case CHAR:
			return new JavaConstantHiveCharObjectInspector((HiveChar) value);
		case VARCHAR:
			return new JavaConstantHiveVarcharObjectInspector((HiveVarchar) value);
		case DATE:
			return new JavaConstantDateObjectInspector((Date) value);
		case TIMESTAMP:
			return new JavaConstantTimestampObjectInspector((Timestamp) value);
		case DECIMAL:
			return new JavaConstantHiveDecimalObjectInspector((HiveDecimal) value);
		case BINARY:
			return new JavaConstantBinaryObjectInspector((byte[]) value);
		case UNKNOWN:
		case VOID:
			// If type is null, we use the Java Constant String to replace
			return new JavaConstantStringObjectInspector((String) value);
		default:
			throw new FlinkHiveUDFException(
				String.format("Cannot find ConstantObjectInspector for %s", typeInfo));
	}
}
 
Example 20
Source Project: flink   Source File: HiveCatalogDataTypeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testVarCharTypeLength() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH + 1)
	};

	exception.expect(CatalogException.class);
	exception.expectMessage("HiveCatalog doesn't support varchar type with length of '65536'. The maximum length is 65535");
	verifyDataTypes(types);
}
 
Example 21
Source Project: pxf   Source File: HiveORCSearchArgumentBuilder.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the type of the given expression node.
 *
 * @param literal the object
 * @return int, string, or float or null if we don't know the type
 */
private PredicateLeaf.Type getType(Object literal) {
    if (literal instanceof Byte ||
            literal instanceof Short ||
            literal instanceof Integer ||
            literal instanceof Long) {
        return PredicateLeaf.Type.LONG;
    } else if (literal instanceof HiveChar ||
            literal instanceof HiveVarchar ||
            literal instanceof String) {
        return PredicateLeaf.Type.STRING;
    } else if (literal instanceof Float ||
            literal instanceof Double) {
        return PredicateLeaf.Type.FLOAT;
    } else if (literal instanceof Date) {
        return PredicateLeaf.Type.DATE;
    } else if (literal instanceof Timestamp) {
        return PredicateLeaf.Type.TIMESTAMP;
    } else if (literal instanceof HiveDecimal ||
            literal instanceof BigDecimal) {
        return PredicateLeaf.Type.DECIMAL;
    } else if (literal instanceof Boolean) {
        return PredicateLeaf.Type.BOOLEAN;
    } else if (literal instanceof List) {
        @SuppressWarnings("unchecked")
        List<Object> l = (List<Object>) literal;
        if (l.size() > 0)
            return getType(l.get(0));
    }
    throw new IllegalArgumentException(String.format("Unknown type for literal %s", literal));
}
 
Example 22
private Object convertStringTypes(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  if (hfsType == HCatFieldSchema.Type.STRING
      || hfsType == HCatFieldSchema.Type.VARCHAR
      || hfsType == HCatFieldSchema.Type.CHAR) {
    String str = val.toString();
    if (doHiveDelimsReplacement) {
      str = FieldFormatter.hiveStringReplaceDelims(str,
        hiveDelimsReplacement, hiveDelimiters);
    }
    if (hfsType == HCatFieldSchema.Type.STRING) {
      return str;
    } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
      VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
      HiveVarchar hvc = new HiveVarchar(str, vti.getLength());
      return hvc;
    } else if (hfsType == HCatFieldSchema.Type.CHAR) {
      CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
      HiveChar hc = new HiveChar(val.toString(), cti.getLength());
      return hc;
    }
  } else if (hfsType == HCatFieldSchema.Type.DECIMAL) {
    BigDecimal bd = new BigDecimal(val.toString(), MathContext.DECIMAL128);
    HiveDecimal hd = HiveDecimal.create(bd);
    return hd;
  }
  return null;
}
 
Example 23
private Object convertBooleanTypes(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  Boolean b = (Boolean) val;
  if (hfsType == HCatFieldSchema.Type.BOOLEAN) {
    return b;
  } else if (hfsType == HCatFieldSchema.Type.TINYINT) {
    return (byte) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.SMALLINT) {
    return (short) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.INT) {
    return (int) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.BIGINT) {
    return (long) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.FLOAT) {
    return (float) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.DOUBLE) {
    return (double) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.STRING) {
    return val.toString();
  } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
    VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
    HiveVarchar hvc = new HiveVarchar(val.toString(), vti.getLength());
    return hvc;
  } else if (hfsType == HCatFieldSchema.Type.CHAR) {
    CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
    HiveChar hChar = new HiveChar(val.toString(), cti.getLength());
    return hChar;
  }
  return null;
}
 
Example 24
Source Project: dremio-oss   Source File: ObjectInspectors.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public HiveVarchar getPrimitiveJavaObject(Object o) {
<#if mode == "Optional">
  if (o == null) {
    return null;
  }
  final NullableVarCharHolder h = (NullableVarCharHolder)o;
<#else>
  final VarCharHolder h = (VarCharHolder)o;
</#if>
  final String s = StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer);
  return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH);
}
 
Example 25
Source Project: Cobol-to-Hive   Source File: CobolStringField.java    License: Apache License 2.0 5 votes vote down vote up
@Override
	public Object deserialize(byte[] rowBytes) {
		byte[] temp = super.transcodeField(super.getBytes(rowBytes));
		String s1 = new String(temp);
//		System.out.println(name+"\t - "+s1+"\t:"+offset+"\[email protected]"+length);
		switch (((PrimitiveTypeInfo) this.typeInfo).getPrimitiveCategory()) {
		case STRING:
			return s1;
		case VARCHAR:
			return new HiveVarchar(s1, this.length);
			//return s1;
		}
		return null;
	}
 
Example 26
@Override
public String extractValue(final Object data, final ObjectInspector objectInspector)
    throws HiveException {
  final Object value = inputObjectInspector.getPrimitiveJavaObject(data);
  if (value instanceof String) {
    return (String) value;
  } else if (value instanceof HiveChar) {
    return ((HiveChar) value).getValue();
  } else if (value instanceof HiveVarchar) {
    return ((HiveVarchar) value).getValue();
  } else {
    throw new UDFArgumentTypeException(0, "unsupported type " + value.getClass().getName());
  }
}
 
Example 27
@Override
public String extractValue(final Object data, final ObjectInspector objectInspector)
    throws HiveException {
  final Object value = inputObjectInspector.getPrimitiveJavaObject(data);
  if (value instanceof String) {
    return (String) value;
  } else if (value instanceof HiveChar) {
    return ((HiveChar) value).getValue();
  } else if (value instanceof HiveVarchar) {
    return ((HiveVarchar) value).getValue();
  } else {
    throw new UDFArgumentTypeException(0, "unsupported type " + value.getClass().getName());
  }
}
 
Example 28
Source Project: flink   Source File: HiveCatalogDataTypeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testVarCharTypeLength() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH + 1)
	};

	exception.expect(CatalogException.class);
	verifyDataTypes(types);
}
 
Example 29
Source Project: HiveRunner   Source File: ConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void otherTypeInfo() {
  assertEquals(HiveDecimal.create("1.234"), Converters.convert("1.234", decimalTypeInfo));
  assertEquals(new HiveChar("foo", -1), Converters.convert("foo", charTypeInfo));
  assertTrue(new HiveVarchar("foo", -1).equals((HiveVarchar) Converters.convert("foo", varcharTypeInfo)));
  assertEquals("foo", Converters.convert("foo", unknownTypeInfo));
  assertEquals("foo", Converters.convert("foo", voidTypeInfo));
}
 
Example 30
Source Project: presto   Source File: TypeConverter.java    License: Apache License 2.0 4 votes vote down vote up
private static TypeInfo toHiveTypeInfo(Type type)
{
    if (BOOLEAN.equals(type)) {
        return HIVE_BOOLEAN.getTypeInfo();
    }
    if (BIGINT.equals(type)) {
        return HIVE_LONG.getTypeInfo();
    }
    if (INTEGER.equals(type)) {
        return HIVE_INT.getTypeInfo();
    }
    if (SMALLINT.equals(type)) {
        return HIVE_SHORT.getTypeInfo();
    }
    if (TINYINT.equals(type)) {
        return HIVE_BYTE.getTypeInfo();
    }
    if (REAL.equals(type)) {
        return HIVE_FLOAT.getTypeInfo();
    }
    if (DOUBLE.equals(type)) {
        return HIVE_DOUBLE.getTypeInfo();
    }
    if (type instanceof VarcharType) {
        VarcharType varcharType = (VarcharType) type;
        if (varcharType.isUnbounded()) {
            return HIVE_STRING.getTypeInfo();
        }
        if (varcharType.getBoundedLength() <= HiveVarchar.MAX_VARCHAR_LENGTH) {
            return getVarcharTypeInfo(varcharType.getBoundedLength());
        }
        throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s. Supported VARCHAR types: VARCHAR(<=%d), VARCHAR.", type, HiveVarchar.MAX_VARCHAR_LENGTH));
    }
    if (type instanceof CharType) {
        CharType charType = (CharType) type;
        int charLength = charType.getLength();
        if (charLength <= HiveChar.MAX_CHAR_LENGTH) {
            return getCharTypeInfo(charLength);
        }
        throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s. Supported CHAR types: CHAR(<=%d).",
                type, HiveChar.MAX_CHAR_LENGTH));
    }
    if (VARBINARY.equals(type)) {
        return HIVE_BINARY.getTypeInfo();
    }
    if (DATE.equals(type)) {
        return HIVE_DATE.getTypeInfo();
    }
    if (TIMESTAMP.equals(type)) {
        return HIVE_TIMESTAMP.getTypeInfo();
    }
    if (TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
        // Hive does not have TIMESTAMP_WITH_TIME_ZONE, this is just a work around for iceberg.
        return HIVE_TIMESTAMP.getTypeInfo();
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale());
    }
    if (isArrayType(type)) {
        TypeInfo elementType = toHiveTypeInfo(type.getTypeParameters().get(0));
        return getListTypeInfo(elementType);
    }
    if (isMapType(type)) {
        TypeInfo keyType = toHiveTypeInfo(type.getTypeParameters().get(0));
        TypeInfo valueType = toHiveTypeInfo(type.getTypeParameters().get(1));
        return getMapTypeInfo(keyType, valueType);
    }
    if (isRowType(type)) {
        ImmutableList.Builder<String> fieldNames = ImmutableList.builder();
        for (TypeSignatureParameter parameter : type.getTypeSignature().getParameters()) {
            if (!parameter.isNamedTypeSignature()) {
                throw new IllegalArgumentException(format("Expected all parameters to be named type, but got %s", parameter));
            }
            NamedTypeSignature namedTypeSignature = parameter.getNamedTypeSignature();
            if (namedTypeSignature.getName().isEmpty()) {
                throw new PrestoException(NOT_SUPPORTED, format("Anonymous row type is not supported in Hive. Please give each field a name: %s", type));
            }
            fieldNames.add(namedTypeSignature.getName().get());
        }
        return getStructTypeInfo(
                fieldNames.build(),
                type.getTypeParameters().stream()
                        .map(TypeConverter::toHiveTypeInfo)
                        .collect(toList()));
    }
    throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s", type));
}