org.apache.hadoop.hive.common.type.HiveVarchar Java Examples

The following examples show how to use org.apache.hadoop.hive.common.type.HiveVarchar. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HiveCatalogDataTypeTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDataTypes() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.TINYINT(),
		DataTypes.SMALLINT(),
		DataTypes.INT(),
		DataTypes.BIGINT(),
		DataTypes.FLOAT(),
		DataTypes.DOUBLE(),
		DataTypes.BOOLEAN(),
		DataTypes.STRING(),
		DataTypes.BYTES(),
		DataTypes.DATE(),
		DataTypes.TIMESTAMP(9),
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH),
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH),
		DataTypes.DECIMAL(5, 3)
	};

	verifyDataTypes(types);
}
 
Example #2
Source File: HCatalogImportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testTableCreationWithStorageStanza() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("1", 20), "1", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("2", 20), "2", KeyType.STATIC_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--hive-partition-key");
  addlArgsArray.add("col1");
  addlArgsArray.add("--hive-partition-value");
  addlArgsArray.add("2");
  addlArgsArray.add("--create-hcatalog-table");
  addlArgsArray.add("--hcatalog-storage-stanza");
  addlArgsArray.add(HCatalogTestUtils.STORED_AS_TEXT);
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
 
Example #3
Source File: HCatalogImportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testCreateTableWithPreExistingTable() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("1", 20), "1", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--create-hcatalog-table");
  setExtraArgs(addlArgsArray);
  try {
    // Precreate table
    utils.createHCatTable(CreateMode.CREATE, TOTAL_RECORDS, table, cols);
    runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
      null, true, false);
    fail("HCatalog job with --create-hcatalog-table and pre-existing"
      + " table should fail");
  } catch (Exception e) {
    LOG.debug("Caught expected exception while running "
      + " create-hcatalog-table with pre-existing table test", e);
  }
}
 
Example #4
Source File: SqoopHCatImportHelper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
private Object convertClobType(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  ClobRef cr = (ClobRef) val;
  String s = cr.isExternal() ? cr.toString() : cr.getData();

  if (hfsType == HCatFieldSchema.Type.STRING) {
    return s;
  } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
    VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
    HiveVarchar hvc = new HiveVarchar(s, vti.getLength());
    return hvc;
  } else if (hfsType == HCatFieldSchema.Type.CHAR) {
    CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
    HiveChar hc = new HiveChar(s, cti.getLength());
    return hc;
  }
  return null;
}
 
Example #5
Source File: HiveORCSearchArgumentBuilder.java    From pxf with Apache License 2.0 6 votes vote down vote up
private static Object boxLiteral(Object literal) {
    if (literal instanceof String ||
            literal instanceof Long ||
            literal instanceof Double ||
            literal instanceof Date ||
            literal instanceof Timestamp ||
            literal instanceof HiveDecimal ||
            literal instanceof BigDecimal ||
            literal instanceof Boolean) {
        return literal;
    } else if (literal instanceof HiveChar ||
            literal instanceof HiveVarchar) {
        return StringUtils.stripEnd(literal.toString(), null);
    } else if (literal instanceof Byte ||
            literal instanceof Short ||
            literal instanceof Integer) {
        return ((Number) literal).longValue();
    } else if (literal instanceof Float) {
        // to avoid change in precision when upcasting float to double
        // we convert the literal to string and parse it as double. (HIVE-8460)
        return Double.parseDouble(literal.toString());
    } else {
        throw new IllegalArgumentException("Unknown type for literal " +
                literal);
    }
}
 
Example #6
Source File: HCatalogExportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testStringTypes() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "char(14)", Types.CHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.CHAR, 14, 0,
        new HiveChar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.VARCHAR, 14, 0,
        new HiveVarchar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
      "longvarchar", Types.LONGVARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
 
Example #7
Source File: CacheablePrimitiveObjectInspectorConverter.java    From transport with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Override
public Object convert(Object input) {
  if (input == null) {
    return null;
  }
  // unfortunately we seem to get instances of varchar object inspectors without params
  // when an old-style UDF has an evaluate() method with varchar arguments.
  // If we disallow varchar in old-style UDFs and only allow GenericUDFs to be defined
  // with varchar arguments, then we might be able to enforce this properly.
  //if (typeParams == null) {
  //  throw new RuntimeException("varchar type used without type params");
  //}
  HiveVarcharWritable hc = new HiveVarcharWritable();
  switch (inputOI.getPrimitiveCategory()) {
    case BOOLEAN:
      return outputOI.set(hc,
          ((BooleanObjectInspector) inputOI).get(input)
              ? new HiveVarchar("TRUE", -1) : new HiveVarchar("FALSE", -1));
    default:
      return outputOI.set(hc, PrimitiveObjectInspectorUtils.getHiveVarchar(input, inputOI));
  }
}
 
Example #8
Source File: HCatalogImportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testStringTypes() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "char(14)", Types.CHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.CHAR, 14, 0,
        new HiveChar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.VARCHAR, 14, 0,
        new HiveVarchar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
      "longvarchar", Types.LONGVARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
 
Example #9
Source File: HiveCatalogDataTypeTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDataTypes() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.TINYINT(),
		DataTypes.SMALLINT(),
		DataTypes.INT(),
		DataTypes.BIGINT(),
		DataTypes.FLOAT(),
		DataTypes.DOUBLE(),
		DataTypes.BOOLEAN(),
		DataTypes.STRING(),
		DataTypes.BYTES(),
		DataTypes.DATE(),
		DataTypes.TIMESTAMP(),
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH),
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH),
		DataTypes.DECIMAL(5, 3)
	};

	verifyDataTypes(types);
}
 
Example #10
Source File: HdfsSerDeImportService.java    From hadoop-etl-udfs with MIT License 6 votes vote down vote up
private static Object getJavaObjectFromPrimitiveData(Object data, ObjectInspector objInsp) {
    assert(objInsp.getCategory() == Category.PRIMITIVE);
    if (data == null) {
        return null;
    }
    if (data instanceof BytesWritable && objInsp instanceof WritableHiveDecimalObjectInspector) {
        // BytesWritable cannot be directly cast to HiveDecimalWritable
        WritableHiveDecimalObjectInspector oi = (WritableHiveDecimalObjectInspector) objInsp;
        data = oi.create(((BytesWritable) data).getBytes(), oi.scale());
    }
    Object obj = ObjectInspectorUtils.copyToStandardJavaObject(data, objInsp);
    if (obj instanceof HiveDecimal) {
        obj = ((HiveDecimal) obj).bigDecimalValue();
    } else if (obj instanceof HiveVarchar || obj instanceof HiveChar) {
        obj = obj.toString();
    } else if (obj instanceof byte[]) {
        obj = Hex.encodeHexString((byte[]) obj);
    }
    return obj;
}
 
Example #11
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeInfo visit(VarCharType varCharType) {
	// Flink's StringType is defined as VARCHAR(Integer.MAX_VALUE)
	// We don't have more information in LogicalTypeRoot to distinguish StringType and a VARCHAR(Integer.MAX_VALUE) instance
	// Thus always treat VARCHAR(Integer.MAX_VALUE) as StringType
	if (varCharType.getLength() == Integer.MAX_VALUE) {
		return TypeInfoFactory.stringTypeInfo;
	}
	if (varCharType.getLength() > HiveVarchar.MAX_VARCHAR_LENGTH) {
		throw new CatalogException(
				String.format("HiveCatalog doesn't support varchar type with length of '%d'. " +
							"The maximum length is %d",
							varCharType.getLength(), HiveVarchar.MAX_VARCHAR_LENGTH));
	}
	return TypeInfoFactory.getVarcharTypeInfo(varCharType.getLength());
}
 
Example #12
Source File: HdfsSerDeImportService.java    From hadoop-etl-udfs with MIT License 6 votes vote down vote up
private static Object getJavaObjectFromFieldData(Object data, ObjectInspector objInsp) {
    if (data == null) {
        return null;
    }
    if (objInsp.getCategory() == Category.PRIMITIVE) {
        Object obj = ObjectInspectorUtils.copyToStandardJavaObject(data, objInsp);
        if (obj instanceof HiveDecimal) {
            obj = ((HiveDecimal) obj).bigDecimalValue();
        } else if (obj instanceof HiveVarchar || obj instanceof HiveChar) {
            obj = obj.toString();
        } else if (obj instanceof byte[]) {
            obj = Hex.encodeHexString((byte[]) obj);
        }
        return obj;
    } else if (objInsp.getCategory() == Category.LIST) {
        return getJsonArrayFromFieldData(data, objInsp, Json.createBuilderFactory(null)).build().toString();
    } else {
        return getJsonObjectFromFieldData(data, objInsp, Json.createBuilderFactory(null)).build().toString();
    }
}
 
Example #13
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeInfo visit(VarCharType varCharType) {
	// Flink's StringType is defined as VARCHAR(Integer.MAX_VALUE)
	// We don't have more information in LogicalTypeRoot to distinguish StringType and a VARCHAR(Integer.MAX_VALUE) instance
	// Thus always treat VARCHAR(Integer.MAX_VALUE) as StringType
	if (varCharType.getLength() == Integer.MAX_VALUE) {
		return TypeInfoFactory.stringTypeInfo;
	}
	// Flink and Hive have different length limit for VARCHAR. Promote it to STRING if it exceeds the limits of
	// Hive and we're told not to check precision. This can be useful when calling Hive UDF to process data.
	if (varCharType.getLength() > HiveVarchar.MAX_VARCHAR_LENGTH || varCharType.getLength() < 1) {
		if (checkPrecision) {
			throw new CatalogException(
					String.format("HiveCatalog doesn't support varchar type with length of '%d'. " +
									"The supported length is [%d, %d]",
							varCharType.getLength(), 1, HiveVarchar.MAX_VARCHAR_LENGTH));
		} else {
			return TypeInfoFactory.stringTypeInfo;
		}
	}
	return TypeInfoFactory.getVarcharTypeInfo(varCharType.getLength());
}
 
Example #14
Source File: HCatalogImportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testTableCreation() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--create-hcatalog-table");
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
    null, true, false);
}
 
Example #15
Source File: HCatalogImportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testTableCreationWithPartition() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
     new HiveVarchar("1", 20), "1", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
     new HiveVarchar("2", 20), "2", KeyType.STATIC_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--hive-partition-key");
  addlArgsArray.add("col1");
  addlArgsArray.add("--hive-partition-value");
  addlArgsArray.add("2");
  addlArgsArray.add("--create-hcatalog-table");
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
 
Example #16
Source File: HCatalogImportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testTableCreationWithMultipleStaticPartKeys() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
      "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.VARCHAR, 20, 0,
      new HiveVarchar("2", 20), "2", KeyType.STATIC_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  addlArgsArray.add("--hcatalog-partition-keys");
  addlArgsArray.add("col0,col1");
  addlArgsArray.add("--hcatalog-partition-values");
  addlArgsArray.add("1,2");
  addlArgsArray.add("--create-hcatalog-table");
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false);
}
 
Example #17
Source File: ObjectInspectors.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Override
public HiveVarchar getPrimitiveJavaObject(Object o) {
<#if mode == "Optional">
  if (o == null) {
    return null;
  }
  final NullableVarCharHolder h = (NullableVarCharHolder)o;
<#else>
  final VarCharHolder h = (VarCharHolder)o;
</#if>
  final String s = StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer);
  return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH);
}
 
Example #18
Source File: SqoopHCatImportHelper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
private Object convertBooleanTypes(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  Boolean b = (Boolean) val;
  if (hfsType == HCatFieldSchema.Type.BOOLEAN) {
    return b;
  } else if (hfsType == HCatFieldSchema.Type.TINYINT) {
    return (byte) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.SMALLINT) {
    return (short) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.INT) {
    return (int) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.BIGINT) {
    return (long) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.FLOAT) {
    return (float) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.DOUBLE) {
    return (double) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.STRING) {
    return val.toString();
  } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
    VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
    HiveVarchar hvc = new HiveVarchar(val.toString(), vti.getLength());
    return hvc;
  } else if (hfsType == HCatFieldSchema.Type.CHAR) {
    CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
    HiveChar hChar = new HiveChar(val.toString(), cti.getLength());
    return hChar;
  }
  return null;
}
 
Example #19
Source File: DataToStringsSketchUDAF.java    From incubator-datasketches-hive with Apache License 2.0 5 votes vote down vote up
@Override
public String extractValue(final Object data, final ObjectInspector objectInspector)
    throws HiveException {
  final Object value = inputObjectInspector.getPrimitiveJavaObject(data);
  if (value instanceof String) {
    return (String) value;
  } else if (value instanceof HiveChar) {
    return ((HiveChar) value).getValue();
  } else if (value instanceof HiveVarchar) {
    return ((HiveVarchar) value).getValue();
  } else {
    throw new UDFArgumentTypeException(0, "unsupported type " + value.getClass().getName());
  }
}
 
Example #20
Source File: DataToStringsSketchUDAF.java    From incubator-datasketches-hive with Apache License 2.0 5 votes vote down vote up
@Override
public String extractValue(final Object data, final ObjectInspector objectInspector)
    throws HiveException {
  final Object value = inputObjectInspector.getPrimitiveJavaObject(data);
  if (value instanceof String) {
    return (String) value;
  } else if (value instanceof HiveChar) {
    return ((HiveChar) value).getValue();
  } else if (value instanceof HiveVarchar) {
    return ((HiveVarchar) value).getValue();
  } else {
    throw new UDFArgumentTypeException(0, "unsupported type " + value.getClass().getName());
  }
}
 
Example #21
Source File: CobolStringField.java    From Cobol-to-Hive with Apache License 2.0 5 votes vote down vote up
@Override
	public Object deserialize(byte[] rowBytes) {
		byte[] temp = super.transcodeField(super.getBytes(rowBytes));
		String s1 = new String(temp);
//		System.out.println(name+"\t - "+s1+"\t:"+offset+"\t@"+length);
		switch (((PrimitiveTypeInfo) this.typeInfo).getPrimitiveCategory()) {
		case STRING:
			return s1;
		case VARCHAR:
			return new HiveVarchar(s1, this.length);
			//return s1;
		}
		return null;
	}
 
Example #22
Source File: SqoopHCatImportHelper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
private Object convertStringTypes(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  if (hfsType == HCatFieldSchema.Type.STRING
      || hfsType == HCatFieldSchema.Type.VARCHAR
      || hfsType == HCatFieldSchema.Type.CHAR) {
    String str = val.toString();
    if (doHiveDelimsReplacement) {
      str = FieldFormatter.hiveStringReplaceDelims(str,
        hiveDelimsReplacement, hiveDelimiters);
    }
    if (hfsType == HCatFieldSchema.Type.STRING) {
      return str;
    } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
      VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
      HiveVarchar hvc = new HiveVarchar(str, vti.getLength());
      return hvc;
    } else if (hfsType == HCatFieldSchema.Type.CHAR) {
      CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
      HiveChar hc = new HiveChar(val.toString(), cti.getLength());
      return hc;
    }
  } else if (hfsType == HCatFieldSchema.Type.DECIMAL) {
    BigDecimal bd = new BigDecimal(val.toString(), MathContext.DECIMAL128);
    HiveDecimal hd = HiveDecimal.create(bd);
    return hd;
  }
  return null;
}
 
Example #23
Source File: HiveORCSearchArgumentBuilder.java    From pxf with Apache License 2.0 5 votes vote down vote up
/**
 * Get the type of the given expression node.
 *
 * @param literal the object
 * @return int, string, or float or null if we don't know the type
 */
private PredicateLeaf.Type getType(Object literal) {
    if (literal instanceof Byte ||
            literal instanceof Short ||
            literal instanceof Integer ||
            literal instanceof Long) {
        return PredicateLeaf.Type.LONG;
    } else if (literal instanceof HiveChar ||
            literal instanceof HiveVarchar ||
            literal instanceof String) {
        return PredicateLeaf.Type.STRING;
    } else if (literal instanceof Float ||
            literal instanceof Double) {
        return PredicateLeaf.Type.FLOAT;
    } else if (literal instanceof Date) {
        return PredicateLeaf.Type.DATE;
    } else if (literal instanceof Timestamp) {
        return PredicateLeaf.Type.TIMESTAMP;
    } else if (literal instanceof HiveDecimal ||
            literal instanceof BigDecimal) {
        return PredicateLeaf.Type.DECIMAL;
    } else if (literal instanceof Boolean) {
        return PredicateLeaf.Type.BOOLEAN;
    } else if (literal instanceof List) {
        @SuppressWarnings("unchecked")
        List<Object> l = (List<Object>) literal;
        if (l.size() > 0)
            return getType(l.get(0));
    }
    throw new IllegalArgumentException(String.format("Unknown type for literal %s", literal));
}
 
Example #24
Source File: HiveCatalogDataTypeTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testVarCharTypeLength() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH + 1)
	};

	exception.expect(CatalogException.class);
	exception.expectMessage("HiveCatalog doesn't support varchar type with length of '65536'. The maximum length is 65535");
	verifyDataTypes(types);
}
 
Example #25
Source File: HiveCatalogDataTypeTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testVarCharTypeLength() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH + 1)
	};

	exception.expect(CatalogException.class);
	verifyDataTypes(types);
}
 
Example #26
Source File: HiveInspectors.java    From flink with Apache License 2.0 5 votes vote down vote up
private static ConstantObjectInspector getPrimitiveJavaConstantObjectInspector(PrimitiveTypeInfo typeInfo, Object value) {
	switch (typeInfo.getPrimitiveCategory()) {
		case BOOLEAN:
			return new JavaConstantBooleanObjectInspector((Boolean) value);
		case BYTE:
			return new JavaConstantByteObjectInspector((Byte) value);
		case SHORT:
			return new JavaConstantShortObjectInspector((Short) value);
		case INT:
			return new JavaConstantIntObjectInspector((Integer) value);
		case LONG:
			return new JavaConstantLongObjectInspector((Long) value);
		case FLOAT:
			return new JavaConstantFloatObjectInspector((Float) value);
		case DOUBLE:
			return new JavaConstantDoubleObjectInspector((Double) value);
		case STRING:
			return new JavaConstantStringObjectInspector((String) value);
		case CHAR:
			return new JavaConstantHiveCharObjectInspector((HiveChar) value);
		case VARCHAR:
			return new JavaConstantHiveVarcharObjectInspector((HiveVarchar) value);
		case DATE:
			return new JavaConstantDateObjectInspector((Date) value);
		case TIMESTAMP:
			return new JavaConstantTimestampObjectInspector((Timestamp) value);
		case DECIMAL:
			return new JavaConstantHiveDecimalObjectInspector((HiveDecimal) value);
		case BINARY:
			return new JavaConstantBinaryObjectInspector((byte[]) value);
		case UNKNOWN:
		case VOID:
			// If type is null, we use the Java Constant String to replace
			return new JavaConstantStringObjectInspector((String) value);
		default:
			throw new FlinkHiveUDFException(
				String.format("Cannot find ConstantObjectInspector for %s", typeInfo));
	}
}
 
Example #27
Source File: TestHiveBucketing.java    From presto with Apache License 2.0 5 votes vote down vote up
public static int getHiveBucketHashCode(BucketingVersion bucketingVersion, List<Entry<ObjectInspector, Object>> columnBindings)
{
    ObjectInspector[] objectInspectors = new ObjectInspector[columnBindings.size()];
    Object[] objects = new Object[columnBindings.size()];

    int i = 0;
    for (Entry<ObjectInspector, Object> entry : columnBindings) {
        objectInspectors[i] = entry.getKey();
        if (entry.getValue() != null && entry.getKey() instanceof JavaHiveVarcharObjectInspector) {
            JavaHiveVarcharObjectInspector varcharObjectInspector = (JavaHiveVarcharObjectInspector) entry.getKey();
            objects[i] = new HiveVarchar(((String) entry.getValue()), varcharObjectInspector.getMaxLength());
        }
        else {
            objects[i] = entry.getValue();
        }
        i++;
    }

    switch (bucketingVersion) {
        case BUCKETING_V1:
            @SuppressWarnings("deprecation")
            int hashCodeOld = ObjectInspectorUtils.getBucketHashCodeOld(objects, objectInspectors);
            return hashCodeOld;
        case BUCKETING_V2:
            return ObjectInspectorUtils.getBucketHashCode(objects, objectInspectors);
        default:
            throw new IllegalArgumentException("Unsupported bucketing version: " + bucketingVersion);
    }
}
 
Example #28
Source File: TestHiveFileFormats.java    From presto with Apache License 2.0 5 votes vote down vote up
@Test
public void testFailForLongVarcharPartitionColumn()
        throws Exception
{
    TestColumn partitionColumn = new TestColumn("partition_column", getPrimitiveJavaObjectInspector(new VarcharTypeInfo(3)), "test", utf8Slice("tes"), true);
    TestColumn varcharColumn = new TestColumn("varchar_column", getPrimitiveJavaObjectInspector(new VarcharTypeInfo(3)), new HiveVarchar("tes", 3), utf8Slice("tes"));

    List<TestColumn> columns = ImmutableList.of(partitionColumn, varcharColumn);

    HiveErrorCode expectedErrorCode = HiveErrorCode.HIVE_INVALID_PARTITION_VALUE;
    String expectedMessage = "Invalid partition value 'test' for varchar(3) partition key: partition_column";

    assertThatFileFormat(RCTEXT)
            .withColumns(columns)
            .isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage)
            .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);

    assertThatFileFormat(RCBINARY)
            .withColumns(columns)
            .isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage)
            .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);

    assertThatFileFormat(ORC)
            .withColumns(columns)
            .isFailingForPageSource(new OrcPageSourceFactory(new OrcReaderOptions(), HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage);

    assertThatFileFormat(PARQUET)
            .withColumns(columns)
            .withSession(PARQUET_SESSION)
            .isFailingForPageSource(new ParquetPageSourceFactory(HDFS_ENVIRONMENT, STATS, new ParquetReaderConfig()), expectedErrorCode, expectedMessage);

    assertThatFileFormat(SEQUENCEFILE)
            .withColumns(columns)
            .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);

    assertThatFileFormat(TEXTFILE)
            .withColumns(columns)
            .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
}
 
Example #29
Source File: ConvertersTest.java    From HiveRunner with Apache License 2.0 5 votes vote down vote up
@Test
public void otherTypeInfo() {
  assertEquals(HiveDecimal.create("1.234"), Converters.convert("1.234", decimalTypeInfo));
  assertEquals(new HiveChar("foo", -1), Converters.convert("foo", charTypeInfo));
  assertTrue(new HiveVarchar("foo", -1).equals((HiveVarchar) Converters.convert("foo", varcharTypeInfo)));
  assertEquals("foo", Converters.convert("foo", unknownTypeInfo));
  assertEquals("foo", Converters.convert("foo", voidTypeInfo));
}
 
Example #30
Source File: SMSerDe.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
/**
   * This method takes an object representing a row of data from Hive, and
   * uses the ObjectInspector to get the data for each column and serialize
   * it.
   */
  //@Override
  public Writable serialize(Object obj, ObjectInspector oi)
          throws SerDeException {
  	ExecRow row = null;
  	int[] execRowFormatIds = null;
      try {
	List<NameType> nameTypes = sqlUtil.getTableStructure(tableName);
	execRowFormatIds = sqlUtil.getExecRowFormatIds(colNames, nameTypes);
	row = sqlUtil.getExecRow(execRowFormatIds);
	if (row == null)
		throw new SerDeException("ExecRow Cannot be Null");
} catch (SQLException | StandardException | IOException e1) {
	throw new SerDeException(e1);
}    	
  	if (Log.isTraceEnabled())
  		SpliceLogUtils.trace(Log, "serialize with obj=%s, oi=%s",obj,oi);
      if (oi.getCategory() != ObjectInspector.Category.STRUCT) {
          throw new SerDeException(getClass().toString()
                  + " can only serialize struct types, but we got: "
                  + oi.getTypeName());
      }

      StructObjectInspector soi = (StructObjectInspector) oi;
      List<? extends StructField> fields = soi.getAllStructFieldRefs();

      try {

      	DataValueDescriptor dvd;
          for (int i = 0; i < fields.size(); i++) {
              StructField field = fields.get(i);
              dvd = row.getColumn(i+1);
              ObjectInspector fieldOI = field.getFieldObjectInspector();
              Object fieldObj = soi.getStructFieldData(obj, field);
              PrimitiveObjectInspector primOI = (PrimitiveObjectInspector) fieldOI;
              Object data = primOI.getPrimitiveJavaObject(fieldObj);

              PrimitiveCategory primitiveCategory = primOI.getPrimitiveCategory();
              switch (primitiveCategory) {
                  case BYTE:
                      dvd.setValue(((Byte) data).byteValue());
                      break;
                  case INT:
                      dvd.setValue(((Integer) data).intValue());
                      break;
                  case VARCHAR:
                      dvd.setValue(((HiveVarchar)data).getValue());
                      break;
                  case CHAR:
                      dvd.setValue(((HiveChar)data).getValue());
                      break;
                  case STRING:
                      dvd.setValue((String) data);
                      break;
                  case BINARY:
                      dvd.setValue((SerializationUtils.serialize((Serializable) data))); // is this right?  Should just be a byte[]
                      break;
                  case BOOLEAN:
                      dvd.setValue(((Boolean) data).booleanValue());
                      break;
                  case DECIMAL:
                      dvd.setValue(((HiveDecimal) data).doubleValue());
                      break;
                  case DOUBLE:
                      dvd.setValue(((Double) data).doubleValue());
                      break;
                  case FLOAT:
                      dvd.setValue(((Float) data).floatValue());
                      break;
                  case LONG:
                      dvd.setValue(((Long) data).longValue());
                      break;
                  case SHORT:
                      dvd.setValue(((Short) data).shortValue());
                      break;
                  case TIMESTAMP:
                      dvd.setValue((Timestamp) data);
                      break;
                  case DATE:
                      dvd.setValue((java.sql.Date) data);
                      break;
                  default:
                      throw new SerDeException(String.format("Hive Type %s Not Supported Yet",primOI.getPrimitiveCategory()));
              }
          }

      } catch (StandardException e) {
          // TODO Auto-generated catch block
          throw new RuntimeException("Serialized Object To Java Type Error");
      }
      ExecRowWritable rowWritable = new ExecRowWritable(WriteReadUtils.getExecRowFromTypeFormatIds(execRowFormatIds));
      rowWritable.set(row);
      return rowWritable;
  }