Java Code Examples for org.apache.flink.table.catalog.hive.client.HiveShim

The following examples show how to use org.apache.flink.table.catalog.hive.client.HiveShim. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: HiveInspectors.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get an array of ObjectInspector from the give array of args and their types.
 */
public static ObjectInspector[] toInspectors(HiveShim hiveShim, Object[] args, DataType[] argTypes) {
	assert args.length == argTypes.length;

	ObjectInspector[] argumentInspectors = new ObjectInspector[argTypes.length];

	for (int i = 0; i < argTypes.length; i++) {
		Object constant = args[i];

		if (constant == null) {
			argumentInspectors[i] =
				TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(
					HiveTypeUtil.toHiveTypeInfo(argTypes[i], false));
		} else {
			PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) HiveTypeUtil.toHiveTypeInfo(argTypes[i], false);
			constant = getConversion(getObjectInspector(primitiveTypeInfo), argTypes[i].getLogicalType(), hiveShim)
					.toHiveObject(constant);
			argumentInspectors[i] = getObjectInspectorForPrimitiveConstant(primitiveTypeInfo, constant, hiveShim);
		}
	}

	return argumentInspectors;
}
 
Example 2
Source Project: flink   Source File: HiveRowDataPartitionComputer.java    License: Apache License 2.0 6 votes vote down vote up
public HiveRowDataPartitionComputer(
		HiveShim hiveShim,
		String defaultPartValue,
		String[] columnNames,
		DataType[] columnTypes,
		String[] partitionColumns) {
	super(defaultPartValue, columnNames, columnTypes, partitionColumns);
	this.partitionConverters = Arrays.stream(partitionTypes)
			.map(TypeConversions::fromLogicalToDataType)
			.map(DataFormatConverters::getConverterForDataType)
			.toArray(DataFormatConverters.DataFormatConverter[]::new);
	this.hiveObjectConversions = new HiveObjectConversion[partitionIndexes.length];
	for (int i = 0; i < hiveObjectConversions.length; i++) {
		DataType partColType = columnTypes[partitionIndexes[i]];
		ObjectInspector objectInspector = HiveInspectors.getObjectInspector(partColType);
		hiveObjectConversions[i] = HiveInspectors.getConversion(objectInspector, partColType.getLogicalType(), hiveShim);
	}
}
 
Example 3
Source Project: flink   Source File: HiveWriterFactory.java    License: Apache License 2.0 6 votes vote down vote up
public HiveWriterFactory(
		JobConf jobConf,
		Class hiveOutputFormatClz,
		SerDeInfo serDeInfo,
		TableSchema schema,
		String[] partitionColumns,
		Properties tableProperties,
		HiveShim hiveShim,
		boolean isCompressed) {
	Preconditions.checkArgument(HiveOutputFormat.class.isAssignableFrom(hiveOutputFormatClz),
			"The output format should be an instance of HiveOutputFormat");
	this.confWrapper = new JobConfWrapper(jobConf);
	this.hiveOutputFormatClz = hiveOutputFormatClz;
	this.serDeInfo = serDeInfo;
	this.allColumns = schema.getFieldNames();
	this.allTypes = schema.getFieldDataTypes();
	this.partitionColumns = partitionColumns;
	this.tableProperties = tableProperties;
	this.hiveShim = hiveShim;
	this.isCompressed = isCompressed;
}
 
Example 4
Source Project: flink   Source File: HiveTableUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Generates a filter string for partition columns from the given filter expressions.
 *
 * @param partColOffset The number of non-partition columns -- used to shift field reference index
 * @param partColNames The names of all partition columns
 * @param expressions  The filter expressions in CNF form
 * @return an Optional filter string equivalent to the expressions, which is empty if the expressions can't be handled
 */
public static Optional<String> makePartitionFilter(
		int partColOffset, List<String> partColNames, List<Expression> expressions, HiveShim hiveShim) {
	List<String> filters = new ArrayList<>(expressions.size());
	ExpressionExtractor extractor = new ExpressionExtractor(partColOffset, partColNames, hiveShim);
	for (Expression expression : expressions) {
		String str = expression.accept(extractor);
		if (str == null) {
			return Optional.empty();
		}
		filters.add(str);
	}
	return Optional.of(String.join(" and ", filters));
}
 
Example 5
Source Project: flink   Source File: HiveReflectionUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Properties getTableMetadata(HiveShim hiveShim, Table table) {
	try {
		Method method = hiveShim.getMetaStoreUtilsClass().getMethod("getTableMetadata", Table.class);
		return (Properties) method.invoke(null, table);
	} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
		throw new CatalogException("Failed to invoke MetaStoreUtils.getTableMetadata()", e);
	}
}
 
Example 6
Source Project: flink   Source File: HiveReflectionUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static List<String> getPvals(HiveShim hiveShim, List<FieldSchema> partCols, Map<String, String> partSpec) {
	try {
		Method method = hiveShim.getMetaStoreUtilsClass().getMethod("getPvals", List.class, Map.class);
		return (List<String>) method.invoke(null, partCols, partSpec);
	} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
		throw new CatalogException("Failed to invoke MetaStoreUtils.getFieldsFromDeserializer", e);
	}
}
 
Example 7
Source Project: flink   Source File: HiveRowPartitionComputer.java    License: Apache License 2.0 5 votes vote down vote up
HiveRowPartitionComputer(HiveShim hiveShim, String defaultPartValue, String[] columnNames,
		DataType[] columnTypes, String[] partitionColumns) {
	super(defaultPartValue, columnNames, partitionColumns);
	partColConversions = new HiveObjectConversion[partitionIndexes.length];
	for (int i = 0; i < partColConversions.length; i++) {
		DataType partColType = columnTypes[partitionIndexes[i]];
		ObjectInspector objectInspector = HiveInspectors.getObjectInspector(partColType);
		partColConversions[i] = HiveInspectors.getConversion(objectInspector, partColType.getLogicalType(), hiveShim);
	}
}
 
Example 8
Source Project: flink   Source File: HiveContinuousMonitoringFunction.java    License: Apache License 2.0 5 votes vote down vote up
public HiveContinuousMonitoringFunction(
		HiveShim hiveShim,
		JobConf conf,
		ObjectPath tablePath,
		CatalogTable catalogTable,
		int readerParallelism,
		ConsumeOrder consumeOrder,
		String consumeOffset,
		String extractorKind,
		String extractorClass,
		String extractorPattern,
		long interval) {
	this.hiveShim = hiveShim;
	this.conf = new JobConfWrapper(conf);
	this.tablePath = tablePath;
	this.partitionKeys = catalogTable.getPartitionKeys();
	this.fieldNames = catalogTable.getSchema().getFieldNames();
	this.fieldTypes = catalogTable.getSchema().getFieldDataTypes();
	this.consumeOrder = consumeOrder;
	this.extractorKind = extractorKind;
	this.extractorClass = extractorClass;
	this.extractorPattern = extractorPattern;
	this.consumeOffset = consumeOffset;

	this.interval = interval;
	this.readerParallelism = Math.max(readerParallelism, 1);
	this.currentReadTime = 0;
}
 
Example 9
Source Project: flink   Source File: HiveTableSource.java    License: Apache License 2.0 5 votes vote down vote up
public static HiveTablePartition toHiveTablePartition(
		List<String> partitionKeys,
		String[] fieldNames,
		DataType[] fieldTypes,
		HiveShim shim,
		Properties tableProps,
		String defaultPartitionName,
		Partition partition) {
	StorageDescriptor sd = partition.getSd();
	Map<String, Object> partitionColValues = new HashMap<>();
	List<String> nameList = Arrays.asList(fieldNames);
	for (int i = 0; i < partitionKeys.size(); i++) {
		String partitionColName = partitionKeys.get(i);
		String partitionValue = partition.getValues().get(i);
		DataType type = fieldTypes[nameList.indexOf(partitionColName)];
		Object partitionObject;
		if (defaultPartitionName.equals(partitionValue)) {
			LogicalTypeRoot typeRoot = type.getLogicalType().getTypeRoot();
			// while this is inline with Hive, seems it should be null for string columns as well
			partitionObject = typeRoot == LogicalTypeRoot.CHAR || typeRoot == LogicalTypeRoot.VARCHAR ? defaultPartitionName : null;
		} else {
			partitionObject = restorePartitionValueFromFromType(shim, partitionValue, type);
		}
		partitionColValues.put(partitionColName, partitionObject);
	}
	return new HiveTablePartition(sd, partitionColValues, tableProps);
}
 
Example 10
Source Project: flink   Source File: HiveTableSource.java    License: Apache License 2.0 5 votes vote down vote up
private static Object restorePartitionValueFromFromType(HiveShim shim, String valStr, DataType type) {
	LogicalTypeRoot typeRoot = type.getLogicalType().getTypeRoot();
	//note: it's not a complete list ofr partition key types that Hive support, we may need add more later.
	switch (typeRoot) {
		case CHAR:
		case VARCHAR:
			return valStr;
		case BOOLEAN:
			return Boolean.parseBoolean(valStr);
		case TINYINT:
			return Integer.valueOf(valStr).byteValue();
		case SMALLINT:
			return Short.valueOf(valStr);
		case INTEGER:
			return Integer.valueOf(valStr);
		case BIGINT:
			return Long.valueOf(valStr);
		case FLOAT:
			return Float.valueOf(valStr);
		case DOUBLE:
			return Double.valueOf(valStr);
		case DATE:
			return HiveInspectors.toFlinkObject(
					HiveInspectors.getObjectInspector(type),
					shim.toHiveDate(Date.valueOf(valStr)),
					shim);
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			return HiveInspectors.toFlinkObject(
					HiveInspectors.getObjectInspector(type),
					shim.toHiveTimestamp(Timestamp.valueOf(valStr)),
					shim);
		default:
			break;
	}
	throw new FlinkHiveException(
			new IllegalArgumentException(String.format("Can not convert %s to type %s for partition value", valStr, type)));
}
 
Example 11
Source Project: flink   Source File: HiveTableUtil.java    License: Apache License 2.0 4 votes vote down vote up
ExpressionExtractor(int partColOffset, List<String> partColNames, HiveShim hiveShim) {
	this.partColOffset = partColOffset;
	this.partColNames = partColNames;
	this.hiveShim = hiveShim;
}
 
Example 12
Source Project: flink   Source File: HiveStatsUtil.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Create Flink ColumnStats from Hive ColumnStatisticsData.
 */
private static CatalogColumnStatisticsDataBase createTableColumnStats(DataType colType, ColumnStatisticsData stats, String hiveVersion) {
	HiveShim hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	if (stats.isSetBinaryStats()) {
		BinaryColumnStatsData binaryStats = stats.getBinaryStats();
		return new CatalogColumnStatisticsDataBinary(
				binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null,
				binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null,
				binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null);
	} else if (stats.isSetBooleanStats()) {
		BooleanColumnStatsData booleanStats = stats.getBooleanStats();
		return new CatalogColumnStatisticsDataBoolean(
				booleanStats.isSetNumTrues() ? booleanStats.getNumTrues() : null,
				booleanStats.isSetNumFalses() ? booleanStats.getNumFalses() : null,
				booleanStats.isSetNumNulls() ? booleanStats.getNumNulls() : null);
	} else if (hiveShim.isDateStats(stats)) {
		return hiveShim.toFlinkDateColStats(stats);
	} else if (stats.isSetDoubleStats()) {
			DoubleColumnStatsData doubleStats = stats.getDoubleStats();
			return new CatalogColumnStatisticsDataDouble(
					doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null,
					doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null,
					doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null,
					doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null);
	} else if (stats.isSetLongStats()) {
			LongColumnStatsData longColStats = stats.getLongStats();
			return new CatalogColumnStatisticsDataLong(
					longColStats.isSetLowValue() ? longColStats.getLowValue() : null,
					longColStats.isSetHighValue() ? longColStats.getHighValue() : null,
					longColStats.isSetNumDVs() ? longColStats.getNumDVs() : null,
					longColStats.isSetNumNulls() ? longColStats.getNumNulls() : null);
	} else if (stats.isSetStringStats()) {
		StringColumnStatsData stringStats = stats.getStringStats();
		return new CatalogColumnStatisticsDataString(
				stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null,
				stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null,
				stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null,
				stringStats.isSetNumDVs() ? stringStats.getNumNulls() : null);
	} else if (stats.isSetDecimalStats()) {
		DecimalColumnStatsData decimalStats = stats.getDecimalStats();
		// for now, just return CatalogColumnStatisticsDataDouble for decimal columns
		Double max = null;
		if (decimalStats.isSetHighValue()) {
			max = toHiveDecimal(decimalStats.getHighValue()).doubleValue();
		}
		Double min = null;
		if (decimalStats.isSetLowValue()) {
			min = toHiveDecimal(decimalStats.getLowValue()).doubleValue();
		}
		Long ndv = decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null;
		Long nullCount = decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null;
		return new CatalogColumnStatisticsDataDouble(min, max, ndv, nullCount);
	} else {
		LOG.warn("Flink does not support converting ColumnStatisticsData '{}' for Hive column type '{}' yet.", stats, colType);
		return null;
	}
}
 
Example 13
Source Project: flink   Source File: HiveFunctionDefinitionFactory.java    License: Apache License 2.0 4 votes vote down vote up
public HiveFunctionDefinitionFactory(HiveShim hiveShim) {
	checkNotNull(hiveShim, "hiveShim cannot be null");
	this.hiveShim = hiveShim;
}
 
Example 14
Source Project: flink   Source File: HiveGenericUDAF.java    License: Apache License 2.0 4 votes vote down vote up
public HiveGenericUDAF(HiveFunctionWrapper funcWrapper, HiveShim hiveShim) {
	this(funcWrapper, false, hiveShim);
}
 
Example 15
Source Project: flink   Source File: HiveGenericUDAF.java    License: Apache License 2.0 4 votes vote down vote up
public HiveGenericUDAF(HiveFunctionWrapper funcWrapper, boolean isUDAFBridgeRequired, HiveShim hiveShim) {
	this.hiveFunctionWrapper = funcWrapper;
	this.isUDAFBridgeRequired = isUDAFBridgeRequired;
	this.hiveShim = hiveShim;
}
 
Example 16
Source Project: flink   Source File: HiveGenericUDTF.java    License: Apache License 2.0 4 votes vote down vote up
public HiveGenericUDTF(HiveFunctionWrapper<GenericUDTF> hiveFunctionWrapper, HiveShim hiveShim) {
	this.hiveFunctionWrapper = hiveFunctionWrapper;
	this.hiveShim = hiveShim;
}
 
Example 17
Source Project: flink   Source File: DeferredObjectAdapter.java    License: Apache License 2.0 4 votes vote down vote up
public DeferredObjectAdapter(ObjectInspector inspector, LogicalType logicalType, HiveShim hiveShim) {
	conversion = HiveInspectors.getConversion(inspector, logicalType, hiveShim);
}
 
Example 18
Source Project: flink   Source File: HiveGenericUDF.java    License: Apache License 2.0 4 votes vote down vote up
public HiveGenericUDF(HiveFunctionWrapper<GenericUDF> hiveFunctionWrapper, HiveShim hiveShim) {
	super(hiveFunctionWrapper);
	this.hiveShim = hiveShim;
	LOG.info("Creating HiveGenericUDF from '{}'", hiveFunctionWrapper.getClassName());
}
 
Example 19
Source Project: flink   Source File: WritableHiveObjectConversion.java    License: Apache License 2.0 4 votes vote down vote up
WritableHiveObjectConversion(HiveObjectConversion flinkToJavaConversion, HiveShim hiveShim) {
	this.flinkToJavaConversion = flinkToJavaConversion;
	this.hiveShim = hiveShim;
}
 
Example 20
Source Project: flink   Source File: HiveSimpleUDF.java    License: Apache License 2.0 4 votes vote down vote up
public HiveSimpleUDF(HiveFunctionWrapper<UDF> hiveFunctionWrapper, HiveShim hiveShim) {
	super(hiveFunctionWrapper);
	this.hiveShim = hiveShim;
	LOG.info("Creating HiveSimpleUDF from '{}'", this.hiveFunctionWrapper.getClassName());
}