org.apache.flink.table.catalog.hive.client.HiveShim Java Examples

public HiveWriterFactory(
		JobConf jobConf,
		Class hiveOutputFormatClz,
		SerDeInfo serDeInfo,
		TableSchema schema,
		String[] partitionColumns,
		Properties tableProperties,
		HiveShim hiveShim,
		boolean isCompressed) {
	Preconditions.checkArgument(HiveOutputFormat.class.isAssignableFrom(hiveOutputFormatClz),
			"The output format should be an instance of HiveOutputFormat");
	this.confWrapper = new JobConfWrapper(jobConf);
	this.hiveOutputFormatClz = hiveOutputFormatClz;
	this.serDeInfo = serDeInfo;
	this.allColumns = schema.getFieldNames();
	this.allTypes = schema.getFieldDataTypes();
	this.partitionColumns = partitionColumns;
	this.tableProperties = tableProperties;
	this.hiveShim = hiveShim;
	this.isCompressed = isCompressed;
}
 

public HiveRowDataPartitionComputer(
		HiveShim hiveShim,
		String defaultPartValue,
		String[] columnNames,
		DataType[] columnTypes,
		String[] partitionColumns) {
	super(defaultPartValue, columnNames, columnTypes, partitionColumns);
	this.partitionConverters = Arrays.stream(partitionTypes)
			.map(TypeConversions::fromLogicalToDataType)
			.map(DataFormatConverters::getConverterForDataType)
			.toArray(DataFormatConverters.DataFormatConverter[]::new);
	this.hiveObjectConversions = new HiveObjectConversion[partitionIndexes.length];
	for (int i = 0; i < hiveObjectConversions.length; i++) {
		DataType partColType = columnTypes[partitionIndexes[i]];
		ObjectInspector objectInspector = HiveInspectors.getObjectInspector(partColType);
		hiveObjectConversions[i] = HiveInspectors.getConversion(objectInspector, partColType.getLogicalType(), hiveShim);
	}
}
 

/**
 * Get an array of ObjectInspector from the give array of args and their types.
 */
public static ObjectInspector[] toInspectors(HiveShim hiveShim, Object[] args, DataType[] argTypes) {
	assert args.length == argTypes.length;

	ObjectInspector[] argumentInspectors = new ObjectInspector[argTypes.length];

	for (int i = 0; i < argTypes.length; i++) {
		Object constant = args[i];

		if (constant == null) {
			argumentInspectors[i] =
				TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(
					HiveTypeUtil.toHiveTypeInfo(argTypes[i], false));
		} else {
			PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) HiveTypeUtil.toHiveTypeInfo(argTypes[i], false);
			constant = getConversion(getObjectInspector(primitiveTypeInfo), argTypes[i].getLogicalType(), hiveShim)
					.toHiveObject(constant);
			argumentInspectors[i] = getObjectInspectorForPrimitiveConstant(primitiveTypeInfo, constant, hiveShim);
		}
	}

	return argumentInspectors;
}
 

private static Object restorePartitionValueFromFromType(HiveShim shim, String valStr, DataType type) {
	LogicalTypeRoot typeRoot = type.getLogicalType().getTypeRoot();
	//note: it's not a complete list ofr partition key types that Hive support, we may need add more later.
	switch (typeRoot) {
		case CHAR:
		case VARCHAR:
			return valStr;
		case BOOLEAN:
			return Boolean.parseBoolean(valStr);
		case TINYINT:
			return Integer.valueOf(valStr).byteValue();
		case SMALLINT:
			return Short.valueOf(valStr);
		case INTEGER:
			return Integer.valueOf(valStr);
		case BIGINT:
			return Long.valueOf(valStr);
		case FLOAT:
			return Float.valueOf(valStr);
		case DOUBLE:
			return Double.valueOf(valStr);
		case DATE:
			return HiveInspectors.toFlinkObject(
					HiveInspectors.getObjectInspector(type),
					shim.toHiveDate(Date.valueOf(valStr)),
					shim);
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			return HiveInspectors.toFlinkObject(
					HiveInspectors.getObjectInspector(type),
					shim.toHiveTimestamp(Timestamp.valueOf(valStr)),
					shim);
		default:
			break;
	}
	throw new FlinkHiveException(
			new IllegalArgumentException(String.format("Can not convert %s to type %s for partition value", valStr, type)));
}
 

public static Properties getTableMetadata(HiveShim hiveShim, Table table) {
	try {
		Method method = hiveShim.getMetaStoreUtilsClass().getMethod("getTableMetadata", Table.class);
		return (Properties) method.invoke(null, table);
	} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
		throw new CatalogException("Failed to invoke MetaStoreUtils.getTableMetadata()", e);
	}
}
 

public static List<String> getPvals(HiveShim hiveShim, List<FieldSchema> partCols, Map<String, String> partSpec) {
	try {
		Method method = hiveShim.getMetaStoreUtilsClass().getMethod("getPvals", List.class, Map.class);
		return (List<String>) method.invoke(null, partCols, partSpec);
	} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
		throw new CatalogException("Failed to invoke MetaStoreUtils.getFieldsFromDeserializer", e);
	}
}
 

public static HiveTablePartition toHiveTablePartition(
		List<String> partitionKeys,
		String[] fieldNames,
		DataType[] fieldTypes,
		HiveShim shim,
		Properties tableProps,
		String defaultPartitionName,
		Partition partition) {
	StorageDescriptor sd = partition.getSd();
	Map<String, Object> partitionColValues = new HashMap<>();
	List<String> nameList = Arrays.asList(fieldNames);
	for (int i = 0; i < partitionKeys.size(); i++) {
		String partitionColName = partitionKeys.get(i);
		String partitionValue = partition.getValues().get(i);
		DataType type = fieldTypes[nameList.indexOf(partitionColName)];
		Object partitionObject;
		if (defaultPartitionName.equals(partitionValue)) {
			LogicalTypeRoot typeRoot = type.getLogicalType().getTypeRoot();
			// while this is inline with Hive, seems it should be null for string columns as well
			partitionObject = typeRoot == LogicalTypeRoot.CHAR || typeRoot == LogicalTypeRoot.VARCHAR ? defaultPartitionName : null;
		} else {
			partitionObject = restorePartitionValueFromFromType(shim, partitionValue, type);
		}
		partitionColValues.put(partitionColName, partitionObject);
	}
	return new HiveTablePartition(sd, partitionColValues, tableProps);
}
 

public HiveContinuousMonitoringFunction(
		HiveShim hiveShim,
		JobConf conf,
		ObjectPath tablePath,
		CatalogTable catalogTable,
		int readerParallelism,
		ConsumeOrder consumeOrder,
		String consumeOffset,
		String extractorKind,
		String extractorClass,
		String extractorPattern,
		long interval) {
	this.hiveShim = hiveShim;
	this.conf = new JobConfWrapper(conf);
	this.tablePath = tablePath;
	this.partitionKeys = catalogTable.getPartitionKeys();
	this.fieldNames = catalogTable.getSchema().getFieldNames();
	this.fieldTypes = catalogTable.getSchema().getFieldDataTypes();
	this.consumeOrder = consumeOrder;
	this.extractorKind = extractorKind;
	this.extractorClass = extractorClass;
	this.extractorPattern = extractorPattern;
	this.consumeOffset = consumeOffset;

	this.interval = interval;
	this.readerParallelism = Math.max(readerParallelism, 1);
	this.currentReadTime = 0;
}
 

HiveRowPartitionComputer(HiveShim hiveShim, String defaultPartValue, String[] columnNames,
		DataType[] columnTypes, String[] partitionColumns) {
	super(defaultPartValue, columnNames, partitionColumns);
	partColConversions = new HiveObjectConversion[partitionIndexes.length];
	for (int i = 0; i < partColConversions.length; i++) {
		DataType partColType = columnTypes[partitionIndexes[i]];
		ObjectInspector objectInspector = HiveInspectors.getObjectInspector(partColType);
		partColConversions[i] = HiveInspectors.getConversion(objectInspector, partColType.getLogicalType(), hiveShim);
	}
}
 

/**
 * Generates a filter string for partition columns from the given filter expressions.
 *
 * @param partColOffset The number of non-partition columns -- used to shift field reference index
 * @param partColNames The names of all partition columns
 * @param expressions  The filter expressions in CNF form
 * @return an Optional filter string equivalent to the expressions, which is empty if the expressions can't be handled
 */
public static Optional<String> makePartitionFilter(
		int partColOffset, List<String> partColNames, List<Expression> expressions, HiveShim hiveShim) {
	List<String> filters = new ArrayList<>(expressions.size());
	ExpressionExtractor extractor = new ExpressionExtractor(partColOffset, partColNames, hiveShim);
	for (Expression expression : expressions) {
		String str = expression.accept(extractor);
		if (str == null) {
			return Optional.empty();
		}
		filters.add(str);
	}
	return Optional.of(String.join(" and ", filters));
}
 

public HiveGenericUDF(HiveFunctionWrapper<GenericUDF> hiveFunctionWrapper, HiveShim hiveShim) {
	super(hiveFunctionWrapper);
	this.hiveShim = hiveShim;
	LOG.info("Creating HiveGenericUDF from '{}'", hiveFunctionWrapper.getClassName());
}
 

public HiveSimpleUDF(HiveFunctionWrapper<UDF> hiveFunctionWrapper, HiveShim hiveShim) {
	super(hiveFunctionWrapper);
	this.hiveShim = hiveShim;
	LOG.info("Creating HiveSimpleUDF from '{}'", this.hiveFunctionWrapper.getClassName());
}
 

WritableHiveObjectConversion(HiveObjectConversion flinkToJavaConversion, HiveShim hiveShim) {
	this.flinkToJavaConversion = flinkToJavaConversion;
	this.hiveShim = hiveShim;
}
 

public DeferredObjectAdapter(ObjectInspector inspector, LogicalType logicalType, HiveShim hiveShim) {
	conversion = HiveInspectors.getConversion(inspector, logicalType, hiveShim);
}
 

public HiveGenericUDTF(HiveFunctionWrapper<GenericUDTF> hiveFunctionWrapper, HiveShim hiveShim) {
	this.hiveFunctionWrapper = hiveFunctionWrapper;
	this.hiveShim = hiveShim;
}
 

public HiveGenericUDAF(HiveFunctionWrapper funcWrapper, boolean isUDAFBridgeRequired, HiveShim hiveShim) {
	this.hiveFunctionWrapper = funcWrapper;
	this.isUDAFBridgeRequired = isUDAFBridgeRequired;
	this.hiveShim = hiveShim;
}
 

public HiveGenericUDAF(HiveFunctionWrapper funcWrapper, HiveShim hiveShim) {
	this(funcWrapper, false, hiveShim);
}
 

public HiveFunctionDefinitionFactory(HiveShim hiveShim) {
	checkNotNull(hiveShim, "hiveShim cannot be null");
	this.hiveShim = hiveShim;
}
 

/**
 * Create Flink ColumnStats from Hive ColumnStatisticsData.
 */
private static CatalogColumnStatisticsDataBase createTableColumnStats(DataType colType, ColumnStatisticsData stats, String hiveVersion) {
	HiveShim hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	if (stats.isSetBinaryStats()) {
		BinaryColumnStatsData binaryStats = stats.getBinaryStats();
		return new CatalogColumnStatisticsDataBinary(
				binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null,
				binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null,
				binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null);
	} else if (stats.isSetBooleanStats()) {
		BooleanColumnStatsData booleanStats = stats.getBooleanStats();
		return new CatalogColumnStatisticsDataBoolean(
				booleanStats.isSetNumTrues() ? booleanStats.getNumTrues() : null,
				booleanStats.isSetNumFalses() ? booleanStats.getNumFalses() : null,
				booleanStats.isSetNumNulls() ? booleanStats.getNumNulls() : null);
	} else if (hiveShim.isDateStats(stats)) {
		return hiveShim.toFlinkDateColStats(stats);
	} else if (stats.isSetDoubleStats()) {
			DoubleColumnStatsData doubleStats = stats.getDoubleStats();
			return new CatalogColumnStatisticsDataDouble(
					doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null,
					doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null,
					doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null,
					doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null);
	} else if (stats.isSetLongStats()) {
			LongColumnStatsData longColStats = stats.getLongStats();
			return new CatalogColumnStatisticsDataLong(
					longColStats.isSetLowValue() ? longColStats.getLowValue() : null,
					longColStats.isSetHighValue() ? longColStats.getHighValue() : null,
					longColStats.isSetNumDVs() ? longColStats.getNumDVs() : null,
					longColStats.isSetNumNulls() ? longColStats.getNumNulls() : null);
	} else if (stats.isSetStringStats()) {
		StringColumnStatsData stringStats = stats.getStringStats();
		return new CatalogColumnStatisticsDataString(
				stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null,
				stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null,
				stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null,
				stringStats.isSetNumDVs() ? stringStats.getNumNulls() : null);
	} else if (stats.isSetDecimalStats()) {
		DecimalColumnStatsData decimalStats = stats.getDecimalStats();
		// for now, just return CatalogColumnStatisticsDataDouble for decimal columns
		Double max = null;
		if (decimalStats.isSetHighValue()) {
			max = toHiveDecimal(decimalStats.getHighValue()).doubleValue();
		}
		Double min = null;
		if (decimalStats.isSetLowValue()) {
			min = toHiveDecimal(decimalStats.getLowValue()).doubleValue();
		}
		Long ndv = decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null;
		Long nullCount = decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null;
		return new CatalogColumnStatisticsDataDouble(min, max, ndv, nullCount);
	} else {
		LOG.warn("Flink does not support converting ColumnStatisticsData '{}' for Hive column type '{}' yet.", stats, colType);
		return null;
	}
}
 

ExpressionExtractor(int partColOffset, List<String> partColNames, HiveShim hiveShim) {
	this.partColOffset = partColOffset;
	this.partColNames = partColNames;
	this.hiveShim = hiveShim;
}