Java Code Examples for org.apache.flink.util.StringUtils#isNullOrWhitespaceOnly()

private Configuration prepareRuntimeConfiguration() {
	// create default configuration from current runtime env (`hbase-site.xml` in classpath) first,
	// and overwrite configuration using serialized configuration from client-side env (`hbase-site.xml` in classpath).
	// user params from client-side have the highest priority
	Configuration runtimeConfig = HBaseConfigurationUtil.deserializeConfiguration(
		serializedConfig,
		HBaseConfigurationUtil.getHBaseConfiguration());

	// do validation: check key option(s) in final runtime configuration
	if (StringUtils.isNullOrWhitespaceOnly(runtimeConfig.get(HConstants.ZOOKEEPER_QUORUM))) {
		LOG.error("can not connect to HBase without {} configuration", HConstants.ZOOKEEPER_QUORUM);
		throw new IllegalArgumentException("check HBase configuration failed, lost: '" + HConstants.ZOOKEEPER_QUORUM + "'!");
	}

	return runtimeConfig;
}
 

private Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        if (!enableQuote || token.charAt(0) != quoteChar) {
            return Tuple2.of(true, token);
        }
        String content;
        if (token.endsWith(quoteChar.toString())) {
            content = token.substring(1, token.length() - 1);
        } else {
            content = token.substring(1, token.length());
        }
        return Tuple2.of(true, content.replace(escapedQuote, quoteString));
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(true, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, fieldDelim.getBytes(), null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}
 

/**
 * Deletes the YARN application files, e.g., Flink binaries, libraries, etc., from the remote
 * filesystem.
 *
 * @param env The environment variables.
 */
public static void deleteApplicationFiles(final Map<String, String> env) {
	final String applicationFilesDir = env.get(YarnConfigKeys.FLINK_YARN_FILES);
	if (!StringUtils.isNullOrWhitespaceOnly(applicationFilesDir)) {
		final org.apache.flink.core.fs.Path path = new org.apache.flink.core.fs.Path(applicationFilesDir);
		try {
			final org.apache.flink.core.fs.FileSystem fileSystem = path.getFileSystem();
			if (!fileSystem.delete(path, true)) {
				LOG.error("Deleting yarn application files under {} was unsuccessful.", applicationFilesDir);
			}
		} catch (final IOException e) {
			LOG.error("Could not properly delete yarn application files directory {}.", applicationFilesDir, e);
		}
	} else {
		LOG.debug("No yarn application files directory set. Therefore, cannot clean up the data.");
	}
}
 

@Override
public Set<GPUInfo> retrieveResourceInfo(long gpuAmount) throws Exception {
	Preconditions.checkArgument(gpuAmount > 0, "The gpuAmount should be positive when retrieving the GPU resource information.");

	final Set<GPUInfo> gpuResources = new HashSet<>();
	String output = executeDiscoveryScript(discoveryScriptFile, gpuAmount, args);
	if (!output.isEmpty()) {
		String[] indexes = output.split(",");
		for (String index : indexes) {
			if (!StringUtils.isNullOrWhitespaceOnly(index)) {
				gpuResources.add(new GPUInfo(index.trim()));
			}
		}
	}
	LOG.info("Discover GPU resources: {}.", gpuResources);
	return Collections.unmodifiableSet(gpuResources);
}
 

private org.apache.hadoop.conf.Configuration prepareRuntimeConfiguration() {
	// create default configuration from current runtime env (`hbase-site.xml` in classpath) first,
	// and overwrite configuration using serialized configuration from client-side env (`hbase-site.xml` in classpath).
	// user params from client-side have the highest priority
	org.apache.hadoop.conf.Configuration runtimeConfig = HBaseConfigurationUtil.deserializeConfiguration(
		serializedConfig,
		HBaseConfigurationUtil.getHBaseConfiguration());

	// do validation: check key option(s) in final runtime configuration
	if (StringUtils.isNullOrWhitespaceOnly(runtimeConfig.get(HConstants.ZOOKEEPER_QUORUM))) {
		LOG.error("can not connect to HBase without {} configuration", HConstants.ZOOKEEPER_QUORUM);
		throw new IllegalArgumentException("check HBase configuration failed, lost: '" + HConstants.ZOOKEEPER_QUORUM + "'!");
	}

	return runtimeConfig;
}
 

/**
 * Deletes the YARN application files, e.g., Flink binaries, libraries, etc., from the remote
 * filesystem.
 *
 * @param env The environment variables.
 */
public static void deleteApplicationFiles(final Map<String, String> env) {
	final String applicationFilesDir = env.get(YarnConfigKeys.FLINK_YARN_FILES);
	if (!StringUtils.isNullOrWhitespaceOnly(applicationFilesDir)) {
		final org.apache.flink.core.fs.Path path = new org.apache.flink.core.fs.Path(applicationFilesDir);
		try {
			final org.apache.flink.core.fs.FileSystem fileSystem = path.getFileSystem();
			if (!fileSystem.delete(path, true)) {
				LOG.error("Deleting yarn application files under {} was unsuccessful.", applicationFilesDir);
			}
		} catch (final IOException e) {
			LOG.error("Could not properly delete yarn application files directory {}.", applicationFilesDir, e);
		}
	} else {
		LOG.debug("No yarn application files directory set. Therefore, cannot clean up the data.");
	}
}
 

/**
 * Formats a Tree of {@link Operation} in a unified way. It prints all the parameters and
 * adds all children formatted and properly indented in the following lines.
 *
 * <p>The format is <pre>
 * {@code
 * <operationName>: [(key1: [value1], key2: [v1, v2])]
 *     <child1>
 *          <child2>
 *     <child3>
 * }
 * </pre>
 *
 * @param operationName The operation name.
 * @param parameters The operation's parameters.
 * @param children The operation's children.
 * @param childToString The function to convert child to String.
 * @param <T> The type of the child.
 * @return String representation of the given operation.
 */
public static <T extends Operation> String formatWithChildren(
		String operationName,
		Map<String, Object> parameters,
		List<T> children,
		Function<T, String> childToString) {
	String description = parameters.entrySet()
		.stream()
		.map(entry -> formatParameter(entry.getKey(), entry.getValue()))
		.collect(Collectors.joining(", "));

	final StringBuilder stringBuilder = new StringBuilder();

	stringBuilder.append(operationName).append(":");

	if (!StringUtils.isNullOrWhitespaceOnly(description)) {
		stringBuilder.append(" (").append(description).append(")");
	}

	String childrenDescription = children.stream()
		.map(child -> OperationUtils.indent(childToString.apply(child)))
		.collect(Collectors.joining());

	return stringBuilder.append(childrenDescription).toString();
}
 

private org.apache.hadoop.conf.Configuration prepareRuntimeConfiguration() {
	// create default configuration from current runtime env (`hbase-site.xml` in classpath) first,
	// and overwrite configuration using serialized configuration from client-side env (`hbase-site.xml` in classpath).
	// user params from client-side have the highest priority
	org.apache.hadoop.conf.Configuration runtimeConfig = HBaseConfigurationUtil.deserializeConfiguration(
		serializedConfig,
		HBaseConfiguration.create());

	// do validation: check key option(s) in final runtime configuration
	if (StringUtils.isNullOrWhitespaceOnly(runtimeConfig.get(HConstants.ZOOKEEPER_QUORUM))) {
		LOG.error("can not connect to HBase without {} configuration", HConstants.ZOOKEEPER_QUORUM);
		throw new IllegalArgumentException("check HBase configuration failed, lost: '" + HConstants.ZOOKEEPER_QUORUM + "'!");
	}

	return runtimeConfig;
}
 

private Partition instantiateHivePartition(Table hiveTable, CatalogPartitionSpec partitionSpec, CatalogPartition catalogPartition)
		throws PartitionSpecInvalidException {
	List<String> partCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partValues = getOrderedFullPartitionValues(
		partitionSpec, partCols, new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName()));
	// validate partition values
	for (int i = 0; i < partCols.size(); i++) {
		if (StringUtils.isNullOrWhitespaceOnly(partValues.get(i))) {
			throw new PartitionSpecInvalidException(getName(), partCols,
				new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName()), partitionSpec);
		}
	}
	// TODO: handle GenericCatalogPartition
	StorageDescriptor sd = hiveTable.getSd().deepCopy();
	sd.setLocation(catalogPartition.getProperties().remove(HiveCatalogConfig.PARTITION_LOCATION));

	Map<String, String> properties = new HashMap<>(catalogPartition.getProperties());
	properties.put(HiveCatalogConfig.COMMENT, catalogPartition.getComment());

	return HiveTableUtil.createHivePartition(
			hiveTable.getDbName(),
			hiveTable.getTableName(),
			partValues,
			sd,
			properties);
}
 

static Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        return Tuple2.of(true, token);
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(false, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, new byte[]{0}, null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}
 

@Override
public void flatMap(Row row, Collector <Row> output) throws Exception {
	if (null == row) {
		output.collect(null);
	}
	bufMap.clear();
	boolean success = formatReader.read(row, bufMap);
	if (success) {
		for (Map.Entry <String, String> entry : bufMap.entrySet()) {
			Tuple2 <Boolean, Object> parsedKey = ColumnsWriter.parseField(parsers[0], entry.getKey(), isString[0]);
			Tuple2 <Boolean, Object> parsedValue = ColumnsWriter.parseField(parsers[1], entry.getValue(), isString[1]);
			if (!StringUtils.isNullOrWhitespaceOnly(entry.getValue())) {
				if (parsedKey.f0 && parsedValue.f0) {
					output.collect(outputColsHelper
						.getResultRow(row, Row.of(parsedKey.f1, parsedValue.f1)));
				} else if (handleInvalid.equals(HandleInvalid.ERROR)) {
					throw new RuntimeException("Fail to write: " + JsonConverter.toJson(bufMap));
				}
			}
		}
	} else if (handleInvalid.equals(HandleInvalid.ERROR)) {
		throw new RuntimeException("Fail to read: " + row);
	}
}
 

public DatahubOutputFormat build() {
	DatahubOutputFormat outputFormat = null;
	if (StringUtils.isNullOrWhitespaceOnly(accessId) || StringUtils.isNullOrWhitespaceOnly(accessKey)) {
		outputFormat = new DatahubOutputFormat(
				endPoint,
				projectName,
				topicName,
				properties,
				rowTypeInfo);
	} else {
		outputFormat = new DatahubOutputFormat(
				endPoint,
				projectName,
				topicName,
				accessId,
				accessKey,
				rowTypeInfo);
	}
	outputFormat.setBatchSize(batchSize)
				.setBatchWriteTimeout(batchWriteTimeout)
				.setMaxRetryTimes(maxRetryTimes)
				.setBatchWriteTimeout(batchWriteTimeout)
				.setPrintRecord(isPrintRecord)
				.setRowTypeInfo(rowTypeInfo)
				.setTimeZone(timeZone)
				.setBufferSize(bufferSize)
				.setRecordResolver(recordResolver);
	return outputFormat;
}
 

@Override
public void flatMap(String s, Collector<Tuple2<String, Integer>> collector) throws Exception {

    if(StringUtils.isNullOrWhitespaceOnly(s)) {
        System.out.println("invalid line");
        return;
    }

    for(String word : s.split(" ")) {
        collector.collect(new Tuple2<String, Integer>(word, 1));
    }
}
 

protected String getNonEmptyJobName(String jobName) {
	if (StringUtils.isNullOrWhitespaceOnly(jobName)) {
		return DEFAULT_JOB_NAME;
	} else {
		return jobName;
	}
}
 

private HiveBatchSource getHiveBatchSource(String tableName, Params parameter) throws Exception {
    ObjectPath objectPath = ObjectPath.fromString(dbName + "." + tableName);
    CatalogTable catalogTable = getCatalogTable(tableName);
    HiveBatchSource hiveTableSource = new HiveBatchSource(new JobConf(catalog.getHiveConf()), objectPath, catalogTable);
    String partitionSpecsStr = parameter.get(HiveSourceParams.PARTITIONS);
    if (!StringUtils.isNullOrWhitespaceOnly(partitionSpecsStr)) {
        String[] partitionSpecs = partitionSpecsStr.trim().split(",");
        List<Map<String, String>> selectedPartitions = getSelectedPartitions(partitionSpecs);
        hiveTableSource = (HiveBatchSource) hiveTableSource.applyPartitionPruning(selectedPartitions);
    }
    return hiveTableSource;
}
 

private HiveTableSource getHiveTableSource(String tableName, Params parameter) throws Exception {
    ObjectPath objectPath = ObjectPath.fromString(dbName + "." + tableName);
    CatalogTable catalogTable = getCatalogTable(tableName);
    HiveTableFactory factory = new HiveTableFactory(getCatalog().getHiveConf());
    HiveTableSource hiveTableSource = (HiveTableSource) factory.createTableSource(objectPath, catalogTable);
    String partitionSpecsStr = parameter.get(HiveSourceParams.PARTITIONS);
    if (!StringUtils.isNullOrWhitespaceOnly(partitionSpecsStr)) {
        String[] partitionSpecs = partitionSpecsStr.split(",");
        hiveTableSource.getPartitions(); // FIXME: This is a workaround to a bug in HiveTableSource in Flink 1.9
        List<Map<String, String>> selectedPartitions = getSelectedPartitions(partitionSpecs);
        hiveTableSource = (HiveTableSource) hiveTableSource.applyPartitionPruning(selectedPartitions);
    }
    return hiveTableSource;
}
 

@Override
public T sinkFrom(StreamOperator in) {
    String topic = getParams().get(KafkaSinkParams.TOPIC);
    String fieldDelimiter = getParams().get(KafkaSinkParams.FIELD_DELIMITER);
    HasDataFormat.DataFormat dataFormat = getParams().get(KafkaSinkParams.DATA_FORMAT);
    String bootstrapServer = getParams().get(KafkaSinkParams.BOOTSTRAP_SERVERS);
    String properties = getParams().get(KafkaSinkParams.PROPERTIES);

    Properties props = new Properties();
    props.setProperty("bootstrap.servers", bootstrapServer);

    if (!StringUtils.isNullOrWhitespaceOnly(properties)) {
        String[] kvPairs = properties.split(",");
        for (String kvPair : kvPairs) {
            int pos = kvPair.indexOf('=');
            Preconditions.checkArgument(pos >= 0, "Invalid properties format, should be \"k1=v1,k2=v2,...\"");
            String key = kvPair.substring(0, pos);
            String value = kvPair.substring(pos + 1);
            props.setProperty(key, value);
        }
    }

    BaseKafkaSinkBuilder builder = getKafkaSinkBuilder();
    builder.setTopic(topic);
    builder.setFieldDelimiter(fieldDelimiter);
    builder.setFieldNames(in.getColNames());
    builder.setFieldTypes(in.getColTypes());
    builder.setFormat(dataFormat);
    builder.setProperties(props);

    RichSinkFunction<Row> sink = builder.build();
    in.getDataStream().addSink(sink).name("kafka");
    return (T) this;
}
 

@Override
public RichParallelSourceFunction<Row> build() {
    FlinkKafkaConsumer<Row> consumer;
    if (!StringUtils.isNullOrWhitespaceOnly(topicPattern)) {
        Pattern pattern = Pattern.compile(topicPattern);
        consumer = new FlinkKafkaConsumer<Row>(pattern, new MessageDeserialization(), properties);
    } else {
        consumer = new FlinkKafkaConsumer<Row>(topic, new MessageDeserialization(), properties);
    }

    switch (super.startupMode) {
        case LATEST: {
            consumer.setStartFromLatest();
            break;
        }
        case EARLIEST: {
            consumer.setStartFromEarliest();
            break;
        }
        case GROUP_OFFSETS: {
            consumer.setStartFromGroupOffsets();
            break;
        }
        case TIMESTAMP: {
            consumer.setStartFromTimestamp(startTimeMs);
            break;
        }
        default: {
            throw new IllegalArgumentException("invalid startupMode.");
        }
    }

    return consumer;
}
 

@Override
public void addPartition(String partition) {
	if (!StringUtils.isNullOrWhitespaceOnly(partition)) {
		this.pendingPartitions.putIfAbsent(partition, procTimeService.getCurrentProcessingTime());
	}
}
 

public static HiveCatalog createHiveCatalog(String name, String hiveVersion) {
	return new HiveCatalog(name, null, createHiveConf(),
			StringUtils.isNullOrWhitespaceOnly(hiveVersion) ? HiveShimLoader.getHiveVersion() : hiveVersion, true);
}