Java Code Examples for org.apache.flink.util.StringUtils#isNullOrWhitespaceOnly()

The following examples show how to use org.apache.flink.util.StringUtils#isNullOrWhitespaceOnly() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private Configuration prepareRuntimeConfiguration() {
	// create default configuration from current runtime env (`hbase-site.xml` in classpath) first,
	// and overwrite configuration using serialized configuration from client-side env (`hbase-site.xml` in classpath).
	// user params from client-side have the highest priority
	Configuration runtimeConfig = HBaseConfigurationUtil.deserializeConfiguration(
		serializedConfig,
		HBaseConfigurationUtil.getHBaseConfiguration());

	// do validation: check key option(s) in final runtime configuration
	if (StringUtils.isNullOrWhitespaceOnly(runtimeConfig.get(HConstants.ZOOKEEPER_QUORUM))) {
		LOG.error("can not connect to HBase without {} configuration", HConstants.ZOOKEEPER_QUORUM);
		throw new IllegalArgumentException("check HBase configuration failed, lost: '" + HConstants.ZOOKEEPER_QUORUM + "'!");
	}

	return runtimeConfig;
}
 
Example 2
Source Project: Alink   File: CsvParser.java    License: Apache License 2.0 6 votes vote down vote up
private Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        if (!enableQuote || token.charAt(0) != quoteChar) {
            return Tuple2.of(true, token);
        }
        String content;
        if (token.endsWith(quoteChar.toString())) {
            content = token.substring(1, token.length() - 1);
        } else {
            content = token.substring(1, token.length());
        }
        return Tuple2.of(true, content.replace(escapedQuote, quoteString));
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(true, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, fieldDelim.getBytes(), null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}
 
Example 3
Source Project: flink   File: GPUDriver.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Set<GPUInfo> retrieveResourceInfo(long gpuAmount) throws Exception {
	Preconditions.checkArgument(gpuAmount > 0, "The gpuAmount should be positive when retrieving the GPU resource information.");

	final Set<GPUInfo> gpuResources = new HashSet<>();
	String output = executeDiscoveryScript(discoveryScriptFile, gpuAmount, args);
	if (!output.isEmpty()) {
		String[] indexes = output.split(",");
		for (String index : indexes) {
			if (!StringUtils.isNullOrWhitespaceOnly(index)) {
				gpuResources.add(new GPUInfo(index.trim()));
			}
		}
	}
	LOG.info("Discover GPU resources: {}.", gpuResources);
	return Collections.unmodifiableSet(gpuResources);
}
 
Example 4
Source Project: flink   File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deletes the YARN application files, e.g., Flink binaries, libraries, etc., from the remote
 * filesystem.
 *
 * @param env The environment variables.
 */
public static void deleteApplicationFiles(final Map<String, String> env) {
	final String applicationFilesDir = env.get(YarnConfigKeys.FLINK_YARN_FILES);
	if (!StringUtils.isNullOrWhitespaceOnly(applicationFilesDir)) {
		final org.apache.flink.core.fs.Path path = new org.apache.flink.core.fs.Path(applicationFilesDir);
		try {
			final org.apache.flink.core.fs.FileSystem fileSystem = path.getFileSystem();
			if (!fileSystem.delete(path, true)) {
				LOG.error("Deleting yarn application files under {} was unsuccessful.", applicationFilesDir);
			}
		} catch (final IOException e) {
			LOG.error("Could not properly delete yarn application files directory {}.", applicationFilesDir, e);
		}
	} else {
		LOG.debug("No yarn application files directory set. Therefore, cannot clean up the data.");
	}
}
 
Example 5
Source Project: flink   File: OperationUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Formats a Tree of {@link Operation} in a unified way. It prints all the parameters and
 * adds all children formatted and properly indented in the following lines.
 *
 * <p>The format is <pre>
 * {@code
 * <operationName>: [(key1: [value1], key2: [v1, v2])]
 *     <child1>
 *          <child2>
 *     <child3>
 * }
 * </pre>
 *
 * @param operationName The operation name.
 * @param parameters The operation's parameters.
 * @param children The operation's children.
 * @param childToString The function to convert child to String.
 * @param <T> The type of the child.
 * @return String representation of the given operation.
 */
public static <T extends Operation> String formatWithChildren(
		String operationName,
		Map<String, Object> parameters,
		List<T> children,
		Function<T, String> childToString) {
	String description = parameters.entrySet()
		.stream()
		.map(entry -> formatParameter(entry.getKey(), entry.getValue()))
		.collect(Collectors.joining(", "));

	final StringBuilder stringBuilder = new StringBuilder();

	stringBuilder.append(operationName).append(":");

	if (!StringUtils.isNullOrWhitespaceOnly(description)) {
		stringBuilder.append(" (").append(description).append(")");
	}

	String childrenDescription = children.stream()
		.map(child -> OperationUtils.indent(childToString.apply(child)))
		.collect(Collectors.joining());

	return stringBuilder.append(childrenDescription).toString();
}
 
Example 6
Source Project: flink   File: HBaseLookupFunction.java    License: Apache License 2.0 6 votes vote down vote up
private org.apache.hadoop.conf.Configuration prepareRuntimeConfiguration() {
	// create default configuration from current runtime env (`hbase-site.xml` in classpath) first,
	// and overwrite configuration using serialized configuration from client-side env (`hbase-site.xml` in classpath).
	// user params from client-side have the highest priority
	org.apache.hadoop.conf.Configuration runtimeConfig = HBaseConfigurationUtil.deserializeConfiguration(
		serializedConfig,
		HBaseConfiguration.create());

	// do validation: check key option(s) in final runtime configuration
	if (StringUtils.isNullOrWhitespaceOnly(runtimeConfig.get(HConstants.ZOOKEEPER_QUORUM))) {
		LOG.error("can not connect to HBase without {} configuration", HConstants.ZOOKEEPER_QUORUM);
		throw new IllegalArgumentException("check HBase configuration failed, lost: '" + HConstants.ZOOKEEPER_QUORUM + "'!");
	}

	return runtimeConfig;
}
 
Example 7
Source Project: flink   File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
private Partition instantiateHivePartition(Table hiveTable, CatalogPartitionSpec partitionSpec, CatalogPartition catalogPartition)
		throws PartitionSpecInvalidException {
	List<String> partCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partValues = getOrderedFullPartitionValues(
		partitionSpec, partCols, new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName()));
	// validate partition values
	for (int i = 0; i < partCols.size(); i++) {
		if (StringUtils.isNullOrWhitespaceOnly(partValues.get(i))) {
			throw new PartitionSpecInvalidException(getName(), partCols,
				new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName()), partitionSpec);
		}
	}
	// TODO: handle GenericCatalogPartition
	StorageDescriptor sd = hiveTable.getSd().deepCopy();
	sd.setLocation(catalogPartition.getProperties().remove(HiveCatalogConfig.PARTITION_LOCATION));

	Map<String, String> properties = new HashMap<>(catalogPartition.getProperties());
	properties.put(HiveCatalogConfig.COMMENT, catalogPartition.getComment());

	return HiveTableUtil.createHivePartition(
			hiveTable.getDbName(),
			hiveTable.getTableName(),
			partValues,
			sd,
			properties);
}
 
Example 8
Source Project: Alink   File: AnyToTripleFlatMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void flatMap(Row row, Collector <Row> output) throws Exception {
	if (null == row) {
		output.collect(null);
	}
	bufMap.clear();
	boolean success = formatReader.read(row, bufMap);
	if (success) {
		for (Map.Entry <String, String> entry : bufMap.entrySet()) {
			Tuple2 <Boolean, Object> parsedKey = ColumnsWriter.parseField(parsers[0], entry.getKey(), isString[0]);
			Tuple2 <Boolean, Object> parsedValue = ColumnsWriter.parseField(parsers[1], entry.getValue(), isString[1]);
			if (!StringUtils.isNullOrWhitespaceOnly(entry.getValue())) {
				if (parsedKey.f0 && parsedValue.f0) {
					output.collect(outputColsHelper
						.getResultRow(row, Row.of(parsedKey.f1, parsedValue.f1)));
				} else if (handleInvalid.equals(HandleInvalid.ERROR)) {
					throw new RuntimeException("Fail to write: " + JsonConverter.toJson(bufMap));
				}
			}
		}
	} else if (handleInvalid.equals(HandleInvalid.ERROR)) {
		throw new RuntimeException("Fail to read: " + row);
	}
}
 
Example 9
Source Project: Alink   File: StringParsers.java    License: Apache License 2.0 6 votes vote down vote up
static Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        return Tuple2.of(true, token);
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(false, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, new byte[]{0}, null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}
 
Example 10
Source Project: flink   File: HBaseLookupFunction.java    License: Apache License 2.0 6 votes vote down vote up
private org.apache.hadoop.conf.Configuration prepareRuntimeConfiguration() {
	// create default configuration from current runtime env (`hbase-site.xml` in classpath) first,
	// and overwrite configuration using serialized configuration from client-side env (`hbase-site.xml` in classpath).
	// user params from client-side have the highest priority
	org.apache.hadoop.conf.Configuration runtimeConfig = HBaseConfigurationUtil.deserializeConfiguration(
		serializedConfig,
		HBaseConfigurationUtil.getHBaseConfiguration());

	// do validation: check key option(s) in final runtime configuration
	if (StringUtils.isNullOrWhitespaceOnly(runtimeConfig.get(HConstants.ZOOKEEPER_QUORUM))) {
		LOG.error("can not connect to HBase without {} configuration", HConstants.ZOOKEEPER_QUORUM);
		throw new IllegalArgumentException("check HBase configuration failed, lost: '" + HConstants.ZOOKEEPER_QUORUM + "'!");
	}

	return runtimeConfig;
}
 
Example 11
Source Project: flink   File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deletes the YARN application files, e.g., Flink binaries, libraries, etc., from the remote
 * filesystem.
 *
 * @param env The environment variables.
 */
public static void deleteApplicationFiles(final Map<String, String> env) {
	final String applicationFilesDir = env.get(YarnConfigKeys.FLINK_YARN_FILES);
	if (!StringUtils.isNullOrWhitespaceOnly(applicationFilesDir)) {
		final org.apache.flink.core.fs.Path path = new org.apache.flink.core.fs.Path(applicationFilesDir);
		try {
			final org.apache.flink.core.fs.FileSystem fileSystem = path.getFileSystem();
			if (!fileSystem.delete(path, true)) {
				LOG.error("Deleting yarn application files under {} was unsuccessful.", applicationFilesDir);
			}
		} catch (final IOException e) {
			LOG.error("Could not properly delete yarn application files directory {}.", applicationFilesDir, e);
		}
	} else {
		LOG.debug("No yarn application files directory set. Therefore, cannot clean up the data.");
	}
}
 
Example 12
public DatahubOutputFormat build() {
	DatahubOutputFormat outputFormat = null;
	if (StringUtils.isNullOrWhitespaceOnly(accessId) || StringUtils.isNullOrWhitespaceOnly(accessKey)) {
		outputFormat = new DatahubOutputFormat(
				endPoint,
				projectName,
				topicName,
				properties,
				rowTypeInfo);
	} else {
		outputFormat = new DatahubOutputFormat(
				endPoint,
				projectName,
				topicName,
				accessId,
				accessKey,
				rowTypeInfo);
	}
	outputFormat.setBatchSize(batchSize)
				.setBatchWriteTimeout(batchWriteTimeout)
				.setMaxRetryTimes(maxRetryTimes)
				.setBatchWriteTimeout(batchWriteTimeout)
				.setPrintRecord(isPrintRecord)
				.setRowTypeInfo(rowTypeInfo)
				.setTimeZone(timeZone)
				.setBufferSize(bufferSize)
				.setRecordResolver(recordResolver);
	return outputFormat;
}
 
Example 13
Source Project: Alink   File: KafkaSourceBuilder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public RichParallelSourceFunction<Row> build() {
    FlinkKafkaConsumer<Row> consumer;
    if (!StringUtils.isNullOrWhitespaceOnly(topicPattern)) {
        Pattern pattern = Pattern.compile(topicPattern);
        consumer = new FlinkKafkaConsumer<Row>(pattern, new MessageDeserialization(), properties);
    } else {
        consumer = new FlinkKafkaConsumer<Row>(topic, new MessageDeserialization(), properties);
    }

    switch (super.startupMode) {
        case LATEST: {
            consumer.setStartFromLatest();
            break;
        }
        case EARLIEST: {
            consumer.setStartFromEarliest();
            break;
        }
        case GROUP_OFFSETS: {
            consumer.setStartFromGroupOffsets();
            break;
        }
        case TIMESTAMP: {
            consumer.setStartFromTimestamp(startTimeMs);
            break;
        }
        default: {
            throw new IllegalArgumentException("invalid startupMode.");
        }
    }

    return consumer;
}
 
Example 14
Source Project: Alink   File: BaseKafkaSinkStreamOp.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public T sinkFrom(StreamOperator in) {
    String topic = getParams().get(KafkaSinkParams.TOPIC);
    String fieldDelimiter = getParams().get(KafkaSinkParams.FIELD_DELIMITER);
    HasDataFormat.DataFormat dataFormat = getParams().get(KafkaSinkParams.DATA_FORMAT);
    String bootstrapServer = getParams().get(KafkaSinkParams.BOOTSTRAP_SERVERS);
    String properties = getParams().get(KafkaSinkParams.PROPERTIES);

    Properties props = new Properties();
    props.setProperty("bootstrap.servers", bootstrapServer);

    if (!StringUtils.isNullOrWhitespaceOnly(properties)) {
        String[] kvPairs = properties.split(",");
        for (String kvPair : kvPairs) {
            int pos = kvPair.indexOf('=');
            Preconditions.checkArgument(pos >= 0, "Invalid properties format, should be \"k1=v1,k2=v2,...\"");
            String key = kvPair.substring(0, pos);
            String value = kvPair.substring(pos + 1);
            props.setProperty(key, value);
        }
    }

    BaseKafkaSinkBuilder builder = getKafkaSinkBuilder();
    builder.setTopic(topic);
    builder.setFieldDelimiter(fieldDelimiter);
    builder.setFieldNames(in.getColNames());
    builder.setFieldTypes(in.getColTypes());
    builder.setFormat(dataFormat);
    builder.setProperties(props);

    RichSinkFunction<Row> sink = builder.build();
    in.getDataStream().addSink(sink).name("kafka");
    return (T) this;
}
 
Example 15
Source Project: Alink   File: HiveDB.java    License: Apache License 2.0 5 votes vote down vote up
private HiveTableSource getHiveTableSource(String tableName, Params parameter) throws Exception {
    ObjectPath objectPath = ObjectPath.fromString(dbName + "." + tableName);
    CatalogTable catalogTable = getCatalogTable(tableName);
    HiveTableFactory factory = new HiveTableFactory(getCatalog().getHiveConf());
    HiveTableSource hiveTableSource = (HiveTableSource) factory.createTableSource(objectPath, catalogTable);
    String partitionSpecsStr = parameter.get(HiveSourceParams.PARTITIONS);
    if (!StringUtils.isNullOrWhitespaceOnly(partitionSpecsStr)) {
        String[] partitionSpecs = partitionSpecsStr.split(",");
        hiveTableSource.getPartitions(); // FIXME: This is a workaround to a bug in HiveTableSource in Flink 1.9
        List<Map<String, String>> selectedPartitions = getSelectedPartitions(partitionSpecs);
        hiveTableSource = (HiveTableSource) hiveTableSource.applyPartitionPruning(selectedPartitions);
    }
    return hiveTableSource;
}
 
Example 16
Source Project: Alink   File: HiveDB.java    License: Apache License 2.0 5 votes vote down vote up
private HiveBatchSource getHiveBatchSource(String tableName, Params parameter) throws Exception {
    ObjectPath objectPath = ObjectPath.fromString(dbName + "." + tableName);
    CatalogTable catalogTable = getCatalogTable(tableName);
    HiveBatchSource hiveTableSource = new HiveBatchSource(new JobConf(catalog.getHiveConf()), objectPath, catalogTable);
    String partitionSpecsStr = parameter.get(HiveSourceParams.PARTITIONS);
    if (!StringUtils.isNullOrWhitespaceOnly(partitionSpecsStr)) {
        String[] partitionSpecs = partitionSpecsStr.trim().split(",");
        List<Map<String, String>> selectedPartitions = getSelectedPartitions(partitionSpecs);
        hiveTableSource = (HiveBatchSource) hiveTableSource.applyPartitionPruning(selectedPartitions);
    }
    return hiveTableSource;
}
 
Example 17
Source Project: flink   File: ExecutorBase.java    License: Apache License 2.0 5 votes vote down vote up
protected String getNonEmptyJobName(String jobName) {
	if (StringUtils.isNullOrWhitespaceOnly(jobName)) {
		return DEFAULT_JOB_NAME;
	} else {
		return jobName;
	}
}
 
Example 18
Source Project: blog_demos   File: Splitter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(String s, Collector<Tuple2<String, Integer>> collector) throws Exception {

    if(StringUtils.isNullOrWhitespaceOnly(s)) {
        System.out.println("invalid line");
        return;
    }

    for(String word : s.split(" ")) {
        collector.collect(new Tuple2<String, Integer>(word, 1));
    }
}
 
Example 19
Source Project: flink   File: ProcTimeCommitTigger.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void addPartition(String partition) {
	if (!StringUtils.isNullOrWhitespaceOnly(partition)) {
		this.pendingPartitions.putIfAbsent(partition, procTimeService.getCurrentProcessingTime());
	}
}
 
Example 20
Source Project: flink   File: HiveTestUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static HiveCatalog createHiveCatalog(String name, String hiveVersion) {
	return new HiveCatalog(name, null, createHiveConf(),
			StringUtils.isNullOrWhitespaceOnly(hiveVersion) ? HiveShimLoader.getHiveVersion() : hiveVersion, true);
}