Java Code Examples for org.apache.flink.util.Preconditions#checkState()

The following examples show how to use org.apache.flink.util.Preconditions#checkState() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Alink   File: EvaluationUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void updateBinaryMetricsSummary(TreeMap<String, Double> labelProbMap,
                                              String label,
                                              BinaryMetricsSummary binaryMetricsSummary) {
    Preconditions.checkState(labelProbMap.size() == BINARY_LABEL_NUMBER,
        "The number of labels must be equal to 2!");
    binaryMetricsSummary.total++;
    binaryMetricsSummary.logLoss += extractLogloss(labelProbMap, label);

    double d = labelProbMap.get(binaryMetricsSummary.labels[0]);
    int idx = d == 1.0 ? ClassificationEvaluationUtil.DETAIL_BIN_NUMBER - 1 :
        (int)Math.floor(d * ClassificationEvaluationUtil.DETAIL_BIN_NUMBER);
    if (idx >= 0 && idx < ClassificationEvaluationUtil.DETAIL_BIN_NUMBER) {
        if (label.equals(binaryMetricsSummary.labels[0])) {
            binaryMetricsSummary.positiveBin[idx] += 1;
        } else if (label.equals(binaryMetricsSummary.labels[1])) {
            binaryMetricsSummary.negativeBin[idx] += 1;
        } else {
            throw new RuntimeException(String
                .format("Label %s not contained in label Array %s in detail column!", label,
                    labelProbMap.keySet().toString()));
        }
    }
}
 
Example 2
@Override
public void start(LeaderRetrievalListener listener) throws Exception {
	Preconditions.checkNotNull(listener, "Listener must not be null.");
	Preconditions.checkState(leaderListener == null, "ZooKeeperLeaderRetrievalService can " +
			"only be started once.");

	LOG.info("Starting ZooKeeperLeaderRetrievalService {}.", retrievalPath);

	synchronized (lock) {
		leaderListener = listener;

		client.getUnhandledErrorListenable().addListener(this);
		cache.getListenable().addListener(this);
		cache.start();

		client.getConnectionStateListenable().addListener(connectionStateListener);

		running = true;
	}
}
 
Example 3
Source Project: Flink-CEPplus   File: Dispatcher.java    License: Apache License 2.0 6 votes vote down vote up
private void registerJobManagerRunnerTerminationFuture(JobID jobId, CompletableFuture<Void> jobManagerRunnerTerminationFuture) {
	Preconditions.checkState(!jobManagerTerminationFutures.containsKey(jobId));

	jobManagerTerminationFutures.put(jobId, jobManagerRunnerTerminationFuture);

	// clean up the pending termination future
	jobManagerRunnerTerminationFuture.thenRunAsync(
		() -> {
			final CompletableFuture<Void> terminationFuture = jobManagerTerminationFutures.remove(jobId);

			//noinspection ObjectEquality
			if (terminationFuture != null && terminationFuture != jobManagerRunnerTerminationFuture) {
				jobManagerTerminationFutures.put(jobId, terminationFuture);
			}
		},
		getUnfencedMainThreadExecutor());
}
 
Example 4
public static Map<JobVertexID, ExecutionJobVertex> includeLegacyJobVertexIDs(
		Map<JobVertexID, ExecutionJobVertex> tasks) {

	Map<JobVertexID, ExecutionJobVertex> expanded = new HashMap<>(2 * tasks.size());
	// first include all new ids
	expanded.putAll(tasks);

	// now expand and add legacy ids
	for (ExecutionJobVertex executionJobVertex : tasks.values()) {
		if (null != executionJobVertex) {
			JobVertex jobVertex = executionJobVertex.getJobVertex();
			if (null != jobVertex) {
				List<JobVertexID> alternativeIds = jobVertex.getIdAlternatives();
				for (JobVertexID jobVertexID : alternativeIds) {
					ExecutionJobVertex old = expanded.put(jobVertexID, executionJobVertex);
					Preconditions.checkState(null == old || old.equals(executionJobVertex),
							"Ambiguous jobvertex id detected during expansion to legacy ids.");
				}
			}
		}
	}

	return expanded;
}
 
Example 5
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {

	ListStateDescriptor<Long> currentKeyDescriptor = new ListStateDescriptor<>("currentKey", Long.class);
	sourceCurrentKeyState = context.getOperatorStateStore().getListState(currentKeyDescriptor);

	currentKey = getRuntimeContext().getIndexOfThisSubtask();
	Iterable<Long> iterable = sourceCurrentKeyState.get();
	if (iterable != null) {
		Iterator<Long> iterator = iterable.iterator();
		if (iterator.hasNext()) {
			currentKey = iterator.next();
			Preconditions.checkState(!iterator.hasNext());
		}
	}
}
 
Example 6
Source Project: flink-connectors   File: SetupUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a stream reader for reading Integer events.
 *
 * @param streamName    Name of the test stream.
 *
 * @return Stream reader instance.
 */
public EventStreamReader<Integer> getIntegerReader(final String streamName) {
    Preconditions.checkState(this.started.get(), "Services not yet started");
    Preconditions.checkNotNull(streamName);

    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(this.scope, getClientConfig());
    final String readerGroup = "testReaderGroup" + this.scope + streamName;
    readerGroupManager.createReaderGroup(
            readerGroup,
            ReaderGroupConfig.builder().stream(Stream.of(this.scope, streamName)).build());

    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(this.scope, getClientConfig());
    final String readerGroupId = UUID.randomUUID().toString();
    return clientFactory.createReader(
            readerGroupId,
            readerGroup,
            new IntegerSerializer(),
            ReaderConfig.builder().build());
}
 
Example 7
Source Project: Flink-CEPplus   File: GridGraph.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Graph<LongValue, NullValue, NullValue> generate() {
	Preconditions.checkState(!dimensions.isEmpty(), "No dimensions added to GridGraph");

	// Vertices
	DataSet<Vertex<LongValue, NullValue>> vertices = GraphGeneratorUtils.vertexSequence(env, parallelism, vertexCount);

	// Edges
	LongValueSequenceIterator iterator = new LongValueSequenceIterator(0, this.vertexCount - 1);

	DataSet<Edge<LongValue, NullValue>> edges = env
		.fromParallelCollection(iterator, LongValue.class)
			.setParallelism(parallelism)
			.name("Edge iterators")
		.flatMap(new LinkVertexToNeighbors(vertexCount, dimensions))
			.setParallelism(parallelism)
			.name("Grid graph edges");

	// Graph
	return Graph.fromDataSet(vertices, edges, env);
}
 
Example 8
Source Project: Flink-CEPplus   File: Dispatcher.java    License: Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Void> runJob(JobGraph jobGraph) {
	Preconditions.checkState(!jobManagerRunnerFutures.containsKey(jobGraph.getJobID()));

	final CompletableFuture<JobManagerRunner> jobManagerRunnerFuture = createJobManagerRunner(jobGraph);

	jobManagerRunnerFutures.put(jobGraph.getJobID(), jobManagerRunnerFuture);

	return jobManagerRunnerFuture
		.thenApply(FunctionUtils.nullFn())
		.whenCompleteAsync(
			(ignored, throwable) -> {
				if (throwable != null) {
					jobManagerRunnerFutures.remove(jobGraph.getJobID());
				}
			},
			getMainThreadExecutor());
}
 
Example 9
Source Project: flink   File: PluginManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
	/*
	 * We setup a plugin directory hierarchy and utilize DirectoryBasedPluginFinder to create the
	 * descriptors:
	 *
	 * <pre>
	 * tmp/plugins-root/
	 *          |-------------A/
	 *          |             |-plugin-a.jar
	 *          |
	 *          |-------------B/
	 *                        |-plugin-b.jar
	 * </pre>
	 */
	final File pluginRootFolder = temporaryFolder.newFolder();
	final Path pluginRootFolderPath = pluginRootFolder.toPath();
	final File pluginAFolder = new File(pluginRootFolder, "A");
	final File pluginBFolder = new File(pluginRootFolder, "B");
	Preconditions.checkState(pluginAFolder.mkdirs());
	Preconditions.checkState(pluginBFolder.mkdirs());
	Files.copy(locateJarFile(PLUGIN_A).toPath(), Paths.get(pluginAFolder.toString(), PLUGIN_A));
	Files.copy(locateJarFile(PLUGIN_B).toPath(), Paths.get(pluginBFolder.toString(), PLUGIN_B));
	final PluginFinder descriptorsFactory = new DirectoryBasedPluginFinder(pluginRootFolderPath);
	descriptors = descriptorsFactory.findPlugins();
	Preconditions.checkState(descriptors.size() == 2);
}
 
Example 10
Source Project: flink   File: StreamTask.java    License: Apache License 2.0 5 votes vote down vote up
private void reportCompletedSnapshotStates(
	TaskStateSnapshot acknowledgedTaskStateSnapshot,
	TaskStateSnapshot localTaskStateSnapshot,
	long asyncDurationMillis) {

	TaskStateManager taskStateManager = owner.getEnvironment().getTaskStateManager();

	boolean hasAckState = acknowledgedTaskStateSnapshot.hasState();
	boolean hasLocalState = localTaskStateSnapshot.hasState();

	Preconditions.checkState(hasAckState || !hasLocalState,
		"Found cached state but no corresponding primary state is reported to the job " +
			"manager. This indicates a problem.");

	// we signal stateless tasks by reporting null, so that there are no attempts to assign empty state
	// to stateless tasks on restore. This enables simple job modifications that only concern
	// stateless without the need to assign them uids to match their (always empty) states.
	taskStateManager.reportTaskStateSnapshots(
		checkpointMetaData,
		checkpointMetrics,
		hasAckState ? acknowledgedTaskStateSnapshot : null,
		hasLocalState ? localTaskStateSnapshot : null);

	LOG.debug("{} - finished asynchronous part of checkpoint {}. Asynchronous duration: {} ms",
		owner.getName(), checkpointMetaData.getCheckpointId(), asyncDurationMillis);

	LOG.trace("{} - reported the following states in snapshot for checkpoint {}: {}.",
		owner.getName(), checkpointMetaData.getCheckpointId(), acknowledgedTaskStateSnapshot);
}
 
Example 11
Source Project: flink   File: FailingCollectionSource.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(
		this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " has not been properly initialized.");

	this.checkpointedState.clear();
	this.checkpointedState.add(this.numElementsEmitted);
	long checkpointId = context.getCheckpointId();
	checkpointedEmittedNums.put(checkpointId, numElementsEmitted);
}
 
Example 12
Source Project: flink   File: TaskManagerRegistration.java    License: Apache License 2.0 5 votes vote down vote up
public void freeSlot() {
	Preconditions.checkState(
		numberFreeSlots < slots.size(),
		"The number of free slots cannot exceed the number of registered slots. This indicates a bug.");
	numberFreeSlots++;

	if (numberFreeSlots == getNumberRegisteredSlots() && idleSince == Long.MAX_VALUE) {
		idleSince = System.currentTimeMillis();
	}
}
 
Example 13
Source Project: flink   File: TaskManagerSlot.java    License: Apache License 2.0 5 votes vote down vote up
public void updateAllocation(AllocationID allocationId, JobID jobId) {
	Preconditions.checkState(state == State.FREE, "The slot has to be free in order to set an allocation id.");

	state = State.ALLOCATED;
	this.allocationId = Preconditions.checkNotNull(allocationId);
	this.jobId = Preconditions.checkNotNull(jobId);
}
 
Example 14
/**
 * Sets the part size above which a part file will have to roll.
 * @param size the allowed part size.
 */
public DefaultRollingPolicy.PolicyBuilder withMaxPartSize(final long size) {
	Preconditions.checkState(size > 0L);
	return new PolicyBuilder(size, rolloverInterval, inactivityInterval);
}
 
Example 15
Source Project: flink   File: PartitionLoader.java    License: Apache License 2.0 4 votes vote down vote up
private void overwriteAndRenameFiles(List<Path> srcDirs, Path destDir) throws Exception {
	boolean dirSuccessExist = fs.exists(destDir) || fs.mkdirs(destDir);
	Preconditions.checkState(dirSuccessExist, "Failed to create dest path " + destDir);
	overwrite(destDir);
	renameFiles(srcDirs, destDir);
}
 
Example 16
@Override
protected DeserializationSchema<T> getDeserializationSchema() {
    Preconditions.checkState(deserializationSchema != null, "Deserialization schema must not be null.");
    return deserializationSchema;
}
 
Example 17
Source Project: Flink-CEPplus   File: TaskManagerSlot.java    License: Apache License 2.0 4 votes vote down vote up
public void assignPendingSlotRequest(PendingSlotRequest pendingSlotRequest) {
	Preconditions.checkState(state == State.FREE, "Slot must be free to be assigned a slot request.");

	state = State.PENDING;
	assignedSlotRequest = Preconditions.checkNotNull(pendingSlotRequest);
}
 
Example 18
public void assignPendingTaskManagerSlot(@Nonnull PendingTaskManagerSlot pendingTaskManagerSlotToAssign) {
	Preconditions.checkState(pendingTaskManagerSlot == null);
	this.pendingTaskManagerSlot = pendingTaskManagerSlotToAssign;
}
 
Example 19
/**
 * Generator for all 3^4 = 81 possible configurations of a OperatorSubtaskState:
 * - 4 different sub-states:
 *      managed/raw + operator/keyed.
 * - 3 different options per sub-state:
 *      empty (simulate no state), single handle (simulate recovery), 2 handles (simulate e.g. rescaling)
 */
private OperatorSubtaskState generateForConfiguration(int conf) {

	Preconditions.checkState(conf >= 0 && conf <= 80); // 3^4
	final int numModes = 3;

	KeyGroupRange keyGroupRange = new KeyGroupRange(0, 4);
	KeyGroupRange keyGroupRange1 = new KeyGroupRange(0, 2);
	KeyGroupRange keyGroupRange2 = new KeyGroupRange(3, 4);

	int div = 1;
	int mode = (conf / div) % numModes;
	StateObjectCollection<OperatorStateHandle> s1 =
		mode == 0 ?
			StateObjectCollection.empty() :
			mode == 1 ?
				new StateObjectCollection<>(
					Collections.singletonList(createNewOperatorStateHandle(2, random))) :
				new StateObjectCollection<>(
					Arrays.asList(
						createNewOperatorStateHandle(2, random),
						createNewOperatorStateHandle(2, random)));
	div *= numModes;
	mode = (conf / div) % numModes;
	StateObjectCollection<OperatorStateHandle> s2 =
		mode == 0 ?
			StateObjectCollection.empty() :
			mode == 1 ?
				new StateObjectCollection<>(
					Collections.singletonList(createNewOperatorStateHandle(2, random))) :
				new StateObjectCollection<>(
					Arrays.asList(
						createNewOperatorStateHandle(2, random),
						createNewOperatorStateHandle(2, random)));

	div *= numModes;
	mode = (conf / div) % numModes;
	StateObjectCollection<KeyedStateHandle> s3 =
		mode == 0 ?
			StateObjectCollection.empty() :
			mode == 1 ?
				new StateObjectCollection<>(
					Collections.singletonList(createNewKeyedStateHandle(keyGroupRange))) :
				new StateObjectCollection<>(
					Arrays.asList(
						createNewKeyedStateHandle(keyGroupRange1),
						createNewKeyedStateHandle(keyGroupRange2)));

	div *= numModes;
	mode = (conf / div) % numModes;
	StateObjectCollection<KeyedStateHandle> s4 =
		mode == 0 ?
			StateObjectCollection.empty() :
			mode == 1 ?
				new StateObjectCollection<>(
					Collections.singletonList(createNewKeyedStateHandle(keyGroupRange))) :
				new StateObjectCollection<>(
					Arrays.asList(
						createNewKeyedStateHandle(keyGroupRange1),
						createNewKeyedStateHandle(keyGroupRange2)));

	return new OperatorSubtaskState(s1, s2, s3, s4);
}
 
Example 20
Source Project: flink   File: DefaultRollingPolicy.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Sets the interval of allowed inactivity after which a part file will have to roll.
 * The frequency at which this is checked is controlled by the
 * {@link org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink.RowFormatBuilder#withBucketCheckInterval(long)}
 * setting.
 * @param interval the allowed inactivity interval.
 */
public DefaultRollingPolicy.PolicyBuilder withInactivityInterval(final long interval) {
	Preconditions.checkState(interval > 0L);
	return new PolicyBuilder(partSize, rolloverInterval, interval);
}