Java Code Examples for org.apache.flink.runtime.util.ReusingMutableToRegularIteratorWrapper

The following examples show how to use org.apache.flink.runtime.util.ReusingMutableToRegularIteratorWrapper. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: GenericWriteAheadSink.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void notifyCheckpointComplete(long checkpointId) throws Exception {
	super.notifyCheckpointComplete(checkpointId);

	synchronized (pendingCheckpoints) {

		Iterator<PendingCheckpoint> pendingCheckpointIt = pendingCheckpoints.iterator();
		while (pendingCheckpointIt.hasNext()) {

			PendingCheckpoint pendingCheckpoint = pendingCheckpointIt.next();

			long pastCheckpointId = pendingCheckpoint.checkpointId;
			int subtaskId = pendingCheckpoint.subtaskId;
			long timestamp = pendingCheckpoint.timestamp;
			StreamStateHandle streamHandle = pendingCheckpoint.stateHandle;

			if (pastCheckpointId <= checkpointId) {
				try {
					if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) {
						try (FSDataInputStream in = streamHandle.openInputStream()) {
							boolean success = sendValues(
									new ReusingMutableToRegularIteratorWrapper<>(
											new InputViewIterator<>(
													new DataInputViewStreamWrapper(
															in),
													serializer),
											serializer),
									pastCheckpointId,
									timestamp);
							if (success) {
								// in case the checkpoint was successfully committed,
								// discard its state from the backend and mark it for removal
								// in case it failed, we retry on the next checkpoint
								committer.commitCheckpoint(subtaskId, pastCheckpointId);
								streamHandle.discardState();
								pendingCheckpointIt.remove();
							}
						}
					} else {
						streamHandle.discardState();
						pendingCheckpointIt.remove();
					}
				} catch (Exception e) {
					// we have to break here to prevent a new (later) checkpoint
					// from being committed before this one
					LOG.error("Could not commit checkpoint.", e);
					break;
				}
			}
		}
	}
}
 
Example 2
Source Project: flink   Source File: GenericWriteAheadSink.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void notifyCheckpointComplete(long checkpointId) throws Exception {
	super.notifyCheckpointComplete(checkpointId);

	synchronized (pendingCheckpoints) {

		Iterator<PendingCheckpoint> pendingCheckpointIt = pendingCheckpoints.iterator();
		while (pendingCheckpointIt.hasNext()) {

			PendingCheckpoint pendingCheckpoint = pendingCheckpointIt.next();

			long pastCheckpointId = pendingCheckpoint.checkpointId;
			int subtaskId = pendingCheckpoint.subtaskId;
			long timestamp = pendingCheckpoint.timestamp;
			StreamStateHandle streamHandle = pendingCheckpoint.stateHandle;

			if (pastCheckpointId <= checkpointId) {
				try {
					if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) {
						try (FSDataInputStream in = streamHandle.openInputStream()) {
							boolean success = sendValues(
									new ReusingMutableToRegularIteratorWrapper<>(
											new InputViewIterator<>(
													new DataInputViewStreamWrapper(
															in),
													serializer),
											serializer),
									pastCheckpointId,
									timestamp);
							if (success) {
								// in case the checkpoint was successfully committed,
								// discard its state from the backend and mark it for removal
								// in case it failed, we retry on the next checkpoint
								committer.commitCheckpoint(subtaskId, pastCheckpointId);
								streamHandle.discardState();
								pendingCheckpointIt.remove();
							}
						}
					} else {
						streamHandle.discardState();
						pendingCheckpointIt.remove();
					}
				} catch (Exception e) {
					// we have to break here to prevent a new (later) checkpoint
					// from being committed before this one
					LOG.error("Could not commit checkpoint.", e);
					break;
				}
			}
		}
	}
}
 
Example 3
Source Project: flink   Source File: GenericWriteAheadSink.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void notifyCheckpointComplete(long checkpointId) throws Exception {
	super.notifyCheckpointComplete(checkpointId);

	synchronized (pendingCheckpoints) {

		Iterator<PendingCheckpoint> pendingCheckpointIt = pendingCheckpoints.iterator();
		while (pendingCheckpointIt.hasNext()) {

			PendingCheckpoint pendingCheckpoint = pendingCheckpointIt.next();

			long pastCheckpointId = pendingCheckpoint.checkpointId;
			int subtaskId = pendingCheckpoint.subtaskId;
			long timestamp = pendingCheckpoint.timestamp;
			StreamStateHandle streamHandle = pendingCheckpoint.stateHandle;

			if (pastCheckpointId <= checkpointId) {
				try {
					if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) {
						try (FSDataInputStream in = streamHandle.openInputStream()) {
							boolean success = sendValues(
									new ReusingMutableToRegularIteratorWrapper<>(
											new InputViewIterator<>(
													new DataInputViewStreamWrapper(
															in),
													serializer),
											serializer),
									pastCheckpointId,
									timestamp);
							if (success) {
								// in case the checkpoint was successfully committed,
								// discard its state from the backend and mark it for removal
								// in case it failed, we retry on the next checkpoint
								committer.commitCheckpoint(subtaskId, pastCheckpointId);
								streamHandle.discardState();
								pendingCheckpointIt.remove();
							}
						}
					} else {
						streamHandle.discardState();
						pendingCheckpointIt.remove();
					}
				} catch (Exception e) {
					// we have to break here to prevent a new (later) checkpoint
					// from being committed before this one
					LOG.error("Could not commit checkpoint.", e);
					break;
				}
			}
		}
	}
}