org.apache.flink.runtime.jobgraph.tasks.InputSplitProviderException Java Examples

The following examples show how to use org.apache.flink.runtime.jobgraph.tasks.InputSplitProviderException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SequenceReader.java    From alibaba-flink-connectors with Apache License 2.0 6 votes vote down vote up 
public void run(SourceFunction.SourceContext<T> ctx) throws InputSplitProviderException, IOException, InterruptedException {

		InputSplit inputSplit;
		inputSplit = inputSplitProvider.getNextInputSplit(sourceFunction.getRuntimeContext().getUserCodeClassLoader());
		while (!isStop && inputSplit != null) {
			RecordReader<T, ?> recordReader = sourceFunction.createReader(config);
			try {
				recordReader.open(inputSplit, sourceFunction.getRuntimeContext());
				while (!isStop && recordReader.next()) {
					if (recordReader.isHeartBeat()) {
						continue;
					}
					synchronized (ctx.getCheckpointLock()) {
						tpsMetric.markEvent();
						ctx.collect(recordReader.getMessage());
					}
				}
			} finally {
				recordReader.close();
			}
			inputSplit = inputSplitProvider.getNextInputSplit(sourceFunction.getRuntimeContext().getUserCodeClassLoader());
		}
	}
Example #2
Source File: RpcInputSplitProvider.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public InputSplit getNextInputSplit(ClassLoader userCodeClassLoader) throws InputSplitProviderException {
	Preconditions.checkNotNull(userCodeClassLoader);

	CompletableFuture<SerializedInputSplit> futureInputSplit = jobMasterGateway.requestNextInputSplit(
		jobVertexID,
		executionAttemptID);

	try {
		SerializedInputSplit serializedInputSplit = futureInputSplit.get(timeout.getSize(), timeout.getUnit());

		if (serializedInputSplit.isEmpty()) {
			return null;
		} else {
			return InstantiationUtil.deserializeObject(serializedInputSplit.getInputSplitData(), userCodeClassLoader);
		}
	} catch (Exception e) {
		throw new InputSplitProviderException("Requesting the next input split failed.", e);
	}
}
Example #3
Source File: TaskInputSplitProviderTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRequestNextInputSplitWithInvalidExecutionID() throws InputSplitProviderException {

	final JobID jobID = new JobID();
	final JobVertexID vertexID = new JobVertexID();
	final ExecutionAttemptID executionID = new ExecutionAttemptID();
	final FiniteDuration timeout = new FiniteDuration(10, TimeUnit.SECONDS);

	final ActorGateway gateway = new NullInputSplitGateway();


	final TaskInputSplitProvider provider = new TaskInputSplitProvider(
		gateway,
		jobID,
		vertexID,
		executionID,
		timeout);

	// The jobManager will return a
	InputSplit nextInputSplit = provider.getNextInputSplit(getClass().getClassLoader());

	assertTrue(nextInputSplit == null);
}
Example #4
Source File: RpcInputSplitProvider.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public InputSplit getNextInputSplit(ClassLoader userCodeClassLoader) throws InputSplitProviderException {
	Preconditions.checkNotNull(userCodeClassLoader);

	CompletableFuture<SerializedInputSplit> futureInputSplit = jobMasterGateway.requestNextInputSplit(
		jobVertexID,
		executionAttemptID);

	try {
		SerializedInputSplit serializedInputSplit = futureInputSplit.get(timeout.getSize(), timeout.getUnit());

		if (serializedInputSplit.isEmpty()) {
			return null;
		} else {
			return InstantiationUtil.deserializeObject(serializedInputSplit.getInputSplitData(), userCodeClassLoader);
		}
	} catch (Exception e) {
		throw new InputSplitProviderException("Requesting the next input split failed.", e);
	}
}
Example #5
Source File: RpcInputSplitProvider.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public InputSplit getNextInputSplit(ClassLoader userCodeClassLoader) throws InputSplitProviderException {
	Preconditions.checkNotNull(userCodeClassLoader);

	CompletableFuture<SerializedInputSplit> futureInputSplit = jobMasterGateway.requestNextInputSplit(
		jobVertexID,
		executionAttemptID);

	try {
		SerializedInputSplit serializedInputSplit = futureInputSplit.get(timeout.getSize(), timeout.getUnit());

		if (serializedInputSplit.isEmpty()) {
			return null;
		} else {
			return InstantiationUtil.deserializeObject(serializedInputSplit.getInputSplitData(), userCodeClassLoader);
		}
	} catch (Exception e) {
		throw new InputSplitProviderException("Requesting the next input split failed.", e);
	}
}
Example #6
Source File: DataSourceTask.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private Iterator<InputSplit> getInputSplits() {

		final InputSplitProvider provider = getEnvironment().getInputSplitProvider();

		return new Iterator<InputSplit>() {

			private InputSplit nextSplit;
			
			private boolean exhausted;

			@Override
			public boolean hasNext() {
				if (exhausted) {
					return false;
				}
				
				if (nextSplit != null) {
					return true;
				}

				final InputSplit split;
				try {
					split = provider.getNextInputSplit(getUserCodeClassLoader());
				} catch (InputSplitProviderException e) {
					throw new RuntimeException("Could not retrieve next input split.", e);
				}

				if (split != null) {
					this.nextSplit = split;
					return true;
				}
				else {
					exhausted = true;
					return false;
				}
			}

			@Override
			public InputSplit next() {
				if (this.nextSplit == null && !hasNext()) {
					throw new NoSuchElementException();
				}

				final InputSplit tmp = this.nextSplit;
				this.nextSplit = null;
				return tmp;
			}

			@Override
			public void remove() {
				throw new UnsupportedOperationException();
			}
		};
	}
Example #7
Source File: InputFormatSourceFunction.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private Iterator<InputSplit> getInputSplits() {

		return new Iterator<InputSplit>() {

			private InputSplit nextSplit;

			private boolean exhausted;

			@Override
			public boolean hasNext() {
				if (exhausted) {
					return false;
				}

				if (nextSplit != null) {
					return true;
				}

				final InputSplit split;
				try {
					split = provider.getNextInputSplit(getRuntimeContext().getUserCodeClassLoader());
				} catch (InputSplitProviderException e) {
					throw new RuntimeException("Could not retrieve next input split.", e);
				}

				if (split != null) {
					this.nextSplit = split;
					return true;
				} else {
					exhausted = true;
					return false;
				}
			}

			@Override
			public InputSplit next() {
				if (this.nextSplit == null && !hasNext()) {
					throw new NoSuchElementException();
				}

				final InputSplit tmp = this.nextSplit;
				this.nextSplit = null;
				return tmp;
			}

			@Override
			public void remove() {
				throw new UnsupportedOperationException();
			}
		};
	}
Example #8
Source File: DataSourceTask.java    From flink with Apache License 2.0 4 votes vote down vote up 
private Iterator<InputSplit> getInputSplits() {

		final InputSplitProvider provider = getEnvironment().getInputSplitProvider();

		return new Iterator<InputSplit>() {

			private InputSplit nextSplit;
			
			private boolean exhausted;

			@Override
			public boolean hasNext() {
				if (exhausted) {
					return false;
				}
				
				if (nextSplit != null) {
					return true;
				}

				final InputSplit split;
				try {
					split = provider.getNextInputSplit(getUserCodeClassLoader());
				} catch (InputSplitProviderException e) {
					throw new RuntimeException("Could not retrieve next input split.", e);
				}

				if (split != null) {
					this.nextSplit = split;
					return true;
				}
				else {
					exhausted = true;
					return false;
				}
			}

			@Override
			public InputSplit next() {
				if (this.nextSplit == null && !hasNext()) {
					throw new NoSuchElementException();
				}

				final InputSplit tmp = this.nextSplit;
				this.nextSplit = null;
				return tmp;
			}

			@Override
			public void remove() {
				throw new UnsupportedOperationException();
			}
		};
	}
Example #9
Source File: InputFormatSourceFunction.java    From flink with Apache License 2.0 4 votes vote down vote up 
private Iterator<InputSplit> getInputSplits() {

		return new Iterator<InputSplit>() {

			private InputSplit nextSplit;

			private boolean exhausted;

			@Override
			public boolean hasNext() {
				if (exhausted) {
					return false;
				}

				if (nextSplit != null) {
					return true;
				}

				final InputSplit split;
				try {
					split = provider.getNextInputSplit(getRuntimeContext().getUserCodeClassLoader());
				} catch (InputSplitProviderException e) {
					throw new RuntimeException("Could not retrieve next input split.", e);
				}

				if (split != null) {
					this.nextSplit = split;
					return true;
				} else {
					exhausted = true;
					return false;
				}
			}

			@Override
			public InputSplit next() {
				if (this.nextSplit == null && !hasNext()) {
					throw new NoSuchElementException();
				}

				final InputSplit tmp = this.nextSplit;
				this.nextSplit = null;
				return tmp;
			}

			@Override
			public void remove() {
				throw new UnsupportedOperationException();
			}
		};
	}
Example #10
Source File: DataSourceTask.java    From flink with Apache License 2.0 4 votes vote down vote up 
private Iterator<InputSplit> getInputSplits() {

		final InputSplitProvider provider = getEnvironment().getInputSplitProvider();

		return new Iterator<InputSplit>() {

			private InputSplit nextSplit;
			
			private boolean exhausted;

			@Override
			public boolean hasNext() {
				if (exhausted) {
					return false;
				}
				
				if (nextSplit != null) {
					return true;
				}

				final InputSplit split;
				try {
					split = provider.getNextInputSplit(getUserCodeClassLoader());
				} catch (InputSplitProviderException e) {
					throw new RuntimeException("Could not retrieve next input split.", e);
				}

				if (split != null) {
					this.nextSplit = split;
					return true;
				}
				else {
					exhausted = true;
					return false;
				}
			}

			@Override
			public InputSplit next() {
				if (this.nextSplit == null && !hasNext()) {
					throw new NoSuchElementException();
				}

				final InputSplit tmp = this.nextSplit;
				this.nextSplit = null;
				return tmp;
			}

			@Override
			public void remove() {
				throw new UnsupportedOperationException();
			}
		};
	}
Example #11
Source File: InputFormatSourceFunction.java    From flink with Apache License 2.0 4 votes vote down vote up 
private Iterator<InputSplit> getInputSplits() {

		return new Iterator<InputSplit>() {

			private InputSplit nextSplit;

			private boolean exhausted;

			@Override
			public boolean hasNext() {
				if (exhausted) {
					return false;
				}

				if (nextSplit != null) {
					return true;
				}

				final InputSplit split;
				try {
					split = provider.getNextInputSplit(getRuntimeContext().getUserCodeClassLoader());
				} catch (InputSplitProviderException e) {
					throw new RuntimeException("Could not retrieve next input split.", e);
				}

				if (split != null) {
					this.nextSplit = split;
					return true;
				} else {
					exhausted = true;
					return false;
				}
			}

			@Override
			public InputSplit next() {
				if (this.nextSplit == null && !hasNext()) {
					throw new NoSuchElementException();
				}

				final InputSplit tmp = this.nextSplit;
				this.nextSplit = null;
				return tmp;
			}

			@Override
			public void remove() {
				throw new UnsupportedOperationException();
			}
		};
	}