org.apache.flink.streaming.runtime.tasks.TwoInputStreamTask Java Examples

The following examples show how to use org.apache.flink.streaming.runtime.tasks.TwoInputStreamTask. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: String2SortMergeJoinOperatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private TwoInputStreamTaskTestHarness<BinaryRow, BinaryRow, JoinedRow> buildSortMergeJoin(StreamOperator operator) throws Exception {
	final TwoInputStreamTaskTestHarness<BinaryRow, BinaryRow, JoinedRow> testHarness =
			new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2,
				new int[]{1, 2}, typeInfo, (TypeInformation) typeInfo, joinedInfo);

	testHarness.memorySize = 36 * 1024 * 1024;
	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getStreamConfig().setStreamOperator(operator);
	testHarness.getStreamConfig().setOperatorID(new OperatorID());

	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processElement(new StreamRecord<>(newRow("a", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("d", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("a", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "1"), initialTime), 0, 1);
	testHarness.processElement(new StreamRecord<>(newRow("c", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "4"), initialTime), 1, 0);
	testHarness.waitForInputProcessing();

	testHarness.endInput();
	return testHarness;
}
Example #2
Source File: StreamGraph.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		TwoInputStreamOperator<IN1, IN2, OUT> taskOperatorObject,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	addNode(vertexID, slotSharingGroup, coLocationGroup, TwoInputStreamTask.class, taskOperatorObject, operatorName);

	TypeSerializer<OUT> outSerializer = (outTypeInfo != null) && !(outTypeInfo instanceof MissingTypeInfo) ?
			outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorObject instanceof OutputTypeConfigurable) {
		@SuppressWarnings("unchecked")
		OutputTypeConfigurable<OUT> outputTypeConfigurable = (OutputTypeConfigurable<OUT>) taskOperatorObject;
		// sets the output type which must be know at StreamGraph creation time
		outputTypeConfigurable.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
Example #3
Source File: String2HashJoinOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void init(boolean leftOut, boolean rightOut, boolean buildLeft) throws Exception {
	HashJoinType type = HashJoinType.of(buildLeft, leftOut, rightOut);
	HashJoinOperator operator = newOperator(33 * 32 * 1024, type, !buildLeft);
	testHarness = new TwoInputStreamTaskTestHarness<>(
			TwoInputStreamTask::new, 2, 2, new int[]{1, 2}, typeInfo, (TypeInformation) typeInfo, joinedInfo);
	testHarness.memorySize = 36 * 1024 * 1024;
	testHarness.getExecutionConfig().enableObjectReuse();
	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getStreamConfig().setStreamOperator(operator);
	testHarness.getStreamConfig().setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();
}
Example #4
Source File: StreamGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<OUT> taskOperatorFactory,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	Class<? extends AbstractInvokable> vertexClass = taskOperatorFactory.isOperatorSelectiveReading() ?
		TwoInputSelectableStreamTask.class : TwoInputStreamTask.class;

	addNode(vertexID, slotSharingGroup, coLocationGroup, vertexClass, taskOperatorFactory, operatorName);

	TypeSerializer<OUT> outSerializer = (outTypeInfo != null) && !(outTypeInfo instanceof MissingTypeInfo) ?
			outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorFactory.isOutputTypeConfigurable()) {
		// sets the output type which must be know at StreamGraph creation time
		taskOperatorFactory.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
Example #5
Source File: String2HashJoinOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void init(boolean leftOut, boolean rightOut, boolean buildLeft) throws Exception {
	HashJoinType type = HashJoinType.of(buildLeft, leftOut, rightOut);
	HashJoinOperator operator = newOperator(33 * 32 * 1024, type, !buildLeft);
	testHarness = new TwoInputStreamTaskTestHarness<>(
			TwoInputStreamTask::new, 2, 2, new int[]{1, 2}, typeInfo, (TypeInformation) typeInfo, joinedInfo);
	testHarness.memorySize = 36 * 1024 * 1024;
	testHarness.getExecutionConfig().enableObjectReuse();
	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getStreamConfig().setStreamOperator(operator);
	testHarness.getStreamConfig().setOperatorID(new OperatorID());
	testHarness.getStreamConfig().setManagedMemoryFraction(0.99);

	testHarness.invoke();
	testHarness.waitForTaskRunning();
}
Example #6
Source File: String2SortMergeJoinOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> buildSortMergeJoin(StreamOperator operator) throws Exception {
	final TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness =
			new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2,
				new int[]{1, 2}, typeInfo, (TypeInformation) typeInfo, joinedInfo);

	testHarness.memorySize = 36 * 1024 * 1024;
	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getStreamConfig().setStreamOperator(operator);
	testHarness.getStreamConfig().setOperatorID(new OperatorID());
	testHarness.getStreamConfig().setManagedMemoryFraction(0.99);

	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processElement(new StreamRecord<>(newRow("a", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("d", "0"), initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<>(newRow("a", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "1"), initialTime), 0, 1);
	testHarness.processElement(new StreamRecord<>(newRow("c", "2"), initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(newRow("b", "4"), initialTime), 1, 0);
	testHarness.waitForInputProcessing();

	testHarness.endInput();
	return testHarness;
}
Example #7
Source File: StreamGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<OUT> taskOperatorFactory,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	Class<? extends AbstractInvokable> vertexClass = TwoInputStreamTask.class;

	addNode(vertexID, slotSharingGroup, coLocationGroup, vertexClass, taskOperatorFactory, operatorName);

	TypeSerializer<OUT> outSerializer = createSerializer(outTypeInfo);

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorFactory.isOutputTypeConfigurable()) {
		// sets the output type which must be know at StreamGraph creation time
		taskOperatorFactory.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
Example #8
Source File: StreamTwoInputProcessor.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
Example #9
Source File: StreamTwoInputProcessor.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);

	this.finishedChannels1 = new BitSet();
	this.finishedChannels2 = new BitSet();
}