org.apache.flink.streaming.api.operators.Output Java Examples

The following examples show how to use org.apache.flink.streaming.api.operators.Output. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: StreamSourceOperatorLatencyMetricsTest.java From flink with Apache License 2.0

6 votes

private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		ProcessingTimeService timeProvider) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setProcessingTimeService(timeProvider)
			.build();

		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #2

Source File: DirectedOutput.java From Flink-CEPplus with Apache License 2.0

6 votes

protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}

Example #3

Source File: AsyncWaitOperator.java From flink with Apache License 2.0

6 votes

@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
	super.setup(containingTask, config, output);

	this.inStreamElementSerializer = new StreamElementSerializer<>(
		getOperatorConfig().<IN>getTypeSerializerIn1(getUserCodeClassloader()));

	switch (outputMode) {
		case ORDERED:
			queue = new OrderedStreamElementQueue<>(capacity);
			break;
		case UNORDERED:
			queue = new UnorderedStreamElementQueue<>(capacity);
			break;
		default:
			throw new IllegalStateException("Unknown async mode: " + outputMode + '.');
	}

	this.timestampedCollector = new TimestampedCollector<>(output);
}

Example #4

Source File: DoFnOperator.java From beam with Apache License 2.0

6 votes

BufferedOutputManager(
    Output<StreamRecord<WindowedValue<OutputT>>> output,
    TupleTag<OutputT> mainTag,
    Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags,
    Map<TupleTag<?>, Integer> tagsToIds,
    Lock bufferLock,
    PushedBackElementsHandler<KV<Integer, WindowedValue<?>>> pushedBackElementsHandler) {
  this.output = output;
  this.mainTag = mainTag;
  this.tagsToOutputTags = tagsToOutputTags;
  this.tagsToIds = tagsToIds;
  this.bufferLock = bufferLock;
  this.idsToTags = new HashMap<>();
  for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) {
    idsToTags.put(entry.getValue(), entry.getKey());
  }
  this.pushedBackElementsHandler = pushedBackElementsHandler;
}

Example #5

Source File: CopyingDirectedOutput.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	if (selectedOutputs.isEmpty()) {
		return;
	}

	Iterator<Output<StreamRecord<OUT>>> it = selectedOutputs.iterator();

	while (true) {
		Output<StreamRecord<OUT>> out = it.next();
		if (it.hasNext()) {
			// we don't have the last output
			// perform a shallow copy
			StreamRecord<OUT> shallowCopy = record.copy(record.getValue());
			out.collect(shallowCopy);
		} else {
			// this is the last output
			out.collect(record);
			break;
		}
	}
}

Example #6

Source File: DirectedOutput.java From flink with Apache License 2.0

6 votes

protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}

Example #7

Source File: DirectedOutput.java From flink with Apache License 2.0

6 votes

protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}

Example #8

Source File: CopyingDirectedOutput.java From flink with Apache License 2.0

6 votes

@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	if (selectedOutputs.isEmpty()) {
		return;
	}

	Iterator<Output<StreamRecord<OUT>>> it = selectedOutputs.iterator();

	while (true) {
		Output<StreamRecord<OUT>> out = it.next();
		if (it.hasNext()) {
			// we don't have the last output
			// perform a shallow copy
			StreamRecord<OUT> shallowCopy = record.copy(record.getValue());
			out.collect(shallowCopy);
		} else {
			// this is the last output
			out.collect(record);
			break;
		}
	}
}

Example #9

Source File: StreamSourceOperatorLatencyMetricsTest.java From Flink-CEPplus with Apache License 2.0

6 votes

private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		ProcessingTimeService timeProvider) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setProcessingTimeService(timeProvider)
			.build();

		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #10

Source File: StreamTaskTest.java From flink with Apache License 2.0

5 votes

@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<Long>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	while (!canceled) {
		try {
			Thread.sleep(500);
		} catch (InterruptedException ignored) {}
	}
}

Example #11

Source File: DirectedOutput.java From flink with Apache License 2.0

5 votes

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}

Example #12

Source File: CodeGenOperatorFactory.java From flink with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
@Override
public <T extends StreamOperator<OUT>> T createStreamOperator(StreamTask<?, ?> containingTask,
		StreamConfig config, Output<StreamRecord<OUT>> output) {
	return (T) generatedClass.newInstance(containingTask.getUserCodeClassLoader(),
			generatedClass.getReferences(), containingTask, config, output);
}

Example #13

Source File: StreamOutputHandler.java From bahir-flink with Apache License 2.0

5 votes

public StreamOutputHandler(TypeInformation<R> typeInfo, AbstractDefinition definition, Output<StreamRecord<R>> output) {
    this.typeInfo = typeInfo;
    this.definition = definition;
    this.output = output;
    this.objectMapper = new ObjectMapper();
    this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}

Example #14

Source File: OperatorChain.java From flink with Apache License 2.0

5 votes

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	if (streamStatusProvider.getStreamStatus().isActive()) {
		for (Output<StreamRecord<T>> output : outputs) {
			output.emitWatermark(mark);
		}
	}
}

Example #15

Source File: DirectedOutput.java From flink with Apache License 2.0

5 votes

@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	for (Output<StreamRecord<OUT>> out : selectedOutputs) {
		out.collect(record);
	}
}

Example #16

Source File: SourceOperatorStreamTask.java From flink with Apache License 2.0

5 votes

AsyncDataOutputToOutput(
		Output<StreamRecord<T>> output,
		StreamStatusMaintainer streamStatusMaintainer) {
	super(streamStatusMaintainer);

	this.output = checkNotNull(output);
}

Example #17

Source File: StreamSourceOperatorLatencyMetricsTest.java From flink with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		TimerService timerService) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setTimerService(timerService)
			.build();

		operator.setProcessingTimeService(mockTask.getProcessingTimeServiceFactory().createProcessingTimeService(null));
		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #18

Source File: DirectedOutput.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}

Example #19

Source File: StreamSourceOperatorWatermarksTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}

Example #20

Source File: StreamSourceOperatorWatermarksTest.java From flink with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(
		StreamSource<T, ?> operator,
		TimeCharacteristic timeChar,
		long watermarkInterval,
		final TimerService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setTimerService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	return mockTask;
}

Example #21

Source File: StreamOutputHandler.java From flink-siddhi with Apache License 2.0

5 votes

public StreamOutputHandler(String outputStreamId, TypeInformation<R> typeInfo, AbstractDefinition definition, Output<StreamRecord<R>> output) {
    this.outputStreamId = outputStreamId;
    this.typeInfo = typeInfo;
    this.definition = definition;
    this.output = output;
    this.objectMapper = new ObjectMapper();
    this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}

Example #22

Source File: DirectedOutput.java From flink with Apache License 2.0

5 votes

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}

Example #23

Source File: DirectedOutput.java From flink with Apache License 2.0

5 votes

@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	for (Output<StreamRecord<OUT>> out : selectedOutputs) {
		out.collect(record);
	}
}

Example #24

Source File: Emitter.java From flink with Apache License 2.0

5 votes

public Emitter(
		final Object checkpointLock,
		final Output<StreamRecord<OUT>> output,
		final StreamElementQueue streamElementQueue,
		final OperatorActions operatorActions) {

	this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "checkpointLock");
	this.output = Preconditions.checkNotNull(output, "output");
	this.streamElementQueue = Preconditions.checkNotNull(streamElementQueue, "streamElementQueue");
	this.operatorActions = Preconditions.checkNotNull(operatorActions, "operatorActions");

	this.timestampedCollector = new TimestampedCollector<>(this.output);
	this.running = true;
}

Example #25

Source File: OperatorChain.java From flink with Apache License 2.0

5 votes

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	if (streamStatusProvider.getStreamStatus().isActive()) {
		for (Output<StreamRecord<T>> output : outputs) {
			output.emitWatermark(mark);
		}
	}
}

Example #26

Source File: IngressRouterOperator.java From flink-statefun with Apache License 2.0

5 votes

DownstreamCollector(
    MessageFactoryType messageFactoryType, Output<StreamRecord<Message>> output) {
  this.factory = MessageFactory.forType(messageFactoryType);
  this.output = Objects.requireNonNull(output);
  this.multiLanguagePayloads =
      messageFactoryType == MessageFactoryType.WITH_PROTOBUF_PAYLOADS_MULTILANG;
}

Example #27

Source File: OperatorChain.java From flink with Apache License 2.0

5 votes

@Override
public void collect(StreamRecord<T> record) {

	for (int i = 0; i < outputs.length - 1; i++) {
		Output<StreamRecord<T>> output = outputs[i];
		StreamRecord<T> shallowCopy = record.copy(record.getValue());
		output.collect(shallowCopy);
	}

	if (outputs.length > 0) {
		// don't copy for the last output
		outputs[outputs.length - 1].collect(record);
	}
}

Example #28

Source File: OperatorChain.java From flink with Apache License 2.0

5 votes

@Override
public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
	for (int i = 0; i < outputs.length - 1; i++) {
		Output<StreamRecord<T>> output = outputs[i];

		StreamRecord<X> shallowCopy = record.copy(record.getValue());
		output.collect(outputTag, shallowCopy);
	}

	if (outputs.length > 0) {
		// don't copy for the last output
		outputs[outputs.length - 1].collect(outputTag, record);
	}
}

Example #29

Source File: IngressRouterOperator.java From stateful-functions with Apache License 2.0

5 votes

DownstreamCollector(
    MessageFactoryType messageFactoryType, Output<StreamRecord<Message>> output) {
  this.factory = MessageFactory.forType(messageFactoryType);
  this.output = Objects.requireNonNull(output);
  this.multiLanguagePayloads =
      messageFactoryType == MessageFactoryType.WITH_PROTOBUF_PAYLOADS_MULTILANG;
}

Example #30

Source File: StreamTaskTest.java From flink with Apache License 2.0

5 votes

@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<Long>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	while (!canceled) {
		try {
			Thread.sleep(500);
		} catch (InterruptedException ignored) {}
	}
}