org.apache.flink.streaming.api.collector.selector.OutputSelector Java Examples
The following examples show how to use
org.apache.flink.streaming.api.collector.selector.OutputSelector.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StreamGraph.java From flink with Apache License 2.0 | 6 votes |
protected StreamNode addNode(Integer vertexID, @Nullable String slotSharingGroup, @Nullable String coLocationGroup, Class<? extends AbstractInvokable> vertexClass, StreamOperatorFactory<?> operatorFactory, String operatorName) { if (streamNodes.containsKey(vertexID)) { throw new RuntimeException("Duplicate vertexID " + vertexID); } StreamNode vertex = new StreamNode( vertexID, slotSharingGroup, coLocationGroup, operatorFactory, operatorName, new ArrayList<OutputSelector<?>>(), vertexClass); streamNodes.put(vertexID, vertex); return vertex; }
Example #2
Source File: StreamTaskTestHarness.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Users of the test harness can call this utility method to setup the stream config * if there will only be a single operator to be tested. The method will setup the * outgoing network connection for the operator. * * <p>For more advanced test cases such as testing chains of multiple operators with the harness, * please manually configure the stream config. */ public void setupOutputForSingletonOperatorChain() { Preconditions.checkState(!setupCalled, "This harness was already setup."); setupCalled = true; streamConfig.setChainStart(); streamConfig.setBufferTimeout(0); streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime); streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList()); streamConfig.setNumberOfOutputs(1); streamConfig.setTypeSerializerOut(outputSerializer); streamConfig.setVertexID(0); streamConfig.setOperatorID(new OperatorID(4711L, 123L)); StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() { private static final long serialVersionUID = 1L; }; List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>(); StreamNode sourceVertexDummy = new StreamNode(null, 0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class); StreamNode targetVertexDummy = new StreamNode(null, 1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class); outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */)); streamConfig.setOutEdgesInOrder(outEdgesInOrder); streamConfig.setNonChainedOutputs(outEdgesInOrder); }
Example #3
Source File: StreamGraph.java From flink with Apache License 2.0 | 6 votes |
protected StreamNode addNode( Integer vertexID, @Nullable String slotSharingGroup, @Nullable String coLocationGroup, Class<? extends AbstractInvokable> vertexClass, StreamOperatorFactory<?> operatorFactory, String operatorName) { if (streamNodes.containsKey(vertexID)) { throw new RuntimeException("Duplicate vertexID " + vertexID); } StreamNode vertex = new StreamNode( vertexID, slotSharingGroup, coLocationGroup, operatorFactory, operatorName, new ArrayList<OutputSelector<?>>(), vertexClass); streamNodes.put(vertexID, vertex); return vertex; }
Example #4
Source File: StreamTaskTestHarness.java From flink with Apache License 2.0 | 6 votes |
/** * Users of the test harness can call this utility method to setup the stream config * if there will only be a single operator to be tested. The method will setup the * outgoing network connection for the operator. * * <p>For more advanced test cases such as testing chains of multiple operators with the harness, * please manually configure the stream config. */ public void setupOutputForSingletonOperatorChain() { Preconditions.checkState(!setupCalled, "This harness was already setup."); setupCalled = true; streamConfig.setChainStart(); streamConfig.setBufferTimeout(0); streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime); streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList()); streamConfig.setNumberOfOutputs(1); streamConfig.setTypeSerializerOut(outputSerializer); streamConfig.setVertexID(0); streamConfig.setOperatorID(new OperatorID(4711L, 123L)); StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() { private static final long serialVersionUID = 1L; }; List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>(); StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class); StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class); outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */)); streamConfig.setOutEdgesInOrder(outEdgesInOrder); streamConfig.setNonChainedOutputs(outEdgesInOrder); }
Example #5
Source File: JavaDataStreamTransformationApp.java From 163-bigdate-note with GNU General Public License v3.0 | 6 votes |
private static void splitFunction(StreamExecutionEnvironment environment) { DataStreamSource<Long> dataStreamSource = environment.addSource(new JavaCustomNonParallelSourceFunction()); SplitStream<Long> splitStream = dataStreamSource.split(new OutputSelector<Long>() { @Override public Iterable<String> select(Long value) { List<String> list = new ArrayList<>(); if (value % 2 == 0) { list.add("event"); } else { list.add("odd"); } return list; } }); splitStream.select("odd").print().setParallelism(1); }
Example #6
Source File: StreamNode.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public StreamNode(StreamExecutionEnvironment env, Integer id, String slotSharingGroup, @Nullable String coLocationGroup, StreamOperator<?> operator, String operatorName, List<OutputSelector<?>> outputSelector, Class<? extends AbstractInvokable> jobVertexClass) { this.env = env; this.id = id; this.operatorName = operatorName; this.operator = operator; this.outputSelectors = outputSelector; this.jobVertexClass = jobVertexClass; this.slotSharingGroup = slotSharingGroup; this.coLocationGroup = coLocationGroup; }
Example #7
Source File: StreamNode.java From flink with Apache License 2.0 | 6 votes |
public StreamNode( Integer id, @Nullable String slotSharingGroup, @Nullable String coLocationGroup, StreamOperatorFactory<?> operatorFactory, String operatorName, List<OutputSelector<?>> outputSelector, Class<? extends AbstractInvokable> jobVertexClass) { this.id = id; this.operatorName = operatorName; this.operatorFactory = operatorFactory; this.outputSelectors = outputSelector; this.jobVertexClass = jobVertexClass; this.slotSharingGroup = slotSharingGroup; this.coLocationGroup = coLocationGroup; }
Example #8
Source File: StreamNode.java From flink with Apache License 2.0 | 6 votes |
public StreamNode( Integer id, @Nullable String slotSharingGroup, @Nullable String coLocationGroup, StreamOperatorFactory<?> operatorFactory, String operatorName, List<OutputSelector<?>> outputSelector, Class<? extends AbstractInvokable> jobVertexClass) { this.id = id; this.operatorName = operatorName; this.operatorFactory = operatorFactory; this.outputSelectors = outputSelector; this.jobVertexClass = jobVertexClass; this.slotSharingGroup = slotSharingGroup; this.coLocationGroup = coLocationGroup; }
Example #9
Source File: StreamGraph.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
protected StreamNode addNode(Integer vertexID, String slotSharingGroup, @Nullable String coLocationGroup, Class<? extends AbstractInvokable> vertexClass, StreamOperator<?> operatorObject, String operatorName) { if (streamNodes.containsKey(vertexID)) { throw new RuntimeException("Duplicate vertexID " + vertexID); } StreamNode vertex = new StreamNode(environment, vertexID, slotSharingGroup, coLocationGroup, operatorObject, operatorName, new ArrayList<OutputSelector<?>>(), vertexClass); streamNodes.put(vertexID, vertex); return vertex; }
Example #10
Source File: StreamConfig.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader userCodeClassloader) { try { List<OutputSelector<T>> selectors = InstantiationUtil.readObjectFromConfig(this.config, OUTPUT_SELECTOR_WRAPPER, userCodeClassloader); return selectors == null ? Collections.<OutputSelector<T>>emptyList() : selectors; } catch (Exception e) { throw new StreamTaskException("Could not read output selectors", e); } }
Example #11
Source File: StreamConfig.java From flink with Apache License 2.0 | 5 votes |
public void setOutputSelectors(List<OutputSelector<?>> outputSelectors) { try { InstantiationUtil.writeObjectToConfig(outputSelectors, this.config, OUTPUT_SELECTOR_WRAPPER); } catch (IOException e) { throw new StreamTaskException("Could not serialize output selectors", e); } }
Example #12
Source File: StreamNode.java From flink with Apache License 2.0 | 5 votes |
@VisibleForTesting public StreamNode( Integer id, @Nullable String slotSharingGroup, @Nullable String coLocationGroup, StreamOperator<?> operator, String operatorName, List<OutputSelector<?>> outputSelector, Class<? extends AbstractInvokable> jobVertexClass) { this(id, slotSharingGroup, coLocationGroup, SimpleOperatorFactory.of(operator), operatorName, outputSelector, jobVertexClass); }
Example #13
Source File: SplitTransformation.java From flink with Apache License 2.0 | 5 votes |
/** * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}. * * @param input The input {@code Transformation} * @param outputSelector The output selector */ public SplitTransformation( Transformation<T> input, OutputSelector<T> outputSelector) { super("Split", input.getOutputType(), input.getParallelism()); this.input = input; this.outputSelector = outputSelector; }
Example #14
Source File: OutputSelectorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testGetOutputs() { OutputSelector<Tuple1<Integer>> selector = new MyOutputSelector(); List<String> expectedOutputs = new ArrayList<String>(); expectedOutputs.add("0"); expectedOutputs.add("1"); assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(2))); expectedOutputs.add("2"); assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(3))); }
Example #15
Source File: SplitTransformation.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}. * * @param input The input {@code StreamTransformation} * @param outputSelector The output selector */ public SplitTransformation(StreamTransformation<T> input, OutputSelector<T> outputSelector) { super("Split", input.getOutputType(), input.getParallelism()); this.input = input; this.outputSelector = outputSelector; }
Example #16
Source File: StreamConfig.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public void setOutputSelectors(List<OutputSelector<?>> outputSelectors) { try { InstantiationUtil.writeObjectToConfig(outputSelectors, this.config, OUTPUT_SELECTOR_WRAPPER); } catch (IOException e) { throw new StreamTaskException("Could not serialize output selectors", e); } }
Example #17
Source File: SingleOutputStreamOperator.java From flink with Apache License 2.0 | 5 votes |
@Override public SplitStream<T> split(OutputSelector<T> outputSelector) { if (requestedSideOutputs.isEmpty()) { wasSplitApplied = true; return super.split(outputSelector); } else { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } }
Example #18
Source File: StreamGraph.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public <T> void addOutputSelector(Integer vertexID, OutputSelector<T> outputSelector) { if (virtualPartitionNodes.containsKey(vertexID)) { addOutputSelector(virtualPartitionNodes.get(vertexID).f0, outputSelector); } else if (virtualSelectNodes.containsKey(vertexID)) { addOutputSelector(virtualSelectNodes.get(vertexID).f0, outputSelector); } else { getStreamNode(vertexID).addOutputSelector(outputSelector); if (LOG.isDebugEnabled()) { LOG.debug("Outputselector set for {}", vertexID); } } }
Example #19
Source File: StreamConfig.java From flink with Apache License 2.0 | 5 votes |
public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader userCodeClassloader) { try { List<OutputSelector<T>> selectors = InstantiationUtil.readObjectFromConfig(this.config, OUTPUT_SELECTOR_WRAPPER, userCodeClassloader); return selectors == null ? Collections.<OutputSelector<T>>emptyList() : selectors; } catch (Exception e) { throw new StreamTaskException("Could not read output selectors", e); } }
Example #20
Source File: StreamGraph.java From flink with Apache License 2.0 | 5 votes |
public <T> void addOutputSelector(Integer vertexID, OutputSelector<T> outputSelector) { if (virtualPartitionNodes.containsKey(vertexID)) { addOutputSelector(virtualPartitionNodes.get(vertexID).f0, outputSelector); } else if (virtualSelectNodes.containsKey(vertexID)) { addOutputSelector(virtualSelectNodes.get(vertexID).f0, outputSelector); } else { getStreamNode(vertexID).addOutputSelector(outputSelector); if (LOG.isDebugEnabled()) { LOG.debug("Outputselector set for {}", vertexID); } } }
Example #21
Source File: OutputSelectorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetOutputs() { OutputSelector<Tuple1<Integer>> selector = new MyOutputSelector(); List<String> expectedOutputs = new ArrayList<String>(); expectedOutputs.add("0"); expectedOutputs.add("1"); assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(2))); expectedOutputs.add("2"); assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(3))); }
Example #22
Source File: SplitEvent.java From flink-learning with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setGlobalJobParameters(params); DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env); //从 Kafka 获取到所有的数据流 SplitStream<MetricEvent> splitData = data.split(new OutputSelector<MetricEvent>() { @Override public Iterable<String> select(MetricEvent metricEvent) { List<String> tags = new ArrayList<>(); String type = metricEvent.getTags().get("type"); switch (type) { case "machine": tags.add("machine"); break; case "docker": tags.add("docker"); break; case "application": tags.add("application"); break; case "middleware": tags.add("middleware"); break; default: break; } return tags; } }); DataStream<MetricEvent> machine = splitData.select("machine"); DataStream<MetricEvent> docker = splitData.select("docker"); DataStream<MetricEvent> application = splitData.select("application"); DataStream<MetricEvent> middleware = splitData.select("middleware"); }
Example #23
Source File: SingleOutputStreamOperator.java From flink with Apache License 2.0 | 5 votes |
@Override public SplitStream<T> split(OutputSelector<T> outputSelector) { if (requestedSideOutputs.isEmpty()) { wasSplitApplied = true; return super.split(outputSelector); } else { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } }
Example #24
Source File: SplitTransformation.java From flink with Apache License 2.0 | 5 votes |
/** * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}. * * @param input The input {@code Transformation} * @param outputSelector The output selector */ public SplitTransformation( Transformation<T> input, OutputSelector<T> outputSelector) { super("Split", input.getOutputType(), input.getParallelism()); this.input = input; this.outputSelector = outputSelector; }
Example #25
Source File: StreamNode.java From flink with Apache License 2.0 | 5 votes |
@VisibleForTesting public StreamNode( Integer id, @Nullable String slotSharingGroup, @Nullable String coLocationGroup, StreamOperator<?> operator, String operatorName, List<OutputSelector<?>> outputSelector, Class<? extends AbstractInvokable> jobVertexClass) { this(id, slotSharingGroup, coLocationGroup, SimpleOperatorFactory.of(operator), operatorName, outputSelector, jobVertexClass); }
Example #26
Source File: StreamConfig.java From flink with Apache License 2.0 | 5 votes |
public void setOutputSelectors(List<OutputSelector<?>> outputSelectors) { try { InstantiationUtil.writeObjectToConfig(outputSelectors, this.config, OUTPUT_SELECTOR_WRAPPER); } catch (IOException e) { throw new StreamTaskException("Could not serialize output selectors", e); } }
Example #27
Source File: StreamConfig.java From flink with Apache License 2.0 | 5 votes |
public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader userCodeClassloader) { try { List<OutputSelector<T>> selectors = InstantiationUtil.readObjectFromConfig(this.config, OUTPUT_SELECTOR_WRAPPER, userCodeClassloader); return selectors == null ? Collections.<OutputSelector<T>>emptyList() : selectors; } catch (Exception e) { throw new StreamTaskException("Could not read output selectors", e); } }
Example #28
Source File: StreamGraph.java From flink with Apache License 2.0 | 5 votes |
public <T> void addOutputSelector(Integer vertexID, OutputSelector<T> outputSelector) { if (virtualPartitionNodes.containsKey(vertexID)) { addOutputSelector(virtualPartitionNodes.get(vertexID).f0, outputSelector); } else if (virtualSelectNodes.containsKey(vertexID)) { addOutputSelector(virtualSelectNodes.get(vertexID).f0, outputSelector); } else { getStreamNode(vertexID).addOutputSelector(outputSelector); if (LOG.isDebugEnabled()) { LOG.debug("Outputselector set for {}", vertexID); } } }
Example #29
Source File: OutputSelectorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetOutputs() { OutputSelector<Tuple1<Integer>> selector = new MyOutputSelector(); List<String> expectedOutputs = new ArrayList<String>(); expectedOutputs.add("0"); expectedOutputs.add("1"); assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(2))); expectedOutputs.add("2"); assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(3))); }
Example #30
Source File: StreamTaskMailboxTestHarnessBuilder.java From flink with Apache License 2.0 | 5 votes |
/** * Users of the test harness can call this utility method to setup the stream config * if there will only be a single operator to be tested. The method will setup the * outgoing network connection for the operator. * * <p>For more advanced test cases such as testing chains of multiple operators with the harness, * please manually configure the stream config. */ public StreamTaskMailboxTestHarnessBuilder<OUT> setupOutputForSingletonOperatorChain( StreamOperatorFactory<?> factory, OperatorID operatorID) { checkState(!setupCalled, "This harness was already setup."); setupCalled = true; streamConfig.setChainStart(); streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime); streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList()); streamConfig.setNumberOfOutputs(1); streamConfig.setTypeSerializerOut(outputSerializer); streamConfig.setVertexID(0); StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() { private static final long serialVersionUID = 1L; }; List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>(); StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class); StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class); outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */)); streamConfig.setOutEdgesInOrder(outEdgesInOrder); streamConfig.setNonChainedOutputs(outEdgesInOrder); streamConfig.setStreamOperatorFactory(factory); streamConfig.setOperatorID(operatorID); return this; }