Java Code Examples for org.apache.flink.util.Collector

The following examples show how to use org.apache.flink.util.Collector. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: Summarization.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void reduce(Iterable<Vertex<K, VV>> values, Collector<VertexGroupItem<K, VV>> out) throws Exception {
	K vertexGroupRepresentativeID = null;
	long vertexGroupCount = 0L;
	VV vertexGroupValue = null;
	boolean isFirstElement = true;

	for (Vertex<K, VV> vertex : values) {
		if (isFirstElement) {
			// take final group representative vertex id from first tuple
			vertexGroupRepresentativeID = vertex.getId();
			vertexGroupValue = vertex.getValue();
			isFirstElement = false;
		}
		// no need to set group value for those tuples
		reuseVertexGroupItem.setVertexId(vertex.getId());
		reuseVertexGroupItem.setGroupRepresentativeId(vertexGroupRepresentativeID);
		out.collect(reuseVertexGroupItem);
		vertexGroupCount++;
	}

	createGroupRepresentativeTuple(vertexGroupRepresentativeID, vertexGroupValue, vertexGroupCount);
	out.collect(reuseVertexGroupItem);
	reuseVertexGroupItem.reset();
}
 
Example 2
@Override
public void process(K k, final Context context, Iterable<T> input, Collector<R> out) throws Exception {

	T curr = null;
	for (T val: input) {
		if (curr == null) {
			curr = val;
		} else {
			curr = reduceFunction.reduce(curr, val);
		}
	}

	this.ctx.window = context.window();
	this.ctx.context = context;
	windowFunction.process(k, ctx, Collections.singletonList(curr), out);
}
 
Example 3
@Override
public void coGroup(Iterable<Tuple2<Long, Long>> candidates, Iterable<Tuple2<Long, Long>> current, Collector<Tuple2<Long, Long>> out) {
	Iterator<Tuple2<Long, Long>> iterator = current.iterator();
	if (!iterator.hasNext()) {
		throw new RuntimeException("Error: Id not encountered before.");
	}

	Tuple2<Long, Long> old = iterator.next();

	long minimumComponentID = Long.MAX_VALUE;

	for (Tuple2<Long, Long> candidate : candidates) {
		long candidateComponentID = candidate.f1;
		if (candidateComponentID < minimumComponentID) {
			minimumComponentID = candidateComponentID;
		}
	}

	if (minimumComponentID < old.f1) {
		old.f1 = minimumComponentID;
		out.collect(old);
	}
}
 
Example 4
Source Project: flink-learning   Source File: Main.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataSource<String> dataSource = env.fromElements(WORDS);

    dataSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
        @Override
        public void flatMap(String line, Collector<Tuple2<String, Integer>> out) throws Exception {
            String[] words = line.split("\\W+");
            for (String word : words) {
                out.collect(new Tuple2<>(word, 1));
            }
        }
    })
            .groupBy(0)
            .sum(1)
            .print();

    long count = dataSource.count();
    System.out.println(count);
}
 
Example 5
Source Project: flink-examples   Source File: TopTweet.java    License: MIT License 6 votes vote down vote up
@Override
public void flatMap(String tweetJsonStr, Collector<Tuple2<String, Integer>> collector) throws Exception {
    JsonNode tweetJson = mapper.readTree(tweetJsonStr);
    JsonNode entities = tweetJson.get("entities");
    if (entities == null) return;

    JsonNode hashtags = entities.get("hashtags");
    if (hashtags == null) return;

    for (Iterator<JsonNode> iter = hashtags.getElements(); iter.hasNext();) {
        JsonNode node = iter.next();
        String hashtag = node.get("text").getTextValue();

        if (hashtag.matches("\\w+")) {
            collector.collect(new Tuple2<>(hashtag, 1));
        }
    }
}
 
Example 6
Source Project: flink-crawler   Source File: UrlDBFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void processElement2(DomainScore domainScore, Context context, Collector<FetchUrl> out)
        throws Exception {
    
    // Ensure we don't wind up with DBZ problems.
    float score = Math.max(0.01f, domainScore.getScore());
    String pld = domainScore.getPld();
    LOGGER.debug("UrlDBFunction ({}/{}) setting '{}' average score to {}",
            _partition, _parallelism, pld, score);
    
    // At this point we might be seeing this PLD for the first time, or we might have seen
    // it before in this method, or we might have seen it via the onTimer call. So it may 
    // or may not have any state set up, and it may or may not be in _domainScores (non-state)
    float summedScores = _averageDomainScore * _scoredDomains.size();
    if (_scoredDomains.contains(pld)) {
        summedScores -= _domainScore.value();
    }
    
    _domainScore.update(score);
    _scoredDomains.add(pld);
    summedScores += score;
    _averageDomainScore = summedScores / _scoredDomains.size();
}
 
Example 7
Source Project: flink   Source File: CoBroadcastWithKeyedOperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void processElement(String value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
	Iterable<Map.Entry<String, Integer>> broadcastStateIt = ctx.getBroadcastState(STATE_DESCRIPTOR).immutableEntries();
	Iterator<Map.Entry<String, Integer>> iter = broadcastStateIt.iterator();

	for (int i = 0; i < expectedBroadcastState.size(); i++) {
		assertTrue(iter.hasNext());

		Map.Entry<String, Integer> entry = iter.next();
		assertTrue(expectedBroadcastState.containsKey(entry.getKey()));
		assertEquals(expectedBroadcastState.get(entry.getKey()), entry.getValue());
	}

	assertFalse(iter.hasNext());

	ctx.timerService().registerEventTimeTimer(timerTs);
}
 
Example 8
Source Project: flink   Source File: CepOperatorBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public static CepOperatorBuilder<Map<String, List<Event>>> createOperatorForNFA(NFA<Event> nfa) {
	return new CepOperatorBuilder<>(
		true,
		new NFACompiler.NFAFactory<Event>() {
			@Override
			public NFA<Event> createNFA() {
				return nfa;
			}
		},
		null,
		null,
		new PatternProcessFunction<Event, Map<String, List<Event>>>() {
			private static final long serialVersionUID = -7143807777582726991L;

			@Override
			public void processMatch(
				Map<String, List<Event>> match,
				Context ctx,
				Collector<Map<String, List<Event>>> out) throws Exception {
				out.collect(match);
			}
		},
		null);
}
 
Example 9
Source Project: flink   Source File: AbstractMergeIterator.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Crosses a single value from the second side with N values, all sharing a common key.
 * Effectively realizes a <i>N:1</i> join.
 *
 * @param val1      The value form the <i>1</i> side.
 * @param firstValN The first of the values from the <i>N</i> side.
 * @param valsN     Iterator over remaining <i>N</i> side values.
 * @throws Exception Forwards all exceptions thrown by the stub.
 */
private void crossSecond1withNValues(T2 val1, T1 firstValN,
									Iterator<T1> valsN, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception {
	T2 copy2 = createCopy(serializer2, val1, this.copy2);
	joinFunction.join(firstValN, copy2, collector);

	// set copy and join first element
	boolean more = true;
	do {
		final T1 nRec = valsN.next();

		if (valsN.hasNext()) {
			copy2 = createCopy(serializer2, val1, this.copy2);
			joinFunction.join(nRec, copy2, collector);
		} else {
			joinFunction.join(nRec, val1, collector);
			more = false;
		}
	}
	while (more);
}
 
Example 10
@Override
public void processElement(
		BaseRow input,
		KeyedProcessFunction<K, BaseRow, BaseRow>.Context ctx,
		Collector<BaseRow> out) throws Exception {
	long currentTime = ctx.timerService().currentProcessingTime();
	// register state-cleanup timer
	registerProcessingCleanupTimer(ctx, currentTime);

	// buffer the event incoming event

	// add current element to the window list of elements with corresponding timestamp
	List<BaseRow> rowList = inputState.get(currentTime);
	// null value means that this si the first event received for this timestamp
	if (rowList == null) {
		rowList = new ArrayList<BaseRow>();
		// register timer to process event once the current millisecond passed
		ctx.timerService().registerProcessingTimeTimer(currentTime + 1);
	}
	rowList.add(input);
	inputState.put(currentTime, rowList);
}
 
Example 11
Source Project: Alink   Source File: BaseComQueue.java    License: Apache License 2.0 6 votes vote down vote up
private DataSet<byte[]> loopStartDataSet(ExecutionEnvironment env) {
	MapPartitionOperator<Integer, byte[]> initial = env
		.fromElements(1)
		.rebalance()
		.mapPartition(new MapPartitionFunction<Integer, byte[]>() {
			@Override
			public void mapPartition(Iterable<Integer> values, Collector<byte[]> out) {
				//pass
			}
		}).name("iterInitialize");

	if (cacheDataRel != null) {
		initial = initial.withBroadcastSet(cacheDataRel, "rel");
	}

	return initial;
}
 
Example 12
Source Project: flink   Source File: ExactlyOnceValidatingConsumerThread.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void flatMap(String value, Collector<String> out) throws Exception {
	LOG.info("Consumed {}", value);

	int id = Integer.parseInt(value.split("-")[0]);
	if (validator.get(id)) {
		throw new RuntimeException("Saw id " + id + " twice!");
	}
	validator.set(id);
	if (id > totalEventCount - 1) {
		throw new RuntimeException("Out of bounds ID observed");
	}

	if (validator.nextClearBit(0) == totalEventCount) {
		throw new SuccessException();
	}
}
 
Example 13
Source Project: OSTMap   Source File: PathCoordGroupReduce.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void reduce(Iterable<Tuple2<String, String>> values, Collector<Tuple2<String, /*TODO POJO*/String>> out) throws Exception {
    coords = "";
    for (Tuple2<String,String> entry: values) {
        if(coordSet.size() == 0){
            user = entry.f0;
            coords = entry.f1.toString();
        }else{
            coords += "|" + entry.f1.toString();
        }
        coordSet.add(entry.f1.toString());
    }
    if(coordSet.size() > 1){
        out.collect(new Tuple2<>(user,coords));
        coordSet.clear();
    }

}
 
Example 14
Source Project: Alink   Source File: AlsPredictBatchOp.java    License: Apache License 2.0 6 votes vote down vote up
private static DataSet<Tuple2<Long, float[]>> getFactors(BatchOperator<?> model, final int identity) {
    return model.getDataSet()
        .flatMap(new FlatMapFunction<Row, Tuple2<Long, float[]>>() {
            @Override
            public void flatMap(Row value, Collector<Tuple2<Long, float[]>> out) throws Exception {
                int w = AlsModelDataConverter.getIsUser(value) ? 0 : 1;
                if (w != identity) {
                    return;
                }

                long idx = AlsModelDataConverter.getVertexId(value);
                float[] factors = AlsModelDataConverter.getFactors(value);
                out.collect(Tuple2.of(idx, factors));
            }
        });
}
 
Example 15
Source Project: Flink-CEPplus   Source File: ScatterGatherIteration.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void coGroup(Iterable<Edge<K, EV>> edges, Iterable<Vertex<K, Tuple3<VV, LongValue, LongValue>>> state,
					Collector<Tuple2<K, Message>> out) throws Exception {

	final Iterator<Vertex<K, Tuple3<VV, LongValue, LongValue>>> stateIter = state.iterator();

	if (stateIter.hasNext()) {
		Vertex<K, Tuple3<VV, LongValue, LongValue>> vertexWithDegrees = stateIter.next();

		nextVertex.f0 = vertexWithDegrees.f0;
		nextVertex.f1 = vertexWithDegrees.f1.f0;

		scatterFunction.setInDegree(vertexWithDegrees.f1.f1.getValue());
		scatterFunction.setOutDegree(vertexWithDegrees.f1.f2.getValue());

		scatterFunction.set(edges.iterator(), out, vertexWithDegrees.getId());
		scatterFunction.sendMessages(nextVertex);
	}
}
 
Example 16
Source Project: flink   Source File: LocalClusteringCoefficient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void flatMap(TriangleListing.Result<T> value, Collector<Tuple2<T, LongValue>> out)
		throws Exception {
	byte bitmask = value.getBitmask().getValue();

	output.f0 = value.getVertexId0();
	output.f1 = ((bitmask & 0b000011) == 0b000011) ? two : one;
	out.collect(output);

	output.f0 = value.getVertexId1();
	output.f1 = ((bitmask & 0b001100) == 0b001100) ? two : one;
	out.collect(output);

	output.f0 = value.getVertexId2();
	output.f1 = ((bitmask & 0b110000) == 0b110000) ? two : one;
	out.collect(output);
}
 
Example 17
Source Project: flink   Source File: CoBroadcastWithKeyedOperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void processBroadcastElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	// put an element in the broadcast state
	ctx.applyToKeyedState(
			listStateDesc,
			new KeyedStateFunction<String, ListState<String>>() {
				@Override
				public void process(String key, ListState<String> state) throws Exception {
					final Iterator<String> it = state.get().iterator();

					final List<String> list = new ArrayList<>();
					while (it.hasNext()) {
						list.add(it.next());
					}
					assertEquals(expectedKeyedStates.get(key), list);
				}
			});
}
 
Example 18
Source Project: flink   Source File: GroupReduceITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGroupReduceWithAtomicValue() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> ds = env.fromElements(1, 1, 2, 3, 4);
	DataSet<Integer> reduceDs = ds.groupBy("*").reduceGroup(new GroupReduceFunction<Integer, Integer>() {
		@Override
		public void reduce(Iterable<Integer> values, Collector<Integer> out) throws Exception {
			out.collect(values.iterator().next());
		}
	});

	List<Integer> result = reduceDs.collect();

	String expected = "1\n" +
			"2\n" +
			"3\n" +
			"4";

	compareResultAsText(result, expected);
}
 
Example 19
Source Project: Alink   Source File: ParseRowModel.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void mapPartition(Iterable<Row> iterable,
                         Collector<Tuple2<DenseVector, double[]>> collector) throws Exception {
    DenseVector coefVector = null;
    double[] lossCurve = null;
    int taskId = getRuntimeContext().getIndexOfThisSubtask();
    if (taskId == 0) {
        for (Row row : iterable) {
            Params params = Params.fromJson((String)row.getField(0));
            coefVector = params.get(ModelParamName.COEF);
            lossCurve = params.get(ModelParamName.LOSS_CURVE);
        }

        if (coefVector != null) {
            collector.collect(Tuple2.of(coefVector, lossCurve));
        }
    }
}
 
Example 20
Source Project: Flink-CEPplus   Source File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Sets the last output {@link Collector} of the collector chain of this {@link BatchTask}.
 * <p>
 * In case of chained tasks, the output collector of the last {@link ChainedDriver} is set. Otherwise it is the
 * single collector of the {@link BatchTask}.
 *
 * @param newOutputCollector new output collector to set as last collector
 */
protected void setLastOutputCollector(Collector<OT> newOutputCollector) {
	int numChained = this.chainedTasks.size();

	if (numChained == 0) {
		output = newOutputCollector;
		return;
	}

	chainedTasks.get(numChained - 1).setOutputCollector(newOutputCollector);
}
 
Example 21
Source Project: flink   Source File: WordCount.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
	// normalize and split the line
	String[] tokens = value.toLowerCase().split("\\W+");

	// emit the pairs
	for (String token : tokens) {
		if (token.length() > 0) {
			out.collect(new Tuple2<>(token, 1));
		}
	}
}
 
Example 22
Source Project: Flink-CEPplus   Source File: UdfAnalyzerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Boolean> out) throws Exception {
	Iterator<Tuple2<Long, Long>> it = values.iterator();
	boolean f = it.hasNext();
	if (!f) {
		System.out.println();
	}
	if (f) {
		System.out.println();
	}
	out.collect(f);
}
 
Example 23
Source Project: Flink-CEPplus   Source File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testProcessEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example 24
Source Project: Flink-CEPplus   Source File: ReduceOnEdgesMethodsITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void iterateEdges(Iterable<Tuple2<Long, Edge<Long, Long>>> edges,
		Collector<Tuple2<Long, Long>> out) throws Exception {

	for (Tuple2<Long, Edge<Long, Long>> edge : edges) {
		out.collect(new Tuple2<>(edge.f0, edge.f1.getSource()));
	}
}
 
Example 25
Source Project: flink   Source File: AggregatorsITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(Tuple2<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> value,
		Collector<Tuple2<Integer, Integer>> out) {

	if (value.f0.f1  > superstep) {
		out.collect(value.f0);
	}
}
 
Example 26
Source Project: flink   Source File: HITS.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void coGroup(Iterable<Tuple2<T, DoubleValue>> vertex, Iterable<Tuple2<T, T>> edges, Collector<Tuple2<T, DoubleValue>> out)
		throws Exception {
	output.f1 = vertex.iterator().next().f1;

	for (Tuple2<T, T> edge : edges) {
		output.f0 = edge.f1;
		out.collect(output);
	}
}
 
Example 27
Source Project: flink   Source File: ReduceTaskExternalITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int cnt = 0;
	
	for (Record next : records) {
		element = next;
		cnt++;
	}
	element.getField(0, this.key);
	this.value.setValue(cnt - this.key.getValue());
	element.setField(1, this.value);
	out.collect(element);
}
 
Example 28
Source Project: flink   Source File: GroupReduceITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<CollectionDataSets.PojoWithCollection> values, Collector<String> out) {
	StringBuilder concat = new StringBuilder();
	concat.append("call");
	for (CollectionDataSets.PojoWithCollection value : values) {
		concat.append("For key ").append(value.key).append(" we got: ");

		for (CollectionDataSets.Pojo1 p :value.pojos) {
			concat.append("pojo.a=").append(p.a);
		}
	}
	out.collect(concat.toString());
}
 
Example 29
@Override
public void processElement(String value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
	Set<String> retrievedKeySet = new HashSet<>();
	for (Map.Entry<String, Integer> entry : ctx.getBroadcastState(STATE_DESCRIPTOR).immutableEntries()) {
		retrievedKeySet.add(entry.getKey());
	}

	Assert.assertEquals(keysToRegister, retrievedKeySet);

	out.collect(value + "WM:" + ctx.currentWatermark() + " TS:" + ctx.timestamp());
}
 
Example 30
@Override
public void process(KEY key, final W window, final InternalWindowContext context, IN input, Collector<OUT> out) throws Exception {
	this.ctx.window = window;
	this.ctx.internalContext = context;

	ProcessWindowFunction<IN, OUT, KEY, W> wrappedFunction = this.wrappedFunction;
	wrappedFunction.process(key, ctx, Collections.singletonList(input), out);
}