Java Code Examples for org.apache.flink.util.Collector

@Override
public void reduce(Iterable<Vertex<K, VV>> values, Collector<VertexGroupItem<K, VV>> out) throws Exception {
	K vertexGroupRepresentativeID = null;
	long vertexGroupCount = 0L;
	VV vertexGroupValue = null;
	boolean isFirstElement = true;

	for (Vertex<K, VV> vertex : values) {
		if (isFirstElement) {
			// take final group representative vertex id from first tuple
			vertexGroupRepresentativeID = vertex.getId();
			vertexGroupValue = vertex.getValue();
			isFirstElement = false;
		}
		// no need to set group value for those tuples
		reuseVertexGroupItem.setVertexId(vertex.getId());
		reuseVertexGroupItem.setGroupRepresentativeId(vertexGroupRepresentativeID);
		out.collect(reuseVertexGroupItem);
		vertexGroupCount++;
	}

	createGroupRepresentativeTuple(vertexGroupRepresentativeID, vertexGroupValue, vertexGroupCount);
	out.collect(reuseVertexGroupItem);
	reuseVertexGroupItem.reset();
}
 

@Override
public void process(K k, final Context context, Iterable<T> input, Collector<R> out) throws Exception {

	T curr = null;
	for (T val: input) {
		if (curr == null) {
			curr = val;
		} else {
			curr = reduceFunction.reduce(curr, val);
		}
	}

	this.ctx.window = context.window();
	this.ctx.context = context;
	windowFunction.process(k, ctx, Collections.singletonList(curr), out);
}
 

@Override
public void coGroup(Iterable<Tuple2<Long, Long>> candidates, Iterable<Tuple2<Long, Long>> current, Collector<Tuple2<Long, Long>> out) {
	Iterator<Tuple2<Long, Long>> iterator = current.iterator();
	if (!iterator.hasNext()) {
		throw new RuntimeException("Error: Id not encountered before.");
	}

	Tuple2<Long, Long> old = iterator.next();

	long minimumComponentID = Long.MAX_VALUE;

	for (Tuple2<Long, Long> candidate : candidates) {
		long candidateComponentID = candidate.f1;
		if (candidateComponentID < minimumComponentID) {
			minimumComponentID = candidateComponentID;
		}
	}

	if (minimumComponentID < old.f1) {
		old.f1 = minimumComponentID;
		out.collect(old);
	}
}
 

public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataSource<String> dataSource = env.fromElements(WORDS);

    dataSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
        @Override
        public void flatMap(String line, Collector<Tuple2<String, Integer>> out) throws Exception {
            String[] words = line.split("\\W+");
            for (String word : words) {
                out.collect(new Tuple2<>(word, 1));
            }
        }
    })
            .groupBy(0)
            .sum(1)
            .print();

    long count = dataSource.count();
    System.out.println(count);
}
 

@Override
public void flatMap(String tweetJsonStr, Collector<Tuple2<String, Integer>> collector) throws Exception {
    JsonNode tweetJson = mapper.readTree(tweetJsonStr);
    JsonNode entities = tweetJson.get("entities");
    if (entities == null) return;

    JsonNode hashtags = entities.get("hashtags");
    if (hashtags == null) return;

    for (Iterator<JsonNode> iter = hashtags.getElements(); iter.hasNext();) {
        JsonNode node = iter.next();
        String hashtag = node.get("text").getTextValue();

        if (hashtag.matches("\\w+")) {
            collector.collect(new Tuple2<>(hashtag, 1));
        }
    }
}
 

@Override
public void processElement2(DomainScore domainScore, Context context, Collector<FetchUrl> out)
        throws Exception {
    
    // Ensure we don't wind up with DBZ problems.
    float score = Math.max(0.01f, domainScore.getScore());
    String pld = domainScore.getPld();
    LOGGER.debug("UrlDBFunction ({}/{}) setting '{}' average score to {}",
            _partition, _parallelism, pld, score);
    
    // At this point we might be seeing this PLD for the first time, or we might have seen
    // it before in this method, or we might have seen it via the onTimer call. So it may 
    // or may not have any state set up, and it may or may not be in _domainScores (non-state)
    float summedScores = _averageDomainScore * _scoredDomains.size();
    if (_scoredDomains.contains(pld)) {
        summedScores -= _domainScore.value();
    }
    
    _domainScore.update(score);
    _scoredDomains.add(pld);
    summedScores += score;
    _averageDomainScore = summedScores / _scoredDomains.size();
}
 

@Override
public void processElement(String value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
	Iterable<Map.Entry<String, Integer>> broadcastStateIt = ctx.getBroadcastState(STATE_DESCRIPTOR).immutableEntries();
	Iterator<Map.Entry<String, Integer>> iter = broadcastStateIt.iterator();

	for (int i = 0; i < expectedBroadcastState.size(); i++) {
		assertTrue(iter.hasNext());

		Map.Entry<String, Integer> entry = iter.next();
		assertTrue(expectedBroadcastState.containsKey(entry.getKey()));
		assertEquals(expectedBroadcastState.get(entry.getKey()), entry.getValue());
	}

	assertFalse(iter.hasNext());

	ctx.timerService().registerEventTimeTimer(timerTs);
}
 

public static CepOperatorBuilder<Map<String, List<Event>>> createOperatorForNFA(NFA<Event> nfa) {
	return new CepOperatorBuilder<>(
		true,
		new NFACompiler.NFAFactory<Event>() {
			@Override
			public NFA<Event> createNFA() {
				return nfa;
			}
		},
		null,
		null,
		new PatternProcessFunction<Event, Map<String, List<Event>>>() {
			private static final long serialVersionUID = -7143807777582726991L;

			@Override
			public void processMatch(
				Map<String, List<Event>> match,
				Context ctx,
				Collector<Map<String, List<Event>>> out) throws Exception {
				out.collect(match);
			}
		},
		null);
}
 

/**
 * Crosses a single value from the second side with N values, all sharing a common key.
 * Effectively realizes a <i>N:1</i> join.
 *
 * @param val1      The value form the <i>1</i> side.
 * @param firstValN The first of the values from the <i>N</i> side.
 * @param valsN     Iterator over remaining <i>N</i> side values.
 * @throws Exception Forwards all exceptions thrown by the stub.
 */
private void crossSecond1withNValues(T2 val1, T1 firstValN,
									Iterator<T1> valsN, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception {
	T2 copy2 = createCopy(serializer2, val1, this.copy2);
	joinFunction.join(firstValN, copy2, collector);

	// set copy and join first element
	boolean more = true;
	do {
		final T1 nRec = valsN.next();

		if (valsN.hasNext()) {
			copy2 = createCopy(serializer2, val1, this.copy2);
			joinFunction.join(nRec, copy2, collector);
		} else {
			joinFunction.join(nRec, val1, collector);
			more = false;
		}
	}
	while (more);
}
 

@Override
public void processElement(
		BaseRow input,
		KeyedProcessFunction<K, BaseRow, BaseRow>.Context ctx,
		Collector<BaseRow> out) throws Exception {
	long currentTime = ctx.timerService().currentProcessingTime();
	// register state-cleanup timer
	registerProcessingCleanupTimer(ctx, currentTime);

	// buffer the event incoming event

	// add current element to the window list of elements with corresponding timestamp
	List<BaseRow> rowList = inputState.get(currentTime);
	// null value means that this si the first event received for this timestamp
	if (rowList == null) {
		rowList = new ArrayList<BaseRow>();
		// register timer to process event once the current millisecond passed
		ctx.timerService().registerProcessingTimeTimer(currentTime + 1);
	}
	rowList.add(input);
	inputState.put(currentTime, rowList);
}
 

private DataSet<byte[]> loopStartDataSet(ExecutionEnvironment env) {
	MapPartitionOperator<Integer, byte[]> initial = env
		.fromElements(1)
		.rebalance()
		.mapPartition(new MapPartitionFunction<Integer, byte[]>() {
			@Override
			public void mapPartition(Iterable<Integer> values, Collector<byte[]> out) {
				//pass
			}
		}).name("iterInitialize");

	if (cacheDataRel != null) {
		initial = initial.withBroadcastSet(cacheDataRel, "rel");
	}

	return initial;
}
 

@Override
public void flatMap(String value, Collector<String> out) throws Exception {
	LOG.info("Consumed {}", value);

	int id = Integer.parseInt(value.split("-")[0]);
	if (validator.get(id)) {
		throw new RuntimeException("Saw id " + id + " twice!");
	}
	validator.set(id);
	if (id > totalEventCount - 1) {
		throw new RuntimeException("Out of bounds ID observed");
	}

	if (validator.nextClearBit(0) == totalEventCount) {
		throw new SuccessException();
	}
}
 

@Override
public void reduce(Iterable<Tuple2<String, String>> values, Collector<Tuple2<String, /*TODO POJO*/String>> out) throws Exception {
    coords = "";
    for (Tuple2<String,String> entry: values) {
        if(coordSet.size() == 0){
            user = entry.f0;
            coords = entry.f1.toString();
        }else{
            coords += "|" + entry.f1.toString();
        }
        coordSet.add(entry.f1.toString());
    }
    if(coordSet.size() > 1){
        out.collect(new Tuple2<>(user,coords));
        coordSet.clear();
    }

}
 

private static DataSet<Tuple2<Long, float[]>> getFactors(BatchOperator<?> model, final int identity) {
    return model.getDataSet()
        .flatMap(new FlatMapFunction<Row, Tuple2<Long, float[]>>() {
            @Override
            public void flatMap(Row value, Collector<Tuple2<Long, float[]>> out) throws Exception {
                int w = AlsModelDataConverter.getIsUser(value) ? 0 : 1;
                if (w != identity) {
                    return;
                }

                long idx = AlsModelDataConverter.getVertexId(value);
                float[] factors = AlsModelDataConverter.getFactors(value);
                out.collect(Tuple2.of(idx, factors));
            }
        });
}
 

@Override
public void coGroup(Iterable<Edge<K, EV>> edges, Iterable<Vertex<K, Tuple3<VV, LongValue, LongValue>>> state,
					Collector<Tuple2<K, Message>> out) throws Exception {

	final Iterator<Vertex<K, Tuple3<VV, LongValue, LongValue>>> stateIter = state.iterator();

	if (stateIter.hasNext()) {
		Vertex<K, Tuple3<VV, LongValue, LongValue>> vertexWithDegrees = stateIter.next();

		nextVertex.f0 = vertexWithDegrees.f0;
		nextVertex.f1 = vertexWithDegrees.f1.f0;

		scatterFunction.setInDegree(vertexWithDegrees.f1.f1.getValue());
		scatterFunction.setOutDegree(vertexWithDegrees.f1.f2.getValue());

		scatterFunction.set(edges.iterator(), out, vertexWithDegrees.getId());
		scatterFunction.sendMessages(nextVertex);
	}
}
 

@Override
public void flatMap(TriangleListing.Result<T> value, Collector<Tuple2<T, LongValue>> out)
		throws Exception {
	byte bitmask = value.getBitmask().getValue();

	output.f0 = value.getVertexId0();
	output.f1 = ((bitmask & 0b000011) == 0b000011) ? two : one;
	out.collect(output);

	output.f0 = value.getVertexId1();
	output.f1 = ((bitmask & 0b001100) == 0b001100) ? two : one;
	out.collect(output);

	output.f0 = value.getVertexId2();
	output.f1 = ((bitmask & 0b110000) == 0b110000) ? two : one;
	out.collect(output);
}
 

@Override
public void processBroadcastElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	// put an element in the broadcast state
	ctx.applyToKeyedState(
			listStateDesc,
			new KeyedStateFunction<String, ListState<String>>() {
				@Override
				public void process(String key, ListState<String> state) throws Exception {
					final Iterator<String> it = state.get().iterator();

					final List<String> list = new ArrayList<>();
					while (it.hasNext()) {
						list.add(it.next());
					}
					assertEquals(expectedKeyedStates.get(key), list);
				}
			});
}
 

@Test
public void testGroupReduceWithAtomicValue() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> ds = env.fromElements(1, 1, 2, 3, 4);
	DataSet<Integer> reduceDs = ds.groupBy("*").reduceGroup(new GroupReduceFunction<Integer, Integer>() {
		@Override
		public void reduce(Iterable<Integer> values, Collector<Integer> out) throws Exception {
			out.collect(values.iterator().next());
		}
	});

	List<Integer> result = reduceDs.collect();

	String expected = "1\n" +
			"2\n" +
			"3\n" +
			"4";

	compareResultAsText(result, expected);
}
 

@Override
public void mapPartition(Iterable<Row> iterable,
                         Collector<Tuple2<DenseVector, double[]>> collector) throws Exception {
    DenseVector coefVector = null;
    double[] lossCurve = null;
    int taskId = getRuntimeContext().getIndexOfThisSubtask();
    if (taskId == 0) {
        for (Row row : iterable) {
            Params params = Params.fromJson((String)row.getField(0));
            coefVector = params.get(ModelParamName.COEF);
            lossCurve = params.get(ModelParamName.LOSS_CURVE);
        }

        if (coefVector != null) {
            collector.collect(Tuple2.of(coefVector, lossCurve));
        }
    }
}
 

/**
 * Sets the last output {@link Collector} of the collector chain of this {@link BatchTask}.
 * <p>
 * In case of chained tasks, the output collector of the last {@link ChainedDriver} is set. Otherwise it is the
 * single collector of the {@link BatchTask}.
 *
 * @param newOutputCollector new output collector to set as last collector
 */
protected void setLastOutputCollector(Collector<OT> newOutputCollector) {
	int numChained = this.chainedTasks.size();

	if (numChained == 0) {
		output = newOutputCollector;
		return;
	}

	chainedTasks.get(numChained - 1).setOutputCollector(newOutputCollector);
}
 

@Override
public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
	// normalize and split the line
	String[] tokens = value.toLowerCase().split("\\W+");

	// emit the pairs
	for (String token : tokens) {
		if (token.length() > 0) {
			out.collect(new Tuple2<>(token, 1));
		}
	}
}
 

@Override
public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Boolean> out) throws Exception {
	Iterator<Tuple2<Long, Long>> it = values.iterator();
	boolean f = it.hasNext();
	if (!f) {
		System.out.println();
	}
	if (f) {
		System.out.println();
	}
	out.collect(f);
}
 

@Test
@SuppressWarnings("rawtypes")
public void testProcessEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 

@Override
public void iterateEdges(Iterable<Tuple2<Long, Edge<Long, Long>>> edges,
		Collector<Tuple2<Long, Long>> out) throws Exception {

	for (Tuple2<Long, Edge<Long, Long>> edge : edges) {
		out.collect(new Tuple2<>(edge.f0, edge.f1.getSource()));
	}
}
 

@Override
public void flatMap(Tuple2<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> value,
		Collector<Tuple2<Integer, Integer>> out) {

	if (value.f0.f1  > superstep) {
		out.collect(value.f0);
	}
}
 

@Override
public void coGroup(Iterable<Tuple2<T, DoubleValue>> vertex, Iterable<Tuple2<T, T>> edges, Collector<Tuple2<T, DoubleValue>> out)
		throws Exception {
	output.f1 = vertex.iterator().next().f1;

	for (Tuple2<T, T> edge : edges) {
		output.f0 = edge.f1;
		out.collect(output);
	}
}
 

@Override
public void reduce(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int cnt = 0;
	
	for (Record next : records) {
		element = next;
		cnt++;
	}
	element.getField(0, this.key);
	this.value.setValue(cnt - this.key.getValue());
	element.setField(1, this.value);
	out.collect(element);
}
 

@Override
public void reduce(Iterable<CollectionDataSets.PojoWithCollection> values, Collector<String> out) {
	StringBuilder concat = new StringBuilder();
	concat.append("call");
	for (CollectionDataSets.PojoWithCollection value : values) {
		concat.append("For key ").append(value.key).append(" we got: ");

		for (CollectionDataSets.Pojo1 p :value.pojos) {
			concat.append("pojo.a=").append(p.a);
		}
	}
	out.collect(concat.toString());
}
 

@Override
public void processElement(String value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
	Set<String> retrievedKeySet = new HashSet<>();
	for (Map.Entry<String, Integer> entry : ctx.getBroadcastState(STATE_DESCRIPTOR).immutableEntries()) {
		retrievedKeySet.add(entry.getKey());
	}

	Assert.assertEquals(keysToRegister, retrievedKeySet);

	out.collect(value + "WM:" + ctx.currentWatermark() + " TS:" + ctx.timestamp());
}
 

@Override
public void process(KEY key, final W window, final InternalWindowContext context, IN input, Collector<OUT> out) throws Exception {
	this.ctx.window = window;
	this.ctx.internalContext = context;

	ProcessWindowFunction<IN, OUT, KEY, W> wrappedFunction = this.wrappedFunction;
	wrappedFunction.process(key, ctx, Collections.singletonList(input), out);
}