Java Code Examples for org.apache.flink.util.Collector#collect()

@Override
public void flatMap(LongValue source, Collector<Edge<LongValue, NullValue>> out)
		throws Exception {
	edge.f0 = source;
	long sourceID = source.getValue();

	for (OffsetRange offsetRange : offsetRanges) {
		long targetID = sourceID + offsetRange.getOffset();

		for (long i = offsetRange.getLength(); i > 0; i--) {
			// add positive offset
			target.setValue(targetID++ % vertexCount);
			out.collect(edge);
		}
	}
}
 

@Override
public void flatMap(Tuple2<ByteArray, Object[]> tuple2, Collector<Tuple2<ByteArray, Object[]>> collector) throws Exception {
    byte[] key = tuple2.f0.array();
    long cuboidId = rowKeySplitter.parseCuboid(key);
    final List<Long> myChildren = cubeSegment.getCuboidScheduler().getSpanningCuboid(cuboidId);

    // if still empty or null
    if (myChildren == null || myChildren.size() == 0) {
        return;
    }
    rowKeySplitter.split(key);
    final Cuboid parentCuboid = Cuboid.findForMandatory(cubeDesc, cuboidId);

    for (Long child : myChildren) {
        Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child);
        ByteArray result = ndCuboidBuilder.buildKey2(parentCuboid, childCuboid,
                rowKeySplitter.getSplitBuffers());

        collector.collect(new Tuple2<>(result, tuple2.f1));
    }
}
 

@Override
public void join(Tuple4<K, K, VV, EV> tripletWithSrcValSet,
				Vertex<K, VV> vertex, Collector<Triplet<K, VV, EV>> collector) throws Exception {

	collector.collect(new Triplet<>(tripletWithSrcValSet.f0, tripletWithSrcValSet.f1,
			tripletWithSrcValSet.f2, vertex.getValue(), tripletWithSrcValSet.f3));
}
 

@Override
public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple2<Integer, Long>> out) {
	int i = 0;
	long l = 0L;

	for (Tuple3<Integer, Long, String> t : values) {
		i += t.f0;
		l = t.f1;
	}

	out.collect(new Tuple2<>(i, l));

}
 

@Override
public void reduce(Iterable<Record> records, Collector<Record> out) throws Exception {
	if (++this.cnt >= 5) {
		throw new RuntimeException("Expected Test Exception");
	}
	
	for (Record r : records) {
		out.collect(r);
	}
}
 

@Override
public void combine(Iterable<Tuple2<Integer, String>> values, Collector<Tuple2<Integer, String>> out) {
	Tuple2<Integer, String> rec = new Tuple2<>();
	int cnt = 0;
	for (Tuple2<Integer, String> next : values) {
		rec = next;
		cnt += Integer.parseInt(rec.f1);
	}

	out.collect(new Tuple2(rec.f0, cnt + ""));
}
 

@Override
public void apply(Tuple key, GlobalWindow window, Iterable<ConnectedCarEvent> events, Collector<StoppedSegment> out) {
	StoppedSegment seg = new StoppedSegment(events);
	if (seg.length > 0) {
		out.collect(seg);
	}
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {

	assertEquals(TimeDomain.EVENT_TIME, ctx.timeDomain());
	out.collect("" + 1777);
}
 

@Override
public void reduce(Iterable<Tuple2<Integer, Integer>> records, Collector<Tuple2<Integer, Integer>> out) {
	int key = 0;
	int sum = 0;

	for (Tuple2<Integer, Integer> next : records) {
		key = next.f0;
		sum += next.f1;
	}
	
	out.collect(new Tuple2<>(key, sum));
}
 

@Override
public void mapPartition(Iterable<T> values, Collector<Object[][]> out) throws Exception {
	final TypeComparator<T> comparator = this.comparatorFactory.createComparator();
	List<T> sampledData = new ArrayList<>();
	for (T value : values) {
		sampledData.add(value);
	}
	Collections.sort(sampledData, new Comparator<T>() {
		@Override
		public int compare(T first, T second) {
			return comparator.compare(first, second);
		}
	});


	int boundarySize = parallelism - 1;
	Object[][] boundaries = new Object[boundarySize][];
	if (sampledData.size() > 0) {
		double avgRange = sampledData.size() / (double) parallelism;
		int numKey = comparator.getFlatComparators().length;
		for (int i = 1; i < parallelism; i++) {
			T record = sampledData.get((int) (i * avgRange));
			Object[] keys = new Object[numKey];
			comparator.extractKeys(record, keys, 0);
			boundaries[i-1] = keys;
		}
	}

	out.collect(boundaries);
}
 

@Override
public void flatMap(Edge<T, TV> value, Collector<Tuple3<T, T, ByteValue>> out)
		throws Exception {
	forward.f0 = value.f0;
	forward.f1 = value.f1;
	out.collect(forward);

	reverse.f0 = value.f1;
	reverse.f1 = value.f0;
	out.collect(reverse);
}
 

@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
	out.collect(value);

	ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
	if (state == null) {
		throw new RuntimeException("Missing key value state for " + value);
	}

	assertEquals(value.f1, state.value());
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void coGroup(Iterable<Edge<K, EV>> edge, Iterable<Edge<K, EV>> edgeToBeRemoved,
					Collector<Edge<K, EV>> out) throws Exception {
	if (!edgeToBeRemoved.iterator().hasNext()) {
		for (Edge<K, EV> next : edge) {
			out.collect(next);
		}
	}
}
 

@Override
public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) {
	neighbors.clear();
	Long id = 0L;

	for (Tuple2<Long, Long> n : values) {
		id = n.f0;
		neighbors.add(n.f1);
	}
	out.collect(new Tuple2<Long, Long[]>(id, neighbors.toArray(new Long[neighbors.size()])));
}
 

@Override
public void flatMap(Tuple2<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> value,
		Collector<Tuple2<Integer, Integer>> out) {

	if (value.f0.f1  > superstep) {
		out.collect(value.f0);
	}
}
 

@Override
public void apply(String key, TimeWindow timeWindow, Iterable<Tuple3<Long, String, Long>> iterable, Collector<Tuple3<Long, String, Long>> collector) throws Exception {
	long count = iterable.iterator().next().f0;
	collector.collect(Tuple3.of(count, key, timeWindow.getStart()));
}
 

@Override
public void readKey(Integer key, KeyedStateReaderFunction.Context ctx, Collector<Integer> out) throws Exception {
	ValueState<Integer> state = getRuntimeContext().getState(stateDescriptor);
	out.collect(state.value());
}
 

@Override
public void join(Tuple2<Integer, Integer> first, Tuple2<Integer, Integer> second, Collector<Tuple2<Integer, Integer>> out) throws Exception {
	out.collect(first != null ? first : second);
}
 

@Override
public void join(Record record1, Record record2, Collector<Record> out) throws Exception {
	out.collect(record1);
}
 

@Override
public void processElement(
		RowData input,
		KeyedProcessFunction<K, RowData, RowData>.Context ctx,
		Collector<RowData> out) throws Exception {
	long currentTime = ctx.timerService().currentProcessingTime();
	// register state-cleanup timer
	registerProcessingCleanupTimer(ctx, currentTime);

	// initialize state for the processed element
	RowData accumulators = accState.value();
	if (accumulators == null) {
		accumulators = function.createAccumulators();
	}
	// set accumulators in context first
	function.setAccumulators(accumulators);

	// get smallest timestamp
	Long smallestTs = smallestTsState.value();
	if (smallestTs == null) {
		smallestTs = currentTime;
		smallestTsState.update(smallestTs);
	}
	// get previous counter value
	Long counter = counterState.value();
	if (counter == null) {
		counter = 0L;
	}

	if (counter == precedingOffset) {
		List<RowData> retractList = inputState.get(smallestTs);
		if (retractList != null) {
			// get oldest element beyond buffer size
			// and if oldest element exist, retract value
			RowData retractRow = retractList.get(0);
			function.retract(retractRow);
			retractList.remove(0);
		} else {
			// Does not retract values which are outside of window if the state is cleared already.
			LOG.warn("The state is cleared because of state ttl. " +
				"This will result in incorrect result. " +
				"You can increase the state ttl to avoid this.");
		}
		// if reference timestamp list not empty, keep the list
		if (retractList != null && !retractList.isEmpty()) {
			inputState.put(smallestTs, retractList);
		} // if smallest timestamp list is empty, remove and find new smallest
		else {
			inputState.remove(smallestTs);
			Iterator<Long> iter = inputState.keys().iterator();
			long currentTs = 0L;
			long newSmallestTs = Long.MAX_VALUE;
			while (iter.hasNext()) {
				currentTs = iter.next();
				if (currentTs < newSmallestTs) {
					newSmallestTs = currentTs;
				}
			}
			smallestTsState.update(newSmallestTs);
		}
	} // we update the counter only while buffer is getting filled
	else {
		counter += 1;
		counterState.update(counter);
	}

	// update map state, counter and timestamp
	List<RowData> currentTimeState = inputState.get(currentTime);
	if (currentTimeState != null) {
		currentTimeState.add(input);
		inputState.put(currentTime, currentTimeState);
	} else { // add new input
		List<RowData> newList = new ArrayList<RowData>();
		newList.add(input);
		inputState.put(currentTime, newList);
	}

	// accumulate current row
	function.accumulate(input);
	// update the value of accumulators for future incremental computation
	accumulators = function.getAccumulators();
	accState.update(accumulators);

	// prepare output row
	RowData aggValue = function.getValue();
	output.replace(input, aggValue);
	out.collect(output);
}