Java Code Examples for org.apache.flink.util.Collector#collect()

The following examples show how to use org.apache.flink.util.Collector#collect() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: CirculantGraph.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void flatMap(LongValue source, Collector<Edge<LongValue, NullValue>> out)
		throws Exception {
	edge.f0 = source;
	long sourceID = source.getValue();

	for (OffsetRange offsetRange : offsetRanges) {
		long targetID = sourceID + offsetRange.getOffset();

		for (long i = offsetRange.getLength(); i > 0; i--) {
			// add positive offset
			target.setValue(targetID++ % vertexCount);
			out.collect(edge);
		}
	}
}
 
Example 2
Source Project: kylin   File: FlinkCubingByLayer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void flatMap(Tuple2<ByteArray, Object[]> tuple2, Collector<Tuple2<ByteArray, Object[]>> collector) throws Exception {
    byte[] key = tuple2.f0.array();
    long cuboidId = rowKeySplitter.parseCuboid(key);
    final List<Long> myChildren = cubeSegment.getCuboidScheduler().getSpanningCuboid(cuboidId);

    // if still empty or null
    if (myChildren == null || myChildren.size() == 0) {
        return;
    }
    rowKeySplitter.split(key);
    final Cuboid parentCuboid = Cuboid.findForMandatory(cubeDesc, cuboidId);

    for (Long child : myChildren) {
        Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child);
        ByteArray result = ndCuboidBuilder.buildKey2(parentCuboid, childCuboid,
                rowKeySplitter.getSplitBuffers());

        collector.collect(new Tuple2<>(result, tuple2.f1));
    }
}
 
Example 3
Source Project: flink   File: AggregatorsITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(Tuple2<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> value,
		Collector<Tuple2<Integer, Integer>> out) {

	if (value.f0.f1  > superstep) {
		out.collect(value.f0);
	}
}
 
Example 4
@Override
public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) {
	neighbors.clear();
	Long id = 0L;

	for (Tuple2<Long, Long> n : values) {
		id = n.f0;
		neighbors.add(n.f1);
	}
	out.collect(new Tuple2<Long, Long[]>(id, neighbors.toArray(new Long[neighbors.size()])));
}
 
Example 5
Source Project: flink   File: Graph.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void coGroup(Iterable<Edge<K, EV>> edge, Iterable<Edge<K, EV>> edgeToBeRemoved,
					Collector<Edge<K, EV>> out) throws Exception {
	if (!edgeToBeRemoved.iterator().hasNext()) {
		for (Edge<K, EV> next : edge) {
			out.collect(next);
		}
	}
}
 
Example 6
@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
	out.collect(value);

	ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
	if (state == null) {
		throw new RuntimeException("Missing key value state for " + value);
	}

	assertEquals(value.f1, state.value());
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
}
 
Example 7
Source Project: flink   File: VertexDegrees.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(Edge<T, TV> value, Collector<Tuple3<T, T, ByteValue>> out)
		throws Exception {
	forward.f0 = value.f0;
	forward.f1 = value.f1;
	out.collect(forward);

	reverse.f0 = value.f1;
	reverse.f1 = value.f0;
	out.collect(reverse);
}
 
Example 8
@Override
public void mapPartition(Iterable<T> values, Collector<Object[][]> out) throws Exception {
	final TypeComparator<T> comparator = this.comparatorFactory.createComparator();
	List<T> sampledData = new ArrayList<>();
	for (T value : values) {
		sampledData.add(value);
	}
	Collections.sort(sampledData, new Comparator<T>() {
		@Override
		public int compare(T first, T second) {
			return comparator.compare(first, second);
		}
	});


	int boundarySize = parallelism - 1;
	Object[][] boundaries = new Object[boundarySize][];
	if (sampledData.size() > 0) {
		double avgRange = sampledData.size() / (double) parallelism;
		int numKey = comparator.getFlatComparators().length;
		for (int i = 1; i < parallelism; i++) {
			T record = sampledData.get((int) (i * avgRange));
			Object[] keys = new Object[numKey];
			comparator.extractKeys(record, keys, 0);
			boundaries[i-1] = keys;
		}
	}

	out.collect(boundaries);
}
 
Example 9
Source Project: flink   File: Graph.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void join(Tuple4<K, K, VV, EV> tripletWithSrcValSet,
				Vertex<K, VV> vertex, Collector<Triplet<K, VV, EV>> collector) throws Exception {

	collector.collect(new Triplet<>(tripletWithSrcValSet.f0, tripletWithSrcValSet.f1,
			tripletWithSrcValSet.f2, vertex.getValue(), tripletWithSrcValSet.f3));
}
 
Example 10
Source Project: Flink-CEPplus   File: CombineTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Tuple2<Integer, Integer>> records, Collector<Tuple2<Integer, Integer>> out) {
	int key = 0;
	int sum = 0;

	for (Tuple2<Integer, Integer> next : records) {
		key = next.f0;
		sum += next.f1;
	}
	
	out.collect(new Tuple2<>(key, sum));
}
 
Example 11
@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {

	assertEquals(TimeDomain.EVENT_TIME, ctx.timeDomain());
	out.collect("" + 1777);
}
 
Example 12
@Override
public void apply(Tuple key, GlobalWindow window, Iterable<ConnectedCarEvent> events, Collector<StoppedSegment> out) {
	StoppedSegment seg = new StoppedSegment(events);
	if (seg.length > 0) {
		out.collect(seg);
	}
}
 
Example 13
@Override
public void combine(Iterable<Tuple2<Integer, String>> values, Collector<Tuple2<Integer, String>> out) {
	Tuple2<Integer, String> rec = new Tuple2<>();
	int cnt = 0;
	for (Tuple2<Integer, String> next : values) {
		rec = next;
		cnt += Integer.parseInt(rec.f1);
	}

	out.collect(new Tuple2(rec.f0, cnt + ""));
}
 
Example 14
Source Project: Flink-CEPplus   File: ChainTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Record> records, Collector<Record> out) throws Exception {
	if (++this.cnt >= 5) {
		throw new RuntimeException("Expected Test Exception");
	}
	
	for (Record r : records) {
		out.collect(r);
	}
}
 
Example 15
Source Project: Flink-CEPplus   File: GroupReduceITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple2<Integer, Long>> out) {
	int i = 0;
	long l = 0L;

	for (Tuple3<Integer, Long, String> t : values) {
		i += t.f0;
		l = t.f1;
	}

	out.collect(new Tuple2<>(i, l));

}
 
Example 16
@Override
public void processElement(
		RowData input,
		KeyedProcessFunction<K, RowData, RowData>.Context ctx,
		Collector<RowData> out) throws Exception {
	long currentTime = ctx.timerService().currentProcessingTime();
	// register state-cleanup timer
	registerProcessingCleanupTimer(ctx, currentTime);

	// initialize state for the processed element
	RowData accumulators = accState.value();
	if (accumulators == null) {
		accumulators = function.createAccumulators();
	}
	// set accumulators in context first
	function.setAccumulators(accumulators);

	// get smallest timestamp
	Long smallestTs = smallestTsState.value();
	if (smallestTs == null) {
		smallestTs = currentTime;
		smallestTsState.update(smallestTs);
	}
	// get previous counter value
	Long counter = counterState.value();
	if (counter == null) {
		counter = 0L;
	}

	if (counter == precedingOffset) {
		List<RowData> retractList = inputState.get(smallestTs);
		if (retractList != null) {
			// get oldest element beyond buffer size
			// and if oldest element exist, retract value
			RowData retractRow = retractList.get(0);
			function.retract(retractRow);
			retractList.remove(0);
		} else {
			// Does not retract values which are outside of window if the state is cleared already.
			LOG.warn("The state is cleared because of state ttl. " +
				"This will result in incorrect result. " +
				"You can increase the state ttl to avoid this.");
		}
		// if reference timestamp list not empty, keep the list
		if (retractList != null && !retractList.isEmpty()) {
			inputState.put(smallestTs, retractList);
		} // if smallest timestamp list is empty, remove and find new smallest
		else {
			inputState.remove(smallestTs);
			Iterator<Long> iter = inputState.keys().iterator();
			long currentTs = 0L;
			long newSmallestTs = Long.MAX_VALUE;
			while (iter.hasNext()) {
				currentTs = iter.next();
				if (currentTs < newSmallestTs) {
					newSmallestTs = currentTs;
				}
			}
			smallestTsState.update(newSmallestTs);
		}
	} // we update the counter only while buffer is getting filled
	else {
		counter += 1;
		counterState.update(counter);
	}

	// update map state, counter and timestamp
	List<RowData> currentTimeState = inputState.get(currentTime);
	if (currentTimeState != null) {
		currentTimeState.add(input);
		inputState.put(currentTime, currentTimeState);
	} else { // add new input
		List<RowData> newList = new ArrayList<RowData>();
		newList.add(input);
		inputState.put(currentTime, newList);
	}

	// accumulate current row
	function.accumulate(input);
	// update the value of accumulators for future incremental computation
	accumulators = function.getAccumulators();
	accState.update(accumulators);

	// prepare output row
	RowData aggValue = function.getValue();
	output.replace(input, aggValue);
	out.collect(output);
}
 
Example 17
Source Project: flink   File: CachedMatchTaskTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void join(Record record1, Record record2, Collector<Record> out) throws Exception {
	out.collect(record1);
}
 
Example 18
@Override
public void apply(String key, TimeWindow timeWindow, Iterable<Tuple3<Long, String, Long>> iterable, Collector<Tuple3<Long, String, Long>> collector) throws Exception {
	long count = iterable.iterator().next().f0;
	collector.collect(Tuple3.of(count, key, timeWindow.getStart()));
}
 
Example 19
Source Project: flink   File: KeyedStateInputFormatTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void readKey(Integer key, KeyedStateReaderFunction.Context ctx, Collector<Integer> out) throws Exception {
	ValueState<Integer> state = getRuntimeContext().getState(stateDescriptor);
	out.collect(state.value());
}
 
Example 20
@Override
public void join(Tuple2<Integer, Integer> first, Tuple2<Integer, Integer> second, Collector<Tuple2<Integer, Integer>> out) throws Exception {
	out.collect(first != null ? first : second);
}