Java Code Examples for org.apache.flink.util.Collector#collect()

The following examples show how to use org.apache.flink.util.Collector#collect() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CirculantGraph.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void flatMap(LongValue source, Collector<Edge<LongValue, NullValue>> out)
		throws Exception {
	edge.f0 = source;
	long sourceID = source.getValue();

	for (OffsetRange offsetRange : offsetRanges) {
		long targetID = sourceID + offsetRange.getOffset();

		for (long i = offsetRange.getLength(); i > 0; i--) {
			// add positive offset
			target.setValue(targetID++ % vertexCount);
			out.collect(edge);
		}
	}
}
 
Example 2
Source File: FlinkCubingByLayer.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void flatMap(Tuple2<ByteArray, Object[]> tuple2, Collector<Tuple2<ByteArray, Object[]>> collector) throws Exception {
    byte[] key = tuple2.f0.array();
    long cuboidId = rowKeySplitter.parseCuboid(key);
    final List<Long> myChildren = cubeSegment.getCuboidScheduler().getSpanningCuboid(cuboidId);

    // if still empty or null
    if (myChildren == null || myChildren.size() == 0) {
        return;
    }
    rowKeySplitter.split(key);
    final Cuboid parentCuboid = Cuboid.findForMandatory(cubeDesc, cuboidId);

    for (Long child : myChildren) {
        Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child);
        ByteArray result = ndCuboidBuilder.buildKey2(parentCuboid, childCuboid,
                rowKeySplitter.getSplitBuffers());

        collector.collect(new Tuple2<>(result, tuple2.f1));
    }
}
 
Example 3
Source File: Graph.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void join(Tuple4<K, K, VV, EV> tripletWithSrcValSet,
				Vertex<K, VV> vertex, Collector<Triplet<K, VV, EV>> collector) throws Exception {

	collector.collect(new Triplet<>(tripletWithSrcValSet.f0, tripletWithSrcValSet.f1,
			tripletWithSrcValSet.f2, vertex.getValue(), tripletWithSrcValSet.f3));
}
 
Example 4
Source File: GroupReduceITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple2<Integer, Long>> out) {
	int i = 0;
	long l = 0L;

	for (Tuple3<Integer, Long, String> t : values) {
		i += t.f0;
		l = t.f1;
	}

	out.collect(new Tuple2<>(i, l));

}
 
Example 5
Source File: ChainTaskTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Record> records, Collector<Record> out) throws Exception {
	if (++this.cnt >= 5) {
		throw new RuntimeException("Expected Test Exception");
	}
	
	for (Record r : records) {
		out.collect(r);
	}
}
 
Example 6
Source File: CombiningUnilateralSortMergerITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void combine(Iterable<Tuple2<Integer, String>> values, Collector<Tuple2<Integer, String>> out) {
	Tuple2<Integer, String> rec = new Tuple2<>();
	int cnt = 0;
	for (Tuple2<Integer, String> next : values) {
		rec = next;
		cnt += Integer.parseInt(rec.f1);
	}

	out.collect(new Tuple2(rec.f0, cnt + ""));
}
 
Example 7
Source File: DrivingSegments.java    From flink-training-exercises with Apache License 2.0 5 votes vote down vote up
@Override
public void apply(Tuple key, GlobalWindow window, Iterable<ConnectedCarEvent> events, Collector<StoppedSegment> out) {
	StoppedSegment seg = new StoppedSegment(events);
	if (seg.length > 0) {
		out.collect(seg);
	}
}
 
Example 8
Source File: KeyedCoProcessOperatorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {

	assertEquals(TimeDomain.EVENT_TIME, ctx.timeDomain());
	out.collect("" + 1777);
}
 
Example 9
Source File: CombineTaskTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Tuple2<Integer, Integer>> records, Collector<Tuple2<Integer, Integer>> out) {
	int key = 0;
	int sum = 0;

	for (Tuple2<Integer, Integer> next : records) {
		key = next.f0;
		sum += next.f1;
	}
	
	out.collect(new Tuple2<>(key, sum));
}
 
Example 10
Source File: RangeBoundaryBuilder.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void mapPartition(Iterable<T> values, Collector<Object[][]> out) throws Exception {
	final TypeComparator<T> comparator = this.comparatorFactory.createComparator();
	List<T> sampledData = new ArrayList<>();
	for (T value : values) {
		sampledData.add(value);
	}
	Collections.sort(sampledData, new Comparator<T>() {
		@Override
		public int compare(T first, T second) {
			return comparator.compare(first, second);
		}
	});


	int boundarySize = parallelism - 1;
	Object[][] boundaries = new Object[boundarySize][];
	if (sampledData.size() > 0) {
		double avgRange = sampledData.size() / (double) parallelism;
		int numKey = comparator.getFlatComparators().length;
		for (int i = 1; i < parallelism; i++) {
			T record = sampledData.get((int) (i * avgRange));
			Object[] keys = new Object[numKey];
			comparator.extractKeys(record, keys, 0);
			boundaries[i-1] = keys;
		}
	}

	out.collect(boundaries);
}
 
Example 11
Source File: VertexDegrees.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(Edge<T, TV> value, Collector<Tuple3<T, T, ByteValue>> out)
		throws Exception {
	forward.f0 = value.f0;
	forward.f1 = value.f1;
	out.collect(forward);

	reverse.f0 = value.f1;
	reverse.f1 = value.f0;
	out.collect(reverse);
}
 
Example 12
Source File: LegacyStatefulJobSavepointMigrationITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
	out.collect(value);

	ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
	if (state == null) {
		throw new RuntimeException("Missing key value state for " + value);
	}

	assertEquals(value.f1, state.value());
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
}
 
Example 13
Source File: Graph.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void coGroup(Iterable<Edge<K, EV>> edge, Iterable<Edge<K, EV>> edgeToBeRemoved,
					Collector<Edge<K, EV>> out) throws Exception {
	if (!edgeToBeRemoved.iterator().hasNext()) {
		for (Edge<K, EV> next : edge) {
			out.collect(next);
		}
	}
}
 
Example 14
Source File: UdfAnalyzerExamplesTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) {
	neighbors.clear();
	Long id = 0L;

	for (Tuple2<Long, Long> n : values) {
		id = n.f0;
		neighbors.add(n.f1);
	}
	out.collect(new Tuple2<Long, Long[]>(id, neighbors.toArray(new Long[neighbors.size()])));
}
 
Example 15
Source File: AggregatorsITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(Tuple2<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> value,
		Collector<Tuple2<Integer, Integer>> out) {

	if (value.f0.f1  > superstep) {
		out.collect(value.f0);
	}
}
 
Example 16
Source File: StreamingETL.java    From flink-streaming-etl with Apache License 2.0 4 votes vote down vote up
@Override
public void apply(String key, TimeWindow timeWindow, Iterable<Tuple3<Long, String, Long>> iterable, Collector<Tuple3<Long, String, Long>> collector) throws Exception {
	long count = iterable.iterator().next().f0;
	collector.collect(Tuple3.of(count, key, timeWindow.getStart()));
}
 
Example 17
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void readKey(Integer key, KeyedStateReaderFunction.Context ctx, Collector<Integer> out) throws Exception {
	ValueState<Integer> state = getRuntimeContext().getState(stateDescriptor);
	out.collect(state.value());
}
 
Example 18
Source File: AbstractOuterJoinTaskExternalITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void join(Tuple2<Integer, Integer> first, Tuple2<Integer, Integer> second, Collector<Tuple2<Integer, Integer>> out) throws Exception {
	out.collect(first != null ? first : second);
}
 
Example 19
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void join(Record record1, Record record2, Collector<Record> out) throws Exception {
	out.collect(record1);
}
 
Example 20
Source File: ProcTimeRowsBoundedPrecedingFunction.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void processElement(
		RowData input,
		KeyedProcessFunction<K, RowData, RowData>.Context ctx,
		Collector<RowData> out) throws Exception {
	long currentTime = ctx.timerService().currentProcessingTime();
	// register state-cleanup timer
	registerProcessingCleanupTimer(ctx, currentTime);

	// initialize state for the processed element
	RowData accumulators = accState.value();
	if (accumulators == null) {
		accumulators = function.createAccumulators();
	}
	// set accumulators in context first
	function.setAccumulators(accumulators);

	// get smallest timestamp
	Long smallestTs = smallestTsState.value();
	if (smallestTs == null) {
		smallestTs = currentTime;
		smallestTsState.update(smallestTs);
	}
	// get previous counter value
	Long counter = counterState.value();
	if (counter == null) {
		counter = 0L;
	}

	if (counter == precedingOffset) {
		List<RowData> retractList = inputState.get(smallestTs);
		if (retractList != null) {
			// get oldest element beyond buffer size
			// and if oldest element exist, retract value
			RowData retractRow = retractList.get(0);
			function.retract(retractRow);
			retractList.remove(0);
		} else {
			// Does not retract values which are outside of window if the state is cleared already.
			LOG.warn("The state is cleared because of state ttl. " +
				"This will result in incorrect result. " +
				"You can increase the state ttl to avoid this.");
		}
		// if reference timestamp list not empty, keep the list
		if (retractList != null && !retractList.isEmpty()) {
			inputState.put(smallestTs, retractList);
		} // if smallest timestamp list is empty, remove and find new smallest
		else {
			inputState.remove(smallestTs);
			Iterator<Long> iter = inputState.keys().iterator();
			long currentTs = 0L;
			long newSmallestTs = Long.MAX_VALUE;
			while (iter.hasNext()) {
				currentTs = iter.next();
				if (currentTs < newSmallestTs) {
					newSmallestTs = currentTs;
				}
			}
			smallestTsState.update(newSmallestTs);
		}
	} // we update the counter only while buffer is getting filled
	else {
		counter += 1;
		counterState.update(counter);
	}

	// update map state, counter and timestamp
	List<RowData> currentTimeState = inputState.get(currentTime);
	if (currentTimeState != null) {
		currentTimeState.add(input);
		inputState.put(currentTime, currentTimeState);
	} else { // add new input
		List<RowData> newList = new ArrayList<RowData>();
		newList.add(input);
		inputState.put(currentTime, newList);
	}

	// accumulate current row
	function.accumulate(input);
	// update the value of accumulators for future incremental computation
	accumulators = function.getAccumulators();
	accState.update(accumulators);

	// prepare output row
	RowData aggValue = function.getValue();
	output.replace(input, aggValue);
	out.collect(output);
}