Java Code Examples for org.apache.flink.util.Collector#collect()
The following examples show how to use
org.apache.flink.util.Collector#collect() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CirculantGraph.java From flink with Apache License 2.0 | 6 votes |
@Override public void flatMap(LongValue source, Collector<Edge<LongValue, NullValue>> out) throws Exception { edge.f0 = source; long sourceID = source.getValue(); for (OffsetRange offsetRange : offsetRanges) { long targetID = sourceID + offsetRange.getOffset(); for (long i = offsetRange.getLength(); i > 0; i--) { // add positive offset target.setValue(targetID++ % vertexCount); out.collect(edge); } } }
Example 2
Source File: FlinkCubingByLayer.java From kylin with Apache License 2.0 | 6 votes |
@Override public void flatMap(Tuple2<ByteArray, Object[]> tuple2, Collector<Tuple2<ByteArray, Object[]>> collector) throws Exception { byte[] key = tuple2.f0.array(); long cuboidId = rowKeySplitter.parseCuboid(key); final List<Long> myChildren = cubeSegment.getCuboidScheduler().getSpanningCuboid(cuboidId); // if still empty or null if (myChildren == null || myChildren.size() == 0) { return; } rowKeySplitter.split(key); final Cuboid parentCuboid = Cuboid.findForMandatory(cubeDesc, cuboidId); for (Long child : myChildren) { Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child); ByteArray result = ndCuboidBuilder.buildKey2(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); collector.collect(new Tuple2<>(result, tuple2.f1)); } }
Example 3
Source File: Graph.java From flink with Apache License 2.0 | 5 votes |
@Override public void join(Tuple4<K, K, VV, EV> tripletWithSrcValSet, Vertex<K, VV> vertex, Collector<Triplet<K, VV, EV>> collector) throws Exception { collector.collect(new Triplet<>(tripletWithSrcValSet.f0, tripletWithSrcValSet.f1, tripletWithSrcValSet.f2, vertex.getValue(), tripletWithSrcValSet.f3)); }
Example 4
Source File: GroupReduceITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple2<Integer, Long>> out) { int i = 0; long l = 0L; for (Tuple3<Integer, Long, String> t : values) { i += t.f0; l = t.f1; } out.collect(new Tuple2<>(i, l)); }
Example 5
Source File: ChainTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void reduce(Iterable<Record> records, Collector<Record> out) throws Exception { if (++this.cnt >= 5) { throw new RuntimeException("Expected Test Exception"); } for (Record r : records) { out.collect(r); } }
Example 6
Source File: CombiningUnilateralSortMergerITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void combine(Iterable<Tuple2<Integer, String>> values, Collector<Tuple2<Integer, String>> out) { Tuple2<Integer, String> rec = new Tuple2<>(); int cnt = 0; for (Tuple2<Integer, String> next : values) { rec = next; cnt += Integer.parseInt(rec.f1); } out.collect(new Tuple2(rec.f0, cnt + "")); }
Example 7
Source File: DrivingSegments.java From flink-training-exercises with Apache License 2.0 | 5 votes |
@Override public void apply(Tuple key, GlobalWindow window, Iterable<ConnectedCarEvent> events, Collector<StoppedSegment> out) { StoppedSegment seg = new StoppedSegment(events); if (seg.length > 0) { out.collect(seg); } }
Example 8
Source File: KeyedCoProcessOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void onTimer( long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception { assertEquals(TimeDomain.EVENT_TIME, ctx.timeDomain()); out.collect("" + 1777); }
Example 9
Source File: CombineTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void reduce(Iterable<Tuple2<Integer, Integer>> records, Collector<Tuple2<Integer, Integer>> out) { int key = 0; int sum = 0; for (Tuple2<Integer, Integer> next : records) { key = next.f0; sum += next.f1; } out.collect(new Tuple2<>(key, sum)); }
Example 10
Source File: RangeBoundaryBuilder.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void mapPartition(Iterable<T> values, Collector<Object[][]> out) throws Exception { final TypeComparator<T> comparator = this.comparatorFactory.createComparator(); List<T> sampledData = new ArrayList<>(); for (T value : values) { sampledData.add(value); } Collections.sort(sampledData, new Comparator<T>() { @Override public int compare(T first, T second) { return comparator.compare(first, second); } }); int boundarySize = parallelism - 1; Object[][] boundaries = new Object[boundarySize][]; if (sampledData.size() > 0) { double avgRange = sampledData.size() / (double) parallelism; int numKey = comparator.getFlatComparators().length; for (int i = 1; i < parallelism; i++) { T record = sampledData.get((int) (i * avgRange)); Object[] keys = new Object[numKey]; comparator.extractKeys(record, keys, 0); boundaries[i-1] = keys; } } out.collect(boundaries); }
Example 11
Source File: VertexDegrees.java From flink with Apache License 2.0 | 5 votes |
@Override public void flatMap(Edge<T, TV> value, Collector<Tuple3<T, T, ByteValue>> out) throws Exception { forward.f0 = value.f0; forward.f1 = value.f1; out.collect(forward); reverse.f0 = value.f1; reverse.f1 = value.f0; out.collect(reverse); }
Example 12
Source File: LegacyStatefulJobSavepointMigrationITCase.java From flink with Apache License 2.0 | 5 votes |
@Override public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception { out.collect(value); ValueState<Long> state = getRuntimeContext().getState(stateDescriptor); if (state == null) { throw new RuntimeException("Missing key value state for " + value); } assertEquals(value.f1, state.value()); getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1); }
Example 13
Source File: Graph.java From flink with Apache License 2.0 | 5 votes |
@Override public void coGroup(Iterable<Edge<K, EV>> edge, Iterable<Edge<K, EV>> edgeToBeRemoved, Collector<Edge<K, EV>> out) throws Exception { if (!edgeToBeRemoved.iterator().hasNext()) { for (Edge<K, EV> next : edge) { out.collect(next); } } }
Example 14
Source File: UdfAnalyzerExamplesTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) { neighbors.clear(); Long id = 0L; for (Tuple2<Long, Long> n : values) { id = n.f0; neighbors.add(n.f1); } out.collect(new Tuple2<Long, Long[]>(id, neighbors.toArray(new Long[neighbors.size()]))); }
Example 15
Source File: AggregatorsITCase.java From flink with Apache License 2.0 | 5 votes |
@Override public void flatMap(Tuple2<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> value, Collector<Tuple2<Integer, Integer>> out) { if (value.f0.f1 > superstep) { out.collect(value.f0); } }
Example 16
Source File: StreamingETL.java From flink-streaming-etl with Apache License 2.0 | 4 votes |
@Override public void apply(String key, TimeWindow timeWindow, Iterable<Tuple3<Long, String, Long>> iterable, Collector<Tuple3<Long, String, Long>> collector) throws Exception { long count = iterable.iterator().next().f0; collector.collect(Tuple3.of(count, key, timeWindow.getStart())); }
Example 17
Source File: KeyedStateInputFormatTest.java From flink with Apache License 2.0 | 4 votes |
@Override public void readKey(Integer key, KeyedStateReaderFunction.Context ctx, Collector<Integer> out) throws Exception { ValueState<Integer> state = getRuntimeContext().getState(stateDescriptor); out.collect(state.value()); }
Example 18
Source File: AbstractOuterJoinTaskExternalITCase.java From flink with Apache License 2.0 | 4 votes |
@Override public void join(Tuple2<Integer, Integer> first, Tuple2<Integer, Integer> second, Collector<Tuple2<Integer, Integer>> out) throws Exception { out.collect(first != null ? first : second); }
Example 19
Source File: CachedMatchTaskTest.java From flink with Apache License 2.0 | 4 votes |
@Override public void join(Record record1, Record record2, Collector<Record> out) throws Exception { out.collect(record1); }
Example 20
Source File: ProcTimeRowsBoundedPrecedingFunction.java From flink with Apache License 2.0 | 4 votes |
@Override public void processElement( RowData input, KeyedProcessFunction<K, RowData, RowData>.Context ctx, Collector<RowData> out) throws Exception { long currentTime = ctx.timerService().currentProcessingTime(); // register state-cleanup timer registerProcessingCleanupTimer(ctx, currentTime); // initialize state for the processed element RowData accumulators = accState.value(); if (accumulators == null) { accumulators = function.createAccumulators(); } // set accumulators in context first function.setAccumulators(accumulators); // get smallest timestamp Long smallestTs = smallestTsState.value(); if (smallestTs == null) { smallestTs = currentTime; smallestTsState.update(smallestTs); } // get previous counter value Long counter = counterState.value(); if (counter == null) { counter = 0L; } if (counter == precedingOffset) { List<RowData> retractList = inputState.get(smallestTs); if (retractList != null) { // get oldest element beyond buffer size // and if oldest element exist, retract value RowData retractRow = retractList.get(0); function.retract(retractRow); retractList.remove(0); } else { // Does not retract values which are outside of window if the state is cleared already. LOG.warn("The state is cleared because of state ttl. " + "This will result in incorrect result. " + "You can increase the state ttl to avoid this."); } // if reference timestamp list not empty, keep the list if (retractList != null && !retractList.isEmpty()) { inputState.put(smallestTs, retractList); } // if smallest timestamp list is empty, remove and find new smallest else { inputState.remove(smallestTs); Iterator<Long> iter = inputState.keys().iterator(); long currentTs = 0L; long newSmallestTs = Long.MAX_VALUE; while (iter.hasNext()) { currentTs = iter.next(); if (currentTs < newSmallestTs) { newSmallestTs = currentTs; } } smallestTsState.update(newSmallestTs); } } // we update the counter only while buffer is getting filled else { counter += 1; counterState.update(counter); } // update map state, counter and timestamp List<RowData> currentTimeState = inputState.get(currentTime); if (currentTimeState != null) { currentTimeState.add(input); inputState.put(currentTime, currentTimeState); } else { // add new input List<RowData> newList = new ArrayList<RowData>(); newList.add(input); inputState.put(currentTime, newList); } // accumulate current row function.accumulate(input); // update the value of accumulators for future incremental computation accumulators = function.getAccumulators(); accState.update(accumulators); // prepare output row RowData aggValue = function.getValue(); output.replace(input, aggValue); out.collect(output); }