org.apache.flink.api.common.functions.util.ListCollector Java Examples

The following examples show how to use org.apache.flink.api.common.functions.util.ListCollector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FilterOperatorBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<T> result = new ArrayList<T>(inputData.size());
	ListCollector<T> collector = new ListCollector<T>(result);

	for (T element : inputData) {
		function.flatMap(element, collector);
	}
	
	FunctionUtils.closeFunction(function);
	
	return result;
}
 
Example #2
Source File: AbstractSortMergeOuterJoinIteratorITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked, rawtypes")
private List<Tuple4<String, String, String, Object>> computeOuterJoin(ResettableMutableObjectIterator<Tuple2<String, String>> input1,
																	  ResettableMutableObjectIterator<Tuple2<String, Integer>> input2,
																	  OuterJoinType outerJoinType) throws Exception {
	input1.reset();
	input2.reset();
	AbstractMergeOuterJoinIterator iterator =
			createOuterJoinIterator(
					outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2,
					pairComp, this.memoryManager, this.ioManager, PAGES_FOR_BNLJN, this.parentTask
			);

	List<Tuple4<String, String, String, Object>> actual = new ArrayList<>();
	ListCollector<Tuple4<String, String, String, Object>> collector = new ListCollector<>(actual);
	while (iterator.callWithNextKey(new SimpleTupleJoinFunction(), collector)) ;
	iterator.close();

	return actual;
}
 
Example #3
Source File: FilterOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<T> result = new ArrayList<T>(inputData.size());
	ListCollector<T> collector = new ListCollector<T>(result);

	for (T element : inputData) {
		function.flatMap(element, collector);
	}
	
	FunctionUtils.closeFunction(function);
	
	return result;
}
 
Example #4
Source File: AbstractSortMergeOuterJoinIteratorITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked, rawtypes")
private List<Tuple4<String, String, String, Object>> computeOuterJoin(ResettableMutableObjectIterator<Tuple2<String, String>> input1,
																	  ResettableMutableObjectIterator<Tuple2<String, Integer>> input2,
																	  OuterJoinType outerJoinType) throws Exception {
	input1.reset();
	input2.reset();
	AbstractMergeOuterJoinIterator iterator =
			createOuterJoinIterator(
					outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2,
					pairComp, this.memoryManager, this.ioManager, PAGES_FOR_BNLJN, this.parentTask
			);

	List<Tuple4<String, String, String, Object>> actual = new ArrayList<>();
	ListCollector<Tuple4<String, String, String, Object>> collector = new ListCollector<>(actual);
	while (iterator.callWithNextKey(new SimpleTupleJoinFunction(), collector)) ;
	iterator.close();

	return actual;
}
 
Example #5
Source File: FilterOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<T> result = new ArrayList<T>(inputData.size());
	ListCollector<T> collector = new ListCollector<T>(result);

	for (T element : inputData) {
		function.flatMap(element, collector);
	}
	
	FunctionUtils.closeFunction(function);
	
	return result;
}
 
Example #6
Source File: AbstractSortMergeOuterJoinIteratorITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked, rawtypes")
private List<Tuple4<String, String, String, Object>> computeOuterJoin(ResettableMutableObjectIterator<Tuple2<String, String>> input1,
																	  ResettableMutableObjectIterator<Tuple2<String, Integer>> input2,
																	  OuterJoinType outerJoinType) throws Exception {
	input1.reset();
	input2.reset();
	AbstractMergeOuterJoinIterator iterator =
			createOuterJoinIterator(
					outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2,
					pairComp, this.memoryManager, this.ioManager, PAGES_FOR_BNLJN, this.parentTask
			);

	List<Tuple4<String, String, String, Object>> actual = new ArrayList<>();
	ListCollector<Tuple4<String, String, String, Object>> collector = new ListCollector<>(actual);
	while (iterator.callWithNextKey(new SimpleTupleJoinFunction(), collector)) ;
	iterator.close();

	return actual;
}
 
Example #7
Source File: TBaseFlatMapperTest.java    From pinpoint with Apache License 2.0 5 votes vote down vote up
@Test
public void flatMapTest() throws Exception {

    ApplicationCache applicationCache = newMockApplicationCache();
    TBaseFlatMapper mapper = new TBaseFlatMapper(new JoinAgentStatBoMapper(), applicationCache, new DefaultTBaseFlatMapperInterceptor());


    TFAgentStatBatch tfAgentStatBatch = createTFAgentStatBatch();
    ArrayList<Tuple3<String, JoinStatBo, Long>> dataList = new ArrayList<>();
    ListCollector<Tuple3<String, JoinStatBo, Long>> collector = new ListCollector<>(dataList);
    RawData rawData = newRawData(tfAgentStatBatch);
    mapper.flatMap(rawData, collector);

    assertEquals(dataList.size(), 2);

    Tuple3<String, JoinStatBo, Long> data1 = dataList.get(0);
    assertEquals(data1.f0, AGENT_ID);
    assertEquals(data1.f2.longValue(), 1491274143454L);
    JoinAgentStatBo joinAgentStatBo = (JoinAgentStatBo) data1.f1;
    assertEquals(joinAgentStatBo.getId(), AGENT_ID);
    assertEquals(joinAgentStatBo.getAgentStartTimestamp(), 1491274142454L);
    assertEquals(joinAgentStatBo.getTimestamp(), 1491274143454L);
    assertJoinCpuLoadBo(joinAgentStatBo.getJoinCpuLoadBoList());

    Tuple3<String, JoinStatBo, Long> data2 = dataList.get(1);
    assertEquals(data2.f0, APPLICATION_ID);
    assertEquals(data2.f2.longValue(), 1491274140000L);
    JoinApplicationStatBo joinApplicationStatBo = (JoinApplicationStatBo) data2.f1;
    assertEquals(joinApplicationStatBo.getId(), APPLICATION_ID);
    assertEquals(joinApplicationStatBo.getTimestamp(), 1491274140000L);
    assertEquals(joinApplicationStatBo.getStatType(), StatType.APP_STST);
    assertJoinCpuLoadBo(joinApplicationStatBo.getJoinCpuLoadBoList());
}
 
Example #8
Source File: TBaseFlatMapperTest.java    From pinpoint with Apache License 2.0 5 votes vote down vote up
@Test
public void flatMap2Test() throws Exception {
    ApplicationCache applicationCache = newMockApplicationCache();
    TBaseFlatMapper mapper = new TBaseFlatMapper(new JoinAgentStatBoMapper(), applicationCache, new DefaultTBaseFlatMapperInterceptor());

    TFAgentStatBatch tfAgentStatBatch = createTFAgentStatBatch2();
    ArrayList<Tuple3<String, JoinStatBo, Long>> dataList = new ArrayList<>();
    ListCollector<Tuple3<String, JoinStatBo, Long>> collector = new ListCollector<>(dataList);
    RawData rawdata = newRawData(tfAgentStatBatch);
    mapper.flatMap(rawdata, collector);

    assertEquals(dataList.size(), 2);

    Tuple3<String, JoinStatBo, Long> data1 = dataList.get(0);
    assertEquals(data1.f0, AGENT_ID);
    assertEquals(data1.f2.longValue(), 1491274143454L);
    JoinAgentStatBo joinAgentStatBo = (JoinAgentStatBo) data1.f1;
    assertEquals(joinAgentStatBo.getId(), AGENT_ID);
    assertEquals(joinAgentStatBo.getAgentStartTimestamp(), 1491274142454L);
    assertEquals(joinAgentStatBo.getTimestamp(), 1491274143454L);
    assertJoinMemoryBo(joinAgentStatBo.getJoinMemoryBoList());

    Tuple3<String, JoinStatBo, Long> data2 = dataList.get(1);
    assertEquals(data2.f0, APPLICATION_ID);
    assertEquals(data2.f2.longValue(), 1491274140000L);
    JoinApplicationStatBo joinApplicationStatBo = (JoinApplicationStatBo) data2.f1;
    assertEquals(joinApplicationStatBo.getId(), APPLICATION_ID);
    assertEquals(joinApplicationStatBo.getTimestamp(), 1491274140000L);
    assertEquals(joinApplicationStatBo.getStatType(), StatType.APP_STST);
    assertJoinMemoryBo(joinApplicationStatBo.getJoinMemoryBoList());
}
 
Example #9
Source File: TBaseFlatMapperTest.java    From pinpoint with Apache License 2.0 5 votes vote down vote up
@Test
public void flatMap3Test() throws Exception {
    ApplicationCache applicationCache = newMockApplicationCache();
    TBaseFlatMapper mapper = new TBaseFlatMapper(new JoinAgentStatBoMapper(), applicationCache, new DefaultTBaseFlatMapperInterceptor());

    TFAgentStatBatch tfAgentStatBatch = createTFAgentStatBatch3();
    ArrayList<Tuple3<String, JoinStatBo, Long>> dataList = new ArrayList<>();
    ListCollector<Tuple3<String, JoinStatBo, Long>> collector = new ListCollector<>(dataList);
    RawData rawData = newRawData(tfAgentStatBatch);
    mapper.flatMap(rawData, collector);

    assertEquals(dataList.size(), 2);

    Tuple3<String, JoinStatBo, Long> data1 = dataList.get(0);
    assertEquals(data1.f0, AGENT_ID);
    assertEquals(data1.f2.longValue(), 1491274143454L);
    JoinAgentStatBo joinAgentStatBo = (JoinAgentStatBo) data1.f1;
    assertEquals(joinAgentStatBo.getId(), AGENT_ID);
    assertEquals(joinAgentStatBo.getAgentStartTimestamp(), 1491274142454L);
    assertEquals(joinAgentStatBo.getTimestamp(), 1491274143454L);
    assertJoinTransactionBo(joinAgentStatBo.getJoinTransactionBoList());

    Tuple3<String, JoinStatBo, Long> data2 = dataList.get(1);
    assertEquals(data2.f0, APPLICATION_ID);
    assertEquals(data2.f2.longValue(), 1491274140000L);
    JoinApplicationStatBo joinApplicationStatBo = (JoinApplicationStatBo) data2.f1;
    assertEquals(joinApplicationStatBo.getId(), APPLICATION_ID);
    assertEquals(joinApplicationStatBo.getTimestamp(), 1491274140000L);
    assertEquals(joinApplicationStatBo.getStatType(), StatType.APP_STST);
    assertJoinTransactionBo(joinApplicationStatBo.getJoinTransactionBoList());
}
 
Example #10
Source File: TBaseFlatMapperTest.java    From pinpoint with Apache License 2.0 5 votes vote down vote up
@Test
public void flatMap4Test() throws Exception {

    ApplicationCache applicationCache = newMockApplicationCache();
    TBaseFlatMapper mapper = new TBaseFlatMapper(new JoinAgentStatBoMapper(), applicationCache, new DefaultTBaseFlatMapperInterceptor());


    TFAgentStatBatch tfAgentStatBatch = createTFAgentStatBatch4();
    ArrayList<Tuple3<String, JoinStatBo, Long>> dataList = new ArrayList<>();
    ListCollector<Tuple3<String, JoinStatBo, Long>> collector = new ListCollector<>(dataList);
    RawData rawData = newRawData(tfAgentStatBatch);
    mapper.flatMap(rawData, collector);

    assertEquals(dataList.size(), 2);

    Tuple3<String, JoinStatBo, Long> data1 = dataList.get(0);
    assertEquals(data1.f0, AGENT_ID);
    assertEquals(data1.f2.longValue(), 1491274143454L);
    JoinAgentStatBo joinAgentStatBo = (JoinAgentStatBo) data1.f1;
    assertEquals(joinAgentStatBo.getId(), AGENT_ID);
    assertEquals(joinAgentStatBo.getAgentStartTimestamp(), 1491274142454L);
    assertEquals(joinAgentStatBo.getTimestamp(), 1491274143454L);
    assertJoinFileDescriptorBo(joinAgentStatBo.getJoinFileDescriptorBoList());

    Tuple3<String, JoinStatBo, Long> data2 = dataList.get(1);
    assertEquals(data2.f0, APPLICATION_ID);
    assertEquals(data2.f2.longValue(), 1491274140000L);
    JoinApplicationStatBo joinApplicationStatBo = (JoinApplicationStatBo) data2.f1;
    assertEquals(joinApplicationStatBo.getId(), APPLICATION_ID);
    assertEquals(joinApplicationStatBo.getTimestamp(), 1491274140000L);
    assertEquals(joinApplicationStatBo.getStatType(), StatType.APP_STST);
    assertJoinFileDescriptorBo(joinApplicationStatBo.getJoinFileDescriptorBoList());
}
 
Example #11
Source File: TBaseFlatMapperTest.java    From pinpoint with Apache License 2.0 5 votes vote down vote up
@Test
public void flatMap5Test() throws Exception {

    ApplicationCache applicationCache = newMockApplicationCache();
    TBaseFlatMapper mapper = new TBaseFlatMapper(new JoinAgentStatBoMapper(), applicationCache, new DefaultTBaseFlatMapperInterceptor());


    TFAgentStatBatch tfAgentStatBatch = createTFAgentStatBatch5();
    ArrayList<Tuple3<String, JoinStatBo, Long>> dataList = new ArrayList<>();
    ListCollector<Tuple3<String, JoinStatBo, Long>> collector = new ListCollector<>(dataList);
    RawData rawData = newRawData(tfAgentStatBatch);
    mapper.flatMap(rawData, collector);

    assertEquals(dataList.size(), 2);

    Tuple3<String, JoinStatBo, Long> data1 = dataList.get(0);
    assertEquals(data1.f0, AGENT_ID);
    assertEquals(data1.f2.longValue(), 1491274143454L);
    JoinAgentStatBo joinAgentStatBo = (JoinAgentStatBo) data1.f1;
    assertEquals(joinAgentStatBo.getId(), AGENT_ID);
    assertEquals(joinAgentStatBo.getAgentStartTimestamp(), 1491274142454L);
    assertEquals(joinAgentStatBo.getTimestamp(), 1491274143454L);
    assertJoinDirectBufferBo(joinAgentStatBo.getJoinDirectBufferBoList());

    Tuple3<String, JoinStatBo, Long> data2 = dataList.get(1);
    assertEquals(data2.f0, APPLICATION_ID);
    assertEquals(data2.f2.longValue(), 1491274140000L);
    JoinApplicationStatBo joinApplicationStatBo = (JoinApplicationStatBo) data2.f1;
    assertEquals(joinApplicationStatBo.getId(), APPLICATION_ID);
    assertEquals(joinApplicationStatBo.getTimestamp(), 1491274140000L);
    assertEquals(joinApplicationStatBo.getStatType(), StatType.APP_STST);
    assertJoinDirectBufferBo(joinApplicationStatBo.getJoinDirectBufferBoList());
}