Java Code Examples for org.apache.flink.api.common.typeinfo.BasicTypeInfo

The following examples show how to use org.apache.flink.api.common.typeinfo.BasicTypeInfo. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: RowSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRowSerializer() {
	TypeInformation<Row> typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);
	Row row1 = new Row(2);
	row1.setKind(RowKind.UPDATE_BEFORE);
	row1.setField(0, 1);
	row1.setField(1, "a");

	Row row2 = new Row(2);
	row2.setKind(RowKind.INSERT);
	row2.setField(0, 2);
	row2.setField(1, null);

	TypeSerializer<Row> serializer = typeInfo.createSerializer(new ExecutionConfig());
	RowSerializerTestInstance instance = new RowSerializerTestInstance(serializer, row1, row2);
	instance.testAll();
}
 
Example 2
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that checkpoints are declined if operators are (partially) closed.
 *
 * <p>See FLINK-16383.
 */
@Test
public void testCheckpointDeclinedOnClosedOperator() throws Throwable {
	ClosingOperator operator = new ClosingOperator();
	MultipleInputStreamTaskTestHarnessBuilder<Integer> builder =
		new MultipleInputStreamTaskTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.INT_TYPE_INFO)
				.addInput(BasicTypeInfo.INT_TYPE_INFO);
	StreamTaskMailboxTestHarness<Integer> harness = builder
		.setupOutputForSingletonOperatorChain(operator)
		.build();
	// keeps the mailbox from suspending
	harness.setAutoProcess(false);
	harness.processElement(new StreamRecord<>(1));

	harness.streamTask.operatorChain.closeOperators(harness.streamTask.getActionExecutor());
	assertEquals(true, operator.closed.get());

	harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(1, 0), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetrics());
	assertEquals(1, harness.getCheckpointResponder().getDeclineReports().size());
}
 
Example 3
Source Project: flink   Source File: StreamSourceOperatorWatermarksTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {
	StreamSource<String, ?> sourceOperator = new StreamSource<>(new InfiniteSource<>());
	StreamTaskTestHarness<String> testHarness = setupSourceStreamTask(
		sourceOperator, BasicTypeInfo.STRING_TYPE_INFO, true);

	testHarness.invoke();
	try {
		testHarness.waitForTaskCompletion();
		fail("should throw an exception");
	} catch (Throwable t) {
		if (!ExceptionUtils.findThrowable(t, CancelTaskException.class).isPresent()) {
			throw t;
		}
	}
	assertTrue(testHarness.getOutput().isEmpty());
}
 
Example 4
Source Project: flink   Source File: LegacyKeyedProcessOperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullOutputTagRefusal() throws Exception {
	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
		new LegacyKeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(17);
	try {
		expectedException.expect(IllegalArgumentException.class);
		testHarness.processElement(new StreamRecord<>(5));
	} finally {
		testHarness.close();
	}
}
 
Example 5
Source Project: flink   Source File: StreamTaskTimerTest.java    License: Apache License 2.0 6 votes vote down vote up
private StreamTaskTestHarness<?> startTestHarness() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setChainIndex(0);
	streamConfig.setStreamOperator(new StreamMap<String, String>(new DummyMapFunction<>()));

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	return testHarness;
}
 
Example 6
Source Project: Flink-CEPplus   Source File: TypeHintITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCombineGroupWithTypeInformationTypeHint() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Integer> resultDs = ds
		.groupBy(0)
		.combineGroup(new GroupCombiner<Tuple3<Integer, Long, String>, Integer>())
		.returns(BasicTypeInfo.INT_TYPE_INFO);
	List<Integer> result = resultDs.collect();

	String expectedResult = "2\n" +
		"3\n" +
		"1\n";

	compareResultAsText(result, expectedResult);
}
 
Example 7
Source Project: flink   Source File: RowTypeInfoTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetFlatFields() {
	RowTypeInfo typeInfo1 = new RowTypeInfo(typeList, new String[]{"int", "row", "string"});
	List<FlatFieldDescriptor> result = new ArrayList<>();
	typeInfo1.getFlatFields("row.*", 0, result);
	assertEquals(2, result.size());
	assertEquals(
		new FlatFieldDescriptor(1, BasicTypeInfo.SHORT_TYPE_INFO).toString(),
		result.get(0).toString());
	assertEquals(
		new FlatFieldDescriptor(2, BasicTypeInfo.BIG_DEC_TYPE_INFO).toString(),
		result.get(1).toString());

	result.clear();
	typeInfo1.getFlatFields("string", 0, result);
	assertEquals(1, result.size());
	assertEquals(
		new FlatFieldDescriptor(3, BasicTypeInfo.STRING_TYPE_INFO).toString(),
		result.get(0).toString());
}
 
Example 8
Source Project: flink   Source File: MultipleInputStreamTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLatencyMarker() throws Exception {
	final Map<String, Metric> metrics = new ConcurrentHashMap<>();
	final TaskMetricGroup taskMetricGroup = new StreamTaskTestHarness.TestTaskMetricGroup(metrics);

	try (StreamTaskMailboxTestHarness<String> testHarness =
			new MultipleInputStreamTaskTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO)
				.addInput(BasicTypeInfo.STRING_TYPE_INFO)
				.addInput(BasicTypeInfo.INT_TYPE_INFO)
				.addInput(BasicTypeInfo.DOUBLE_TYPE_INFO)
				.setupOutputForSingletonOperatorChain(new MapToStringMultipleInputOperatorFactory())
				.setTaskMetricGroup(taskMetricGroup)
				.build()) {
		ArrayDeque<Object> expectedOutput = new ArrayDeque<>();

		OperatorID sourceId = new OperatorID();
		LatencyMarker latencyMarker = new LatencyMarker(42L, sourceId, 0);
		testHarness.processElement(latencyMarker);
		expectedOutput.add(latencyMarker);

		assertThat(testHarness.getOutput(), contains(expectedOutput.toArray()));

		testHarness.endInput();
		testHarness.waitForTaskCompletion();
	}
}
 
Example 9
Source Project: flink   Source File: TypeHintITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUnsortedGroupReduceWithTypeInformationTypeHint() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Integer> resultDs = ds
		.groupBy(0)
		.reduceGroup(new GroupReducer<Tuple3<Integer, Long, String>, Integer>())
		.returns(BasicTypeInfo.INT_TYPE_INFO);
	List<Integer> result = resultDs.collect();

	String expectedResult = "2\n" +
		"3\n" +
		"1\n";

	compareResultAsText(result, expectedResult);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: EitherSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testEitherWithTuple() {

Either<Tuple2<Long, Long>, Double>[] testData = new Either[] {
		Either.Left(new Tuple2<>(2L, 9L)),
		new Left<>(new Tuple2<>(Long.MIN_VALUE, Long.MAX_VALUE)),
		new Right<>(32.0),
		Right(Double.MIN_VALUE),
		Right(Double.MAX_VALUE)};

EitherTypeInfo<Tuple2<Long, Long>, Double> eitherTypeInfo = (EitherTypeInfo<Tuple2<Long, Long>, Double>)
		new EitherTypeInfo<Tuple2<Long, Long>, Double>(
		new TupleTypeInfo<Tuple2<Long, Long>>(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO),
		BasicTypeInfo.DOUBLE_TYPE_INFO);
EitherSerializer<Tuple2<Long, Long>, Double> eitherSerializer =
		(EitherSerializer<Tuple2<Long, Long>, Double>) eitherTypeInfo.createSerializer(new ExecutionConfig());
SerializerTestInstance<Either<Tuple2<Long, Long>, Double>> testInstance =
		new EitherSerializerTestInstance<Either<Tuple2<Long, Long>, Double>>(
				eitherSerializer, eitherTypeInfo.getTypeClass(), -1, testData);
testInstance.testAll();
}
 
Example 11
Source Project: Flink-CEPplus   Source File: PojoTypeExtractionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGenericPojoTypeInference1() {
	MyMapper<String> function = new MyMapper<>();

	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(
			function,
			TypeInformation.of(new TypeHint<PojoWithGenerics<Long, String>>(){}));
	
	Assert.assertTrue(ti instanceof PojoTypeInfo<?>);
	PojoTypeInfo<?> pti = (PojoTypeInfo<?>) ti;
	for(int i = 0; i < pti.getArity(); i++) {
		PojoField field = pti.getPojoFieldAt(i);
		String name = field.getField().getName();
		if(name.equals("field1")) {
			Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.getTypeInformation());
		} else if (name.equals("field2")) {
			Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.getTypeInformation());
		} else if (name.equals("key")) {
			Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.getTypeInformation());
		} else {
			Assert.fail("Unexpected field "+field);
		}
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: FromElementsFunctionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStrings() {
	try {
		String[] data = { "Oh", "boy", "what", "a", "show", "!"};

		FromElementsFunction<String> source = new FromElementsFunction<String>(
				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), data);

		List<String> result = new ArrayList<String>();
		source.run(new ListSourceContext<String>(result));

		assertEquals(Arrays.asList(data), result);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 13
Source Project: flink   Source File: ExpressionKeysTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStandardTupleKeys() {
	TupleTypeInfo<Tuple7<String, String, String, String, String, String, String>> typeInfo = new TupleTypeInfo<>(
			BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO);
	
	ExpressionKeys<Tuple7<String, String, String, String, String, String, String>> ek;
	
	for( int i = 1; i < 8; i++) {
		int[] ints = new int[i];
		for( int j = 0; j < i; j++) {
			ints[j] = j;
		}
		int[] inInts = Arrays.copyOf(ints, ints.length); // copy, just to make sure that the code is not cheating by changing the ints.
		ek = new ExpressionKeys<>(inInts, typeInfo);
		Assert.assertArrayEquals(ints, ek.computeLogicalKeyPositions());
		Assert.assertEquals(ints.length, ek.computeLogicalKeyPositions().length);
		
		ArrayUtils.reverse(ints);
		inInts = Arrays.copyOf(ints, ints.length);
		ek = new ExpressionKeys<>(inInts, typeInfo);
		Assert.assertArrayEquals(ints, ek.computeLogicalKeyPositions());
		Assert.assertEquals(ints.length, ek.computeLogicalKeyPositions().length);
	}
}
 
Example 14
Source Project: flink   Source File: ImmutableMapStateTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	if (!mapStateDesc.isSerializerInitialized()) {
		mapStateDesc.initializeSerializerUnlessSet(new ExecutionConfig());
	}

	Map<Long, Long> initMap = new HashMap<>();
	initMap.put(1L, 5L);
	initMap.put(2L, 5L);

	byte[] initSer = KvStateSerializer.serializeMap(
			initMap.entrySet(),
			BasicTypeInfo.LONG_TYPE_INFO.createSerializer(new ExecutionConfig()),
			BasicTypeInfo.LONG_TYPE_INFO.createSerializer(new ExecutionConfig()));

	mapState = ImmutableMapState.createState(mapStateDesc, initSer);
}
 
Example 15
Source Project: flink   Source File: SelectorFunctionKeysTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAreCompatible4() throws Keys.IncompatibleKeysException {
	TypeInformation<Tuple3<String, Long, Integer>> t1 = new TupleTypeInfo<>(
		BasicTypeInfo.STRING_TYPE_INFO,
		BasicTypeInfo.LONG_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO
	);
	TypeInformation<PojoWithMultiplePojos> t2 = TypeExtractor.getForClass(PojoWithMultiplePojos.class);

	Keys.ExpressionKeys<Tuple3<String, Long, Integer>> ek1 = new Keys.ExpressionKeys<>(new int[]{2,0}, t1);
	Keys<PojoWithMultiplePojos> sk2 = new Keys.SelectorFunctionKeys<>(
		new KeySelector3(),
		t2,
		new TupleTypeInfo<Tuple2<Integer, String>>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO)
	);

	Assert.assertTrue(sk2.areCompatible(ek1));
}
 
Example 16
Source Project: Flink-CEPplus   Source File: PojoTypeExtractionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test if the TypeExtractor is accepting untyped generics,
 * making them GenericTypes
 */
@Test
public void testPojoWithGenericsSomeFieldsGeneric() {
	TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(PojoWithGenerics.class);
	Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
	PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
	for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
		PojoField field = pojoTypeForClass.getPojoFieldAt(i);
		String name = field.getField().getName();
		if(name.equals("field1")) {
			Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.getTypeInformation());
		} else if (name.equals("field2")) {
			Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.getTypeInformation());
		} else if (name.equals("key")) {
			Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.getTypeInformation());
		} else {
			Assert.fail("Unexpected field "+field);
		}
	}
}
 
Example 17
Source Project: Flink-CEPplus   Source File: WindowTranslationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testReduceProcessingTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.reduce(new DummyReducer());

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example 18
Source Project: flink   Source File: SourceStreamTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test verifies that open() and close() are correctly called by the StreamTask.
 */
@Test
@SuppressWarnings("unchecked")
public void testOpenClose() throws Exception {
	final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(
			SourceStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource());
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskCompletion();

	assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled);

	List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
	Assert.assertEquals(10, resultElements.size());
}
 
Example 19
Source Project: Flink-CEPplus   Source File: SelectorFunctionKeysTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAreCompatible2() throws Keys.IncompatibleKeysException {
	TypeInformation<PojoWithMultiplePojos> t1 = TypeExtractor.getForClass(PojoWithMultiplePojos.class);
	TypeInformation<Tuple3<Long, Pojo1, Integer>> t2 = new TupleTypeInfo<>(
		BasicTypeInfo.LONG_TYPE_INFO,
		TypeExtractor.getForClass(Pojo1.class),
		BasicTypeInfo.INT_TYPE_INFO);
	TypeInformation<Tuple2<Integer, String>> kt = new TupleTypeInfo<>(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO
	);

	Keys<PojoWithMultiplePojos> k1 = new Keys.SelectorFunctionKeys<>(
		new KeySelector3(),
		t1,
		kt
	);
	Keys<Tuple3<Long, Pojo1, Integer>> k2 = new Keys.SelectorFunctionKeys<>(
		new KeySelector4(),
		t2,
		kt
	);

	Assert.assertTrue(k1.areCompatible(k2));
	Assert.assertTrue(k2.areCompatible(k1));
}
 
Example 20
Source Project: flink   Source File: CollectionDataSets.java    License: Apache License 2.0 6 votes vote down vote up
public static DataSet<Tuple5<Integer, Long, Integer, String, Long>> getSmall5TupleDataSet(ExecutionEnvironment env) {

		List<Tuple5<Integer, Long, Integer, String, Long>> data = new ArrayList<>();
		data.add(new Tuple5<>(1, 1L, 0, "Hallo", 1L));
		data.add(new Tuple5<>(2, 2L, 1, "Hallo Welt", 2L));
		data.add(new Tuple5<>(2, 3L, 2, "Hallo Welt wie", 1L));

		Collections.shuffle(data);

		TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>> type = new TupleTypeInfo<>(
				BasicTypeInfo.INT_TYPE_INFO,
				BasicTypeInfo.LONG_TYPE_INFO,
				BasicTypeInfo.INT_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.LONG_TYPE_INFO
		);

		return env.fromCollection(data, type);
	}
 
Example 21
Source Project: flink   Source File: TypeExtractorTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testSameGenericVariable() {
	RichMapFunction<?, ?> function = new RichMapFunction<SameTypeVariable<String>, SameTypeVariable<String>>() {
		private static final long serialVersionUID = 1L;

		@Override
		public SameTypeVariable<String> map(SameTypeVariable<String> value) throws Exception {
			return null;
		}
	};

	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, (TypeInformation) TypeInformation.of(new TypeHint<Tuple2<String, String>>(){}));

	Assert.assertTrue(ti.isTupleType());
	Assert.assertEquals(2, ti.getArity());
	
	TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
	Assert.assertEquals(SameTypeVariable.class, tti.getTypeClass());
	
	Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(1));
}
 
Example 22
Source Project: Flink-CEPplus   Source File: RowTypeInfoTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaEquals() {
	final RowTypeInfo row1 = new RowTypeInfo(
		new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO},
		new String[] {"field1", "field2"});
	final RowTypeInfo row2 = new RowTypeInfo(
		new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO},
		new String[] {"field1", "field2"});
	assertTrue(row1.schemaEquals(row2));

	final RowTypeInfo other1 = new RowTypeInfo(
		new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO},
		new String[] {"otherField", "field2"});
	final RowTypeInfo other2 = new RowTypeInfo(
		new TypeInformation[]{BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO},
		new String[] {"field1", "field2"});
	assertFalse(row1.schemaEquals(other1));
	assertFalse(row1.schemaEquals(other2));
}
 
Example 23
Source Project: flink   Source File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testReduceWithProcessWindowFunctionProcessingTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window = source
			.keyBy(new TupleKeySelector())
			.window(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.reduce(new DummyReducer(), new ProcessWindowFunction<Tuple2<String, Integer>, Tuple3<String, String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String tuple,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple3<String, String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(new Tuple3<>(in.f0, in.f0, in.f1));
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor);

	processElementAndEnsureOutput(operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example 24
Source Project: flink   Source File: RowCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReadSparseWithMask() throws Exception {
	String fileContent = "111&&222&&333&&444&&555&&666&&777&&888&&999&&000&&\n" +
		"000&&999&&888&&777&&666&&555&&444&&333&&222&&111&&";

	FileInputSplit split = RowCsvInputFormatTest.createTempFile(fileContent);

	TypeInformation[] fieldTypes = new TypeInformation[]{
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO};

	RowCsvInputFormat format = new RowCsvInputFormat(
		PATH,
		fieldTypes,
		new int[]{0, 3, 7});
	format.setFieldDelimiter("&&");
	format.configure(new Configuration());
	format.open(split);

	Row result = new Row(3);

	result = format.nextRecord(result);
	assertNotNull(result);
	assertEquals(111, result.getField(0));
	assertEquals(444, result.getField(1));
	assertEquals(888, result.getField(2));

	result = format.nextRecord(result);
	assertNotNull(result);
	assertEquals(0, result.getField(0));
	assertEquals(777, result.getField(1));
	assertEquals(333, result.getField(2));

	result = format.nextRecord(result);
	assertNull(result);
	assertTrue(format.reachedEnd());
}
 
Example 25
Source Project: flink   Source File: KeyedCoProcessOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	KeyedCoProcessOperator<String, Integer, String, String> operator =
			new KeyedCoProcessOperator<>(new WatermarkQueryingProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					operator,
					new IntToStringKeySelector<>(),
					new IdentityKeySelector<String>(),
					BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark1(new Watermark(17));
	testHarness.processWatermark2(new Watermark(17));
	testHarness.processElement1(new StreamRecord<>(5, 12L));

	testHarness.processWatermark1(new Watermark(42));
	testHarness.processWatermark2(new Watermark(42));
	testHarness.processElement2(new StreamRecord<>("6", 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 
Example 26
Source Project: flink   Source File: RowSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLargeRowSerializer() {
	TypeInformation<Row> typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	Row row = new Row(13);
	row.setField(0, 2);
	row.setField(1, null);
	row.setField(3, null);
	row.setField(4, null);
	row.setField(5, null);
	row.setField(6, null);
	row.setField(7, null);
	row.setField(8, null);
	row.setField(9, null);
	row.setField(10, null);
	row.setField(11, null);
	row.setField(12, "Test");

	TypeSerializer<Row> serializer = typeInfo.createSerializer(new ExecutionConfig());
	RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, row);
	testInstance.testAll();
}
 
Example 27
Source Project: flink   Source File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAggregateWithWindowFunctionEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple3<String, String, Integer>> source = env.fromElements(
		Tuple3.of("hello", "hallo", 1),
		Tuple3.of("hello", "hallo", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<String> window = source
			.keyBy(new Tuple3KeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.aggregate(new DummyAggregationFunction(), new TestWindowFunction());

	final OneInputTransformation<Tuple3<String, String, Integer>, String> transform =
		(OneInputTransformation<Tuple3<String, String, Integer>, String>) window.getTransformation();

	final OneInputStreamOperator<Tuple3<String, String, Integer>, String> operator = transform.getOperator();

	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?> winOperator =
		(WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?>) operator;

	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof AggregatingStateDescriptor);

	processElementAndEnsureOutput(
			operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple3<>("hello", "hallo", 1));
}
 
Example 28
Source Project: flink   Source File: TupleTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@PublicEvolving
public static <X extends Tuple> TupleTypeInfo<X> getBasicAndBasicValueTupleTypeInfo(Class<?>... basicTypes) {
	if (basicTypes == null || basicTypes.length == 0) {
		throw new IllegalArgumentException();
	}

	TypeInformation<?>[] infos = new TypeInformation<?>[basicTypes.length];
	for (int i = 0; i < infos.length; i++) {
		Class<?> type = basicTypes[i];
		if (type == null) {
			throw new IllegalArgumentException("Type at position " + i + " is null.");
		}

		TypeInformation<?> info = BasicTypeInfo.getInfoFor(type);
		if (info == null) {
			try {
				info = ValueTypeInfo.getValueTypeInfo((Class<Value>) type);
				if (!((ValueTypeInfo<?>) info).isBasicValueType()) {
					throw new IllegalArgumentException("Type at position " + i + " is not a basic or value type.");
				}
			} catch (ClassCastException | InvalidTypesException e) {
				throw new IllegalArgumentException("Type at position " + i + " is not a basic or value type.", e);
			}
		}
		infos[i] = info;
	}


	return (TupleTypeInfo<X>) new TupleTypeInfo<>(infos);
}
 
Example 29
Source Project: Flink-CEPplus   Source File: LambdaExtractionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitionerLambda() {
	Partitioner<Tuple2<Integer, String>> partitioner = (key, numPartitions) -> key.f1.length() % numPartitions;
	final TypeInformation<?> ti = TypeExtractor.getPartitionerTypes(partitioner, null, true);

	if (!(ti instanceof MissingTypeInfo)) {
		assertTrue(ti.isTupleType());
		assertEquals(2, ti.getArity());
		assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(0), BasicTypeInfo.INT_TYPE_INFO);
		assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO);
	}
}
 
Example 30
Source Project: flink   Source File: InnerJoinOperatorBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJoinPlain(){
	final FlatJoinFunction<String, String, Integer> joiner = new FlatJoinFunction<String, String, Integer>() {

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	@SuppressWarnings({ "rawtypes", "unchecked" })
	InnerJoinOperatorBase<String, String, Integer,
					FlatJoinFunction<String, String,Integer> > base = new InnerJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
					BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], "TestJoiner");

	List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6 ,6));

	try {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2, null, executionConfig);
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2, null, executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}