Java Code Examples for org.apache.flink.api.common.typeinfo.TypeInformation#createSerializer()

The following examples show how to use org.apache.flink.api.common.typeinfo.TypeInformation#createSerializer() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: CollectStreamResult.java    License: Apache License 2.0 6 votes vote down vote up
public CollectStreamResult(
		TableSchema tableSchema,
		ExecutionConfig config,
		InetAddress gatewayAddress,
		int gatewayPort,
		ClassLoader classLoader) {
	resultLock = new Object();

	// create socket stream iterator
	final TypeInformation<Tuple2<Boolean, Row>> socketType = Types.TUPLE(Types.BOOLEAN, tableSchema.toRowType());
	final TypeSerializer<Tuple2<Boolean, Row>> serializer = socketType.createSerializer(config);
	try {
		// pass gateway port and address such that iterator knows where to bind to
		iterator = new SocketStreamIterator<>(gatewayPort, gatewayAddress, serializer);
	} catch (IOException e) {
		throw new SqlClientException("Could not start socket for result retrieval.", e);
	}

	// create table sink
	// pass binding address and port such that sink knows where to send to
	collectTableSink = new CollectStreamTableSink(iterator.getBindAddress(), iterator.getPort(), serializer, tableSchema);
	retrievalThread = new ResultRetrievalThread();

	this.classLoader = checkNotNull(classLoader);
}
 
Example 2
Source Project: Flink-CEPplus   File: FieldAccessor.java    License: Apache License 2.0 6 votes vote down vote up
RecursiveProductFieldAccessor(int pos, TypeInformation<T> typeInfo, FieldAccessor<R, F> innerAccessor, ExecutionConfig config) {
	int arity = ((TupleTypeInfoBase) typeInfo).getArity();
	if (pos < 0 || pos >= arity) {
		throw new CompositeType.InvalidFieldReferenceException(
			"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
				typeInfo.toString() + "\", which is an invalid index.");
	}
	checkNotNull(typeInfo, "typeInfo must not be null.");
	checkNotNull(innerAccessor, "innerAccessor must not be null.");

	this.pos = pos;
	this.fieldType = ((TupleTypeInfoBase<T>) typeInfo).getTypeAt(pos);
	this.serializer = (TupleSerializerBase<T>) typeInfo.createSerializer(config);
	this.length = this.serializer.getArity();
	this.fields = new Object[this.length];
	this.innerAccessor = innerAccessor;
}
 
Example 3
Source Project: flink   File: RowSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLargeRowSerializer() {
	TypeInformation<Row> typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	Row row = new Row(13);
	row.setField(0, 2);
	row.setField(1, null);
	row.setField(3, null);
	row.setField(4, null);
	row.setField(5, null);
	row.setField(6, null);
	row.setField(7, null);
	row.setField(8, null);
	row.setField(9, null);
	row.setField(10, null);
	row.setField(11, null);
	row.setField(12, "Test");

	TypeSerializer<Row> serializer = typeInfo.createSerializer(new ExecutionConfig());
	RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, row);
	testInstance.testAll();
}
 
Example 4
Source Project: flink   File: WindowOperator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public <S extends Serializable> ValueState<S> getKeyValueState(String name,
	TypeInformation<S> stateType,
	S defaultState) {

	checkNotNull(name, "The name of the state must not be null");
	checkNotNull(stateType, "The state type information must not be null");

	ValueStateDescriptor<S> stateDesc = new ValueStateDescriptor<>(name, stateType.createSerializer(getExecutionConfig()), defaultState);
	return getPartitionedState(stateDesc);
}
 
Example 5
Source Project: flink   File: OuterJoinOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected List<OUT> executeOnCollections(List<IN1> leftInput, List<IN2> rightInput, RuntimeContext runtimeContext, ExecutionConfig executionConfig) throws Exception {
	TypeInformation<IN1> leftInformation = getOperatorInfo().getFirstInputType();
	TypeInformation<IN2> rightInformation = getOperatorInfo().getSecondInputType();
	TypeInformation<OUT> outInformation = getOperatorInfo().getOutputType();

	TypeComparator<IN1> leftComparator = buildComparatorFor(0, executionConfig, leftInformation);
	TypeComparator<IN2> rightComparator = buildComparatorFor(1, executionConfig, rightInformation);

	TypeSerializer<IN1> leftSerializer = leftInformation.createSerializer(executionConfig);
	TypeSerializer<IN2> rightSerializer = rightInformation.createSerializer(executionConfig);

	OuterJoinListIterator<IN1, IN2> outerJoinIterator =
			new OuterJoinListIterator<>(leftInput, leftSerializer, leftComparator,
					rightInput, rightSerializer, rightComparator, outerJoinType);

	// --------------------------------------------------------------------
	// Run UDF
	// --------------------------------------------------------------------
	FlatJoinFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject();

	FunctionUtils.setFunctionRuntimeContext(function, runtimeContext);
	FunctionUtils.openFunction(function, this.parameters);

	List<OUT> result = new ArrayList<>();
	Collector<OUT> collector = new CopyingListCollector<>(result, outInformation.createSerializer(executionConfig));

	while (outerJoinIterator.next()) {
		IN1 left = outerJoinIterator.getLeft();
		IN2 right = outerJoinIterator.getRight();
		function.join(left == null ? null : leftSerializer.copy(left), right == null ? null : rightSerializer.copy(right), collector);
	}

	FunctionUtils.closeFunction(function);

	return result;
}
 
Example 6
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testGenericObjectArrays() {
	MyGenericPojo<String>[][] array = (MyGenericPojo<String>[][]) new MyGenericPojo[][]{
		{ new MyGenericPojo<String>(new String[][]{{"a", "b"},{"c", "d"}}), null}
	};

	TypeInformation<MyGenericPojo<String>[][]> ti =
			TypeInformation.of(new TypeHint<MyGenericPojo<String>[][]>(){});

	SerializerTestInstance testInstance = new SerializerTestInstance(ti.createSerializer(new ExecutionConfig()), MyGenericPojo[][].class, -1, (Object) array);
	testInstance.testAll();
}
 
Example 7
@Test
public void testObjectArrays() {
	Integer[][] array = new Integer[][]{{0,1}, null, {null, 42}};
	TypeInformation<Integer[][]> ti = TypeExtractor.getForClass(Integer[][].class);

	SerializerTestInstance<Integer[][]> testInstance = new SerializerTestInstance<Integer[][]>(ti.createSerializer(new ExecutionConfig()), Integer[][].class, -1, array);
	testInstance.testAll();

	MyPojo[][] array2 = new MyPojo[][]{{new MyPojo(null, 42), new MyPojo("test2", -1)}, {null, null}, null};
	TypeInformation<MyPojo[][]> ti2 = TypeExtractor.getForClass(MyPojo[][].class);

	SerializerTestInstance<MyPojo[][]> testInstance2 = new SerializerTestInstance<MyPojo[][]>(ti2.createSerializer(new ExecutionConfig()), MyPojo[][].class, -1, array2);
	testInstance2.testAll();
}
 
Example 8
@Override
protected <T> TypeSerializer<T> createSerializer(Class<T> type) {
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerTypeWithKryoSerializer(LocalDate.class, LocalDateSerializer.class);
	TypeInformation<T> typeInfo = new GenericTypeInfo<T>(type);
	return typeInfo.createSerializer(conf);
}
 
Example 9
Source Project: flink   File: StreamGraph.java    License: Apache License 2.0 5 votes vote down vote up
public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<OUT> taskOperatorFactory,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	Class<? extends AbstractInvokable> vertexClass = taskOperatorFactory.isOperatorSelectiveReading() ?
		TwoInputSelectableStreamTask.class : TwoInputStreamTask.class;

	addNode(vertexID, slotSharingGroup, coLocationGroup, vertexClass, taskOperatorFactory, operatorName);

	TypeSerializer<OUT> outSerializer = (outTypeInfo != null) && !(outTypeInfo instanceof MissingTypeInfo) ?
			outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorFactory.isOutputTypeConfigurable()) {
		// sets the output type which must be know at StreamGraph creation time
		taskOperatorFactory.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
 
Example 10
@Test
public void testStringArray() {
	String[][] array = new String[][]{{null,"b"},{"c","d"},{"e","f"},{"g","h"},null};
	TypeInformation<String[][]> ti = TypeExtractor.getForClass(String[][].class);

	SerializerTestInstance<String[][]> testInstance = new SerializerTestInstance<String[][]>(ti.createSerializer(new ExecutionConfig()), String[][].class, -1, array);
	testInstance.testAll();
}
 
Example 11
Source Project: flink   File: Event.java    License: Apache License 2.0 4 votes vote down vote up
public static TypeSerializer<Event> createTypeSerializer() {
	TypeInformation<Event> typeInformation = (TypeInformation<Event>) TypeExtractor.createTypeInfo(Event.class);

	return typeInformation.createSerializer(new ExecutionConfig());
}
 
Example 12
@Override
@SuppressWarnings("unchecked")
public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) {
	serializer = (TypeSerializer<T>) type.createSerializer(executionConfig);
}
 
Example 13
@Test
public void testSerializability() {
	try (ByteArrayOutputStream buffer = new ByteArrayOutputStream();
		ObjectOutputStream out = new ObjectOutputStream(buffer)) {

		Collection<ElementType> inputCollection = new ArrayList<ElementType>();
		ElementType element1 = new ElementType(1);
		ElementType element2 = new ElementType(2);
		ElementType element3 = new ElementType(3);
		inputCollection.add(element1);
		inputCollection.add(element2);
		inputCollection.add(element3);

		@SuppressWarnings("unchecked")
		TypeInformation<ElementType> info = (TypeInformation<ElementType>) TypeExtractor.createTypeInfo(ElementType.class);

		CollectionInputFormat<ElementType> inputFormat = new CollectionInputFormat<ElementType>(inputCollection,
				info.createSerializer(new ExecutionConfig()));

		out.writeObject(inputFormat);

		ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(buffer.toByteArray()));

		Object serializationResult = in.readObject();

		assertNotNull(serializationResult);
		assertTrue(serializationResult instanceof CollectionInputFormat<?>);

		@SuppressWarnings("unchecked")
		CollectionInputFormat<ElementType> result = (CollectionInputFormat<ElementType>) serializationResult;

		GenericInputSplit inputSplit = new GenericInputSplit(0, 1);
		inputFormat.open(inputSplit);
		result.open(inputSplit);

		while (!inputFormat.reachedEnd() && !result.reachedEnd()){
			ElementType expectedElement = inputFormat.nextRecord(null);
			ElementType actualElement = result.nextRecord(null);

			assertEquals(expectedElement, actualElement);
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.toString());
	}

}
 
Example 14
Source Project: flink   File: CoGroupRawOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected List<OUT> executeOnCollections(List<IN1> input1, List<IN2> input2, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	// --------------------------------------------------------------------
	// Setup
	// --------------------------------------------------------------------
	TypeInformation<IN1> inputType1 = getOperatorInfo().getFirstInputType();
	TypeInformation<IN2> inputType2 = getOperatorInfo().getSecondInputType();

	int[] inputKeys1 = getKeyColumns(0);
	int[] inputKeys2 = getKeyColumns(1);

	boolean[] inputSortDirections1 = new boolean[inputKeys1.length];
	boolean[] inputSortDirections2 = new boolean[inputKeys2.length];

	Arrays.fill(inputSortDirections1, true);
	Arrays.fill(inputSortDirections2, true);

	final TypeSerializer<IN1> inputSerializer1 = inputType1.createSerializer(executionConfig);
	final TypeSerializer<IN2> inputSerializer2 = inputType2.createSerializer(executionConfig);

	final TypeComparator<IN1> inputComparator1 = getTypeComparator(executionConfig, inputType1, inputKeys1, inputSortDirections1);
	final TypeComparator<IN2> inputComparator2 = getTypeComparator(executionConfig, inputType2, inputKeys2, inputSortDirections2);

	SimpleListIterable<IN1> iterator1 = new SimpleListIterable<IN1>(input1, inputComparator1, inputSerializer1);
	SimpleListIterable<IN2> iterator2 = new SimpleListIterable<IN2>(input2, inputComparator2, inputSerializer2);

	// --------------------------------------------------------------------
	// Run UDF
	// --------------------------------------------------------------------
	CoGroupFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject();

	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, parameters);

	List<OUT> result = new ArrayList<OUT>();
	Collector<OUT> resultCollector = new CopyingListCollector<OUT>(result, getOperatorInfo().getOutputType().createSerializer(executionConfig));

	function.coGroup(iterator1, iterator2, resultCollector);

	FunctionUtils.closeFunction(function);

	return result;
}
 
Example 15
@Test
public void testRestoreKryoSerializedKeysWindows() throws Exception {
	final int windowSize = 3;

	TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();

	ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer<>(),
		inputType.createSerializer(new ExecutionConfig()));

	TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
	assertTrue(keySerializer instanceof KryoSerializer);

	WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector<>(),
		keySerializer,
		stateDesc,
		new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
		EventTimeTrigger.create(),
		0,
		null /* late data output tag */);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));

	testHarness.setup();

	testHarness.initializeState(
		OperatorSnapshotUtil.getResourceFilename(
			"win-op-migration-test-kryo-serialized-key-flink" + testMigrateVersion + "-snapshot"));

	testHarness.open();

	testHarness.processWatermark(new Watermark(2999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 3), 2999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 3), 2999));
	expectedOutput.add(new Watermark(2999));

	testHarness.processWatermark(new Watermark(3999));
	expectedOutput.add(new Watermark(3999));

	testHarness.processWatermark(new Watermark(4999));
	expectedOutput.add(new Watermark(4999));

	testHarness.processWatermark(new Watermark(5999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 2), 5999));
	expectedOutput.add(new Watermark(5999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
	testHarness.close();
}
 
Example 16
@Override
public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig executionConfig) {
	this.serializer = outTypeInfo.createSerializer(executionConfig);
}
 
Example 17
@Override
public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig executionConfig) {
	this.serializer = outTypeInfo.createSerializer(executionConfig);
}
 
Example 18
/**
 * Manually run this to write binary snapshot data.
 */
@Ignore
@Test
public void writeWindowsWithKryoSerializedKeysSnapshot() throws Exception {
	final int windowSize = 3;

	TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();

	ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer<>(),
		inputType.createSerializer(new ExecutionConfig()));

	TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
	assertTrue(keySerializer instanceof KryoSerializer);

	WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector<>(),
		keySerializer,
		stateDesc,
		new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
		EventTimeTrigger.create(),
		0,
		null /* late data output tag */);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));

	testHarness.setup();
	testHarness.open();

	// add elements out-of-order
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 0));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1998));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1000));

	testHarness.processWatermark(new Watermark(999));
	expectedOutput.add(new Watermark(999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());

	testHarness.processWatermark(new Watermark(1999));
	expectedOutput.add(new Watermark(1999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());

	// do snapshot and save to file
	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
	OperatorSnapshotUtil.writeStateHandle(
		snapshot,
		"src/test/resources/win-op-migration-test-kryo-serialized-key-flink" + flinkGenerateSavepointVersion + "-snapshot");

	testHarness.close();
}
 
Example 19
/**
 * Creates a new de-/serialization schema for the given type.
 *
 * @param typeInfo The type information for the type de-/serialized by this schema.
 * @param ec The execution config, which is used to parametrize the type serializers.
 */
public TypeInformationSerializationSchema(TypeInformation<T> typeInfo, ExecutionConfig ec) {
	this.typeInfo = checkNotNull(typeInfo, "typeInfo");
	this.serializer = typeInfo.createSerializer(ec);
}
 
Example 20
/**
 * Creates a new de-/serialization schema for the given type.
 *
 * @param typeInfo The type information for the type de-/serialized by this schema.
 * @param ec The execution config, which is used to parametrize the type serializers.
 */
public TypeInformationSerializationSchema(TypeInformation<T> typeInfo, ExecutionConfig ec) {
	this.typeInfo = checkNotNull(typeInfo, "typeInfo");
	this.serializer = typeInfo.createSerializer(ec);
}