Java Code Examples for org.apache.flink.api.java.typeutils.runtime.TupleSerializer

The following examples show how to use org.apache.flink.api.java.typeutils.runtime.TupleSerializer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: TupleTypeInfo.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
@PublicEvolving
public TupleSerializer<T> createSerializer(ExecutionConfig executionConfig) {
	if (getTypeClass() == Tuple0.class) {
		return (TupleSerializer<T>) Tuple0Serializer.INSTANCE;
	}

	TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[getArity()];
	for (int i = 0; i < types.length; i++) {
		fieldSerializers[i] = types[i].createSerializer(executionConfig);
	}
	
	Class<T> tupleClass = getTypeClass();
	
	return new TupleSerializer<T>(tupleClass, fieldSerializers);
}
 
Example 2
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 3
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 4
Source Project: flink   Source File: TupleTypeInfo.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
@PublicEvolving
public TupleSerializer<T> createSerializer(ExecutionConfig executionConfig) {
	if (getTypeClass() == Tuple0.class) {
		return (TupleSerializer<T>) Tuple0Serializer.INSTANCE;
	}

	TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[getArity()];
	for (int i = 0; i < types.length; i++) {
		fieldSerializers[i] = types[i].createSerializer(executionConfig);
	}
	
	Class<T> tupleClass = getTypeClass();
	
	return new TupleSerializer<T>(tupleClass, fieldSerializers);
}
 
Example 5
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 6
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 7
Source Project: flink   Source File: TupleTypeInfo.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
@PublicEvolving
public TupleSerializer<T> createSerializer(ExecutionConfig executionConfig) {
	if (getTypeClass() == Tuple0.class) {
		return (TupleSerializer<T>) Tuple0Serializer.INSTANCE;
	}

	TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[getArity()];
	for (int i = 0; i < types.length; i++) {
		fieldSerializers[i] = types[i].createSerializer(executionConfig);
	}
	
	Class<T> tupleClass = getTypeClass();
	
	return new TupleSerializer<T>(tupleClass, fieldSerializers);
}
 
Example 8
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
	
	this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
	this.ioManager = new IOManagerAsync();
}
 
Example 9
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);

	this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
	this.ioManager = new IOManagerAsync();
}
 
Example 10
public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	fields = new Object[((TupleSerializer<IN>) serializer).getArity()];
}
 
Example 11
Source Project: Flink-CEPplus   Source File: HashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
public HashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, BytePrimitiveArraySerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, byte[]>> clazz = (Class<Tuple2<Long, byte[]>>) (Class<?>) Tuple2.class;
	this.buildSerializer = new TupleSerializer<Tuple2<Long, byte[]>>(clazz, fieldSerializers);
	
	this.probeSerializer = LongSerializer.INSTANCE;

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.buildComparator = new TupleComparator<Tuple2<Long, byte[]>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, byte[]>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, byte[]> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, byte[]> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 
Example 12
Source Project: Flink-CEPplus   Source File: InPlaceMutableHashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
public InPlaceMutableHashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, StringSerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, String>> clazz = (Class<Tuple2<Long, String>>) (Class<?>) Tuple2.class;
	this.serializer = new TupleSerializer<Tuple2<Long, String>>(clazz, fieldSerializers);

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.comparator = new TupleComparator<Tuple2<Long, String>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, String>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, String> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, String> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 
Example 13
Source Project: Flink-CEPplus   Source File: TtlMapStateVerifier.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
@Nonnull
public TypeSerializer<Tuple2<String, String>> getUpdateSerializer() {
	return new TupleSerializer(
		Tuple2.class, new TypeSerializer[] {StringSerializer.INSTANCE, StringSerializer.INSTANCE});
}
 
Example 14
Source Project: Flink-CEPplus   Source File: StreamProjectTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 
Example 15
Source Project: flink   Source File: CassandraTupleWriteAheadSink.java    License: Apache License 2.0 5 votes vote down vote up
public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	fields = new Object[((TupleSerializer<IN>) serializer).getArity()];
}
 
Example 16
Source Project: flink   Source File: HashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
public HashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, BytePrimitiveArraySerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, byte[]>> clazz = (Class<Tuple2<Long, byte[]>>) (Class<?>) Tuple2.class;
	this.buildSerializer = new TupleSerializer<Tuple2<Long, byte[]>>(clazz, fieldSerializers);
	
	this.probeSerializer = LongSerializer.INSTANCE;

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.buildComparator = new TupleComparator<Tuple2<Long, byte[]>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, byte[]>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, byte[]> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, byte[]> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 
Example 17
Source Project: flink   Source File: InPlaceMutableHashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
public InPlaceMutableHashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, StringSerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, String>> clazz = (Class<Tuple2<Long, String>>) (Class<?>) Tuple2.class;
	this.serializer = new TupleSerializer<Tuple2<Long, String>>(clazz, fieldSerializers);

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.comparator = new TupleComparator<Tuple2<Long, String>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, String>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, String> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, String> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 
Example 18
Source Project: flink   Source File: TtlMapStateVerifier.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
@Nonnull
public TypeSerializer<Tuple2<String, String>> getUpdateSerializer() {
	return new TupleSerializer(
		Tuple2.class, new TypeSerializer[] {StringSerializer.INSTANCE, StringSerializer.INSTANCE});
}
 
Example 19
Source Project: flink   Source File: StreamProjectTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 
Example 20
Source Project: flink   Source File: CassandraTupleWriteAheadSink.java    License: Apache License 2.0 5 votes vote down vote up
public void open() throws Exception {
	super.open();
	if (!getRuntimeContext().isCheckpointingEnabled()) {
		throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled.");
	}
	cluster = builder.getCluster();
	session = cluster.connect();
	preparedStatement = session.prepare(insertQuery);

	fields = new Object[((TupleSerializer<IN>) serializer).getArity()];
}
 
Example 21
Source Project: flink   Source File: FlinkKafkaConsumerBase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates state serializer for kafka topic partition to offset tuple.
 * Using of the explicit state serializer with KryoSerializer is needed because otherwise
 * users cannot use 'disableGenericTypes' properties with KafkaConsumer.
 */
@VisibleForTesting
static TupleSerializer<Tuple2<KafkaTopicPartition, Long>> createStateSerializer(ExecutionConfig executionConfig) {
	// explicit serializer will keep the compatibility with GenericTypeInformation and allow to disableGenericTypes for users
	TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[]{
		new KryoSerializer<>(KafkaTopicPartition.class, executionConfig),
		LongSerializer.INSTANCE
	};
	@SuppressWarnings("unchecked")
	Class<Tuple2<KafkaTopicPartition, Long>> tupleClass = (Class<Tuple2<KafkaTopicPartition, Long>>) (Class<?>) Tuple2.class;
	return new TupleSerializer<>(tupleClass, fieldSerializers);
}
 
Example 22
Source Project: flink   Source File: FlinkKafkaConsumerBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Before using an explicit TypeSerializer for the partition state the {@link
 * FlinkKafkaConsumerBase} was creating a serializer using a {@link TypeHint}. Here, we verify
 * that the two methods create compatible serializers.
 */
@Test
public void testExplicitStateSerializerCompatibility() throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();

	Tuple2<KafkaTopicPartition, Long> tuple =
			new Tuple2<>(new KafkaTopicPartition("dummy", 0), 42L);

	// This is how the KafkaConsumerBase used to create the TypeSerializer
	TypeInformation<Tuple2<KafkaTopicPartition, Long>> originalTypeHintTypeInfo =
			new TypeHint<Tuple2<KafkaTopicPartition, Long>>() {}.getTypeInfo();
	TypeSerializer<Tuple2<KafkaTopicPartition, Long>> serializerFromTypeHint =
			originalTypeHintTypeInfo.createSerializer(executionConfig);
	byte[] bytes = InstantiationUtil.serializeToByteArray(serializerFromTypeHint, tuple);

	// Directly use the Consumer to create the TypeSerializer (using the new method)
	TupleSerializer<Tuple2<KafkaTopicPartition, Long>> kafkaConsumerSerializer =
			FlinkKafkaConsumerBase.createStateSerializer(executionConfig);
	Tuple2<KafkaTopicPartition, Long> actualTuple =
			InstantiationUtil.deserializeFromByteArray(kafkaConsumerSerializer, bytes);

	Assert.assertEquals(
			"Explicit Serializer is not compatible with previous method of creating Serializer using TypeHint.",
			tuple,
			actualTuple
	);
}
 
Example 23
Source Project: flink   Source File: AbstractArrowSourceFunction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState == null,
		"The " + getClass().getSimpleName() + " has already been initialized.");

	this.checkpointedState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>(
			"arrow-source-state",
			new TupleSerializer<>(
				(Class<Tuple2<Integer, Integer>>) (Class<?>) Tuple2.class,
				new TypeSerializer[]{IntSerializer.INSTANCE, IntSerializer.INSTANCE})
		)
	);

	this.indexesToEmit = new ArrayDeque<>();
	if (context.isRestored()) {
		// upon restoring
		for (Tuple2<Integer, Integer> v : this.checkpointedState.get()) {
			this.indexesToEmit.add(v);
		}
		LOG.info("Subtask {} restored state: {}.", getRuntimeContext().getIndexOfThisSubtask(), indexesToEmit);
	} else {
		// the first time the job is executed
		final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks();
		final int taskIdx = getRuntimeContext().getIndexOfThisSubtask();

		for (int i = taskIdx; i < arrowData.length; i += stepSize) {
			this.indexesToEmit.add(Tuple2.of(i, 0));
		}
		LOG.info("Subtask {} has no restore state, initialized with {}.", taskIdx, indexesToEmit);
	}
}
 
Example 24
Source Project: flink   Source File: HashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
public HashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, BytePrimitiveArraySerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, byte[]>> clazz = (Class<Tuple2<Long, byte[]>>) (Class<?>) Tuple2.class;
	this.buildSerializer = new TupleSerializer<Tuple2<Long, byte[]>>(clazz, fieldSerializers);
	
	this.probeSerializer = LongSerializer.INSTANCE;

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.buildComparator = new TupleComparator<Tuple2<Long, byte[]>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, byte[]>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, byte[]> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, byte[]> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 
Example 25
Source Project: flink   Source File: InPlaceMutableHashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
public InPlaceMutableHashTableTest() {
	TypeSerializer<?>[] fieldSerializers = { LongSerializer.INSTANCE, StringSerializer.INSTANCE };
	@SuppressWarnings("unchecked")
	Class<Tuple2<Long, String>> clazz = (Class<Tuple2<Long, String>>) (Class<?>) Tuple2.class;
	this.serializer = new TupleSerializer<Tuple2<Long, String>>(clazz, fieldSerializers);

	TypeComparator<?>[] comparators = { new LongComparator(true) };
	TypeSerializer<?>[] comparatorSerializers = { LongSerializer.INSTANCE };

	this.comparator = new TupleComparator<Tuple2<Long, String>>(new int[] {0}, comparators, comparatorSerializers);

	this.probeComparator = new LongComparator(true);

	this.pairComparator = new TypePairComparator<Long, Tuple2<Long, String>>() {

		private long ref;

		@Override
		public void setReference(Long reference) {
			ref = reference;
		}

		@Override
		public boolean equalToReference(Tuple2<Long, String> candidate) {
			//noinspection UnnecessaryUnboxing
			return candidate.f0.longValue() == ref;
		}

		@Override
		public int compareToReference(Tuple2<Long, String> candidate) {
			long x = ref;
			long y = candidate.f0;
			return (x < y) ? -1 : ((x == y) ? 0 : 1);
		}
	};
}
 
Example 26
Source Project: flink   Source File: TtlMapStateVerifier.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
@Nonnull
public TypeSerializer<Tuple2<String, String>> getUpdateSerializer() {
	return new TupleSerializer(
		Tuple2.class, new TypeSerializer[] {StringSerializer.INSTANCE, StringSerializer.INSTANCE});
}
 
Example 27
Source Project: flink   Source File: StreamProjectTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 
Example 28
Source Project: Flink-CEPplus   Source File: TupleComparatorTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected abstract TupleSerializer<T> createSerializer();
 
Example 29
Source Project: Flink-CEPplus   Source File: WindowOperator.java    License: Apache License 2.0 4 votes vote down vote up
@Override
	public void open() throws Exception {
		super.open();

		this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
		timestampedCollector = new TimestampedCollector<>(output);

		internalTimerService =
				getInternalTimerService("window-timers", windowSerializer, this);

		triggerContext = new Context(null, null);
		processContext = new WindowContext(null);

		windowAssignerContext = new WindowAssigner.WindowAssignerContext() {
			@Override
			public long getCurrentProcessingTime() {
				return internalTimerService.currentProcessingTime();
			}
		};

		// create (or restore) the state that hold the actual window contents
		// NOTE - the state may be null in the case of the overriding evicting window operator
		if (windowStateDescriptor != null) {
			windowState = (InternalAppendingState<K, W, IN, ACC, ACC>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
		}

		// create the typed and helper states for merging windows
		if (windowAssigner instanceof MergingWindowAssigner) {

			// store a typed reference for the state of merging windows - sanity check
			if (windowState instanceof InternalMergingState) {
				windowMergingState = (InternalMergingState<K, W, IN, ACC, ACC>) windowState;
			}
			// TODO this sanity check should be here, but is prevented by an incorrect test (pending validation)
			// TODO see WindowOperatorTest.testCleanupTimerWithEmptyFoldingStateForSessionWindows()
			// TODO activate the sanity check once resolved
//			else if (windowState != null) {
//				throw new IllegalStateException(
//						"The window uses a merging assigner, but the window state is not mergeable.");
//			}

			@SuppressWarnings("unchecked")
			final Class<Tuple2<W, W>> typedTuple = (Class<Tuple2<W, W>>) (Class<?>) Tuple2.class;

			final TupleSerializer<Tuple2<W, W>> tupleSerializer = new TupleSerializer<>(
					typedTuple,
					new TypeSerializer[] {windowSerializer, windowSerializer});

			final ListStateDescriptor<Tuple2<W, W>> mergingSetsStateDescriptor =
					new ListStateDescriptor<>("merging-window-set", tupleSerializer);

			// get the state that stores the merging sets
			mergingSetsState = (InternalListState<K, VoidNamespace, Tuple2<W, W>>)
					getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, mergingSetsStateDescriptor);
			mergingSetsState.setCurrentNamespace(VoidNamespace.INSTANCE);
		}
	}
 
Example 30
Source Project: flink   Source File: TupleComparatorTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected abstract TupleSerializer<T> createSerializer();