org.apache.flink.api.common.typeutils.base.VoidSerializer Java Examples

The following examples show how to use org.apache.flink.api.common.typeutils.base.VoidSerializer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private OperatorSubtaskState createOperatorSubtaskState(OneInputStreamOperator<Integer, Void> operator) throws Exception {
	try (KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, id -> id, Types.INT, 128, 1, 0)) {

		testHarness.setup(VoidSerializer.INSTANCE);
		testHarness.open();

		testHarness.processElement(1, 0);
		testHarness.processElement(2, 0);
		testHarness.processElement(3, 0);

		return testHarness.snapshot(0, 0);
	}
}
 
Example #2
Source File: FlinkStateInternals.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public <T> SetState<T> bindSet(String id, StateSpec<SetState<T>> spec, Coder<T> elemCoder) {
  try {
    keyedStateBackend.getOrCreateKeyedState(
        StringSerializer.INSTANCE,
        new MapStateDescriptor<>(
            id, new CoderTypeSerializer<>(elemCoder), VoidSerializer.INSTANCE));
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
  return null;
}
 
Example #3
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private OperatorSubtaskState createOperatorSubtaskState(OneInputStreamOperator<Integer, Void> operator) throws Exception {
	try (KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, id -> id, Types.INT, 128, 1, 0)) {

		testHarness.setup(VoidSerializer.INSTANCE);
		testHarness.open();

		testHarness.processElement(1, 0);
		testHarness.processElement(2, 0);
		testHarness.processElement(3, 0);

		return testHarness.snapshot(0, 0);
	}
}
 
Example #4
Source File: RocksDBAsyncSnapshotTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Test that the snapshot files are cleaned up in case of a failure during the snapshot
 * procedure.
 */
@Test
public void testCleanupOfSnapshotsInFailureCase() throws Exception {
	long checkpointId = 1L;
	long timestamp = 42L;

	Environment env = new DummyEnvironment("test task", 1, 0);

	final IOException testException = new IOException("Test exception");
	CheckpointStateOutputStream outputStream = spy(new FailingStream(testException));

	RocksDBStateBackend backend = new RocksDBStateBackend((StateBackend) new MemoryStateBackend());

	backend.setDbStoragePath(temporaryFolder.newFolder().toURI().toString());

	AbstractKeyedStateBackend<Void> keyedStateBackend = backend.createKeyedStateBackend(
		env,
		new JobID(),
		"test operator",
		VoidSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		null,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());

	try {
		// register a state so that the state backend has to checkpoint something
		keyedStateBackend.getPartitionedState(
			"namespace",
			StringSerializer.INSTANCE,
			new ValueStateDescriptor<>("foobar", String.class));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotFuture = keyedStateBackend.snapshot(
			checkpointId, timestamp,
			new TestCheckpointStreamFactory(() -> outputStream),
			CheckpointOptions.forCheckpointWithDefaultLocation());

		try {
			FutureUtils.runIfNotDoneAndGet(snapshotFuture);
			fail("Expected an exception to be thrown here.");
		} catch (ExecutionException e) {
			Assert.assertEquals(testException, e.getCause());
		}

		verify(outputStream).close();
	} finally {
		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
	}
}
 
Example #5
Source File: TwoPhaseCommitSinkFunctionTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public ContentDumpSinkFunction() {
	super(
		new KryoSerializer<>(ContentTransaction.class, new ExecutionConfig()),
		VoidSerializer.INSTANCE, clock);
}
 
Example #6
Source File: DataTypesTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Parameters(name = "{index}: {0}=[Logical: {1}, Class: {2}]")
public static List<Object[]> dataTypes() {
	return Arrays.asList(
		new Object[][]{
			{CHAR(2), new CharType(2), String.class},

			{VARCHAR(2), new VarCharType(2), String.class},

			{STRING(), new VarCharType(VarCharType.MAX_LENGTH), String.class},

			{BOOLEAN(), new BooleanType(), Boolean.class},

			{BINARY(42), new BinaryType(42), byte[].class},

			{VARBINARY(42), new VarBinaryType(42), byte[].class},

			{BYTES(), new VarBinaryType(VarBinaryType.MAX_LENGTH), byte[].class},

			{DECIMAL(10, 10), new DecimalType(10, 10), BigDecimal.class},

			{TINYINT(), new TinyIntType(), Byte.class},

			{SMALLINT(), new SmallIntType(), Short.class},

			{INT(), new IntType(), Integer.class},

			{BIGINT(), new BigIntType(), Long.class},

			{FLOAT(), new FloatType(), Float.class},

			{DOUBLE(), new DoubleType(), Double.class},

			{DATE(), new DateType(), java.time.LocalDate.class},

			{TIME(3), new TimeType(3), java.time.LocalTime.class},

			{TIME(), new TimeType(0), java.time.LocalTime.class},

			{TIMESTAMP(3), new TimestampType(3), java.time.LocalDateTime.class},

			{TIMESTAMP(), new TimestampType(6), java.time.LocalDateTime.class},

			{TIMESTAMP_WITH_TIME_ZONE(3),
				new ZonedTimestampType(3),
				java.time.OffsetDateTime.class},

			{TIMESTAMP_WITH_TIME_ZONE(),
				new ZonedTimestampType(6),
				java.time.OffsetDateTime.class},

			{TIMESTAMP_WITH_LOCAL_TIME_ZONE(3),
				new LocalZonedTimestampType(3),
				java.time.Instant.class},

			{TIMESTAMP_WITH_LOCAL_TIME_ZONE(),
				new LocalZonedTimestampType(6),
				java.time.Instant.class},

			{INTERVAL(MINUTE(), SECOND(3)),
				new DayTimeIntervalType(MINUTE_TO_SECOND, DEFAULT_DAY_PRECISION, 3),
				java.time.Duration.class},

			{INTERVAL(MONTH()),
				new YearMonthIntervalType(YearMonthResolution.MONTH),
				java.time.Period.class},

			{ARRAY(ARRAY(INT())),
				new ArrayType(new ArrayType(new IntType())),
				Integer[][].class},

			{MULTISET(MULTISET(INT())),
				new MultisetType(new MultisetType(new IntType())),
				Map.class},

			{MAP(INT(), SMALLINT()),
				new MapType(new IntType(), new SmallIntType()),
				Map.class},

			{ROW(FIELD("field1", CHAR(2)), FIELD("field2", BOOLEAN())),
				new RowType(
					Arrays.asList(
						new RowType.RowField("field1", new CharType(2)),
						new RowType.RowField("field2", new BooleanType()))),
				Row.class},

			{NULL(), new NullType(), Object.class},

			{ANY(Types.GENERIC(DataTypesTest.class)),
				new TypeInformationAnyType<>(Types.GENERIC(DataTypesTest.class)),
				DataTypesTest.class},

			{ANY(Void.class, VoidSerializer.INSTANCE),
				new AnyType<>(Void.class, VoidSerializer.INSTANCE),
				Void.class}
		}
	);
}
 
Example #7
Source File: RocksDBAsyncSnapshotTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Test that the snapshot files are cleaned up in case of a failure during the snapshot
 * procedure.
 */
@Test
public void testCleanupOfSnapshotsInFailureCase() throws Exception {
	long checkpointId = 1L;
	long timestamp = 42L;

	Environment env = new DummyEnvironment("test task", 1, 0);

	final IOException testException = new IOException("Test exception");
	CheckpointStateOutputStream outputStream = spy(new FailingStream(testException));

	RocksDBStateBackend backend = new RocksDBStateBackend((StateBackend) new MemoryStateBackend());

	backend.setDbStoragePath(temporaryFolder.newFolder().toURI().toString());

	AbstractKeyedStateBackend<Void> keyedStateBackend = backend.createKeyedStateBackend(
		env,
		new JobID(),
		"test operator",
		VoidSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		null,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());

	try {
		// register a state so that the state backend has to checkpoint something
		keyedStateBackend.getPartitionedState(
			"namespace",
			StringSerializer.INSTANCE,
			new ValueStateDescriptor<>("foobar", String.class));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotFuture = keyedStateBackend.snapshot(
			checkpointId, timestamp,
			new TestCheckpointStreamFactory(() -> outputStream),
			CheckpointOptions.forCheckpointWithDefaultLocation());

		try {
			FutureUtils.runIfNotDoneAndGet(snapshotFuture);
			fail("Expected an exception to be thrown here.");
		} catch (ExecutionException e) {
			Assert.assertEquals(testException, e.getCause());
		}

		verify(outputStream).close();
	} finally {
		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
	}
}
 
Example #8
Source File: TwoPhaseCommitSinkFunctionTest.java    From flink with Apache License 2.0 4 votes vote down vote up
public ContentDumpSinkFunction() {
	super(
		new KryoSerializer<>(ContentTransaction.class, new ExecutionConfig()),
		VoidSerializer.INSTANCE, clock);
}
 
Example #9
Source File: RocksDBAsyncSnapshotTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Test that the snapshot files are cleaned up in case of a failure during the snapshot
 * procedure.
 */
@Test
public void testCleanupOfSnapshotsInFailureCase() throws Exception {
	long checkpointId = 1L;
	long timestamp = 42L;

	MockEnvironment env = MockEnvironment.builder().build();

	final IOException testException = new IOException("Test exception");
	CheckpointStateOutputStream outputStream = spy(new FailingStream(testException));

	RocksDBStateBackend backend = new RocksDBStateBackend((StateBackend) new MemoryStateBackend());

	backend.setDbStoragePath(temporaryFolder.newFolder().toURI().toString());

	AbstractKeyedStateBackend<Void> keyedStateBackend = backend.createKeyedStateBackend(
		env,
		new JobID(),
		"test operator",
		VoidSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		null,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());

	try {
		// register a state so that the state backend has to checkpoint something
		keyedStateBackend.getPartitionedState(
			"namespace",
			StringSerializer.INSTANCE,
			new ValueStateDescriptor<>("foobar", String.class));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotFuture = keyedStateBackend.snapshot(
			checkpointId, timestamp,
			new TestCheckpointStreamFactory(() -> outputStream),
			CheckpointOptions.forCheckpointWithDefaultLocation());

		try {
			FutureUtils.runIfNotDoneAndGet(snapshotFuture);
			fail("Expected an exception to be thrown here.");
		} catch (ExecutionException e) {
			Assert.assertEquals(testException, e.getCause());
		}

		verify(outputStream).close();
	} finally {
		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
		IOUtils.closeQuietly(env);
	}
}
 
Example #10
Source File: TwoPhaseCommitSinkFunctionTest.java    From flink with Apache License 2.0 4 votes vote down vote up
public ContentDumpSinkFunction() {
	super(
		new KryoSerializer<>(ContentTransaction.class, new ExecutionConfig()),
		VoidSerializer.INSTANCE, clock);
}