org.apache.flink.runtime.state.PriorityQueueSetFactory Java Examples
The following examples show how to use
org.apache.flink.runtime.state.PriorityQueueSetFactory.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 6 votes |
|
private static <K, N> InternalTimerServiceImpl<K, N> createInternalTimerService(
KeyGroupRange keyGroupsList,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
TypeSerializer<K> keySerializer,
TypeSerializer<N> namespaceSerializer,
PriorityQueueSetFactory priorityQueueSetFactory) {
TimerSerializer<K, N> timerSerializer = new TimerSerializer<>(keySerializer, namespaceSerializer);
return new InternalTimerServiceImpl<>(
keyGroupsList,
keyContext,
processingTimeService,
createTimerQueue("__test_processing_timers", timerSerializer, priorityQueueSetFactory),
createTimerQueue("__test_event_timers", timerSerializer, priorityQueueSetFactory));
}
Example #2
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 6 votes |
|
private static InternalTimerServiceImpl<Integer, String> createAndStartInternalTimerService(
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
KeyGroupRange keyGroupList,
PriorityQueueSetFactory priorityQueueSetFactory) {
InternalTimerServiceImpl<Integer, String> service = createInternalTimerService(
keyGroupList,
keyContext,
processingTimeService,
IntSerializer.INSTANCE,
StringSerializer.INSTANCE,
priorityQueueSetFactory);
service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
return service;
}
Example #3
| Source File: InternalTimerServiceImplTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
private static InternalTimerServiceImpl<Integer, String> createAndStartInternalTimerService(
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
KeyGroupRange keyGroupList,
PriorityQueueSetFactory priorityQueueSetFactory) {
InternalTimerServiceImpl<Integer, String> service = createInternalTimerService(
keyGroupList,
keyContext,
processingTimeService,
IntSerializer.INSTANCE,
StringSerializer.INSTANCE,
priorityQueueSetFactory);
service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
return service;
}
Example #4
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 6 votes |
|
private static <K, N> InternalTimerServiceImpl<K, N> createInternalTimerService(
KeyGroupRange keyGroupsList,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
TypeSerializer<K> keySerializer,
TypeSerializer<N> namespaceSerializer,
PriorityQueueSetFactory priorityQueueSetFactory) {
TimerSerializer<K, N> timerSerializer = new TimerSerializer<>(keySerializer, namespaceSerializer);
return new InternalTimerServiceImpl<>(
keyGroupsList,
keyContext,
processingTimeService,
createTimerQueue("__test_processing_timers", timerSerializer, priorityQueueSetFactory),
createTimerQueue("__test_event_timers", timerSerializer, priorityQueueSetFactory));
}
Example #5
| Source File: InternalTimerServiceImplTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
private static <K, N> InternalTimerServiceImpl<K, N> createInternalTimerService(
KeyGroupRange keyGroupsList,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
TypeSerializer<K> keySerializer,
TypeSerializer<N> namespaceSerializer,
PriorityQueueSetFactory priorityQueueSetFactory) {
TimerSerializer<K, N> timerSerializer = new TimerSerializer<>(keySerializer, namespaceSerializer);
return new InternalTimerServiceImpl<>(
keyGroupsList,
keyContext,
processingTimeService,
createTimerQueue("__test_processing_timers", timerSerializer, priorityQueueSetFactory),
createTimerQueue("__test_event_timers", timerSerializer, priorityQueueSetFactory));
}
Example #6
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 6 votes |
|
private static InternalTimerServiceImpl<Integer, String> createAndStartInternalTimerService(
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
KeyGroupRange keyGroupList,
PriorityQueueSetFactory priorityQueueSetFactory) {
InternalTimerServiceImpl<Integer, String> service = createInternalTimerService(
keyGroupList,
keyContext,
processingTimeService,
IntSerializer.INSTANCE,
StringSerializer.INSTANCE,
priorityQueueSetFactory);
service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
return service;
}
Example #7
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 5 votes |
|
private static <K, N> KeyGroupedInternalPriorityQueue<TimerHeapInternalTimer<K, N>> createTimerQueue(
String name,
TimerSerializer<K, N> timerSerializer,
PriorityQueueSetFactory priorityQueueSetFactory) {
return priorityQueueSetFactory.create(
name,
timerSerializer);
}
Example #8
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 5 votes |
|
private static InternalTimerServiceImpl<Integer, String> restoreTimerService(
Map<Integer, byte[]> state,
int snapshotVersion,
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
KeyGroupRange keyGroupsList,
PriorityQueueSetFactory priorityQueueSetFactory) throws Exception {
// create an empty service
InternalTimerServiceImpl<Integer, String> service = createInternalTimerService(
keyGroupsList,
keyContext,
processingTimeService,
IntSerializer.INSTANCE,
StringSerializer.INSTANCE,
priorityQueueSetFactory);
// restore the timers
for (Integer keyGroupIndex : keyGroupsList) {
if (state.containsKey(keyGroupIndex)) {
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(state.get(keyGroupIndex))) {
InternalTimersSnapshot<?, ?> restoredTimersSnapshot =
InternalTimersSnapshotReaderWriters
.getReaderForVersion(snapshotVersion, InternalTimerServiceImplTest.class.getClassLoader())
.readTimersSnapshot(new DataInputViewStreamWrapper(inputStream));
service.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIndex);
}
}
}
// initialize the service
service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
return service;
}
Example #9
| Source File: InternalTimeServiceManager.java From flink with Apache License 2.0 | 5 votes |
|
InternalTimeServiceManager(
KeyGroupRange localKeyGroupRange,
KeyContext keyContext,
PriorityQueueSetFactory priorityQueueSetFactory,
ProcessingTimeService processingTimeService, boolean useLegacySynchronousSnapshots) {
this.localKeyGroupRange = Preconditions.checkNotNull(localKeyGroupRange);
this.priorityQueueSetFactory = Preconditions.checkNotNull(priorityQueueSetFactory);
this.keyContext = Preconditions.checkNotNull(keyContext);
this.processingTimeService = Preconditions.checkNotNull(processingTimeService);
this.useLegacySynchronousSnapshots = useLegacySynchronousSnapshots;
this.timerServices = new HashMap<>();
}
Example #10
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 5 votes |
|
private static <K, N> KeyGroupedInternalPriorityQueue<TimerHeapInternalTimer<K, N>> createTimerQueue(
String name,
TimerSerializer<K, N> timerSerializer,
PriorityQueueSetFactory priorityQueueSetFactory) {
return priorityQueueSetFactory.create(
name,
timerSerializer);
}
Example #11
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 5 votes |
|
private static InternalTimerServiceImpl<Integer, String> restoreTimerService(
Map<Integer, byte[]> state,
int snapshotVersion,
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
KeyGroupRange keyGroupsList,
PriorityQueueSetFactory priorityQueueSetFactory) throws Exception {
// create an empty service
InternalTimerServiceImpl<Integer, String> service = createInternalTimerService(
keyGroupsList,
keyContext,
processingTimeService,
IntSerializer.INSTANCE,
StringSerializer.INSTANCE,
priorityQueueSetFactory);
// restore the timers
for (Integer keyGroupIndex : keyGroupsList) {
if (state.containsKey(keyGroupIndex)) {
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(state.get(keyGroupIndex))) {
InternalTimersSnapshot<?, ?> restoredTimersSnapshot =
InternalTimersSnapshotReaderWriters
.getReaderForVersion(snapshotVersion, InternalTimerServiceImplTest.class.getClassLoader())
.readTimersSnapshot(new DataInputViewStreamWrapper(inputStream));
service.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIndex);
}
}
}
// initialize the service
service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
return service;
}
Example #12
| Source File: InternalTimeServiceManager.java From flink with Apache License 2.0 | 5 votes |
|
InternalTimeServiceManager(
KeyGroupRange localKeyGroupRange,
KeyContext keyContext,
PriorityQueueSetFactory priorityQueueSetFactory,
ProcessingTimeService processingTimeService, boolean useLegacySynchronousSnapshots) {
this.localKeyGroupRange = Preconditions.checkNotNull(localKeyGroupRange);
this.priorityQueueSetFactory = Preconditions.checkNotNull(priorityQueueSetFactory);
this.keyContext = Preconditions.checkNotNull(keyContext);
this.processingTimeService = Preconditions.checkNotNull(processingTimeService);
this.useLegacySynchronousSnapshots = useLegacySynchronousSnapshots;
this.timerServices = new HashMap<>();
}
Example #13
| Source File: InternalTimerServiceImplTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private static <K, N> KeyGroupedInternalPriorityQueue<TimerHeapInternalTimer<K, N>> createTimerQueue(
String name,
TimerSerializer<K, N> timerSerializer,
PriorityQueueSetFactory priorityQueueSetFactory) {
return priorityQueueSetFactory.create(
name,
timerSerializer);
}
Example #14
| Source File: InternalTimeServiceManager.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
InternalTimeServiceManager(
KeyGroupRange localKeyGroupRange,
KeyContext keyContext,
PriorityQueueSetFactory priorityQueueSetFactory,
ProcessingTimeService processingTimeService, boolean useLegacySynchronousSnapshots) {
this.localKeyGroupRange = Preconditions.checkNotNull(localKeyGroupRange);
this.priorityQueueSetFactory = Preconditions.checkNotNull(priorityQueueSetFactory);
this.keyContext = Preconditions.checkNotNull(keyContext);
this.processingTimeService = Preconditions.checkNotNull(processingTimeService);
this.useLegacySynchronousSnapshots = useLegacySynchronousSnapshots;
this.timerServices = new HashMap<>();
}
Example #15
| Source File: InternalTimerServiceImplTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private static InternalTimerServiceImpl<Integer, String> restoreTimerService(
Map<Integer, byte[]> state,
int snapshotVersion,
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
KeyGroupRange keyGroupsList,
PriorityQueueSetFactory priorityQueueSetFactory) throws Exception {
// create an empty service
InternalTimerServiceImpl<Integer, String> service = createInternalTimerService(
keyGroupsList,
keyContext,
processingTimeService,
IntSerializer.INSTANCE,
StringSerializer.INSTANCE,
priorityQueueSetFactory);
// restore the timers
for (Integer keyGroupIndex : keyGroupsList) {
if (state.containsKey(keyGroupIndex)) {
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(state.get(keyGroupIndex))) {
InternalTimersSnapshot<?, ?> restoredTimersSnapshot =
InternalTimersSnapshotReaderWriters
.getReaderForVersion(snapshotVersion, InternalTimerServiceImplTest.class.getClassLoader())
.readTimersSnapshot(new DataInputViewStreamWrapper(inputStream));
service.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIndex);
}
}
}
// initialize the service
service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
return service;
}
Example #16
| Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0 | 4 votes |
|
public RocksDBKeyedStateBackend(
ClassLoader userCodeClassLoader,
File instanceBasePath,
RocksDBResourceContainer optionsContainer,
Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
TaskKvStateRegistry kvStateRegistry,
TypeSerializer<K> keySerializer,
ExecutionConfig executionConfig,
TtlTimeProvider ttlTimeProvider,
RocksDB db,
LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
int keyGroupPrefixBytes,
CloseableRegistry cancelStreamRegistry,
StreamCompressionDecorator keyGroupCompressionDecorator,
ResourceGuard rocksDBResourceGuard,
RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
RocksDBWriteBatchWrapper writeBatchWrapper,
ColumnFamilyHandle defaultColumnFamilyHandle,
RocksDBNativeMetricMonitor nativeMetricMonitor,
RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
PriorityQueueSetFactory priorityQueueFactory,
RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
InternalKeyContext<K> keyContext,
@Nonnegative long writeBatchSize) {
super(
kvStateRegistry,
keySerializer,
userCodeClassLoader,
executionConfig,
ttlTimeProvider,
cancelStreamRegistry,
keyGroupCompressionDecorator,
keyContext);
this.ttlCompactFiltersManager = ttlCompactFiltersManager;
// ensure that we use the right merge operator, because other code relies on this
this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);
this.optionsContainer = Preconditions.checkNotNull(optionsContainer);
this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);
this.keyGroupPrefixBytes = keyGroupPrefixBytes;
this.kvStateInformation = kvStateInformation;
this.writeOptions = optionsContainer.getWriteOptions();
this.readOptions = optionsContainer.getReadOptions();
checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative value.");
this.writeBatchSize = writeBatchSize;
this.db = db;
this.rocksDBResourceGuard = rocksDBResourceGuard;
this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
this.savepointSnapshotStrategy = savepointSnapshotStrategy;
this.writeBatchWrapper = writeBatchWrapper;
this.defaultColumnFamily = defaultColumnFamilyHandle;
this.nativeMetricMonitor = nativeMetricMonitor;
this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
this.priorityQueueFactory = priorityQueueFactory;
}
Example #17
| Source File: RocksDBKeyedStateBackend.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
public RocksDBKeyedStateBackend(
ClassLoader userCodeClassLoader,
File instanceBasePath,
DBOptions dbOptions,
Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
TaskKvStateRegistry kvStateRegistry,
StateSerializerProvider<K> keySerializerProvider,
int numberOfKeyGroups,
KeyGroupRange keyGroupRange,
ExecutionConfig executionConfig,
TtlTimeProvider ttlTimeProvider,
RocksDB db,
LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
int keyGroupPrefixBytes,
CloseableRegistry cancelStreamRegistry,
StreamCompressionDecorator keyGroupCompressionDecorator,
ResourceGuard rocksDBResourceGuard,
RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
RocksDBWriteBatchWrapper writeBatchWrapper,
ColumnFamilyHandle defaultColumnFamilyHandle,
RocksDBNativeMetricMonitor nativeMetricMonitor,
RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
PriorityQueueSetFactory priorityQueueFactory,
RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
super(kvStateRegistry, keySerializerProvider, userCodeClassLoader, numberOfKeyGroups,
keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry, keyGroupCompressionDecorator);
this.ttlCompactFiltersManager = ttlCompactFiltersManager;
// ensure that we use the right merge operator, because other code relies on this
this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);
this.dbOptions = Preconditions.checkNotNull(dbOptions);
this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);
this.keyGroupPrefixBytes = keyGroupPrefixBytes;
this.kvStateInformation = kvStateInformation;
this.writeOptions = new WriteOptions().setDisableWAL(true);
this.db = db;
this.rocksDBResourceGuard = rocksDBResourceGuard;
this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
this.savepointSnapshotStrategy = savepointSnapshotStrategy;
this.writeBatchWrapper = writeBatchWrapper;
this.defaultColumnFamily = defaultColumnFamilyHandle;
this.nativeMetricMonitor = nativeMetricMonitor;
this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
this.priorityQueueFactory = priorityQueueFactory;
}
Example #18
| Source File: RocksDBKeyedStateBackend.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
PriorityQueueSetFactory getPriorityQueueFactory() {
return priorityQueueFactory;
}
Example #19
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 4 votes |
|
protected PriorityQueueSetFactory createQueueFactory(KeyGroupRange keyGroupRange, int numKeyGroups) {
return new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128);
}
Example #20
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 4 votes |
|
private PriorityQueueSetFactory createQueueFactory() {
return createQueueFactory(testKeyGroupRange, maxParallelism);
}
Example #21
| Source File: InternalTimerServiceImplTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testTimerAssignmentToKeyGroups() {
int totalNoOfTimers = 100;
int totalNoOfKeyGroups = 100;
int startKeyGroupIdx = 0;
int endKeyGroupIdx = totalNoOfKeyGroups - 1; // we have 0 to 99
@SuppressWarnings("unchecked")
Set<TimerHeapInternalTimer<Integer, String>>[] expectedNonEmptyTimerSets = new HashSet[totalNoOfKeyGroups];
TestKeyContext keyContext = new TestKeyContext();
final KeyGroupRange keyGroupRange = new KeyGroupRange(startKeyGroupIdx, endKeyGroupIdx);
final PriorityQueueSetFactory priorityQueueSetFactory =
createQueueFactory(keyGroupRange, totalNoOfKeyGroups);
InternalTimerServiceImpl<Integer, String> timerService = createInternalTimerService(
keyGroupRange,
keyContext,
new TestProcessingTimeService(),
IntSerializer.INSTANCE,
StringSerializer.INSTANCE,
priorityQueueSetFactory);
timerService.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, mock(Triggerable.class));
for (int i = 0; i < totalNoOfTimers; i++) {
// create the timer to be registered
TimerHeapInternalTimer<Integer, String> timer = new TimerHeapInternalTimer<>(10 + i, i, "hello_world_" + i);
int keyGroupIdx = KeyGroupRangeAssignment.assignToKeyGroup(timer.getKey(), totalNoOfKeyGroups);
// add it in the adequate expected set of timers per keygroup
Set<TimerHeapInternalTimer<Integer, String>> timerSet = expectedNonEmptyTimerSets[keyGroupIdx];
if (timerSet == null) {
timerSet = new HashSet<>();
expectedNonEmptyTimerSets[keyGroupIdx] = timerSet;
}
timerSet.add(timer);
// register the timer as both processing and event time one
keyContext.setCurrentKey(timer.getKey());
timerService.registerEventTimeTimer(timer.getNamespace(), timer.getTimestamp());
timerService.registerProcessingTimeTimer(timer.getNamespace(), timer.getTimestamp());
}
List<Set<TimerHeapInternalTimer<Integer, String>>> eventTimeTimers =
timerService.getEventTimeTimersPerKeyGroup();
List<Set<TimerHeapInternalTimer<Integer, String>>> processingTimeTimers =
timerService.getProcessingTimeTimersPerKeyGroup();
// finally verify that the actual timers per key group sets are the expected ones.
for (int i = 0; i < expectedNonEmptyTimerSets.length; i++) {
Set<TimerHeapInternalTimer<Integer, String>> expected = expectedNonEmptyTimerSets[i];
Set<TimerHeapInternalTimer<Integer, String>> actualEvent = eventTimeTimers.get(i);
Set<TimerHeapInternalTimer<Integer, String>> actualProcessing = processingTimeTimers.get(i);
if (expected == null) {
Assert.assertTrue(actualEvent.isEmpty());
Assert.assertTrue(actualProcessing.isEmpty());
} else {
Assert.assertEquals(expected, actualEvent);
Assert.assertEquals(expected, actualProcessing);
}
}
}
Example #22
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Verify that we only ever have one processing-time task registered at the
* {@link ProcessingTimeService}.
*/
@Test
public void testOnlySetsOnePhysicalProcessingTimeTimer() throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
PriorityQueueSetFactory priorityQueueSetFactory =
new HeapPriorityQueueSetFactory(testKeyGroupRange, maxParallelism, 128);
InternalTimerServiceImpl<Integer, String> timerService =
createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, priorityQueueSetFactory);
int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
keyContext.setCurrentKey(key);
timerService.registerProcessingTimeTimer("ciao", 10);
timerService.registerProcessingTimeTimer("ciao", 20);
timerService.registerProcessingTimeTimer("ciao", 30);
timerService.registerProcessingTimeTimer("hello", 10);
timerService.registerProcessingTimeTimer("hello", 20);
assertEquals(5, timerService.numProcessingTimeTimers());
assertEquals(2, timerService.numProcessingTimeTimers("hello"));
assertEquals(3, timerService.numProcessingTimeTimers("ciao"));
assertEquals(1, processingTimeService.getNumActiveTimers());
assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L));
processingTimeService.setCurrentTime(10);
assertEquals(3, timerService.numProcessingTimeTimers());
assertEquals(1, timerService.numProcessingTimeTimers("hello"));
assertEquals(2, timerService.numProcessingTimeTimers("ciao"));
assertEquals(1, processingTimeService.getNumActiveTimers());
assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L));
processingTimeService.setCurrentTime(20);
assertEquals(1, timerService.numProcessingTimeTimers());
assertEquals(0, timerService.numProcessingTimeTimers("hello"));
assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
assertEquals(1, processingTimeService.getNumActiveTimers());
assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L));
processingTimeService.setCurrentTime(30);
assertEquals(0, timerService.numProcessingTimeTimers());
assertEquals(0, processingTimeService.getNumActiveTimers());
timerService.registerProcessingTimeTimer("ciao", 40);
assertEquals(1, processingTimeService.getNumActiveTimers());
}
Example #23
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testTimerAssignmentToKeyGroups() {
int totalNoOfTimers = 100;
int totalNoOfKeyGroups = 100;
int startKeyGroupIdx = 0;
int endKeyGroupIdx = totalNoOfKeyGroups - 1; // we have 0 to 99
@SuppressWarnings("unchecked")
Set<TimerHeapInternalTimer<Integer, String>>[] expectedNonEmptyTimerSets = new HashSet[totalNoOfKeyGroups];
TestKeyContext keyContext = new TestKeyContext();
final KeyGroupRange keyGroupRange = new KeyGroupRange(startKeyGroupIdx, endKeyGroupIdx);
final PriorityQueueSetFactory priorityQueueSetFactory =
createQueueFactory(keyGroupRange, totalNoOfKeyGroups);
InternalTimerServiceImpl<Integer, String> timerService = createInternalTimerService(
keyGroupRange,
keyContext,
new TestProcessingTimeService(),
IntSerializer.INSTANCE,
StringSerializer.INSTANCE,
priorityQueueSetFactory);
timerService.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, mock(Triggerable.class));
for (int i = 0; i < totalNoOfTimers; i++) {
// create the timer to be registered
TimerHeapInternalTimer<Integer, String> timer = new TimerHeapInternalTimer<>(10 + i, i, "hello_world_" + i);
int keyGroupIdx = KeyGroupRangeAssignment.assignToKeyGroup(timer.getKey(), totalNoOfKeyGroups);
// add it in the adequate expected set of timers per keygroup
Set<TimerHeapInternalTimer<Integer, String>> timerSet = expectedNonEmptyTimerSets[keyGroupIdx];
if (timerSet == null) {
timerSet = new HashSet<>();
expectedNonEmptyTimerSets[keyGroupIdx] = timerSet;
}
timerSet.add(timer);
// register the timer as both processing and event time one
keyContext.setCurrentKey(timer.getKey());
timerService.registerEventTimeTimer(timer.getNamespace(), timer.getTimestamp());
timerService.registerProcessingTimeTimer(timer.getNamespace(), timer.getTimestamp());
}
List<Set<TimerHeapInternalTimer<Integer, String>>> eventTimeTimers =
timerService.getEventTimeTimersPerKeyGroup();
List<Set<TimerHeapInternalTimer<Integer, String>>> processingTimeTimers =
timerService.getProcessingTimeTimersPerKeyGroup();
// finally verify that the actual timers per key group sets are the expected ones.
for (int i = 0; i < expectedNonEmptyTimerSets.length; i++) {
Set<TimerHeapInternalTimer<Integer, String>> expected = expectedNonEmptyTimerSets[i];
Set<TimerHeapInternalTimer<Integer, String>> actualEvent = eventTimeTimers.get(i);
Set<TimerHeapInternalTimer<Integer, String>> actualProcessing = processingTimeTimers.get(i);
if (expected == null) {
Assert.assertTrue(actualEvent.isEmpty());
Assert.assertTrue(actualProcessing.isEmpty());
} else {
Assert.assertEquals(expected, actualEvent);
Assert.assertEquals(expected, actualProcessing);
}
}
}
Example #24
| Source File: InternalTimerServiceImplTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Verify that we only ever have one processing-time task registered at the
* {@link ProcessingTimeService}.
*/
@Test
public void testOnlySetsOnePhysicalProcessingTimeTimer() throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
PriorityQueueSetFactory priorityQueueSetFactory =
new HeapPriorityQueueSetFactory(testKeyGroupRange, maxParallelism, 128);
InternalTimerServiceImpl<Integer, String> timerService =
createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, priorityQueueSetFactory);
int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
keyContext.setCurrentKey(key);
timerService.registerProcessingTimeTimer("ciao", 10);
timerService.registerProcessingTimeTimer("ciao", 20);
timerService.registerProcessingTimeTimer("ciao", 30);
timerService.registerProcessingTimeTimer("hello", 10);
timerService.registerProcessingTimeTimer("hello", 20);
assertEquals(5, timerService.numProcessingTimeTimers());
assertEquals(2, timerService.numProcessingTimeTimers("hello"));
assertEquals(3, timerService.numProcessingTimeTimers("ciao"));
assertEquals(1, processingTimeService.getNumActiveTimers());
assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L));
processingTimeService.setCurrentTime(10);
assertEquals(3, timerService.numProcessingTimeTimers());
assertEquals(1, timerService.numProcessingTimeTimers("hello"));
assertEquals(2, timerService.numProcessingTimeTimers("ciao"));
assertEquals(1, processingTimeService.getNumActiveTimers());
assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L));
processingTimeService.setCurrentTime(20);
assertEquals(1, timerService.numProcessingTimeTimers());
assertEquals(0, timerService.numProcessingTimeTimers("hello"));
assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
assertEquals(1, processingTimeService.getNumActiveTimers());
assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L));
processingTimeService.setCurrentTime(30);
assertEquals(0, timerService.numProcessingTimeTimers());
assertEquals(0, processingTimeService.getNumActiveTimers());
timerService.registerProcessingTimeTimer("ciao", 40);
assertEquals(1, processingTimeService.getNumActiveTimers());
}
Example #25
| Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
PriorityQueueSetFactory getPriorityQueueFactory() {
return priorityQueueFactory;
}
Example #26
| Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
PriorityQueueSetFactory getPriorityQueueFactory() {
return priorityQueueFactory;
}
Example #27
| Source File: InternalTimerServiceImplTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
private PriorityQueueSetFactory createQueueFactory() {
return createQueueFactory(testKeyGroupRange, maxParallelism);
}
Example #28
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 4 votes |
|
protected PriorityQueueSetFactory createQueueFactory(KeyGroupRange keyGroupRange, int numKeyGroups) {
return new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128);
}
Example #29
| Source File: InternalTimerServiceImplTest.java From flink with Apache License 2.0 | 4 votes |
|
private PriorityQueueSetFactory createQueueFactory() {
return createQueueFactory(testKeyGroupRange, maxParallelism);
}
Example #30
| Source File: InternalTimerServiceImplTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
protected PriorityQueueSetFactory createQueueFactory(KeyGroupRange keyGroupRange, int numKeyGroups) {
return new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128);
}
