Java Code Examples for java.util.concurrent.ConcurrentLinkedQueue#addAll()

The following examples show how to use java.util.concurrent.ConcurrentLinkedQueue#addAll() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: BotLogger.java From TelegramApi with MIT License

6 votes

@Override
public void run() {
    while(true) {
        final ConcurrentLinkedQueue<String> stringsToLog = new ConcurrentLinkedQueue<>();
        synchronized (logsToFile) {
            if (logsToFile.isEmpty()) {
                try {
                    logsToFile.wait();
                } catch (InterruptedException e) {
                    return;
                }
                if (logsToFile.isEmpty()) {
                    continue;
                }
            }
            stringsToLog.addAll(logsToFile);
            logsToFile.clear();
        }

        stringsToLog.stream().forEach(logginFile::println);
        logginFile.flush();
    }
}

Example 2

Source File: WriteBatcherImpl.java From java-client-api with Apache License 2.0

6 votes

public ConcurrentLinkedQueue<Runnable> snapshotQueuedAndExecutingTasks() {
  ConcurrentLinkedQueue<Runnable> snapshot = new ConcurrentLinkedQueue<>();
  activeSnapshots.put( Thread.currentThread(), snapshot );
  snapshot.addAll(queuedAndExecutingTasks);
  // There is inconsistency between queuedAndExecutingTasks and snapshot
  // taken here. If there are a large number of tasks, by the time we
  // iterate queuedAndExecutingTasks and get all the tasks into the snapshot
  // queue, some tasks complete and are removed from
  // queuedAndExecutingTasks. So, iterate over the snapshot again and remove
  // those completed tasks so that they both are consistent.
  for (Runnable task : snapshot) {
    if ( !(queuedAndExecutingTasks.contains(task)) )
      snapshot.remove(task);
  }
  return snapshot;
}

Example 3

Source File: ConcurrentLinkedQueueTest.java From j2objc with Apache License 2.0

5 votes

/**
 * addAll(this) throws IAE
 */
public void testAddAllSelf() {
    ConcurrentLinkedQueue q = populatedQueue(SIZE);
    try {
        q.addAll(q);
        shouldThrow();
    } catch (IllegalArgumentException success) {}
}

Example 4

Source File: ConcurrentLinkedQueueTest.java From openjdk-jdk9 with GNU General Public License v2.0

5 votes

/**
 * addAll of a collection with any null elements throws NPE after
 * possibly adding some elements
 */
public void testAddAll3() {
    ConcurrentLinkedQueue q = new ConcurrentLinkedQueue();
    Integer[] ints = new Integer[SIZE];
    for (int i = 0; i < SIZE - 1; ++i)
        ints[i] = new Integer(i);
    try {
        q.addAll(Arrays.asList(ints));
        shouldThrow();
    } catch (NullPointerException success) {}
}

Example 5

Source File: ConcurrentLinkedQueueTest.java From openjdk-jdk9 with GNU General Public License v2.0

5 votes

/**
 * addAll of a collection with null elements throws NPE
 */
public void testAddAll2() {
    ConcurrentLinkedQueue q = new ConcurrentLinkedQueue();
    try {
        q.addAll(Arrays.asList(new Integer[SIZE]));
        shouldThrow();
    } catch (NullPointerException success) {}
}

Example 6

Source File: ConcurrentLinkedQueueTest.java From openjdk-jdk9 with GNU General Public License v2.0

5 votes

/**
 * addAll(this) throws IAE
 */
public void testAddAllSelf() {
    ConcurrentLinkedQueue q = populatedQueue(SIZE);
    try {
        q.addAll(q);
        shouldThrow();
    } catch (IllegalArgumentException success) {}
}

Example 7

Source File: ConcurrentLinkedQueueTest.java From j2objc with Apache License 2.0

5 votes

/**
 * addAll of a collection with any null elements throws NPE after
 * possibly adding some elements
 */
public void testAddAll3() {
    ConcurrentLinkedQueue q = new ConcurrentLinkedQueue();
    Integer[] ints = new Integer[SIZE];
    for (int i = 0; i < SIZE - 1; ++i)
        ints[i] = new Integer(i);
    try {
        q.addAll(Arrays.asList(ints));
        shouldThrow();
    } catch (NullPointerException success) {}
}

Example 8

Source File: ConcurrentLinkedQueueTest.java From j2objc with Apache License 2.0

5 votes

/**
 * addAll of a collection with null elements throws NPE
 */
public void testAddAll2() {
    ConcurrentLinkedQueue q = new ConcurrentLinkedQueue();
    try {
        q.addAll(Arrays.asList(new Integer[SIZE]));
        shouldThrow();
    } catch (NullPointerException success) {}
}

Example 9

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

/**
 * This tests a custom Session window assigner that assigns some elements to "point windows",
 * windows that have the same timestamp for start and end.
 *
 * <p>In this test, elements that have 33 as the second tuple field will be put into a point
 * window.
 */
@Test
@SuppressWarnings("unchecked")
public void testPointSessions() throws Exception {
	closeCalled.set(0);

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.assigner(new PointSessionWindowAssigner(3000))
			.withEventTime(2)
			.aggregateAndBuild(getTimeWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness = createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	// add elements out-of-order
	testHarness.processElement(record("key2", 1, 0L));
	testHarness.processElement(record("key2", 33, 1000L));

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
	testHarness.close();

	testHarness = createTestHarness(operator);
	testHarness.setup();
	testHarness.initializeState(snapshot);
	testHarness.open();

	testHarness.processElement(record("key2", 33, 2500L));

	testHarness.processElement(record("key1", 1, 10L));
	testHarness.processElement(record("key1", 2, 1000L));
	testHarness.processElement(record("key1", 33, 2500L));

	testHarness.processWatermark(new Watermark(12000));

	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key1", 36L, 3L, 10L, 4000L, 3999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key2", 67L, 3L, 0L, 3000L, 2999L)));
	expectedOutput.add(new Watermark(12000));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();

	// we close once in the rest...
	assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 10

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testTumblingCountWindow() throws Exception {
	closeCalled.set(0);
	final int windowSize = 3;
	LogicalType[] windowTypes = new LogicalType[] { new BigIntType() };

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.countWindow(windowSize)
			.aggregateAndBuild(getCountWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness = createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(record("key2", 1, 0L));
	testHarness.processElement(record("key2", 2, 1000L));
	testHarness.processElement(record("key2", 3, 2500L));
	testHarness.processElement(record("key1", 1, 10L));
	testHarness.processElement(record("key1", 2, 1000L));

	testHarness.processWatermark(new Watermark(12000));
	testHarness.setProcessingTime(12000L);
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key2", 6L, 3L, 0L)));
	expectedOutput.add(new Watermark(12000));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshotV2 = testHarness.snapshot(0L, 0);
	testHarness.close();
	expectedOutput.clear();

	testHarness = createTestHarness(operator);
	testHarness.setup();
	testHarness.initializeState(snapshotV2);
	testHarness.open();

	testHarness.processElement(record("key1", 2, 2500L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key1", 5L, 3L, 0L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(record("key2", 4, 5501L));
	testHarness.processElement(record("key2", 5, 6000L));
	testHarness.processElement(record("key2", 5, 6000L));
	testHarness.processElement(record("key2", 6, 6050L));

	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key2", 14L, 3L, 1L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(record("key1", 3, 4000L));
	testHarness.processElement(record("key2", 10, 15000L));
	testHarness.processElement(record("key2", 20, 15000L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key2", 36L, 3L, 2L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(record("key1", 2, 2500L));
	testHarness.processElement(record("key1", 2, 2500L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key1", 7L, 3L, 1L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();

	// we close once in the rest...
	assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 11

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testSlidingCountWindow() throws Exception {
	closeCalled.set(0);
	final int windowSize = 5;
	final int windowSlide = 3;
	LogicalType[] windowTypes = new LogicalType[] { new BigIntType() };

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.countWindow(windowSize, windowSlide)
			.aggregateAndBuild(getCountWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness = createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(record("key2", 1, 0L));
	testHarness.processElement(record("key2", 2, 1000L));
	testHarness.processElement(record("key2", 3, 2500L));
	testHarness.processElement(record("key2", 4, 2500L));
	testHarness.processElement(record("key2", 5, 2500L));
	testHarness.processElement(record("key1", 1, 10L));
	testHarness.processElement(record("key1", 2, 1000L));

	testHarness.processWatermark(new Watermark(12000));
	testHarness.setProcessingTime(12000L);
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key2", 15L, 5L, 0L)));
	expectedOutput.add(new Watermark(12000));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshotV2 = testHarness.snapshot(0L, 0);
	testHarness.close();
	expectedOutput.clear();

	testHarness = createTestHarness(operator);
	testHarness.setup();
	testHarness.initializeState(snapshotV2);
	testHarness.open();

	testHarness.processElement(record("key1", 3, 2500L));
	testHarness.processElement(record("key1", 4, 2500L));
	testHarness.processElement(record("key1", 5, 2500L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key1", 15L, 5L, 0L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(record("key2", 6, 6000L));
	testHarness.processElement(record("key2", 7, 6000L));
	testHarness.processElement(record("key2", 8, 6050L));
	testHarness.processElement(record("key2", 9, 6050L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key2", 30L, 5L, 1L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(record("key1", 6, 4000L));
	testHarness.processElement(record("key1", 7, 4000L));
	testHarness.processElement(record("key1", 8, 4000L));
	testHarness.processElement(record("key2", 10, 15000L));
	testHarness.processElement(record("key2", 11, 15000L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key1", 30L, 5L, 1L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, record("key2", 45L, 5L, 2L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();

	// we close once in the rest...
	assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 12

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testProcessingTimeTumblingWindows() throws Exception {
	closeCalled.set(0);

	WindowOperator operator = WindowOperatorBuilder.builder()
			.withInputFields(inputFieldTypes)
			.tumble(Duration.ofSeconds(3))
			.withProcessingTime()
			.aggregateAndBuild(getTimeWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.setProcessingTime(3);

	// timestamp is ignored in processing time
	testHarness.processElement(insertRecord("key2", 1, Long.MAX_VALUE));
	testHarness.processElement(insertRecord("key2", 1, 7000L));
	testHarness.processElement(insertRecord("key2", 1, 7000L));

	testHarness.processElement(insertRecord("key1", 1, 7000L));
	testHarness.processElement(insertRecord("key1", 1, 7000L));

	testHarness.setProcessingTime(5000);

	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 3L, 3L, 0L, 3000L, 2999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 2L, 2L, 0L, 3000L, 2999L)));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key1", 1, 7000L));
	testHarness.processElement(insertRecord("key1", 1, 7000L));
	testHarness.processElement(insertRecord("key1", 1, 7000L));

	testHarness.setProcessingTime(7000);

	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 3L, 3L, 3000L, 6000L, 5999L)));

	assertEquals(0L, operator.getWatermarkLatency().getValue());
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}

Example 13

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testEventTimeTumblingWindows() throws Exception {
	closeCalled.set(0);

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.tumble(Duration.ofSeconds(3))
			.withEventTime(2)
			.aggregateAndBuild(getTimeWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	// add elements out-of-order
	testHarness.processElement(insertRecord("key2", 1, 3999L));
	testHarness.processElement(insertRecord("key2", 1, 3000L));

	testHarness.processElement(insertRecord("key1", 1, 20L));
	testHarness.processElement(insertRecord("key1", 1, 0L));
	testHarness.processElement(insertRecord("key1", 1, 999L));

	testHarness.processElement(insertRecord("key2", 1, 1998L));
	testHarness.processElement(insertRecord("key2", 1, 1999L));
	testHarness.processElement(insertRecord("key2", 1, 1000L));

	testHarness.processWatermark(new Watermark(999));
	expectedOutput.add(new Watermark(999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processWatermark(new Watermark(1999));
	expectedOutput.add(new Watermark(1999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
	testHarness.close();
	expectedOutput.clear();

	testHarness = createTestHarness(operator);
	testHarness.setup();
	testHarness.initializeState(snapshot);
	testHarness.open();

	testHarness.processWatermark(new Watermark(2999));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 3L, 3L, 0L, 3000L, 2999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 3L, 3L, 0L, 3000L, 2999L)));
	expectedOutput.add(new Watermark(2999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processWatermark(new Watermark(3999));
	expectedOutput.add(new Watermark(3999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processWatermark(new Watermark(4999));
	expectedOutput.add(new Watermark(4999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processWatermark(new Watermark(5999));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 2L, 2L, 3000L, 6000L, 5999L)));
	expectedOutput.add(new Watermark(5999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	// those don't have any effect...
	testHarness.processWatermark(new Watermark(6999));
	testHarness.processWatermark(new Watermark(7999));
	expectedOutput.add(new Watermark(6999));
	expectedOutput.add(new Watermark(7999));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();

	// we close once in the rest...
	assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 14

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testEventTimeSlidingWindows() throws Exception {
	closeCalled.set(0);

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.sliding(Duration.ofSeconds(3), Duration.ofSeconds(1))
			.withEventTime(2)
			.aggregateAndBuild(getTimeWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);

	testHarness.open();

	// process elements
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	// add elements out-of-order
	testHarness.processElement(insertRecord("key2", 1, 3999L));
	testHarness.processElement(insertRecord("key2", 1, 3000L));

	testHarness.processElement(insertRecord("key1", 1, 20L));
	testHarness.processElement(insertRecord("key1", 1, 0L));
	testHarness.processElement(insertRecord("key1", 1, 999L));

	testHarness.processElement(insertRecord("key2", 1, 1998L));
	testHarness.processElement(insertRecord("key2", 1, 1999L));
	testHarness.processElement(insertRecord("key2", 1, 1000L));

	testHarness.processWatermark(new Watermark(999));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 3L, 3L, -2000L, 1000L, 999L)));
	expectedOutput.add(new Watermark(999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processWatermark(new Watermark(1999));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 3L, 3L, -1000L, 2000L, 1999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 3L, 3L, -1000L, 2000L, 1999L)));
	expectedOutput.add(new Watermark(1999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processWatermark(new Watermark(2999));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 3L, 3L, 0L, 3000L, 2999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 3L, 3L, 0L, 3000L, 2999L)));
	expectedOutput.add(new Watermark(2999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
	testHarness.close();
	expectedOutput.clear();

	testHarness = createTestHarness(operator);
	testHarness.setup();
	testHarness.initializeState(snapshot);
	testHarness.open();

	testHarness.processWatermark(new Watermark(3999));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 5L, 5L, 1000L, 4000L, 3999L)));
	expectedOutput.add(new Watermark(3999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processWatermark(new Watermark(4999));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 2L, 2L, 2000L, 5000L, 4999L)));
	expectedOutput.add(new Watermark(4999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processWatermark(new Watermark(5999));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 2L, 2L, 3000L, 6000L, 5999L)));
	expectedOutput.add(new Watermark(5999));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	// those don't have any effect...
	testHarness.processWatermark(new Watermark(6999));
	testHarness.processWatermark(new Watermark(7999));
	expectedOutput.add(new Watermark(6999));
	expectedOutput.add(new Watermark(7999));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();

	// we close once in the rest...
	assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 15

Source File: AsyncWaitOperatorTest.java From flink with Apache License 2.0

4 votes

private void testAsyncTimeout(
		LazyAsyncFunction lazyAsyncFunction,
		Optional<Class<? extends Throwable>> expectedException,
		StreamRecord<Integer>... expectedRecords) throws Exception {
	final long timeout = 10L;

	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		createTestHarness(lazyAsyncFunction, timeout, 2, AsyncDataStream.OutputMode.ORDERED);

	final MockEnvironment mockEnvironment = testHarness.getEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.setProcessingTime(initialTime);

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime));
		testHarness.setProcessingTime(initialTime + 5L);
		testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
	}

	// trigger the timeout of the first stream record
	testHarness.setProcessingTime(initialTime + timeout + 1L);

	// allow the second async stream record to be processed
	lazyAsyncFunction.countDown();

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	expectedOutput.addAll(Arrays.asList(expectedRecords));

	TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());

	if (expectedException.isPresent()) {
		assertTrue(mockEnvironment.getActualExternalFailureCause().isPresent());
		assertTrue(ExceptionUtils.findThrowable(
			mockEnvironment.getActualExternalFailureCause().get(),
			expectedException.get()).isPresent());
	}
}

Example 16

Source File: ExecutorServiceParallelExecutor.java From beam with Apache License 2.0

4 votes

@Override
// TODO: [BEAM-4563] Pass Future back to consumer to check for async errors
@SuppressWarnings("FutureReturnValueIgnored")
public void start(DirectGraph graph, RootProviderRegistry rootProviderRegistry) {
  int numTargetSplits = Math.max(3, targetParallelism);
  ImmutableMap.Builder<AppliedPTransform<?, ?, ?>, ConcurrentLinkedQueue<CommittedBundle<?>>>
      pendingRootBundles = ImmutableMap.builder();
  for (AppliedPTransform<?, ?, ?> root : graph.getRootTransforms()) {
    ConcurrentLinkedQueue<CommittedBundle<?>> pending = new ConcurrentLinkedQueue<>();
    try {
      Collection<CommittedBundle<?>> initialInputs =
          rootProviderRegistry.getInitialInputs(root, numTargetSplits);
      pending.addAll(initialInputs);
    } catch (Exception e) {
      throw UserCodeException.wrap(e);
    }
    pendingRootBundles.put(root, pending);
  }
  evaluationContext.initialize(pendingRootBundles.build());
  final ExecutionDriver executionDriver =
      QuiescenceDriver.create(
          evaluationContext, graph, this, visibleUpdates, pendingRootBundles.build());
  executorService.submit(
      new Runnable() {
        @Override
        public void run() {
          DriverState drive = executionDriver.drive();
          if (drive.isTermainal()) {
            State newPipelineState = State.UNKNOWN;
            switch (drive) {
              case FAILED:
                newPipelineState = State.FAILED;
                break;
              case SHUTDOWN:
                newPipelineState = State.DONE;
                break;
              case CONTINUE:
                throw new IllegalStateException(
                    String.format("%s should not be a terminal state", DriverState.CONTINUE));
              default:
                throw new IllegalArgumentException(
                    String.format("Unknown %s %s", DriverState.class.getSimpleName(), drive));
            }
            shutdownIfNecessary(newPipelineState);
          } else {
            executorService.submit(this);
          }
        }
      });
}

Example 17

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testSlidingCountWindow() throws Exception {
	closeCalled.set(0);
	final int windowSize = 5;
	final int windowSlide = 3;
	LogicalType[] windowTypes = new LogicalType[] { new BigIntType() };

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.countWindow(windowSize, windowSlide)
			.aggregateAndBuild(getCountWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(insertRecord("key2", 1, 0L));
	testHarness.processElement(insertRecord("key2", 2, 1000L));
	testHarness.processElement(insertRecord("key2", 3, 2500L));
	testHarness.processElement(insertRecord("key2", 4, 2500L));
	testHarness.processElement(insertRecord("key2", 5, 2500L));
	testHarness.processElement(insertRecord("key1", 1, 10L));
	testHarness.processElement(insertRecord("key1", 2, 1000L));

	testHarness.processWatermark(new Watermark(12000));
	testHarness.setProcessingTime(12000L);
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 15L, 5L, 0L)));
	expectedOutput.add(new Watermark(12000));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshotV2 = testHarness.snapshot(0L, 0);
	testHarness.close();
	expectedOutput.clear();

	testHarness = createTestHarness(operator);
	testHarness.setup();
	testHarness.initializeState(snapshotV2);
	testHarness.open();

	testHarness.processElement(insertRecord("key1", 3, 2500L));
	testHarness.processElement(insertRecord("key1", 4, 2500L));
	testHarness.processElement(insertRecord("key1", 5, 2500L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 15L, 5L, 0L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key2", 6, 6000L));
	testHarness.processElement(insertRecord("key2", 7, 6000L));
	testHarness.processElement(insertRecord("key2", 8, 6050L));
	testHarness.processElement(insertRecord("key2", 9, 6050L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 30L, 5L, 1L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key1", 6, 4000L));
	testHarness.processElement(insertRecord("key1", 7, 4000L));
	testHarness.processElement(insertRecord("key1", 8, 4000L));
	testHarness.processElement(insertRecord("key2", 10, 15000L));
	testHarness.processElement(insertRecord("key2", 11, 15000L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 30L, 5L, 1L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 45L, 5L, 2L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();

	// we close once in the rest...
	assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 18

Source File: WindowOperatorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testProcessingTimeSessionWindows() throws Throwable {
	closeCalled.set(0);

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.session(Duration.ofSeconds(3))
			.withProcessingTime()
			.aggregateAndBuild(getTimeWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);

	RowDataHarnessAssertor assertor = new RowDataHarnessAssertor(
			outputType.getFieldTypes(), new GenericRowRecordSortComparator(0, new VarCharType(VarCharType.MAX_LENGTH)));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	// timestamp is ignored in processing time
	testHarness.setProcessingTime(3);
	testHarness.processElement(insertRecord("key2", 1, 1L));

	testHarness.setProcessingTime(1000);
	testHarness.processElement(insertRecord("key2", 1, 1002L));

	testHarness.setProcessingTime(5000);

	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 2L, 2L, 3L, 4000L, 3999L)));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key2", 1, 5000L));
	testHarness.processElement(insertRecord("key2", 1, 5000L));
	testHarness.processElement(insertRecord("key1", 1, 5000L));
	testHarness.processElement(insertRecord("key1", 1, 5000L));
	testHarness.processElement(insertRecord("key1", 1, 5000L));

	testHarness.setProcessingTime(10000);

	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 2L, 2L, 5000L, 8000L, 7999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 3L, 3L, 5000L, 8000L, 7999L)));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}

Example 19

Source File: TransferenceManager.java From megabasterd with GNU General Public License v3.0

3 votes

protected void sortTransferenceQueue(ConcurrentLinkedQueue<Transference> queue) {

        synchronized (_transference_queue_sort_lock) {

            ArrayList<Transference> trans_list = new ArrayList(queue);

            trans_list.sort((Transference o1, Transference o2) -> MiscTools.naturalCompare(o1.getFile_name(), o2.getFile_name(), true));

            queue.clear();

            queue.addAll(trans_list);
        }
    }

Example 20

Source File: WindowOperatorTest.java From flink with Apache License 2.0

2 votes

@Test
public void testProcessingTimeSlidingWindows() throws Throwable {
	closeCalled.set(0);

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.sliding(Duration.ofSeconds(3), Duration.ofSeconds(1))
			.withProcessingTime()
			.aggregateAndBuild(getTimeWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	// timestamp is ignored in processing time
	testHarness.setProcessingTime(3);
	testHarness.processElement(insertRecord("key2", 1, Long.MAX_VALUE));

	testHarness.setProcessingTime(1000);

	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 1L, 1L, -2000L, 1000L, 999L)));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key2", 1, Long.MAX_VALUE));
	testHarness.processElement(insertRecord("key2", 1, Long.MAX_VALUE));

	testHarness.setProcessingTime(2000);

	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 3L, 3L, -1000L, 2000L, 1999L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key1", 1, Long.MAX_VALUE));
	testHarness.processElement(insertRecord("key1", 1, Long.MAX_VALUE));

	testHarness.setProcessingTime(3000);

	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 3L, 3L, 0L, 3000L, 2999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 2L, 2L, 0L, 3000L, 2999L)));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key1", 1, Long.MAX_VALUE));
	testHarness.processElement(insertRecord("key1", 1, Long.MAX_VALUE));
	testHarness.processElement(insertRecord("key1", 1, Long.MAX_VALUE));

	testHarness.setProcessingTime(7000);

	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 2L, 2L, 1000L, 4000L, 3999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 5L, 5L, 1000L, 4000L, 3999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 5L, 5L, 2000L, 5000L, 4999L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 3L, 3L, 3000L, 6000L, 5999L)));

	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}