org.apache.flink.core.fs.local.LocalDataOutputStream Java Examples

The following examples show how to use org.apache.flink.core.fs.local.LocalDataOutputStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: InitOutputPathTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private void runTest(final boolean useAwaits) throws Exception {
	final File tempFile = tempDir.newFile();
	final Path path1 = new Path(tempFile.getAbsolutePath(), "1");
	final Path path2 = new Path(tempFile.getAbsolutePath(), "2");

	final OneShotLatch deleteAwaitLatch1 = new OneShotLatch();
	final OneShotLatch deleteAwaitLatch2 = new OneShotLatch();
	final OneShotLatch mkdirsAwaitLatch1 = new OneShotLatch();
	final OneShotLatch mkdirsAwaitLatch2 = new OneShotLatch();

	final OneShotLatch deleteTriggerLatch1 = new OneShotLatch();
	final OneShotLatch deletetriggerLatch2 = new OneShotLatch();
	final OneShotLatch mkdirsTriggerLatch1 = new OneShotLatch();
	final OneShotLatch mkdirsTriggerLatch2 = new OneShotLatch();

	final OneShotLatch createAwaitLatch = new OneShotLatch();
	final OneShotLatch createTriggerLatch = new OneShotLatch();

	// this "new LocalDataOutputStream()" is in the end called by the async threads
	whenNew(LocalDataOutputStream.class).withAnyArguments().thenAnswer(new Answer<LocalDataOutputStream>() {

		@Override
		public LocalDataOutputStream answer(InvocationOnMock invocation) throws Throwable {
			createAwaitLatch.trigger();
			createTriggerLatch.await();

			final File file = (File) invocation.getArguments()[0];
			return new LocalDataOutputStream(file);
		}
	});

	final LocalFileSystem fs1 = new SyncedFileSystem(
			deleteAwaitLatch1, mkdirsAwaitLatch1,
			deleteTriggerLatch1, mkdirsTriggerLatch1);

	final LocalFileSystem fs2 = new SyncedFileSystem(
			deleteAwaitLatch2, mkdirsAwaitLatch2,
			deletetriggerLatch2, mkdirsTriggerLatch2);

	// start the concurrent file creators
	FileCreator thread1 = new FileCreator(fs1, path1);
	FileCreator thread2 = new FileCreator(fs2, path2);
	thread1.start();
	thread2.start();

	// wait until they both decide to delete the directory
	if (useAwaits) {
		deleteAwaitLatch1.await();
		deleteAwaitLatch2.await();
	} else {
		Thread.sleep(5);
	}

	// now send off #1 to delete the directory (it will pass the 'mkdirs' fast) and wait to create the file
	mkdirsTriggerLatch1.trigger();
	deleteTriggerLatch1.trigger();

	if (useAwaits) {
		createAwaitLatch.await();
	} else {
		// this needs a bit more sleep time, because here mockito is working
		Thread.sleep(100);
	}

	// now send off #2 to delete the directory - it waits at 'mkdirs'
	deletetriggerLatch2.trigger();
	if (useAwaits) {
		mkdirsAwaitLatch2.await();
	} else {
		Thread.sleep(5);
	}

	// let #1 try to create the file and see if it succeeded
	createTriggerLatch.trigger();
	if (useAwaits) {
		thread1.sync();
	} else {
		Thread.sleep(5);
	}

	// now let #1 finish up
	mkdirsTriggerLatch2.trigger();

	thread1.sync();
	thread2.sync();
}
Example #2
Source File: InitOutputPathTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void runTest(final boolean useAwaits) throws Exception {
	final File tempFile = tempDir.newFile();
	final Path path1 = new Path(tempFile.getAbsolutePath(), "1");
	final Path path2 = new Path(tempFile.getAbsolutePath(), "2");

	final OneShotLatch deleteAwaitLatch1 = new OneShotLatch();
	final OneShotLatch deleteAwaitLatch2 = new OneShotLatch();
	final OneShotLatch mkdirsAwaitLatch1 = new OneShotLatch();
	final OneShotLatch mkdirsAwaitLatch2 = new OneShotLatch();

	final OneShotLatch deleteTriggerLatch1 = new OneShotLatch();
	final OneShotLatch deletetriggerLatch2 = new OneShotLatch();
	final OneShotLatch mkdirsTriggerLatch1 = new OneShotLatch();
	final OneShotLatch mkdirsTriggerLatch2 = new OneShotLatch();

	final OneShotLatch createAwaitLatch = new OneShotLatch();
	final OneShotLatch createTriggerLatch = new OneShotLatch();

	// this "new LocalDataOutputStream()" is in the end called by the async threads
	whenNew(LocalDataOutputStream.class).withAnyArguments().thenAnswer(new Answer<LocalDataOutputStream>() {

		@Override
		public LocalDataOutputStream answer(InvocationOnMock invocation) throws Throwable {
			createAwaitLatch.trigger();
			createTriggerLatch.await();

			final File file = (File) invocation.getArguments()[0];
			return new LocalDataOutputStream(file);
		}
	});

	final LocalFileSystem fs1 = new SyncedFileSystem(
			deleteAwaitLatch1, mkdirsAwaitLatch1,
			deleteTriggerLatch1, mkdirsTriggerLatch1);

	final LocalFileSystem fs2 = new SyncedFileSystem(
			deleteAwaitLatch2, mkdirsAwaitLatch2,
			deletetriggerLatch2, mkdirsTriggerLatch2);

	// start the concurrent file creators
	FileCreator thread1 = new FileCreator(fs1, path1);
	FileCreator thread2 = new FileCreator(fs2, path2);
	thread1.start();
	thread2.start();

	// wait until they both decide to delete the directory
	if (useAwaits) {
		deleteAwaitLatch1.await();
		deleteAwaitLatch2.await();
	} else {
		Thread.sleep(5);
	}

	// now send off #1 to delete the directory (it will pass the 'mkdirs' fast) and wait to create the file
	mkdirsTriggerLatch1.trigger();
	deleteTriggerLatch1.trigger();

	if (useAwaits) {
		createAwaitLatch.await();
	} else {
		// this needs a bit more sleep time, because here mockito is working
		Thread.sleep(100);
	}

	// now send off #2 to delete the directory - it waits at 'mkdirs'
	deletetriggerLatch2.trigger();
	if (useAwaits) {
		mkdirsAwaitLatch2.await();
	} else {
		Thread.sleep(5);
	}

	// let #1 try to create the file and see if it succeeded
	createTriggerLatch.trigger();
	if (useAwaits) {
		thread1.sync();
	} else {
		Thread.sleep(5);
	}

	// now let #1 finish up
	mkdirsTriggerLatch2.trigger();

	thread1.sync();
	thread2.sync();
}
Example #3
Source File: InitOutputPathTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void runTest(final boolean useAwaits) throws Exception {
	final File tempFile = tempDir.newFile();
	final Path path1 = new Path(tempFile.getAbsolutePath(), "1");
	final Path path2 = new Path(tempFile.getAbsolutePath(), "2");

	final OneShotLatch deleteAwaitLatch1 = new OneShotLatch();
	final OneShotLatch deleteAwaitLatch2 = new OneShotLatch();
	final OneShotLatch mkdirsAwaitLatch1 = new OneShotLatch();
	final OneShotLatch mkdirsAwaitLatch2 = new OneShotLatch();

	final OneShotLatch deleteTriggerLatch1 = new OneShotLatch();
	final OneShotLatch deletetriggerLatch2 = new OneShotLatch();
	final OneShotLatch mkdirsTriggerLatch1 = new OneShotLatch();
	final OneShotLatch mkdirsTriggerLatch2 = new OneShotLatch();

	final OneShotLatch createAwaitLatch = new OneShotLatch();
	final OneShotLatch createTriggerLatch = new OneShotLatch();

	// this "new LocalDataOutputStream()" is in the end called by the async threads
	whenNew(LocalDataOutputStream.class).withAnyArguments().thenAnswer(new Answer<LocalDataOutputStream>() {

		@Override
		public LocalDataOutputStream answer(InvocationOnMock invocation) throws Throwable {
			createAwaitLatch.trigger();
			createTriggerLatch.await();

			final File file = (File) invocation.getArguments()[0];
			return new LocalDataOutputStream(file);
		}
	});

	final LocalFileSystem fs1 = new SyncedFileSystem(
			deleteAwaitLatch1, mkdirsAwaitLatch1,
			deleteTriggerLatch1, mkdirsTriggerLatch1);

	final LocalFileSystem fs2 = new SyncedFileSystem(
			deleteAwaitLatch2, mkdirsAwaitLatch2,
			deletetriggerLatch2, mkdirsTriggerLatch2);

	// start the concurrent file creators
	FileCreator thread1 = new FileCreator(fs1, path1);
	FileCreator thread2 = new FileCreator(fs2, path2);
	thread1.start();
	thread2.start();

	// wait until they both decide to delete the directory
	if (useAwaits) {
		deleteAwaitLatch1.await();
		deleteAwaitLatch2.await();
	} else {
		Thread.sleep(5);
	}

	// now send off #1 to delete the directory (it will pass the 'mkdirs' fast) and wait to create the file
	mkdirsTriggerLatch1.trigger();
	deleteTriggerLatch1.trigger();

	if (useAwaits) {
		createAwaitLatch.await();
	} else {
		// this needs a bit more sleep time, because here mockito is working
		Thread.sleep(100);
	}

	// now send off #2 to delete the directory - it waits at 'mkdirs'
	deletetriggerLatch2.trigger();
	if (useAwaits) {
		mkdirsAwaitLatch2.await();
	} else {
		Thread.sleep(5);
	}

	// let #1 try to create the file and see if it succeeded
	createTriggerLatch.trigger();
	if (useAwaits) {
		thread1.sync();
	} else {
		Thread.sleep(5);
	}

	// now let #1 finish up
	mkdirsTriggerLatch2.trigger();

	thread1.sync();
	thread2.sync();
}