Java Code Examples for org.apache.flink.core.fs.Path#fromLocalFile()
The following examples show how to use
org.apache.flink.core.fs.Path#fromLocalFile() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tests that the underlying stream file is deleted upon calling close. */ @Test public void testCleanupWhenClosingStream() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); stream.close(); verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example 2
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that the underlying stream file is deleted upon calling close. */ @Test public void testCleanupWhenClosingStream() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); stream.close(); verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example 3
Source File: FsCheckpointStorageTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testSavepointsInOneDirectoryDefaultLocation() throws Exception { final Path defaultSavepointDir = Path.fromLocalFile(tmp.newFolder()); final FsCheckpointStorage storage = new FsCheckpointStorage( Path.fromLocalFile(tmp.newFolder()), defaultSavepointDir, new JobID(), FILE_SIZE_THRESHOLD); final FsCheckpointStorageLocation savepointLocation = (FsCheckpointStorageLocation) storage.initializeLocationForSavepoint(52452L, null); // all state types should be in the expected location assertParent(defaultSavepointDir, savepointLocation.getCheckpointDirectory()); assertParent(defaultSavepointDir, savepointLocation.getSharedStateDirectory()); assertParent(defaultSavepointDir, savepointLocation.getTaskOwnedStateDirectory()); // cleanup savepointLocation.disposeOnFailure(); }
Example 4
Source File: CompressionFactoryITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testWriteCompressedFile() throws Exception { final File folder = TEMPORARY_FOLDER.newFolder(); final Path testPath = Path.fromLocalFile(folder); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); env.enableCheckpointing(100); DataStream<String> stream = env.addSource( new FiniteTestSource<>(testData), TypeInformation.of(String.class) ); stream.map(str -> str).addSink( StreamingFileSink.forBulkFormat( testPath, CompressWriters.forExtractor(new DefaultExtractor<String>()).withHadoopCompression(TEST_CODEC_NAME) ).build()); env.execute(); validateResults(folder, testData, new CompressionCodecFactory(configuration).getCodecByName(TEST_CODEC_NAME)); }
Example 5
Source File: FileCache.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public Path call() throws IOException { final File file = blobService.getFile(jobID, blobKey); if (isDirectory) { Path directory = FileUtils.expandDirectory(new Path(file.getAbsolutePath()), target); return directory; } else { //noinspection ResultOfMethodCallIgnored file.setExecutable(isExecutable); return Path.fromLocalFile(file); } }
Example 6
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * This test checks that the stream does not check and clean the parent directory * when encountering a write error. */ @Test public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception { final File directory = tempDir.newFolder(); // prevent creation of files in that directory assertTrue(directory.setWritable(false, true)); checkDirectoryNotWritable(directory); FileSystem fs = spy(FileSystem.getLocalFileSystem()); FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream( Path.fromLocalFile(directory), fs, 1024, 1); FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream( Path.fromLocalFile(directory), fs, 1024, 1); stream1.write(new byte[61]); stream2.write(new byte[61]); try { stream1.closeAndGetHandle(); fail("this should fail with an exception"); } catch (IOException ignored) {} stream2.close(); // no delete call must have happened verify(fs, times(0)).delete(any(Path.class), anyBoolean()); // the directory must still exist as a proper directory assertTrue(directory.exists()); assertTrue(directory.isDirectory()); }
Example 7
Source File: FileCache.java From flink with Apache License 2.0 | 5 votes |
@Override public Path call() throws IOException { final File file = blobService.getFile(jobID, blobKey); if (isDirectory) { Path directory = FileUtils.expandDirectory(new Path(file.getAbsolutePath()), target); return directory; } else { //noinspection ResultOfMethodCallIgnored file.setExecutable(isExecutable); return Path.fromLocalFile(file); } }
Example 8
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that the underlying stream file is deleted upon calling close. */ @Test public void testCleanupWhenClosingStream() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0, relativePaths); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); stream.close(); verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example 9
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testEmptyState() throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512, relativePaths); StreamStateHandle handle = stream.closeAndGetHandle(); assertNull(handle); }
Example 10
Source File: AbstractFileCheckpointStorageTestBase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPointerPathResolution() throws Exception { final FileSystem fs = FileSystem.getLocalFileSystem(); final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME); final String basePointer = metadataFile.getParent().toString(); final String pointer1 = metadataFile.toString(); final String pointer2 = metadataFile.getParent().toString(); final String pointer3 = metadataFile.getParent().toString() + '/'; // create the storage for some random checkpoint directory final CheckpointStorage storage = createCheckpointStorage(randomTempPath()); final byte[] data = new byte[23686]; new Random().nextBytes(data); try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) { out.write(data); } CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1); CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2); CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3); assertEquals(basePointer, completed1.getExternalPointer()); assertEquals(basePointer, completed2.getExternalPointer()); assertEquals(basePointer, completed3.getExternalPointer()); StreamStateHandle handle1 = completed1.getMetadataHandle(); StreamStateHandle handle2 = completed2.getMetadataHandle(); StreamStateHandle handle3 = completed3.getMetadataHandle(); assertNotNull(handle1); assertNotNull(handle2); assertNotNull(handle3); validateContents(handle1, data); validateContents(handle2, data); validateContents(handle3, data); }
Example 11
Source File: FileStateHandleTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testDisposeDeletesFile() throws Exception { File file = tempFolder.newFile(); writeTestData(file); assertTrue(file.exists()); FileStateHandle handle = new FileStateHandle(Path.fromLocalFile(file), file.length()); handle.discardState(); assertFalse(file.exists()); }
Example 12
Source File: AbstractFileCheckpointStorageTestBase.java From flink with Apache License 2.0 | 5 votes |
private void testSavepoint( @Nullable Path savepointDir, @Nullable Path customDir, Path expectedParent) throws Exception { final CheckpointStorage storage = savepointDir == null ? createCheckpointStorage(randomTempPath()) : createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir); final String customLocation = customDir == null ? null : customDir.toString(); final CheckpointStorageLocation savepointLocation = storage.initializeLocationForSavepoint(52452L, customLocation); final byte[] data = {77, 66, 55, 99, 88}; final CompletedCheckpointStorageLocation completed; try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) { out.write(data); completed = out.closeAndFinalizeCheckpoint(); } // we need to do this step to make sure we have a slash (or not) in the same way as the // expected path has it final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath())); assertEquals(expectedParent, normalizedWithSlash); validateContents(completed.getMetadataHandle(), data); // validate that the correct directory was used FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle(); // we need to recreate that path in the same way as the "expected path" (via File and URI) to make // sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity) Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI()); assertEquals(expectedParent, usedSavepointDir); }
Example 13
Source File: SequenceStreamingFileSinkITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testWriteSequenceFile() throws Exception { final File folder = TEMPORARY_FOLDER.newFolder(); final Path testPath = Path.fromLocalFile(folder); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); env.enableCheckpointing(100); DataStream<Tuple2<Long, String>> stream = env.addSource( new FiniteTestSource<>(testData), TypeInformation.of(new TypeHint<Tuple2<Long, String>>() { }) ); stream.map(new MapFunction<Tuple2<Long, String>, Tuple2<LongWritable, Text>>() { @Override public Tuple2<LongWritable, Text> map(Tuple2<Long, String> value) throws Exception { return new Tuple2<>(new LongWritable(value.f0), new Text(value.f1)); } }).addSink( StreamingFileSink.forBulkFormat( testPath, new SequenceFileWriterFactory<>(configuration, LongWritable.class, Text.class, "BZip2") ).build()); env.execute(); validateResults(folder, testData); }
Example 14
Source File: AbstractFileCheckpointStorageTestBase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPointerPathResolution() throws Exception { final FileSystem fs = FileSystem.getLocalFileSystem(); final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME); final String basePointer = metadataFile.getParent().toString(); final String pointer1 = metadataFile.toString(); final String pointer2 = metadataFile.getParent().toString(); final String pointer3 = metadataFile.getParent().toString() + '/'; // create the storage for some random checkpoint directory final CheckpointStorage storage = createCheckpointStorage(randomTempPath()); final byte[] data = new byte[23686]; new Random().nextBytes(data); try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) { out.write(data); } CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1); CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2); CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3); assertEquals(basePointer, completed1.getExternalPointer()); assertEquals(basePointer, completed2.getExternalPointer()); assertEquals(basePointer, completed3.getExternalPointer()); StreamStateHandle handle1 = completed1.getMetadataHandle(); StreamStateHandle handle2 = completed2.getMetadataHandle(); StreamStateHandle handle3 = completed3.getMetadataHandle(); assertNotNull(handle1); assertNotNull(handle2); assertNotNull(handle3); validateContents(handle1, data); validateContents(handle2, data); validateContents(handle3, data); }
Example 15
Source File: SequenceStreamingFileSinkITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testWriteSequenceFile() throws Exception { final File folder = TEMPORARY_FOLDER.newFolder(); final Path testPath = Path.fromLocalFile(folder); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); env.enableCheckpointing(100); DataStream<Tuple2<Long, String>> stream = env.addSource( new FiniteTestSource<>(testData), TypeInformation.of(new TypeHint<Tuple2<Long, String>>() { }) ); stream.map(new MapFunction<Tuple2<Long, String>, Tuple2<LongWritable, Text>>() { @Override public Tuple2<LongWritable, Text> map(Tuple2<Long, String> value) throws Exception { return new Tuple2<>(new LongWritable(value.f0), new Text(value.f1)); } }).addSink( StreamingFileSink.forBulkFormat( testPath, new SequenceFileWriterFactory<>(configuration, LongWritable.class, Text.class, "BZip2") ).build()); env.execute(); validateResults(folder, testData); }
Example 16
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testMixedBelowAndAboveThreshold() throws Exception { final byte[] state1 = new byte[1274673]; final byte[] state2 = new byte[1]; final byte[] state3 = new byte[0]; final byte[] state4 = new byte[177]; final Random rnd = new Random(); rnd.nextBytes(state1); rnd.nextBytes(state2); rnd.nextBytes(state3); rnd.nextBytes(state4); final File directory = tempDir.newFolder(); final Path basePath = Path.fromLocalFile(directory); final Supplier<CheckpointStateOutputStream> factory = () -> new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15, relativePaths); CheckpointStateOutputStream stream1 = factory.get(); CheckpointStateOutputStream stream2 = factory.get(); CheckpointStateOutputStream stream3 = factory.get(); stream1.write(state1); stream2.write(state2); stream3.write(state3); FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle(); ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle(); ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle(); // use with try-with-resources StreamStateHandle handle4; try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) { stream4.write(state4); handle4 = stream4.closeAndGetHandle(); } // close before accessing handle CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get(); stream5.write(state4); stream5.close(); try { stream5.closeAndGetHandle(); fail(); } catch (IOException e) { // uh-huh } validateBytesInStream(handle1.openInputStream(), state1); handle1.discardState(); assertFalse(isDirectoryEmpty(directory)); ensureLocalFileDeleted(handle1.getFilePath()); validateBytesInStream(handle2.openInputStream(), state2); handle2.discardState(); assertFalse(isDirectoryEmpty(directory)); // nothing was written to the stream, so it will return nothing assertNull(handle3); assertFalse(isDirectoryEmpty(directory)); validateBytesInStream(handle4.openInputStream(), state4); handle4.discardState(); assertTrue(isDirectoryEmpty(directory)); }
Example 17
Source File: AbstractFileCheckpointStorageTestBase.java From flink with Apache License 2.0 | 4 votes |
public Path randomTempPath() throws IOException { return Path.fromLocalFile(tmp.newFolder()); }
Example 18
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 4 votes |
@Test(expected = IllegalArgumentException.class) public void testWrongParameters() throws Exception { // this should fail new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 4000, 5000, relativePaths); }
Example 19
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testMixedBelowAndAboveThreshold() throws Exception { final byte[] state1 = new byte[1274673]; final byte[] state2 = new byte[1]; final byte[] state3 = new byte[0]; final byte[] state4 = new byte[177]; final Random rnd = new Random(); rnd.nextBytes(state1); rnd.nextBytes(state2); rnd.nextBytes(state3); rnd.nextBytes(state4); final File directory = tempDir.newFolder(); final Path basePath = Path.fromLocalFile(directory); final Supplier<CheckpointStateOutputStream> factory = () -> new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15); CheckpointStateOutputStream stream1 = factory.get(); CheckpointStateOutputStream stream2 = factory.get(); CheckpointStateOutputStream stream3 = factory.get(); stream1.write(state1); stream2.write(state2); stream3.write(state3); FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle(); ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle(); ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle(); // use with try-with-resources StreamStateHandle handle4; try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) { stream4.write(state4); handle4 = stream4.closeAndGetHandle(); } // close before accessing handle CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get(); stream5.write(state4); stream5.close(); try { stream5.closeAndGetHandle(); fail(); } catch (IOException e) { // uh-huh } validateBytesInStream(handle1.openInputStream(), state1); handle1.discardState(); assertFalse(isDirectoryEmpty(directory)); ensureLocalFileDeleted(handle1.getFilePath()); validateBytesInStream(handle2.openInputStream(), state2); handle2.discardState(); assertFalse(isDirectoryEmpty(directory)); // nothing was written to the stream, so it will return nothing assertNull(handle3); assertFalse(isDirectoryEmpty(directory)); validateBytesInStream(handle4.openInputStream(), state4); handle4.discardState(); assertTrue(isDirectoryEmpty(directory)); }
Example 20
Source File: FsCheckpointStreamFactoryTest.java From flink with Apache License 2.0 | 4 votes |
@Before public void createStateDirectories() throws IOException { exclusiveStateDir = Path.fromLocalFile(TMP.newFolder("exclusive")); sharedStateDir = Path.fromLocalFile(TMP.newFolder("shared")); }