Java Code Examples for org.apache.flink.core.fs.FileSystem#get()

The following examples show how to use org.apache.flink.core.fs.FileSystem#get() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path[] filePaths, ArrayList<FileStatus> files) throws IOException {

		long totalLength = 0;
		long latestModTime = 0;

		for (Path path : filePaths) {
			final FileSystem fs = FileSystem.get(path.toUri());
			final FileBaseStatistics stats = getFileStats(cachedStats, path, fs, files);

			if (stats.getTotalInputSize() == BaseStatistics.SIZE_UNKNOWN) {
				totalLength = BaseStatistics.SIZE_UNKNOWN;
			} else if (totalLength != BaseStatistics.SIZE_UNKNOWN) {
				totalLength += stats.getTotalInputSize();
			}
			latestModTime = Math.max(latestModTime, stats.getLastModificationTime());
		}

		// check whether the cached statistics are still valid, if we have any
		if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) {
			return cachedStats;
		}

		return new FileBaseStatistics(latestModTime, totalLength, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
	}
 
Example 2
Source File: FilesystemSchemeConfigTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testExplicitlySetToOther() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	try {
		FileSystem.get(justPath);
		fail("should have failed with an exception");
	}
	catch (UnsupportedFileSystemSchemeException e) {
		assertTrue(e.getMessage().contains("otherFS"));
	}
}
 
Example 3
Source File: FileInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
	try {
		final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
		this.fdis = fs.open(this.split.getPath());
		
		// check for canceling and close the stream in that case, because no one will obtain it
		if (this.aborted) {
			final FSDataInputStream f = this.fdis;
			this.fdis = null;
			f.close();
		}
	}
	catch (Throwable t) {
		this.error = t;
	}
}
 
Example 4
Source File: PrestoS3FileSystemTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigPropagationAlternateStyle() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", "test_access_key_id");
	conf.setString("s3.secret.key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 5
Source File: BucketStateSerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}
 
Example 6
Source File: FilesystemSchemeConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testDefaultsToLocal() throws Exception {
	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 7
Source File: FileMonitoringFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void run(SourceContext<Tuple3<String, Long, Long>> ctx) throws Exception {
	FileSystem fileSystem = FileSystem.get(new URI(path));

	while (isRunning) {
		List<String> files = listNewFiles(fileSystem);
		for (String filePath : files) {
			if (watchType == WatchType.ONLY_NEW_FILES
					|| watchType == WatchType.REPROCESS_WITH_APPENDED) {
				ctx.collect(new Tuple3<String, Long, Long>(filePath, 0L, -1L));
				offsetOfFiles.put(filePath, -1L);
			} else if (watchType == WatchType.PROCESS_ONLY_APPENDED) {
				long offset = 0;
				long fileSize = fileSystem.getFileStatus(new Path(filePath)).getLen();
				if (offsetOfFiles.containsKey(filePath)) {
					offset = offsetOfFiles.get(filePath);
				}

				ctx.collect(new Tuple3<String, Long, Long>(filePath, offset, fileSize));
				offsetOfFiles.put(filePath, fileSize);

				LOG.info("File processed: {}, {}, {}", filePath, offset, fileSize);
			}
		}

		Thread.sleep(interval);
	}
}
 
Example 8
Source File: FilesystemSchemeConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testExplicitlySetToLocal() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, LocalFileSystem.getLocalFsURI().toString());
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 9
Source File: PrestoS3FileSystemTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigPropagationWithPrestoPrefix() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("presto.s3.access-key", "test_access_key_id");
	conf.setString("presto.s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 10
Source File: PrestoS3FileSystemTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigPropagationWithPrestoPrefix() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("presto.s3.access-key", "test_access_key_id");
	conf.setString("presto.s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 11
Source File: PrestoS3FileSystemTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigPropagation() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", "test_access_key_id");
	conf.setString("s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 12
Source File: LimitedConnectionsConfigurationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfiguration() throws Exception {

	// nothing configured, we should get a regular file system
	FileSystem hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c"));
	FileSystem ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));

	assertFalse(hdfs instanceof LimitedConnectionsFileSystem);
	assertFalse(ftpfs instanceof LimitedConnectionsFileSystem);

	// configure some limits, which should cause "fsScheme" to be limited

	final Configuration config = new Configuration();
	config.setInteger("fs.hdfs.limit.total", 40);
	config.setInteger("fs.hdfs.limit.input", 39);
	config.setInteger("fs.hdfs.limit.output", 38);
	config.setInteger("fs.hdfs.limit.timeout", 23456);
	config.setInteger("fs.hdfs.limit.stream-timeout", 34567);

	try {
		FileSystem.initialize(config);

		hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c"));
		ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));

		assertTrue(hdfs instanceof LimitedConnectionsFileSystem);
		assertFalse(ftpfs instanceof LimitedConnectionsFileSystem);

		LimitedConnectionsFileSystem limitedFs = (LimitedConnectionsFileSystem) hdfs;
		assertEquals(40, limitedFs.getMaxNumOpenStreamsTotal());
		assertEquals(39, limitedFs.getMaxNumOpenInputStreams());
		assertEquals(38, limitedFs.getMaxNumOpenOutputStreams());
		assertEquals(23456, limitedFs.getStreamOpenTimeout());
		assertEquals(34567, limitedFs.getStreamInactivityTimeout());
	}
	finally {
		// clear all settings
		FileSystem.initialize(new Configuration());
	}
}
 
Example 13
Source File: FilesystemSchemeConfigTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testExplicitlyPathTakesPrecedence() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI pathAndScheme = tempFolder.newFile().toURI();
	assertNotNull(pathAndScheme.getScheme());

	FileSystem fs = FileSystem.get(pathAndScheme);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 14
Source File: PrestoS3FileSystemTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigPropagation() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", "test_access_key_id");
	conf.setString("s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 15
Source File: FilesystemSchemeConfigTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testExplicitlySetToLocal() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, LocalFileSystem.getLocalFsURI().toString());
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 16
Source File: BucketStateSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}
 
Example 17
Source File: FilesystemSchemeConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testExplicitlyPathTakesPrecedence() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI pathAndScheme = tempFolder.newFile().toURI();
	assertNotNull(pathAndScheme.getScheme());

	FileSystem fs = FileSystem.get(pathAndScheme);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 18
Source File: PrestoS3RecoverableWriterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test(expected = UnsupportedOperationException.class)
public void requestingRecoverableWriterShouldThroughException() throws Exception {
	URI s3Uri = URI.create(S3TestCredentials.getTestBucketUri());
	FlinkS3FileSystem fileSystem = (FlinkS3FileSystem) FileSystem.get(s3Uri);
	fileSystem.createRecoverableWriter();
}
 
Example 19
Source File: FileSystemStateStorageHelper.java    From flink with Apache License 2.0 4 votes vote down vote up
public FileSystemStateStorageHelper(Path rootPath, String prefix) throws IOException {
	this.rootPath = Preconditions.checkNotNull(rootPath, "Root path");
	this.prefix = Preconditions.checkNotNull(prefix, "Prefix");

	fs = FileSystem.get(rootPath.toUri());
}
 
Example 20
Source File: FileStateHandle.java    From flink with Apache License 2.0 2 votes vote down vote up
/**
 * Gets the file system that stores the file state.
 *
 * @return The file system that stores the file state.
 * @throws IOException Thrown if the file system cannot be accessed.
 */
private FileSystem getFileSystem() throws IOException {
	return FileSystem.get(filePath.toUri());
}