Java Code Examples for org.apache.flink.core.fs.FileSystem#get()

protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path[] filePaths, ArrayList<FileStatus> files) throws IOException {

		long totalLength = 0;
		long latestModTime = 0;

		for (Path path : filePaths) {
			final FileSystem fs = FileSystem.get(path.toUri());
			final FileBaseStatistics stats = getFileStats(cachedStats, path, fs, files);

			if (stats.getTotalInputSize() == BaseStatistics.SIZE_UNKNOWN) {
				totalLength = BaseStatistics.SIZE_UNKNOWN;
			} else if (totalLength != BaseStatistics.SIZE_UNKNOWN) {
				totalLength += stats.getTotalInputSize();
			}
			latestModTime = Math.max(latestModTime, stats.getLastModificationTime());
		}

		// check whether the cached statistics are still valid, if we have any
		if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) {
			return cachedStats;
		}

		return new FileBaseStatistics(latestModTime, totalLength, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
	}
 

@Test
public void testExplicitlySetToOther() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	try {
		FileSystem.get(justPath);
		fail("should have failed with an exception");
	}
	catch (UnsupportedFileSystemSchemeException e) {
		assertTrue(e.getMessage().contains("otherFS"));
	}
}
 

@Override
public void run() {
	try {
		final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
		this.fdis = fs.open(this.split.getPath());
		
		// check for canceling and close the stream in that case, because no one will obtain it
		if (this.aborted) {
			final FSDataInputStream f = this.fdis;
			this.fdis = null;
			f.close();
		}
	}
	catch (Throwable t) {
		this.error = t;
	}
}
 

@Test
public void testConfigPropagationAlternateStyle() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", "test_access_key_id");
	conf.setString("s3.secret.key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}
 

@Test
public void testDefaultsToLocal() throws Exception {
	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 

@Override
public void run(SourceContext<Tuple3<String, Long, Long>> ctx) throws Exception {
	FileSystem fileSystem = FileSystem.get(new URI(path));

	while (isRunning) {
		List<String> files = listNewFiles(fileSystem);
		for (String filePath : files) {
			if (watchType == WatchType.ONLY_NEW_FILES
					|| watchType == WatchType.REPROCESS_WITH_APPENDED) {
				ctx.collect(new Tuple3<String, Long, Long>(filePath, 0L, -1L));
				offsetOfFiles.put(filePath, -1L);
			} else if (watchType == WatchType.PROCESS_ONLY_APPENDED) {
				long offset = 0;
				long fileSize = fileSystem.getFileStatus(new Path(filePath)).getLen();
				if (offsetOfFiles.containsKey(filePath)) {
					offset = offsetOfFiles.get(filePath);
				}

				ctx.collect(new Tuple3<String, Long, Long>(filePath, offset, fileSize));
				offsetOfFiles.put(filePath, fileSize);

				LOG.info("File processed: {}, {}, {}", filePath, offset, fileSize);
			}
		}

		Thread.sleep(interval);
	}
}
 

@Test
public void testExplicitlySetToLocal() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, LocalFileSystem.getLocalFsURI().toString());
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 

@Test
public void testConfigPropagationWithPrestoPrefix() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("presto.s3.access-key", "test_access_key_id");
	conf.setString("presto.s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

@Test
public void testConfigPropagationWithPrestoPrefix() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("presto.s3.access-key", "test_access_key_id");
	conf.setString("presto.s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

@Test
public void testConfigPropagation() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", "test_access_key_id");
	conf.setString("s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

@Test
public void testConfiguration() throws Exception {

	// nothing configured, we should get a regular file system
	FileSystem hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c"));
	FileSystem ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));

	assertFalse(hdfs instanceof LimitedConnectionsFileSystem);
	assertFalse(ftpfs instanceof LimitedConnectionsFileSystem);

	// configure some limits, which should cause "fsScheme" to be limited

	final Configuration config = new Configuration();
	config.setInteger("fs.hdfs.limit.total", 40);
	config.setInteger("fs.hdfs.limit.input", 39);
	config.setInteger("fs.hdfs.limit.output", 38);
	config.setInteger("fs.hdfs.limit.timeout", 23456);
	config.setInteger("fs.hdfs.limit.stream-timeout", 34567);

	try {
		FileSystem.initialize(config);

		hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c"));
		ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));

		assertTrue(hdfs instanceof LimitedConnectionsFileSystem);
		assertFalse(ftpfs instanceof LimitedConnectionsFileSystem);

		LimitedConnectionsFileSystem limitedFs = (LimitedConnectionsFileSystem) hdfs;
		assertEquals(40, limitedFs.getMaxNumOpenStreamsTotal());
		assertEquals(39, limitedFs.getMaxNumOpenInputStreams());
		assertEquals(38, limitedFs.getMaxNumOpenOutputStreams());
		assertEquals(23456, limitedFs.getStreamOpenTimeout());
		assertEquals(34567, limitedFs.getStreamInactivityTimeout());
	}
	finally {
		// clear all settings
		FileSystem.initialize(new Configuration());
	}
}
 

@Test
public void testExplicitlyPathTakesPrecedence() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI pathAndScheme = tempFolder.newFile().toURI();
	assertNotNull(pathAndScheme.getScheme());

	FileSystem fs = FileSystem.get(pathAndScheme);
	assertEquals("file", fs.getUri().getScheme());
}
 

@Test
public void testConfigPropagation() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", "test_access_key_id");
	conf.setString("s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 

@Test
public void testExplicitlySetToLocal() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, LocalFileSystem.getLocalFsURI().toString());
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}
 

@Test
public void testExplicitlyPathTakesPrecedence() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI pathAndScheme = tempFolder.newFile().toURI();
	assertNotNull(pathAndScheme.getScheme());

	FileSystem fs = FileSystem.get(pathAndScheme);
	assertEquals("file", fs.getUri().getScheme());
}
 

@Test(expected = UnsupportedOperationException.class)
public void requestingRecoverableWriterShouldThroughException() throws Exception {
	URI s3Uri = URI.create(S3TestCredentials.getTestBucketUri());
	FlinkS3FileSystem fileSystem = (FlinkS3FileSystem) FileSystem.get(s3Uri);
	fileSystem.createRecoverableWriter();
}
 

public FileSystemStateStorageHelper(Path rootPath, String prefix) throws IOException {
	this.rootPath = Preconditions.checkNotNull(rootPath, "Root path");
	this.prefix = Preconditions.checkNotNull(prefix, "Prefix");

	fs = FileSystem.get(rootPath.toUri());
}
 

/**
 * Gets the file system that stores the file state.
 *
 * @return The file system that stores the file state.
 * @throws IOException Thrown if the file system cannot be accessed.
 */
private FileSystem getFileSystem() throws IOException {
	return FileSystem.get(filePath.toUri());
}