Java Code Examples for org.apache.flink.core.fs.FileSystem#get()

The following examples show how to use org.apache.flink.core.fs.FileSystem#get() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: FileInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
	try {
		final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
		this.fdis = fs.open(this.split.getPath());
		
		// check for canceling and close the stream in that case, because no one will obtain it
		if (this.aborted) {
			final FSDataInputStream f = this.fdis;
			this.fdis = null;
			f.close();
		}
	}
	catch (Throwable t) {
		this.error = t;
	}
}
 
Example 2
@Test
public void testExplicitlySetToOther() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	try {
		FileSystem.get(justPath);
		fail("should have failed with an exception");
	}
	catch (UnsupportedFileSystemSchemeException e) {
		assertTrue(e.getMessage().contains("otherFS"));
	}
}
 
Example 3
Source Project: flink   File: FileInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path[] filePaths, ArrayList<FileStatus> files) throws IOException {

		long totalLength = 0;
		long latestModTime = 0;

		for (Path path : filePaths) {
			final FileSystem fs = FileSystem.get(path.toUri());
			final FileBaseStatistics stats = getFileStats(cachedStats, path, fs, files);

			if (stats.getTotalInputSize() == BaseStatistics.SIZE_UNKNOWN) {
				totalLength = BaseStatistics.SIZE_UNKNOWN;
			} else if (totalLength != BaseStatistics.SIZE_UNKNOWN) {
				totalLength += stats.getTotalInputSize();
			}
			latestModTime = Math.max(latestModTime, stats.getLastModificationTime());
		}

		// check whether the cached statistics are still valid, if we have any
		if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) {
			return cachedStats;
		}

		return new FileBaseStatistics(latestModTime, totalLength, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
	}
 
Example 4
@Test
public void testExplicitlyPathTakesPrecedence() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI pathAndScheme = tempFolder.newFile().toURI();
	assertNotNull(pathAndScheme.getScheme());

	FileSystem fs = FileSystem.get(pathAndScheme);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 5
Source Project: flink   File: PrestoS3FileSystemTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigPropagationAlternateStyle() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", "test_access_key_id");
	conf.setString("s3.secret.key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 6
Source Project: flink   File: BucketStateSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}
 
Example 7
@Test
public void testExplicitlySetToLocal() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, LocalFileSystem.getLocalFsURI().toString());
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 8
Source Project: flink   File: PrestoS3FileSystemTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigPropagation() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", "test_access_key_id");
	conf.setString("s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 9
@Test
public void testExplicitlyPathTakesPrecedence() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/");
	FileSystem.initialize(conf);

	URI pathAndScheme = tempFolder.newFile().toURI();
	assertNotNull(pathAndScheme.getScheme());

	FileSystem fs = FileSystem.get(pathAndScheme);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 10
@Test
public void testConfiguration() throws Exception {

	// nothing configured, we should get a regular file system
	FileSystem hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c"));
	FileSystem ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));

	assertFalse(hdfs instanceof LimitedConnectionsFileSystem);
	assertFalse(ftpfs instanceof LimitedConnectionsFileSystem);

	// configure some limits, which should cause "fsScheme" to be limited

	final Configuration config = new Configuration();
	config.setInteger("fs.hdfs.limit.total", 40);
	config.setInteger("fs.hdfs.limit.input", 39);
	config.setInteger("fs.hdfs.limit.output", 38);
	config.setInteger("fs.hdfs.limit.timeout", 23456);
	config.setInteger("fs.hdfs.limit.stream-timeout", 34567);

	try {
		FileSystem.initialize(config);

		hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c"));
		ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));

		assertTrue(hdfs instanceof LimitedConnectionsFileSystem);
		assertFalse(ftpfs instanceof LimitedConnectionsFileSystem);

		LimitedConnectionsFileSystem limitedFs = (LimitedConnectionsFileSystem) hdfs;
		assertEquals(40, limitedFs.getMaxNumOpenStreamsTotal());
		assertEquals(39, limitedFs.getMaxNumOpenInputStreams());
		assertEquals(38, limitedFs.getMaxNumOpenOutputStreams());
		assertEquals(23456, limitedFs.getStreamOpenTimeout());
		assertEquals(34567, limitedFs.getStreamInactivityTimeout());
	}
	finally {
		// clear all settings
		FileSystem.initialize(new Configuration());
	}
}
 
Example 11
@Test
public void testConfigPropagation() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("s3.access-key", "test_access_key_id");
	conf.setString("s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 12
@Test
public void testConfigPropagationWithPrestoPrefix() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("presto.s3.access-key", "test_access_key_id");
	conf.setString("presto.s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 13
Source Project: flink   File: PrestoS3FileSystemTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigPropagationWithPrestoPrefix() throws Exception{
	final Configuration conf = new Configuration();
	conf.setString("presto.s3.access-key", "test_access_key_id");
	conf.setString("presto.s3.secret-key", "test_secret_access_key");

	FileSystem.initialize(conf);

	FileSystem fs = FileSystem.get(new URI("s3://test"));
	validateBasicCredentials(fs);
}
 
Example 14
@Test
public void testExplicitlySetToLocal() throws Exception {
	final Configuration conf = new Configuration();
	conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, LocalFileSystem.getLocalFsURI().toString());
	FileSystem.initialize(conf);

	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 15
Source Project: flink   File: FileMonitoringFunction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run(SourceContext<Tuple3<String, Long, Long>> ctx) throws Exception {
	FileSystem fileSystem = FileSystem.get(new URI(path));

	while (isRunning) {
		List<String> files = listNewFiles(fileSystem);
		for (String filePath : files) {
			if (watchType == WatchType.ONLY_NEW_FILES
					|| watchType == WatchType.REPROCESS_WITH_APPENDED) {
				ctx.collect(new Tuple3<String, Long, Long>(filePath, 0L, -1L));
				offsetOfFiles.put(filePath, -1L);
			} else if (watchType == WatchType.PROCESS_ONLY_APPENDED) {
				long offset = 0;
				long fileSize = fileSystem.getFileStatus(new Path(filePath)).getLen();
				if (offsetOfFiles.containsKey(filePath)) {
					offset = offsetOfFiles.get(filePath);
				}

				ctx.collect(new Tuple3<String, Long, Long>(filePath, offset, fileSize));
				offsetOfFiles.put(filePath, fileSize);

				LOG.info("File processed: {}, {}, {}", filePath, offset, fileSize);
			}
		}

		Thread.sleep(interval);
	}
}
 
Example 16
@Test
public void testDefaultsToLocal() throws Exception {
	URI justPath = new URI(tempFolder.newFile().toURI().getPath());
	assertNull(justPath.getScheme());

	FileSystem fs = FileSystem.get(justPath);
	assertEquals("file", fs.getUri().getScheme());
}
 
Example 17
@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}
 
Example 18
@Test(expected = UnsupportedOperationException.class)
public void requestingRecoverableWriterShouldThroughException() throws Exception {
	URI s3Uri = URI.create(S3TestCredentials.getTestBucketUri());
	FlinkS3FileSystem fileSystem = (FlinkS3FileSystem) FileSystem.get(s3Uri);
	fileSystem.createRecoverableWriter();
}
 
Example 19
public FileSystemStateStorageHelper(Path rootPath, String prefix) throws IOException {
	this.rootPath = Preconditions.checkNotNull(rootPath, "Root path");
	this.prefix = Preconditions.checkNotNull(prefix, "Prefix");

	fs = FileSystem.get(rootPath.toUri());
}
 
Example 20
Source Project: flink   File: FileStateHandle.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Gets the file system that stores the file state.
 *
 * @return The file system that stores the file state.
 * @throws IOException Thrown if the file system cannot be accessed.
 */
private FileSystem getFileSystem() throws IOException {
	return FileSystem.get(filePath.toUri());
}