Java Code Examples for org.apache.flink.core.fs.Path#toUri()

The following examples show how to use org.apache.flink.core.fs.Path#toUri() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: AbstractFileStateBackend.java    License: Apache License 2.0 7 votes vote down vote up
/**
 * Checks the validity of the path's scheme and path.
 *
 * @param path The path to check.
 * @return The URI as a Path.
 *
 * @throws IllegalArgumentException Thrown, if the URI misses scheme or path.
 */
private static Path validatePath(Path path) {
	final URI uri = path.toUri();
	final String scheme = uri.getScheme();
	final String pathPart = uri.getPath();

	// some validity checks
	if (scheme == null) {
		throw new IllegalArgumentException("The scheme (hdfs://, file://, etc) is null. " +
				"Please specify the file system scheme explicitly in the URI.");
	}
	if (pathPart == null) {
		throw new IllegalArgumentException("The path to store the checkpoint data in is null. " +
				"Please specify a directory path for the checkpoint data.");
	}
	if (pathPart.length() == 0 || pathPart.equals("/")) {
		throw new IllegalArgumentException("Cannot use the root directory for checkpoints.");
	}

	return path;
}
 
Example 2
@Override
public String map(Integer value) throws Exception {
	final Path actualFile = new Path(getRuntimeContext().getDistributedCache().getFile("test_data").toURI());

	Path path = new Path(actualFile.toUri());
	assertFalse(path.getFileSystem().isDistributedFS());

	DataInputStream in = new DataInputStream(actualFile.getFileSystem().open(actualFile));
	String contents = in.readUTF();

	assertEquals(testFileContent, contents);

	final Path actualDir = new Path(getRuntimeContext().getDistributedCache().getFile("test_dir").toURI());
	FileStatus fileStatus = actualDir.getFileSystem().getFileStatus(actualDir);
	assertTrue(fileStatus.isDir());
	FileStatus[] fileStatuses = actualDir.getFileSystem().listStatus(actualDir);
	assertEquals(2, fileStatuses.length);

	return contents;
}
 
Example 3
/**
 * Checks the validity of the path's scheme and path.
 *
 * @param path The path to check.
 * @return The URI as a Path.
 *
 * @throws IllegalArgumentException Thrown, if the URI misses scheme or path.
 */
private static Path validatePath(Path path) {
	final URI uri = path.toUri();
	final String scheme = uri.getScheme();
	final String pathPart = uri.getPath();

	// some validity checks
	if (scheme == null) {
		throw new IllegalArgumentException("The scheme (hdfs://, file://, etc) is null. " +
				"Please specify the file system scheme explicitly in the URI.");
	}
	if (pathPart == null) {
		throw new IllegalArgumentException("The path to store the checkpoint data in is null. " +
				"Please specify a directory path for the checkpoint data.");
	}
	if (pathPart.length() == 0 || pathPart.equals("/")) {
		throw new IllegalArgumentException("Cannot use the root directory for checkpoints.");
	}

	return path;
}
 
Example 4
Source Project: flink   File: DistributedCacheDfsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String map(Integer value) throws Exception {
	final Path actualFile = new Path(getRuntimeContext().getDistributedCache().getFile("test_data").toURI());

	Path path = new Path(actualFile.toUri());
	assertFalse(path.getFileSystem().isDistributedFS());

	DataInputStream in = new DataInputStream(actualFile.getFileSystem().open(actualFile));
	String contents = in.readUTF();

	assertEquals(testFileContent, contents);

	final Path actualDir = new Path(getRuntimeContext().getDistributedCache().getFile("test_dir").toURI());
	FileStatus fileStatus = actualDir.getFileSystem().getFileStatus(actualDir);
	assertTrue(fileStatus.isDir());
	FileStatus[] fileStatuses = actualDir.getFileSystem().listStatus(actualDir);
	assertEquals(2, fileStatuses.length);

	return contents;
}
 
Example 5
Source Project: flink   File: DistributedCacheDfsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String map(Integer value) throws Exception {
	final Path actualFile = new Path(getRuntimeContext().getDistributedCache().getFile("test_data").toURI());

	Path path = new Path(actualFile.toUri());
	assertFalse(path.getFileSystem().isDistributedFS());

	DataInputStream in = new DataInputStream(actualFile.getFileSystem().open(actualFile));
	String contents = in.readUTF();

	assertEquals(testFileContent, contents);

	final Path actualDir = new Path(getRuntimeContext().getDistributedCache().getFile("test_dir").toURI());
	FileStatus fileStatus = actualDir.getFileSystem().getFileStatus(actualDir);
	assertTrue(fileStatus.isDir());
	FileStatus[] fileStatuses = actualDir.getFileSystem().listStatus(actualDir);
	assertEquals(2, fileStatuses.length);

	return contents;
}
 
Example 6
Source Project: flink   File: AbstractFileStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks the validity of the path's scheme and path.
 *
 * @param path The path to check.
 * @return The URI as a Path.
 *
 * @throws IllegalArgumentException Thrown, if the URI misses scheme or path.
 */
private static Path validatePath(Path path) {
	final URI uri = path.toUri();
	final String scheme = uri.getScheme();
	final String pathPart = uri.getPath();

	// some validity checks
	if (scheme == null) {
		throw new IllegalArgumentException("The scheme (hdfs://, file://, etc) is null. " +
				"Please specify the file system scheme explicitly in the URI.");
	}
	if (pathPart == null) {
		throw new IllegalArgumentException("The path to store the checkpoint data in is null. " +
				"Please specify a directory path for the checkpoint data.");
	}
	if (pathPart.length() == 0 || pathPart.equals("/")) {
		throw new IllegalArgumentException("Cannot use the root directory for checkpoints.");
	}

	return path;
}
 
Example 7
private static void ensureLocalFileDeleted(Path path) {
	URI uri = path.toUri();
	if ("file".equals(uri.getScheme())) {
		File file = new File(uri.getPath());
		assertFalse("file not properly deleted", file.exists());
	}
	else {
		throw new IllegalArgumentException("not a local path");
	}
}
 
Example 8
private static void ensureLocalFileDeleted(Path path) {
	URI uri = path.toUri();
	if ("file".equals(uri.getScheme())) {
		File file = new File(uri.getPath());
		assertFalse("file not properly deleted", file.exists());
	}
	else {
		throw new IllegalArgumentException("not a local path");
	}
}
 
Example 9
private static void ensureLocalFileDeleted(Path path) {
	URI uri = path.toUri();
	if ("file".equals(uri.getScheme())) {
		File file = new File(uri.getPath());
		assertFalse("file not properly deleted", file.exists());
	}
	else {
		throw new IllegalArgumentException("not a local path");
	}
}
 
Example 10
Source Project: Flink-CEPplus   File: HadoopFileSystem.java    License: Apache License 2.0 4 votes vote down vote up
public static org.apache.hadoop.fs.Path toHadoopPath(Path path) {
	return new org.apache.hadoop.fs.Path(path.toUri());
}
 
Example 11
Source Project: flink   File: HadoopFileSystem.java    License: Apache License 2.0 4 votes vote down vote up
public static org.apache.hadoop.fs.Path toHadoopPath(Path path) {
	return new org.apache.hadoop.fs.Path(path.toUri());
}
 
Example 12
Source Project: flink-dataflow   File: WriteSinkITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void open(String uId) throws Exception {
	Path path = new Path(resultPath + "/" + uId);
	FileSystem.get(new URI("file:///")).create(path, false);
	internalWriter = new PrintWriter(new File(path.toUri()));
}
 
Example 13
Source Project: flink   File: ParquetSplitReaderUtil.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Util for generating partitioned {@link ParquetColumnarRowSplitReader}.
 */
public static ParquetColumnarRowSplitReader genPartColumnarRowReader(
		boolean utcTimestamp,
		boolean caseSensitive,
		Configuration conf,
		String[] fullFieldNames,
		DataType[] fullFieldTypes,
		Map<String, Object> partitionSpec,
		int[] selectedFields,
		int batchSize,
		Path path,
		long splitStart,
		long splitLength) throws IOException {
	List<String> nonPartNames = Arrays.stream(fullFieldNames)
			.filter(n -> !partitionSpec.containsKey(n))
			.collect(Collectors.toList());

	List<String> selNonPartNames = Arrays.stream(selectedFields)
			.mapToObj(i -> fullFieldNames[i])
			.filter(nonPartNames::contains).collect(Collectors.toList());

	int[] selParquetFields = selNonPartNames.stream()
			.mapToInt(nonPartNames::indexOf)
			.toArray();

	ParquetColumnarRowSplitReader.ColumnBatchGenerator gen = readVectors -> {
		// create and initialize the row batch
		ColumnVector[] vectors = new ColumnVector[selectedFields.length];
		for (int i = 0; i < vectors.length; i++) {
			String name = fullFieldNames[selectedFields[i]];
			LogicalType type = fullFieldTypes[selectedFields[i]].getLogicalType();
			vectors[i] = partitionSpec.containsKey(name) ?
					createVectorFromConstant(type, partitionSpec.get(name), batchSize) :
					readVectors[selNonPartNames.indexOf(name)];
		}
		return new VectorizedColumnBatch(vectors);
	};

	return new ParquetColumnarRowSplitReader(
			utcTimestamp,
			caseSensitive,
			conf,
			Arrays.stream(selParquetFields)
					.mapToObj(i -> fullFieldTypes[i].getLogicalType())
					.toArray(LogicalType[]::new),
			selNonPartNames.toArray(new String[0]),
			gen,
			batchSize,
			new org.apache.hadoop.fs.Path(path.toUri()),
			splitStart,
			splitLength);
}
 
Example 14
Source Project: flink   File: HadoopFileSystem.java    License: Apache License 2.0 4 votes vote down vote up
public static org.apache.hadoop.fs.Path toHadoopPath(Path path) {
	return new org.apache.hadoop.fs.Path(path.toUri());
}
 
Example 15
Source Project: Flink-CEPplus   File: FsStateBackend.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 */
public FsStateBackend(Path checkpointDataUri) {
	this(checkpointDataUri.toUri());
}
 
Example 16
Source Project: Flink-CEPplus   File: FsStateBackend.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(Path checkpointDataUri, boolean asynchronousSnapshots) {
	this(checkpointDataUri.toUri(), asynchronousSnapshots);
}
 
Example 17
Source Project: flink   File: FsStateBackend.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 */
public FsStateBackend(Path checkpointDataUri) {
	this(checkpointDataUri.toUri());
}
 
Example 18
Source Project: flink   File: FsStateBackend.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(Path checkpointDataUri, boolean asynchronousSnapshots) {
	this(checkpointDataUri.toUri(), asynchronousSnapshots);
}
 
Example 19
Source Project: flink   File: FsStateBackend.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 */
public FsStateBackend(Path checkpointDataUri) {
	this(checkpointDataUri.toUri());
}
 
Example 20
Source Project: flink   File: FsStateBackend.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(Path checkpointDataUri, boolean asynchronousSnapshots) {
	this(checkpointDataUri.toUri(), asynchronousSnapshots);
}