Java Code Examples for org.apache.flink.core.fs.Path#getPath()

The following examples show how to use org.apache.flink.core.fs.Path#getPath() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: GlobFilePathFilter.java    License: Apache License 2.0 7 votes vote down vote up
@Override
public boolean filterPath(Path filePath) {
	if (getIncludeMatchers().isEmpty() && getExcludeMatchers().isEmpty()) {
		return false;
	}

	// compensate for the fact that Flink paths are slashed
	final String path = filePath.hasWindowsDrive() ?
			filePath.getPath().substring(1) :
			filePath.getPath();

	final java.nio.file.Path nioPath = Paths.get(path);

	for (PathMatcher matcher : getIncludeMatchers()) {
		if (matcher.matches(nioPath)) {
			return shouldExclude(nioPath);
		}
	}

	return true;
}
 
Example 2
@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
}
 
Example 3
@Override
public boolean filterPath(Path filePath) {
	if (getIncludeMatchers().isEmpty() && getExcludeMatchers().isEmpty()) {
		return false;
	}

	// compensate for the fact that Flink paths are slashed
	final String path = filePath.hasWindowsDrive() ?
			filePath.getPath().substring(1) :
			filePath.getPath();

	final java.nio.file.Path nioPath = Paths.get(path);

	for (PathMatcher matcher : getIncludeMatchers()) {
		if (matcher.matches(nioPath)) {
			return shouldExclude(nioPath);
		}
	}

	return true;
}
 
Example 4
/**
 * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
 * a local state.
 */
private void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {

	FileSystem fileSystem = source.getFileSystem();

	final FileStatus[] fileStatuses = fileSystem.listStatus(source);

	if (fileStatuses == null) {
		throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
	}

	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		File restoreFile = new File(source.getPath(), fileName);
		File targetFile = new File(instanceRocksDBPath, fileName);
		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			// hardlink'ing the immutable sst-files.
			Files.createLink(targetFile.toPath(), restoreFile.toPath());
		} else {
			// true copy for all other files.
			Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
		}
	}
}
 
Example 5
/**
 * This test checks that no mkdirs is called by the checkpoint storage location when resolved.
 */
@Test
public void testStorageLocationDoesNotMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD);

	File baseDir =  new File(storage.getCheckpointsDirectory().getPath());
	assertTrue(baseDir.exists());

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example 6
Source Project: flink   File: AvroOutputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
}
 
Example 7
/**
 * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
 * a local state.
 */
private void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {

	FileSystem fileSystem = source.getFileSystem();

	final FileStatus[] fileStatuses = fileSystem.listStatus(source);

	if (fileStatuses == null) {
		throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
	}

	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		File restoreFile = new File(source.getPath(), fileName);
		File targetFile = new File(instanceRocksDBPath, fileName);
		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			// hardlink'ing the immutable sst-files.
			Files.createLink(targetFile.toPath(), restoreFile.toPath());
		} else {
			// true copy for all other files.
			Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
		}
	}
}
 
Example 8
Source Project: flink   File: FsCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test checks that no mkdirs is called by the checkpoint storage location when resolved.
 */
@Test
public void testStorageLocationDoesNotMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	File baseDir =  new File(storage.getCheckpointsDirectory().getPath());
	assertTrue(baseDir.exists());

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example 9
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	super.open(taskNumber, numTasks);
	if (config != null) {
		TSFileConfigUtil.setGlobalTSFileConfig(config);
	}
	// Use TsFile API to write instead of FSDataOutputStream.
	this.stream.close();
	Path actualFilePath = getAcutalFilePath();
	TsFileOutput out;
	try {
		if (actualFilePath.getFileSystem().isDistributedFS()) {
			// HDFS
			out = new HDFSOutput(
				new org.apache.hadoop.fs.Path(new URI(actualFilePath.getPath())), hadoopConf, true);
		} else {
			// Local File System
			out = new LocalTsFileOutput(new FileOutputStream(actualFilePath.getPath()));
		}
	} catch (URISyntaxException e) {
		throw new RuntimeException(e);
	}
	writer = new TsFileWriter(out, schema);
}
 
Example 10
Source Project: flink   File: AvroOutputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
}
 
Example 11
Source Project: flink   File: GlobFilePathFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean filterPath(Path filePath) {
	if (getIncludeMatchers().isEmpty() && getExcludeMatchers().isEmpty()) {
		return false;
	}

	// compensate for the fact that Flink paths are slashed
	final String path = filePath.hasWindowsDrive() ?
			filePath.getPath().substring(1) :
			filePath.getPath();

	final java.nio.file.Path nioPath = Paths.get(path);

	for (PathMatcher matcher : getIncludeMatchers()) {
		if (matcher.matches(nioPath)) {
			return shouldExclude(nioPath);
		}
	}

	return true;
}
 
Example 12
Source Project: flink   File: FsJobArchivist.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads the given archive file and returns a {@link Collection} of contained {@link ArchivedJson}.
 *
 * @param file archive to extract
 * @return collection of archived jsons
 * @throws IOException if the file can't be opened, read or doesn't contain valid json
 */
public static Collection<ArchivedJson> getArchivedJsons(Path file) throws IOException {
	try (FSDataInputStream input = file.getFileSystem().open(file);
		ByteArrayOutputStream output = new ByteArrayOutputStream()) {
		IOUtils.copyBytes(input, output);

		try {
			JsonNode archive = mapper.readTree(output.toByteArray());

			Collection<ArchivedJson> archives = new ArrayList<>();
			for (JsonNode archivePart : archive.get(ARCHIVE)) {
				String path = archivePart.get(PATH).asText();
				String json = archivePart.get(JSON).asText();
				archives.add(new ArchivedJson(path, json));
			}
			return archives;
		} catch (NullPointerException npe) {
			// occurs if the archive is empty or any of the expected fields are not present
			throw new IOException("Job archive (" + file.getPath() + ") did not conform to expected format.");
		}
	}
}
 
Example 13
Source Project: flink   File: FsCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test checks that the expected mkdirs action for checkpoint storage,
 * only be called when initializeBaseLocations and not called when resolveCheckpointStorageLocation.
 */
@Test
public void testStorageLocationMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	File baseDir = new File(storage.getCheckpointsDirectory().getPath());
	assertFalse(baseDir.exists());

	// mkdirs would only be called when initializeBaseLocations
	storage.initializeBaseLocations();
	assertTrue(baseDir.exists());

	// mkdir would not be called when resolveCheckpointStorageLocation
	storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example 14
@Test
public void testProgram() throws Exception {
	Path testEntryPoint = new Path(getBaseTestPythonDir(), "run_all_tests.py");
	List<String> testFiles = findTestFiles();

	Preconditions.checkState(testFiles.size() > 0, "No test files were found in {}.", getBaseTestPythonDir());

	String[] arguments = new String[1 + 1 + testFiles.size()];
	arguments[0] = testEntryPoint.getPath();
	arguments[1] = findUtilsModule().getPath();
	int index = 2;
	for (String testFile : testFiles) {
		arguments[index] = testFile;
		index++;
	}
	try {
		new PythonStreamBinder(new Configuration())
			.runPlan(arguments);
	} catch (PyException e) {
		if (e.getCause() instanceof JobExecutionException) {
			// JobExecutionExceptions are wrapped again by the jython interpreter resulting in horrible stacktraces
			throw (JobExecutionException) e.getCause();
		} else {
			// probably caused by some issue in the main script itself
			throw e;
		}
	}
}
 
Example 15
@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
	if (key.equals(permanentBlobKey)) {
		final java.nio.file.Path directory = temporaryFolder.newFolder("zipArchive").toPath();
		final java.nio.file.Path containedFile = directory.resolve("cacheFile");
		Files.copy(new ByteArrayInputStream(testFileContent.getBytes(StandardCharsets.UTF_8)), containedFile);
		Path zipPath = FileUtils.compressDirectory(new Path(directory.toString()), new Path(directory + ".zip"));
		return new File(zipPath.getPath());
	} else {
		throw new IllegalArgumentException("This service contains only entry for " + permanentBlobKey);
	}
}
 
Example 16
Source Project: flink   File: FileCacheDirectoriesTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
	if (key.equals(permanentBlobKey)) {
		final java.nio.file.Path directory = temporaryFolder.newFolder("zipArchive").toPath();
		final java.nio.file.Path containedFile = directory.resolve("cacheFile");
		Files.copy(new ByteArrayInputStream(testFileContent.getBytes(StandardCharsets.UTF_8)), containedFile);
		Path zipPath = FileUtils.compressDirectory(new Path(directory.toString()), new Path(directory + ".zip"));
		return new File(zipPath.getPath());
	} else {
		throw new IllegalArgumentException("This service contains only entry for " + permanentBlobKey);
	}
}
 
Example 17
Source Project: flink   File: FileCacheDirectoriesTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
	if (key.equals(permanentBlobKey)) {
		final java.nio.file.Path directory = temporaryFolder.newFolder("zipArchive").toPath();
		final java.nio.file.Path containedFile = directory.resolve("cacheFile");
		Files.copy(new ByteArrayInputStream(testFileContent.getBytes(StandardCharsets.UTF_8)), containedFile);
		Path zipPath = FileUtils.compressDirectory(new Path(directory.toString()), new Path(directory + ".zip"));
		return new File(zipPath.getPath());
	} else {
		throw new IllegalArgumentException("This service contains only entry for " + permanentBlobKey);
	}
}
 
Example 18
private int readSplitAndCheck(
		int start,
		long seekToRow,
		Path testPath,
		long splitStart,
		long splitLength,
		List<Integer> values) throws IOException {
	LogicalType[] fieldTypes = new LogicalType[]{
			new VarCharType(VarCharType.MAX_LENGTH),
			new BooleanType(),
			new TinyIntType(),
			new SmallIntType(),
			new IntType(),
			new BigIntType(),
			new FloatType(),
			new DoubleType(),
			new TimestampType(9),
			new DecimalType(5, 0),
			new DecimalType(15, 0),
			new DecimalType(20, 0),
			new DecimalType(5, 0),
			new DecimalType(15, 0),
			new DecimalType(20, 0)};

	ParquetColumnarRowSplitReader reader = new ParquetColumnarRowSplitReader(
			false,
			true,
			new Configuration(),
			fieldTypes,
			new String[] {
					"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
					"f8", "f9", "f10", "f11", "f12", "f13", "f14"},
			VectorizedColumnBatch::new,
			500,
			new org.apache.hadoop.fs.Path(testPath.getPath()),
			splitStart,
			splitLength);
	reader.seekToRow(seekToRow);

	int i = start;
	while (!reader.reachedEnd()) {
		ColumnarRowData row = reader.nextRecord();
		Integer v = values.get(i);
		if (v == null) {
			assertTrue(row.isNullAt(0));
			assertTrue(row.isNullAt(1));
			assertTrue(row.isNullAt(2));
			assertTrue(row.isNullAt(3));
			assertTrue(row.isNullAt(4));
			assertTrue(row.isNullAt(5));
			assertTrue(row.isNullAt(6));
			assertTrue(row.isNullAt(7));
			assertTrue(row.isNullAt(8));
			assertTrue(row.isNullAt(9));
			assertTrue(row.isNullAt(10));
			assertTrue(row.isNullAt(11));
			assertTrue(row.isNullAt(12));
			assertTrue(row.isNullAt(13));
			assertTrue(row.isNullAt(14));
		} else {
			assertEquals("" + v, row.getString(0).toString());
			assertEquals(v % 2 == 0, row.getBoolean(1));
			assertEquals(v.byteValue(), row.getByte(2));
			assertEquals(v.shortValue(), row.getShort(3));
			assertEquals(v.intValue(), row.getInt(4));
			assertEquals(v.longValue(), row.getLong(5));
			assertEquals(v.floatValue(), row.getFloat(6), 0);
			assertEquals(v.doubleValue(), row.getDouble(7), 0);
			assertEquals(
					toDateTime(v),
					row.getTimestamp(8, 9).toLocalDateTime());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(9, 5, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(10, 15, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(11, 20, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(12, 5, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(13, 15, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(14, 20, 0).toBigDecimal());
		}
		i++;
	}
	reader.close();
	return i - start;
}
 
Example 19
Source Project: flink   File: LocalFileSystem.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Converts the given Path to a File for this file system.
 */
public File pathToFile(Path path) {
	return new File(path.getPath());
}