Java Code Examples for org.apache.flink.core.fs.Path#toString()

The following examples show how to use org.apache.flink.core.fs.Path#toString() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testParametrizationDirectories() throws Exception {
	final JobID jid = new JobID();
	final Path checkpointPath = new Path(tmp.newFolder().toURI().toString());
	final Path savepointPath = new Path(tmp.newFolder().toURI().toString());

	MemoryStateBackend backend = new MemoryStateBackend(
			checkpointPath.toString(), savepointPath.toString());

	MemoryBackendCheckpointStorage storage =
			(MemoryBackendCheckpointStorage) backend.createCheckpointStorage(jid);

	assertTrue(storage.supportsHighlyAvailableStorage());
	assertTrue(storage.hasDefaultSavepointLocation());
	assertNotNull(storage.getDefaultSavepointDirectory());

	assertEquals(savepointPath, storage.getDefaultSavepointDirectory());
}
 
Example 2
@Test
public void testParametrizationDirectories() throws Exception {
	final JobID jid = new JobID();
	final Path checkpointPath = new Path(tmp.newFolder().toURI().toString());
	final Path savepointPath = new Path(tmp.newFolder().toURI().toString());

	MemoryStateBackend backend = new MemoryStateBackend(
			checkpointPath.toString(), savepointPath.toString());

	MemoryBackendCheckpointStorage storage =
			(MemoryBackendCheckpointStorage) backend.createCheckpointStorage(jid);

	assertTrue(storage.supportsHighlyAvailableStorage());
	assertTrue(storage.hasDefaultSavepointLocation());
	assertNotNull(storage.getDefaultSavepointDirectory());

	assertEquals(savepointPath, storage.getDefaultSavepointDirectory());
}
 
Example 3
@Test
public void testParametrizationDirectories() throws Exception {
	final JobID jid = new JobID();
	final Path checkpointPath = new Path(tmp.newFolder().toURI().toString());
	final Path savepointPath = new Path(tmp.newFolder().toURI().toString());

	MemoryStateBackend backend = new MemoryStateBackend(
			checkpointPath.toString(), savepointPath.toString());

	MemoryBackendCheckpointStorage storage =
			(MemoryBackendCheckpointStorage) backend.createCheckpointStorage(jid);

	assertTrue(storage.supportsHighlyAvailableStorage());
	assertTrue(storage.hasDefaultSavepointLocation());
	assertNotNull(storage.getDefaultSavepointDirectory());

	assertEquals(savepointPath, storage.getDefaultSavepointDirectory());
}
 
Example 4
public VirtualFileServerHandler(Path path) throws IOException {
	this.path = path;
	if (!path.isAbsolute()) {
		throw new IllegalArgumentException("path must be absolute: " + path.toString());
	}
	this.fs = path.getFileSystem();
	if (!fs.exists(path) || fs.getFileStatus(path).isDir()) {
		throw new IllegalArgumentException("no such file: " + path.toString());
	}
}
 
Example 5
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example 6
private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 
Example 7
Source Project: flink   File: PythonEnvUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStartPythonProcess() {
	PythonEnvUtils.PythonEnvironment pythonEnv = new PythonEnvUtils.PythonEnvironment();
	pythonEnv.workingDirectory = tmpDirPath.toString();
	pythonEnv.pythonPath = tmpDirPath.toString();
	List<String> commands = new ArrayList<>();
	Path pyPath = new Path(tmpDirPath, "word_count.py");
	try {
		tmpDirFs.create(pyPath, FileSystem.WriteMode.OVERWRITE);
		File pyFile = new File(pyPath.toString());
		String pyProgram = "#!/usr/bin/python\n" +
			"# -*- coding: UTF-8 -*-\n" +
			"import sys\n" +
			"\n" +
			"if __name__=='__main__':\n" +
			"\tfilename = sys.argv[1]\n" +
			"\tfo = open(filename, \"w\")\n" +
			"\tfo.write( \"hello world\")\n" +
			"\tfo.close()";
		Files.write(pyFile.toPath(), pyProgram.getBytes(), StandardOpenOption.WRITE);
		Path result = new Path(tmpDirPath, "word_count_result.txt");
		commands.add(pyFile.getName());
		commands.add(result.getName());
		Process pythonProcess = PythonEnvUtils.startPythonProcess(pythonEnv, commands);
		int exitCode = pythonProcess.waitFor();
		if (exitCode != 0) {
			throw new RuntimeException("Python process exits with code: " + exitCode);
		}
		String cmdResult = new String(Files.readAllBytes(new File(result.toString()).toPath()));
		Assert.assertEquals(cmdResult, "hello world");
		pythonProcess.destroyForcibly();
		tmpDirFs.delete(pyPath, true);
		tmpDirFs.delete(result, true);
	} catch (IOException | InterruptedException e) {
		throw new RuntimeException("test start Python process failed " + e.getMessage());
	}
}
 
Example 8
Source Project: flink   File: MesosArtifactServer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Adds a path to the artifact server.
 * @param path the qualified FS path to serve (local, hdfs, etc).
 * @param remoteFile the remote path with which to locate the file.
 * @return the fully-qualified remote path to the file.
 * @throws MalformedURLException if the remote path is invalid.
 */
public synchronized URL addPath(Path path, Path remoteFile) throws IOException, MalformedURLException {
	if (paths.containsKey(remoteFile)) {
		throw new IllegalArgumentException("duplicate path registered");
	}
	if (remoteFile.isAbsolute()) {
		throw new IllegalArgumentException("not expecting an absolute path");
	}
	URL fileURL = new URL(baseURL, remoteFile.toString());
	router.addAny(fileURL.getPath(), new VirtualFileServerHandler(path));

	paths.put(remoteFile, fileURL);

	return fileURL;
}
 
Example 9
Source Project: flink   File: MesosArtifactServer.java    License: Apache License 2.0 5 votes vote down vote up
public VirtualFileServerHandler(Path path) throws IOException {
	this.path = path;
	if (!path.isAbsolute()) {
		throw new IllegalArgumentException("path must be absolute: " + path.toString());
	}
	this.fs = path.getFileSystem();
	if (!fs.exists(path) || fs.getFileStatus(path).isDir()) {
		throw new IllegalArgumentException("no such file: " + path.toString());
	}
}
 
Example 10
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example 11
private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 
Example 12
Source Project: flink   File: MesosArtifactServer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Adds a path to the artifact server.
 * @param path the qualified FS path to serve (local, hdfs, etc).
 * @param remoteFile the remote path with which to locate the file.
 * @return the fully-qualified remote path to the file.
 * @throws MalformedURLException if the remote path is invalid.
 */
public synchronized URL addPath(Path path, Path remoteFile) throws IOException, MalformedURLException {
	if (paths.containsKey(remoteFile)) {
		throw new IllegalArgumentException("duplicate path registered");
	}
	if (remoteFile.isAbsolute()) {
		throw new IllegalArgumentException("not expecting an absolute path");
	}
	URL fileURL = new URL(baseURL, remoteFile.toString());
	router.addAny(fileURL.getPath(), new VirtualFileServerHandler(path));

	paths.put(remoteFile, fileURL);

	return fileURL;
}
 
Example 13
Source Project: flink   File: MesosArtifactServer.java    License: Apache License 2.0 5 votes vote down vote up
public VirtualFileServerHandler(Path path) throws IOException {
	this.path = path;
	if (!path.isAbsolute()) {
		throw new IllegalArgumentException("path must be absolute: " + path.toString());
	}
	this.fs = path.getFileSystem();
	if (!fs.exists(path) || fs.getFileStatus(path).isDir()) {
		throw new IllegalArgumentException("no such file: " + path.toString());
	}
}
 
Example 14
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example 15
private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 
Example 16
Source Project: Flink-CEPplus   File: FileOutputFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	if (taskNumber < 0 || numTasks < 1) {
		throw new IllegalArgumentException("TaskNumber: " + taskNumber + ", numTasks: " + numTasks);
	}
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Opening stream for output (" + (taskNumber+1) + "/" + numTasks + "). WriteMode=" + writeMode +
				", OutputDirectoryMode=" + outputDirectoryMode);
	}
	
	Path p = this.outputFilePath;
	if (p == null) {
		throw new IOException("The file path is null.");
	}
	
	final FileSystem fs = p.getFileSystem();

	// if this is a local file system, we need to initialize the local output directory here
	if (!fs.isDistributedFS()) {
		
		if (numTasks == 1 && outputDirectoryMode == OutputDirectoryMode.PARONLY) {
			// output should go to a single file
			
			// prepare local output path. checks for write mode and removes existing files in case of OVERWRITE mode
			if(!fs.initOutPathLocalFS(p, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path '" + p.toString() + "' could not be initialized. Canceling task...");
			}
		}
		else {
			// numTasks > 1 || outDirMode == OutputDirectoryMode.ALWAYS
			
			if(!fs.initOutPathLocalFS(p, writeMode, true)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output directory '" + p.toString() + "' could not be created. Canceling task...");
			}
		}
	}



	// Suffix the path with the parallel instance index, if needed
	this.actualFilePath = (numTasks > 1 || outputDirectoryMode == OutputDirectoryMode.ALWAYS) ? p.suffix("/" + getDirectoryFileName(taskNumber)) : p;

	// create output file
	this.stream = fs.create(this.actualFilePath, writeMode);
	
	// at this point, the file creation must have succeeded, or an exception has been thrown
	this.fileCreated = true;
}
 
Example 17
Source Project: flink   File: PythonEnvUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Prepares PythonEnvironment to start python process.
 *
 * @param pythonLibFiles The dependent Python files.
 * @return PythonEnvironment the Python environment which will be executed in Python process.
 */
public static PythonEnvironment preparePythonEnvironment(List<Path> pythonLibFiles) throws IOException {
	PythonEnvironment env = new PythonEnvironment();

	// 1. setup temporary local directory for the user files
	String tmpDir = System.getProperty("java.io.tmpdir") +
		File.separator + "pyflink" + File.separator + UUID.randomUUID();

	Path tmpDirPath = new Path(tmpDir);
	FileSystem fs = tmpDirPath.getFileSystem();
	if (fs.exists(tmpDirPath)) {
		fs.delete(tmpDirPath, true);
	}
	fs.mkdirs(tmpDirPath);

	env.workingDirectory = tmpDirPath.toString();

	StringBuilder pythonPathEnv = new StringBuilder();

	pythonPathEnv.append(env.workingDirectory);

	// 2. create symbolLink in the working directory for the pyflink dependency libs.
	List<java.nio.file.Path> pythonLibs = getLibFiles(FLINK_OPT_DIR_PYTHON);
	for (java.nio.file.Path libPath : pythonLibs) {
		java.nio.file.Path symbolicLinkFilePath = FileSystems.getDefault().getPath(env.workingDirectory,
			libPath.getFileName().toString());
		createSymbolicLinkForPyflinkLib(libPath, symbolicLinkFilePath);
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(symbolicLinkFilePath.toString());
	}

	// 3. copy relevant python files to tmp dir and set them in PYTHONPATH.
	for (Path pythonFile : pythonLibFiles) {
		String sourceFileName = pythonFile.getName();
		Path targetPath = new Path(tmpDirPath, sourceFileName);
		FileUtils.copy(pythonFile, targetPath, true);
		String targetFileNames = Files.walk(Paths.get(targetPath.toString()))
			.filter(Files::isRegularFile)
			.filter(f -> !f.toString().endsWith(".py"))
			.map(java.nio.file.Path::toString)
			.collect(Collectors.joining(File.pathSeparator));
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(targetFileNames);
	}

	// 4. add the parent directory to PYTHONPATH for files suffixed with .py
	String pyFileParents = Files.walk(Paths.get(tmpDirPath.toString()))
		.filter(file -> file.toString().endsWith(".py"))
		.map(java.nio.file.Path::getParent)
		.distinct()
		.map(java.nio.file.Path::toString)
		.collect(Collectors.joining(File.pathSeparator));
	if (!StringUtils.isNullOrWhitespaceOnly(pyFileParents)) {
		pythonPathEnv.append(File.pathSeparator);
		pythonPathEnv.append(pyFileParents);
	}

	env.pythonPath = pythonPathEnv.toString();
	return env;
}
 
Example 18
Source Project: flink   File: FileOutputFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	if (taskNumber < 0 || numTasks < 1) {
		throw new IllegalArgumentException("TaskNumber: " + taskNumber + ", numTasks: " + numTasks);
	}
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Opening stream for output (" + (taskNumber+1) + "/" + numTasks + "). WriteMode=" + writeMode +
				", OutputDirectoryMode=" + outputDirectoryMode);
	}
	
	Path p = this.outputFilePath;
	if (p == null) {
		throw new IOException("The file path is null.");
	}
	
	final FileSystem fs = p.getFileSystem();

	// if this is a local file system, we need to initialize the local output directory here
	if (!fs.isDistributedFS()) {
		
		if (numTasks == 1 && outputDirectoryMode == OutputDirectoryMode.PARONLY) {
			// output should go to a single file
			
			// prepare local output path. checks for write mode and removes existing files in case of OVERWRITE mode
			if(!fs.initOutPathLocalFS(p, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path '" + p.toString() + "' could not be initialized. Canceling task...");
			}
		}
		else {
			// numTasks > 1 || outDirMode == OutputDirectoryMode.ALWAYS
			
			if(!fs.initOutPathLocalFS(p, writeMode, true)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output directory '" + p.toString() + "' could not be created. Canceling task...");
			}
		}
	}



	// Suffix the path with the parallel instance index, if needed
	this.actualFilePath = (numTasks > 1 || outputDirectoryMode == OutputDirectoryMode.ALWAYS) ? p.suffix("/" + getDirectoryFileName(taskNumber)) : p;

	// create output file
	this.stream = fs.create(this.actualFilePath, writeMode);
	
	// at this point, the file creation must have succeeded, or an exception has been thrown
	this.fileCreated = true;
}
 
Example 19
Source Project: flink   File: FileOutputFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	if (taskNumber < 0 || numTasks < 1) {
		throw new IllegalArgumentException("TaskNumber: " + taskNumber + ", numTasks: " + numTasks);
	}
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Opening stream for output (" + (taskNumber+1) + "/" + numTasks + "). WriteMode=" + writeMode +
				", OutputDirectoryMode=" + outputDirectoryMode);
	}
	
	Path p = this.outputFilePath;
	if (p == null) {
		throw new IOException("The file path is null.");
	}
	
	final FileSystem fs = p.getFileSystem();

	// if this is a local file system, we need to initialize the local output directory here
	if (!fs.isDistributedFS()) {
		
		if (numTasks == 1 && outputDirectoryMode == OutputDirectoryMode.PARONLY) {
			// output should go to a single file
			
			// prepare local output path. checks for write mode and removes existing files in case of OVERWRITE mode
			if(!fs.initOutPathLocalFS(p, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path '" + p.toString() + "' could not be initialized. Canceling task...");
			}
		}
		else {
			// numTasks > 1 || outDirMode == OutputDirectoryMode.ALWAYS
			
			if(!fs.initOutPathLocalFS(p, writeMode, true)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output directory '" + p.toString() + "' could not be created. Canceling task...");
			}
		}
	}



	// Suffix the path with the parallel instance index, if needed
	this.actualFilePath = (numTasks > 1 || outputDirectoryMode == OutputDirectoryMode.ALWAYS) ? p.suffix("/" + getDirectoryFileName(taskNumber)) : p;

	// create output file
	this.stream = fs.create(this.actualFilePath, writeMode);
	
	// at this point, the file creation must have succeeded, or an exception has been thrown
	this.fileCreated = true;
}
 
Example 20
private static void assertPathsEqual(Path expected, Path actual) {
	final Path reNormalizedExpected = new Path(expected.toString());
	final Path reNormalizedActual = new Path(actual.toString());
	assertEquals(reNormalizedExpected, reNormalizedActual);
}