Java Code Examples for org.apache.flink.util.FileUtils#deleteDirectory()

The following examples show how to use org.apache.flink.util.FileUtils#deleteDirectory() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: IOManager.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Close method, marks the I/O manager as closed
 * and removed all temporary files.
 */
public void shutdown() {
	// remove all of our temp directories
	for (File path : paths) {
		try {
			if (path != null) {
				if (path.exists()) {
					FileUtils.deleteDirectory(path);
					LOG.info("I/O manager removed spill file directory {}", path.getAbsolutePath());
				}
			}
		} catch (Throwable t) {
			LOG.error("IOManager failed to properly clean up temp file directory: " + path, t);
		}
	}
}
 
Example 2
Source File: AbstractBlobCache.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void close() throws IOException {
	cancelCleanupTask();

	if (shutdownRequested.compareAndSet(false, true)) {
		log.info("Shutting down BLOB cache");

		// Clean up the storage directory
		try {
			FileUtils.deleteDirectory(storageDir);
		} finally {
			// Remove shutdown hook to prevent resource leaks
			ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), log);
		}
	}
}
 
Example 3
Source File: FileCache.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
	try {
		synchronized (lock) {

			Set<ExecutionAttemptID> jobRefs = jobRefHolders.get(jobID);
			if (jobRefs != null && jobRefs.isEmpty()) {
				// abort the copy
				for (Future<Path> fileFuture : entries.get(jobID).values()) {
					fileFuture.cancel(true);
				}

				//remove job specific entries in maps
				entries.remove(jobID);
				jobRefHolders.remove(jobID);

				// remove the job wide temp directories
				for (File storageDirectory : storageDirectories) {
					File tempDir = new File(storageDirectory, jobID.toString());
					FileUtils.deleteDirectory(tempDir);
				}
			}
		}
	} catch (IOException e) {
		LOG.error("Could not delete file from local file cache.", e);
	}
}
 
Example 4
Source File: FileUploadHandler.java    From flink with Apache License 2.0 5 votes vote down vote up
private void deleteUploadedFiles() {
	if (currentUploadDir != null) {
		try {
			FileUtils.deleteDirectory(currentUploadDir.toFile());
		} catch (IOException e) {
			LOG.warn("Could not cleanup uploaded files.", e);
		}
	}
}
 
Example 5
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private SnapshotDirectory prepareLocalSnapshotDirectory(long checkpointId) throws IOException {

	if (localRecoveryConfig.isLocalRecoveryEnabled()) {
		// create a "permanent" snapshot directory for local recovery.
		LocalRecoveryDirectoryProvider directoryProvider = localRecoveryConfig.getLocalStateDirectoryProvider();
		File directory = directoryProvider.subtaskSpecificCheckpointDirectory(checkpointId);

		if (!directory.exists() && !directory.mkdirs()) {
			throw new IOException("Local state base directory for checkpoint " + checkpointId +
				" already exists: " + directory);
		}

		// introduces an extra directory because RocksDB wants a non-existing directory for native checkpoints.
		// append localDirectoryName here to solve directory collision problem when two stateful operators chained in one task.
		File rdbSnapshotDir = new File(directory, localDirectoryName);
		if (rdbSnapshotDir.exists()) {
			FileUtils.deleteDirectory(rdbSnapshotDir);
		}

		Path path = new Path(rdbSnapshotDir.toURI());
		// create a "permanent" snapshot directory because local recovery is active.
		try {
			return SnapshotDirectory.permanent(path);
		} catch (IOException ex) {
			try {
				FileUtils.deleteDirectory(directory);
			} catch (IOException delEx) {
				ex = ExceptionUtils.firstOrSuppressed(delEx, ex);
			}
			throw ex;
		}
	} else {
		// create a "temporary" snapshot directory because local recovery is inactive.
		File snapshotDir = new File(instanceBasePath, "chk-" + checkpointId);
		return SnapshotDirectory.temporary(snapshotDir);
	}
}
 
Example 6
Source File: FileUploadHandler.java    From flink with Apache License 2.0 5 votes vote down vote up
private void deleteUploadedFiles() {
	if (currentUploadDir != null) {
		try {
			FileUtils.deleteDirectory(currentUploadDir.toFile());
		} catch (IOException e) {
			LOG.warn("Could not cleanup uploaded files.", e);
		}
	}
}
 
Example 7
Source File: FileUploadHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testUploadDirectoryRegeneration() throws Exception {
	OkHttpClient client = createOkHttpClientWithNoTimeouts();

	MultipartUploadResource.MultipartFileHandler fileHandler = MULTIPART_UPLOAD_RESOURCE.getFileHandler();

	FileUtils.deleteDirectory(MULTIPART_UPLOAD_RESOURCE.getUploadDirectory().toFile());

	Request fileRequest = buildFileRequest(fileHandler.getMessageHeaders().getTargetRestEndpointURL());
	try (Response response = client.newCall(fileRequest).execute()) {
		assertEquals(fileHandler.getMessageHeaders().getResponseStatusCode().code(), response.code());
	}

	verifyNoFileIsRegisteredToDeleteOnExitHook();
}
 
Example 8
Source File: FileCache.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
	try {
		synchronized (lock) {

			Set<ExecutionAttemptID> jobRefs = jobRefHolders.get(jobID);
			if (jobRefs != null && jobRefs.isEmpty()) {
				// abort the copy
				for (Future<Path> fileFuture : entries.get(jobID).values()) {
					fileFuture.cancel(true);
				}

				//remove job specific entries in maps
				entries.remove(jobID);
				jobRefHolders.remove(jobID);

				// remove the job wide temp directories
				for (File storageDirectory : storageDirectories) {
					File tempDir = new File(storageDirectory, jobID.toString());
					FileUtils.deleteDirectory(tempDir);
				}
			}
		}
	} catch (IOException e) {
		LOG.error("Could not delete file from local file cache.", e);
	}
}
 
Example 9
Source File: FileChannelManagerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
private static AutoCloseable getFileCloser(File path) {
	return () -> {
		try {
			FileUtils.deleteDirectory(path);
			LOG.info("FileChannelManager removed spill file directory {}", path.getAbsolutePath());
		} catch (IOException e) {
			String errorMessage = String.format("FileChannelManager failed to properly clean up temp file directory: %s", path);
			throw new IOException(errorMessage, e);
		}
	};
}
 
Example 10
Source File: BlobServer.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Removes all BLOBs from local and HA store belonging to the given job ID.
 *
 * @param jobId
 * 		ID of the job this blob belongs to
 * @param cleanupBlobStoreFiles
 * 		True if the corresponding blob store files shall be cleaned up as well. Otherwise false.
 *
 * @return  <tt>true</tt> if the job directory is successfully deleted or non-existing;
 *          <tt>false</tt> otherwise
 */
public boolean cleanupJob(JobID jobId, boolean cleanupBlobStoreFiles) {
	checkNotNull(jobId);

	final File jobDir =
		new File(BlobUtils.getStorageLocationPath(storageDir.getAbsolutePath(), jobId));

	readWriteLock.writeLock().lock();

	try {
		// delete locally
		boolean deletedLocally = false;
		try {
			FileUtils.deleteDirectory(jobDir);

			// NOTE: Instead of going through blobExpiryTimes, keep lingering entries - they
			//       will be cleaned up by the timer task which tolerates non-existing files
			//       If inserted again with the same IDs (via put()), the TTL will be updated
			//       again.

			deletedLocally = true;
		} catch (IOException e) {
			LOG.warn("Failed to locally delete BLOB storage directory at " +
				jobDir.getAbsolutePath(), e);
		}

		// delete in HA blob store files
		final boolean deletedHA = !cleanupBlobStoreFiles || blobStore.deleteAll(jobId);

		return deletedLocally && deletedHA;
	} finally {
		readWriteLock.writeLock().unlock();
	}
}
 
Example 11
Source File: FileUploadHandler.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void deleteUploadedFiles() {
	if (currentUploadDir != null) {
		try {
			FileUtils.deleteDirectory(currentUploadDir.toFile());
		} catch (IOException e) {
			LOG.warn("Could not cleanup uploaded files.", e);
		}
	}
}
 
Example 12
Source File: RocksIncrementalSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Nonnull
private SnapshotDirectory prepareLocalSnapshotDirectory(long checkpointId) throws IOException {

	if (localRecoveryConfig.isLocalRecoveryEnabled()) {
		// create a "permanent" snapshot directory for local recovery.
		LocalRecoveryDirectoryProvider directoryProvider = localRecoveryConfig.getLocalStateDirectoryProvider();
		File directory = directoryProvider.subtaskSpecificCheckpointDirectory(checkpointId);

		if (!directory.exists() && !directory.mkdirs()) {
			throw new IOException("Local state base directory for checkpoint " + checkpointId +
				" already exists: " + directory);
		}

		// introduces an extra directory because RocksDB wants a non-existing directory for native checkpoints.
		// append localDirectoryName here to solve directory collision problem when two stateful operators chained in one task.
		File rdbSnapshotDir = new File(directory, localDirectoryName);
		if (rdbSnapshotDir.exists()) {
			FileUtils.deleteDirectory(rdbSnapshotDir);
		}

		Path path = new Path(rdbSnapshotDir.toURI());
		// create a "permanent" snapshot directory because local recovery is active.
		try {
			return SnapshotDirectory.permanent(path);
		} catch (IOException ex) {
			try {
				FileUtils.deleteDirectory(directory);
			} catch (IOException delEx) {
				ex = ExceptionUtils.firstOrSuppressed(delEx, ex);
			}
			throw ex;
		}
	} else {
		// create a "temporary" snapshot directory because local recovery is inactive.
		File snapshotDir = new File(instanceBasePath, "chk-" + checkpointId);
		return SnapshotDirectory.temporary(snapshotDir);
	}
}
 
Example 13
Source File: RocksDBKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void cleanInstanceBasePath() {
	LOG.info("Deleting existing instance base directory {}.", instanceBasePath);

	try {
		FileUtils.deleteDirectory(instanceBasePath);
	} catch (IOException ex) {
		LOG.warn("Could not delete instance base path for RocksDB: " + instanceBasePath, ex);
	}
}
 
Example 14
Source File: BlobServer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Shuts down the BLOB server.
 */
@Override
public void close() throws IOException {
	cleanupTimer.cancel();

	if (shutdownRequested.compareAndSet(false, true)) {
		Exception exception = null;

		try {
			this.serverSocket.close();
		}
		catch (IOException ioe) {
			exception = ioe;
		}

		// wake the thread up, in case it is waiting on some operation
		interrupt();

		try {
			join();
		}
		catch (InterruptedException ie) {
			Thread.currentThread().interrupt();

			LOG.debug("Error while waiting for this thread to die.", ie);
		}

		synchronized (activeConnections) {
			if (!activeConnections.isEmpty()) {
				for (BlobServerConnection conn : activeConnections) {
					LOG.debug("Shutting down connection {}.", conn.getName());
					conn.close();
				}
				activeConnections.clear();
			}
		}

		// Clean up the storage directory
		try {
			FileUtils.deleteDirectory(storageDir);
		}
		catch (IOException e) {
			exception = ExceptionUtils.firstOrSuppressed(e, exception);
		}

		// Remove shutdown hook to prevent resource leaks
		ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), LOG);

		if (LOG.isInfoEnabled()) {
			LOG.info("Stopped BLOB server at {}:{}", serverSocket.getInetAddress().getHostAddress(), getPort());
		}

		ExceptionUtils.tryRethrowIOException(exception);
	}
}
 
Example 15
Source File: FileUploads.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public void close() throws IOException {
	if (uploadDirectory != null) {
		FileUtils.deleteDirectory(uploadDirectory.toFile());
	}
}
 
Example 16
Source File: ModelSaveAndLoadTest.java    From Alink with Apache License 2.0 4 votes vote down vote up
@After
public void clear() throws Exception {
    FileUtils.deleteDirectory(new File(MODEL_PATH));
}
 
Example 17
Source File: PermanentBlobCache.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Cleans up BLOBs which are not referenced anymore.
 */
@Override
public void run() {
	synchronized (jobRefCounters) {
		Iterator<Map.Entry<JobID, RefCount>> entryIter = jobRefCounters.entrySet().iterator();
		final long currentTimeMillis = System.currentTimeMillis();

		while (entryIter.hasNext()) {
			Map.Entry<JobID, RefCount> entry = entryIter.next();
			RefCount ref = entry.getValue();

			if (ref.references <= 0 && ref.keepUntil > 0 && currentTimeMillis >= ref.keepUntil) {
				JobID jobId = entry.getKey();

				final File localFile =
					new File(BlobUtils.getStorageLocationPath(storageDir.getAbsolutePath(), jobId));

			/*
			 * NOTE: normally it is not required to acquire the write lock to delete the job's
			 *       storage directory since there should be no one accessing it with the ref
			 *       counter being 0 - acquire it just in case, to always be on the safe side
			 */
				readWriteLock.writeLock().lock();

				boolean success = false;
				try {
					FileUtils.deleteDirectory(localFile);
					success = true;
				} catch (Throwable t) {
					log.warn("Failed to locally delete job directory " + localFile.getAbsolutePath(), t);
				} finally {
					readWriteLock.writeLock().unlock();
				}

				// let's only remove this directory from cleanup if the cleanup was successful
				// (does not need the write lock)
				if (success) {
					entryIter.remove();
				}
			}
		}
	}
}
 
Example 18
Source File: FileUploads.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void close() throws IOException {
	if (uploadDirectory != null) {
		FileUtils.deleteDirectory(uploadDirectory.toFile());
	}
}
 
Example 19
Source File: DirectoryStateHandle.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void discardState() throws IOException {
	ensurePath();
	FileUtils.deleteDirectory(directory.toFile());
}
 
Example 20
Source File: SnapshotDirectory.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Calling this method will attempt delete the underlying snapshot directory recursively, if the state is
 * "ongoing". In this case, the state will be set to "deleted" as a result of this call.
 *
 * @return <code>true</code> if delete is successful, <code>false</code> otherwise.
 * @throws IOException if an exception happens during the delete.
 */
public boolean cleanup() throws IOException {
	if (state.compareAndSet(State.ONGOING, State.DELETED)) {
		FileUtils.deleteDirectory(directory.toFile());
	}
	return true;
}