/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.blob;

import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.BlobServerOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.net.SSLUtils;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FileUtils;
import org.apache.flink.util.NetUtils;
import org.apache.flink.util.ShutdownHookUtil;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import javax.net.ServerSocketFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.Timer;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import static org.apache.flink.runtime.blob.BlobKey.BlobType.PERMANENT_BLOB;
import static org.apache.flink.runtime.blob.BlobKey.BlobType.TRANSIENT_BLOB;
import static org.apache.flink.runtime.blob.BlobServerProtocol.BUFFER_SIZE;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;

/**
 * This class implements the BLOB server. The BLOB server is responsible for listening for incoming requests and
 * spawning threads to handle these requests. Furthermore, it takes care of creating the directory structure to store
 * the BLOBs or temporarily cache them.
 */
public class BlobServer extends Thread implements BlobService, BlobWriter, PermanentBlobService, TransientBlobService {

	/** The log object used for debugging. */
	private static final Logger LOG = LoggerFactory.getLogger(BlobServer.class);

	/** Counter to generate unique names for temporary files. */
	private final AtomicLong tempFileCounter = new AtomicLong(0);

	/** The server socket listening for incoming connections. */
	private final ServerSocket serverSocket;

	/** Blob Server configuration. */
	private final Configuration blobServiceConfiguration;

	/** Indicates whether a shutdown of server component has been requested. */
	private final AtomicBoolean shutdownRequested = new AtomicBoolean();

	/** Root directory for local file storage. */
	private final File storageDir;

	/** Blob store for distributed file storage, e.g. in HA. */
	private final BlobStore blobStore;

	/** Set of currently running threads. */
	private final Set<BlobServerConnection> activeConnections = new HashSet<>();

	/** The maximum number of concurrent connections. */
	private final int maxConnections;

	/** Lock guarding concurrent file accesses. */
	private final ReadWriteLock readWriteLock;

	/**
	 * Shutdown hook thread to ensure deletion of the local storage directory.
	 */
	private final Thread shutdownHook;

	// --------------------------------------------------------------------------------------------

	/**
	 * Map to store the TTL of each element stored in the local storage, i.e. via one of the {@link
	 * #getFile} methods.
	 **/
	private final ConcurrentHashMap<Tuple2<JobID, TransientBlobKey>, Long> blobExpiryTimes =
		new ConcurrentHashMap<>();

	/** Time interval (ms) to run the cleanup task; also used as the default TTL. */
	private final long cleanupInterval;

	/**
	 * Timer task to execute the cleanup at regular intervals.
	 */
	private final Timer cleanupTimer;

	/**
	 * Instantiates a new BLOB server and binds it to a free network port.
	 *
	 * @param config Configuration to be used to instantiate the BlobServer
	 * @param blobStore BlobStore to store blobs persistently
	 *
	 * @throws IOException
	 * 		thrown if the BLOB server cannot bind to a free network port or if the
	 * 		(local or distributed) file storage cannot be created or is not usable
	 */
	public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
		this.blobServiceConfiguration = checkNotNull(config);
		this.blobStore = checkNotNull(blobStore);
		this.readWriteLock = new ReentrantReadWriteLock();

		// configure and create the storage directory
		this.storageDir = BlobUtils.initLocalStorageDirectory(config);
		LOG.info("Created BLOB server storage directory {}", storageDir);

		// configure the maximum number of concurrent connections
		final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
		if (maxConnections >= 1) {
			this.maxConnections = maxConnections;
		}
		else {
			LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
					maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
			this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
		}

		// configure the backlog of connections
		int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
		if (backlog < 1) {
			LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
					backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
			backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
		}

		// Initializing the clean up task
		this.cleanupTimer = new Timer(true);

		this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
		this.cleanupTimer
			.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
				storageDir, LOG), cleanupInterval, cleanupInterval);

		this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

		//  ----------------------- start the server -------------------

		final String serverPortRange = config.getString(BlobServerOptions.PORT);
		final Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);

		final ServerSocketFactory socketFactory;
		if (SSLUtils.isInternalSSLEnabled(config) && config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
			try {
				socketFactory = SSLUtils.createSSLServerSocketFactory(config);
			}
			catch (Exception e) {
				throw new IOException("Failed to initialize SSL for the blob server", e);
			}
		}
		else {
			socketFactory = ServerSocketFactory.getDefault();
		}

		final int finalBacklog = backlog;
		this.serverSocket = NetUtils.createSocketFromPorts(ports,
				(port) -> socketFactory.createServerSocket(port, finalBacklog));

		if (serverSocket == null) {
			throw new IOException("Unable to open BLOB Server in specified port range: " + serverPortRange);
		}

		// start the server thread
		setName("BLOB Server listener at " + getPort());
		setDaemon(true);

		if (LOG.isInfoEnabled()) {
			LOG.info("Started BLOB server at {}:{} - max concurrent requests: {} - max backlog: {}",
					serverSocket.getInetAddress().getHostAddress(), getPort(), maxConnections, backlog);
		}
	}

	// --------------------------------------------------------------------------------------------
	//  Path Accessors
	// --------------------------------------------------------------------------------------------

	/**
	 * Returns a file handle to the file associated with the given blob key on the blob
	 * server.
	 *
	 * <p><strong>This is only called from {@link BlobServerConnection} or unit tests.</strong>
	 *
	 * @param jobId ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
	 * @param key identifying the file
	 * @return file handle to the file
	 *
	 * @throws IOException
	 * 		if creating the directory fails
	 */
	@VisibleForTesting
	public File getStorageLocation(@Nullable JobID jobId, BlobKey key) throws IOException {
		return BlobUtils.getStorageLocation(storageDir, jobId, key);
	}

	/**
	 * Returns a temporary file inside the BLOB server's incoming directory.
	 *
	 * @return a temporary file inside the BLOB server's incoming directory
	 *
	 * @throws IOException
	 * 		if creating the directory fails
	 */
	File createTemporaryFilename() throws IOException {
		return new File(BlobUtils.getIncomingDirectory(storageDir),
				String.format("temp-%08d", tempFileCounter.getAndIncrement()));
	}

	/**
	 * Returns the lock used to guard file accesses.
	 */
	ReadWriteLock getReadWriteLock() {
		return readWriteLock;
	}

	@Override
	public void run() {
		try {
			while (!this.shutdownRequested.get()) {
				BlobServerConnection conn = new BlobServerConnection(serverSocket.accept(), this);
				try {
					synchronized (activeConnections) {
						while (activeConnections.size() >= maxConnections) {
							activeConnections.wait(2000);
						}
						activeConnections.add(conn);
					}

					conn.start();
					conn = null;
				}
				finally {
					if (conn != null) {
						conn.close();
						synchronized (activeConnections) {
							activeConnections.remove(conn);
						}
					}
				}
			}
		}
		catch (Throwable t) {
			if (!this.shutdownRequested.get()) {
				LOG.error("BLOB server stopped working. Shutting down", t);

				try {
					close();
				} catch (Throwable closeThrowable) {
					LOG.error("Could not properly close the BlobServer.", closeThrowable);
				}
			}
		}
	}

	/**
	 * Shuts down the BLOB server.
	 */
	@Override
	public void close() throws IOException {
		cleanupTimer.cancel();

		if (shutdownRequested.compareAndSet(false, true)) {
			Exception exception = null;

			try {
				this.serverSocket.close();
			}
			catch (IOException ioe) {
				exception = ioe;
			}

			// wake the thread up, in case it is waiting on some operation
			interrupt();

			try {
				join();
			}
			catch (InterruptedException ie) {
				Thread.currentThread().interrupt();

				LOG.debug("Error while waiting for this thread to die.", ie);
			}

			synchronized (activeConnections) {
				if (!activeConnections.isEmpty()) {
					for (BlobServerConnection conn : activeConnections) {
						LOG.debug("Shutting down connection {}.", conn.getName());
						conn.close();
					}
					activeConnections.clear();
				}
			}

			// Clean up the storage directory
			try {
				FileUtils.deleteDirectory(storageDir);
			}
			catch (IOException e) {
				exception = ExceptionUtils.firstOrSuppressed(e, exception);
			}

			// Remove shutdown hook to prevent resource leaks
			ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), LOG);

			if (LOG.isInfoEnabled()) {
				LOG.info("Stopped BLOB server at {}:{}", serverSocket.getInetAddress().getHostAddress(), getPort());
			}

			ExceptionUtils.tryRethrowIOException(exception);
		}
	}

	protected BlobClient createClient() throws IOException {
		return new BlobClient(new InetSocketAddress(serverSocket.getInetAddress(), getPort()),
			blobServiceConfiguration);
	}

	/**
	 * Retrieves the local path of a (job-unrelated) file associated with a job and a blob key.
	 *
	 * <p>The blob server looks the blob key up in its local storage. If the file exists, it is
	 * returned. If the file does not exist, it is retrieved from the HA blob store (if available)
	 * or a {@link FileNotFoundException} is thrown.
	 *
	 * @param key
	 * 		blob key associated with the requested file
	 *
	 * @return file referring to the local storage location of the BLOB
	 *
	 * @throws IOException
	 * 		Thrown if the file retrieval failed.
	 */
	@Override
	public File getFile(TransientBlobKey key) throws IOException {
		return getFileInternal(null, key);
	}

	/**
	 * Retrieves the local path of a file associated with a job and a blob key.
	 *
	 * <p>The blob server looks the blob key up in its local storage. If the file exists, it is
	 * returned. If the file does not exist, it is retrieved from the HA blob store (if available)
	 * or a {@link FileNotFoundException} is thrown.
	 *
	 * @param jobId
	 * 		ID of the job this blob belongs to
	 * @param key
	 * 		blob key associated with the requested file
	 *
	 * @return file referring to the local storage location of the BLOB
	 *
	 * @throws IOException
	 * 		Thrown if the file retrieval failed.
	 */
	@Override
	public File getFile(JobID jobId, TransientBlobKey key) throws IOException {
		checkNotNull(jobId);
		return getFileInternal(jobId, key);
	}

	/**
	 * Returns the path to a local copy of the file associated with the provided job ID and blob
	 * key.
	 *
	 * <p>We will first attempt to serve the BLOB from the local storage. If the BLOB is not in
	 * there, we will try to download it from the HA store.
	 *
	 * @param jobId
	 * 		ID of the job this blob belongs to
	 * @param key
	 * 		blob key associated with the requested file
	 *
	 * @return The path to the file.
	 *
	 * @throws java.io.FileNotFoundException
	 * 		if the BLOB does not exist;
	 * @throws IOException
	 * 		if any other error occurs when retrieving the file
	 */
	@Override
	public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
		checkNotNull(jobId);
		return getFileInternal(jobId, key);
	}

	/**
	 * Retrieves the local path of a file associated with a job and a blob key.
	 *
	 * <p>The blob server looks the blob key up in its local storage. If the file exists, it is
	 * returned. If the file does not exist, it is retrieved from the HA blob store (if available)
	 * or a {@link FileNotFoundException} is thrown.
	 *
	 * @param jobId
	 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
	 * @param blobKey
	 * 		blob key associated with the requested file
	 *
	 * @return file referring to the local storage location of the BLOB
	 *
	 * @throws IOException
	 * 		Thrown if the file retrieval failed.
	 */
	private File getFileInternal(@Nullable JobID jobId, BlobKey blobKey) throws IOException {
		checkArgument(blobKey != null, "BLOB key cannot be null.");

		final File localFile = BlobUtils.getStorageLocation(storageDir, jobId, blobKey);
		readWriteLock.readLock().lock();

		try {
			getFileInternal(jobId, blobKey, localFile);
			return localFile;
		} finally {
			readWriteLock.readLock().unlock();
		}
	}

	/**
	 * Helper to retrieve the local path of a file associated with a job and a blob key.
	 *
	 * <p>The blob server looks the blob key up in its local storage. If the file exists, it is
	 * returned. If the file does not exist, it is retrieved from the HA blob store (if available)
	 * or a {@link FileNotFoundException} is thrown.
	 *
	 * <p><strong>Assumes the read lock has already been acquired.</strong>
	 *
	 * @param jobId
	 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
	 * @param blobKey
	 * 		blob key associated with the requested file
	 * @param localFile
	 *      (local) file where the blob is/should be stored
	 *
	 * @throws IOException
	 * 		Thrown if the file retrieval failed.
	 */
	void getFileInternal(@Nullable JobID jobId, BlobKey blobKey, File localFile) throws IOException {
		// assume readWriteLock.readLock() was already locked (cannot really check that)

		if (localFile.exists()) {
			// update TTL for transient BLOBs:
			if (blobKey instanceof TransientBlobKey) {
				// regarding concurrent operations, it is not really important which timestamp makes
				// it into the map as they are close to each other anyway, also we can simply
				// overwrite old values as long as we are in the read (or write) lock
				blobExpiryTimes
					.put(Tuple2.of(jobId, (TransientBlobKey) blobKey),
						System.currentTimeMillis() + cleanupInterval);
			}
			return;
		} else if (blobKey instanceof PermanentBlobKey) {
			// Try the HA blob store
			// first we have to release the read lock in order to acquire the write lock
			readWriteLock.readLock().unlock();

			// use a temporary file (thread-safe without locking)
			File incomingFile = null;
			try {
				incomingFile = createTemporaryFilename();
				blobStore.get(jobId, blobKey, incomingFile);

				readWriteLock.writeLock().lock();
				try {
					BlobUtils.moveTempFileToStore(
						incomingFile, jobId, blobKey, localFile, LOG, null);
				} finally {
					readWriteLock.writeLock().unlock();
				}

				return;
			} finally {
				// delete incomingFile from a failed download
				if (incomingFile != null && !incomingFile.delete() && incomingFile.exists()) {
					LOG.warn("Could not delete the staging file {} for blob key {} and job {}.",
						incomingFile, blobKey, jobId);
				}

				// re-acquire lock so that it can be unlocked again outside
				readWriteLock.readLock().lock();
			}
		}

		throw new FileNotFoundException("Local file " + localFile + " does not exist " +
			"and failed to copy from blob store.");
	}

	@Override
	public TransientBlobKey putTransient(byte[] value) throws IOException {
		return (TransientBlobKey) putBuffer(null, value, TRANSIENT_BLOB);
	}

	@Override
	public TransientBlobKey putTransient(JobID jobId, byte[] value) throws IOException {
		checkNotNull(jobId);
		return (TransientBlobKey) putBuffer(jobId, value, TRANSIENT_BLOB);
	}

	@Override
	public TransientBlobKey putTransient(InputStream inputStream) throws IOException {
		return (TransientBlobKey) putInputStream(null, inputStream, TRANSIENT_BLOB);
	}

	@Override
	public TransientBlobKey putTransient(JobID jobId, InputStream inputStream) throws IOException {
		checkNotNull(jobId);
		return (TransientBlobKey) putInputStream(jobId, inputStream, TRANSIENT_BLOB);
	}

	@Override
	public PermanentBlobKey putPermanent(JobID jobId, byte[] value) throws IOException {
		checkNotNull(jobId);
		return (PermanentBlobKey) putBuffer(jobId, value, PERMANENT_BLOB);
	}

	@Override
	public PermanentBlobKey putPermanent(JobID jobId, InputStream inputStream) throws IOException {
		checkNotNull(jobId);
		return (PermanentBlobKey) putInputStream(jobId, inputStream, PERMANENT_BLOB);
	}

	/**
	 * Uploads the data of the given byte array for the given job to the BLOB server.
	 *
	 * @param jobId
	 * 		the ID of the job the BLOB belongs to
	 * @param value
	 * 		the buffer to upload
	 * @param blobType
	 * 		whether to make the data permanent or transient
	 *
	 * @return the computed BLOB key identifying the BLOB on the server
	 *
	 * @throws IOException
	 * 		thrown if an I/O error occurs while writing it to a local file, or uploading it to the HA
	 * 		store
	 */
	private BlobKey putBuffer(@Nullable JobID jobId, byte[] value, BlobKey.BlobType blobType)
			throws IOException {

		if (LOG.isDebugEnabled()) {
			LOG.debug("Received PUT call for BLOB of job {}.", jobId);
		}

		File incomingFile = createTemporaryFilename();
		MessageDigest md = BlobUtils.createMessageDigest();
		BlobKey blobKey = null;
		try (FileOutputStream fos = new FileOutputStream(incomingFile)) {
			md.update(value);
			fos.write(value);
		} catch (IOException ioe) {
			// delete incomingFile from a failed download
			if (!incomingFile.delete() && incomingFile.exists()) {
				LOG.warn("Could not delete the staging file {} for job {}.",
					incomingFile, jobId);
			}
			throw ioe;
		}

		try {
			// persist file
			blobKey = moveTempFileToStore(incomingFile, jobId, md.digest(), blobType);

			return blobKey;
		} finally {
			// delete incomingFile from a failed download
			if (!incomingFile.delete() && incomingFile.exists()) {
				LOG.warn("Could not delete the staging file {} for blob key {} and job {}.",
					incomingFile, blobKey, jobId);
			}
		}
	}


	/**
	 * Uploads the data from the given input stream for the given job to the BLOB server.
	 *
	 * @param jobId
	 * 		the ID of the job the BLOB belongs to
	 * @param inputStream
	 * 		the input stream to read the data from
	 * @param blobType
	 * 		whether to make the data permanent or transient
	 *
	 * @return the computed BLOB key identifying the BLOB on the server
	 *
	 * @throws IOException
	 * 		thrown if an I/O error occurs while reading the data from the input stream, writing it to a
	 * 		local file, or uploading it to the HA store
	 */
	private BlobKey putInputStream(
			@Nullable JobID jobId, InputStream inputStream, BlobKey.BlobType blobType)
			throws IOException {

		if (LOG.isDebugEnabled()) {
			LOG.debug("Received PUT call for BLOB of job {}.", jobId);
		}

		File incomingFile = createTemporaryFilename();
		MessageDigest md = BlobUtils.createMessageDigest();
		BlobKey blobKey = null;
		try (FileOutputStream fos = new FileOutputStream(incomingFile)) {
			// read stream
			byte[] buf = new byte[BUFFER_SIZE];
			while (true) {
				final int bytesRead = inputStream.read(buf);
				if (bytesRead == -1) {
					// done
					break;
				}
				fos.write(buf, 0, bytesRead);
				md.update(buf, 0, bytesRead);
			}

			// persist file
			blobKey = moveTempFileToStore(incomingFile, jobId, md.digest(), blobType);

			return blobKey;
		} finally {
			// delete incomingFile from a failed download
			if (!incomingFile.delete() && incomingFile.exists()) {
				LOG.warn("Could not delete the staging file {} for blob key {} and job {}.",
					incomingFile, blobKey, jobId);
			}
		}
	}

	/**
	 * Moves the temporary <tt>incomingFile</tt> to its permanent location where it is available for
	 * use.
	 *
	 * @param incomingFile
	 * 		temporary file created during transfer
	 * @param jobId
	 * 		ID of the job this blob belongs to or <tt>null</tt> if job-unrelated
	 * @param digest
	 * 		BLOB content digest, i.e. hash
	 * @param blobType
	 * 		whether this file is a permanent or transient BLOB
	 *
	 * @return unique BLOB key that identifies the BLOB on the server
	 *
	 * @throws IOException
	 * 		thrown if an I/O error occurs while moving the file or uploading it to the HA store
	 */
	BlobKey moveTempFileToStore(
			File incomingFile, @Nullable JobID jobId, byte[] digest, BlobKey.BlobType blobType)
			throws IOException {

		int retries = 10;

		int attempt = 0;
		while (true) {
			// add unique component independent of the BLOB content
			BlobKey blobKey = BlobKey.createKey(blobType, digest);
			File storageFile = BlobUtils.getStorageLocation(storageDir, jobId, blobKey);

			// try again until the key is unique (put the existence check into the lock!)
			readWriteLock.writeLock().lock();
			try {
				if (!storageFile.exists()) {
					BlobUtils.moveTempFileToStore(
						incomingFile, jobId, blobKey, storageFile, LOG,
						blobKey instanceof PermanentBlobKey ? blobStore : null);
					// add TTL for transient BLOBs:
					if (blobKey instanceof TransientBlobKey) {
						// must be inside read or write lock to add a TTL
						blobExpiryTimes
							.put(Tuple2.of(jobId, (TransientBlobKey) blobKey),
								System.currentTimeMillis() + cleanupInterval);
					}
					return blobKey;
				}
			} finally {
				readWriteLock.writeLock().unlock();
			}

			++attempt;
			if (attempt >= retries) {
				String message = "Failed to find a unique key for BLOB of job " + jobId + " (last tried " + storageFile.getAbsolutePath() + ".";
				LOG.error(message + " No retries left.");
				throw new IOException(message);
			} else {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Trying to find a unique key for BLOB of job {} (retry {}, last tried {})",
						jobId, attempt, storageFile.getAbsolutePath());
				}
			}
		}
	}

	/**
	 * Deletes the (job-unrelated) file associated with the blob key in the local storage of the
	 * blob server.
	 *
	 * @param key
	 * 		blob key associated with the file to be deleted
	 *
	 * @return  <tt>true</tt> if the given blob is successfully deleted or non-existing;
	 *          <tt>false</tt> otherwise
	 */
	@Override
	public boolean deleteFromCache(TransientBlobKey key) {
		return deleteInternal(null, key);
	}

	/**
	 * Deletes the file associated with the blob key in the local storage of the blob server.
	 *
	 * @param jobId
	 * 		ID of the job this blob belongs to
	 * @param key
	 * 		blob key associated with the file to be deleted
	 *
	 * @return  <tt>true</tt> if the given blob is successfully deleted or non-existing;
	 *          <tt>false</tt> otherwise
	 */
	@Override
	public boolean deleteFromCache(JobID jobId, TransientBlobKey key) {
		checkNotNull(jobId);
		return deleteInternal(jobId, key);
	}

	/**
	 * Deletes the file associated with the blob key in the local storage of the blob server.
	 *
	 * @param jobId
	 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
	 * @param key
	 * 		blob key associated with the file to be deleted
	 *
	 * @return  <tt>true</tt> if the given blob is successfully deleted or non-existing;
	 *          <tt>false</tt> otherwise
	 */
	boolean deleteInternal(@Nullable JobID jobId, TransientBlobKey key) {
		final File localFile =
			new File(BlobUtils.getStorageLocationPath(storageDir.getAbsolutePath(), jobId, key));

		readWriteLock.writeLock().lock();

		try {
			if (!localFile.delete() && localFile.exists()) {
				LOG.warn("Failed to locally delete BLOB " + key + " at " + localFile.getAbsolutePath());
				return false;
			}
			// this needs to happen inside the write lock in case of concurrent getFile() calls
			blobExpiryTimes.remove(Tuple2.of(jobId, key));
			return true;
		} finally {
			readWriteLock.writeLock().unlock();
		}
	}

	/**
	 * Removes all BLOBs from local and HA store belonging to the given job ID.
	 *
	 * @param jobId
	 * 		ID of the job this blob belongs to
	 * @param cleanupBlobStoreFiles
	 * 		True if the corresponding blob store files shall be cleaned up as well. Otherwise false.
	 *
	 * @return  <tt>true</tt> if the job directory is successfully deleted or non-existing;
	 *          <tt>false</tt> otherwise
	 */
	public boolean cleanupJob(JobID jobId, boolean cleanupBlobStoreFiles) {
		checkNotNull(jobId);

		final File jobDir =
			new File(BlobUtils.getStorageLocationPath(storageDir.getAbsolutePath(), jobId));

		readWriteLock.writeLock().lock();

		try {
			// delete locally
			boolean deletedLocally = false;
			try {
				FileUtils.deleteDirectory(jobDir);

				// NOTE: Instead of going through blobExpiryTimes, keep lingering entries - they
				//       will be cleaned up by the timer task which tolerates non-existing files
				//       If inserted again with the same IDs (via put()), the TTL will be updated
				//       again.

				deletedLocally = true;
			} catch (IOException e) {
				LOG.warn("Failed to locally delete BLOB storage directory at " +
					jobDir.getAbsolutePath(), e);
			}

			// delete in HA blob store files
			final boolean deletedHA = !cleanupBlobStoreFiles || blobStore.deleteAll(jobId);

			return deletedLocally && deletedHA;
		} finally {
			readWriteLock.writeLock().unlock();
		}
	}

	@Override
	public PermanentBlobService getPermanentBlobService() {
		return this;
	}

	@Override
	public TransientBlobService getTransientBlobService() {
		return this;
	}

	/**
	 * Returns the configuration used by the BLOB server.
	 *
	 * @return configuration
	 */
	@Override
	public final int getMinOffloadingSize() {
		return blobServiceConfiguration.getInteger(BlobServerOptions.OFFLOAD_MINSIZE);
	}

	/**
	 * Returns the port on which the server is listening.
	 *
	 * @return port on which the server is listening
	 */
	@Override
	public int getPort() {
		return this.serverSocket.getLocalPort();
	}

	/**
	 * Returns the blob expiry times - for testing purposes only!
	 *
	 * @return blob expiry times (internal state!)
	 */
	@VisibleForTesting
	ConcurrentMap<Tuple2<JobID, TransientBlobKey>, Long> getBlobExpiryTimes() {
		return blobExpiryTimes;
	}

	/**
	 * Tests whether the BLOB server has been requested to shut down.
	 *
	 * @return True, if the server has been requested to shut down, false otherwise.
	 */
	public boolean isShutdown() {
		return this.shutdownRequested.get();
	}

	/**
	 * Access to the server socket, for testing.
	 */
	ServerSocket getServerSocket() {
		return this.serverSocket;
	}

	void unregisterConnection(BlobServerConnection conn) {
		synchronized (activeConnections) {
			activeConnections.remove(conn);
			activeConnections.notifyAll();
		}
	}

	/**
	 * Returns all the current active connections in the BlobServer.
	 *
	 * @return the list of all the active in current BlobServer
	 */
	List<BlobServerConnection> getCurrentActiveConnections() {
		synchronized (activeConnections) {
			return new ArrayList<>(activeConnections);
		}
	}
}