/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.azure;
import java.io.DataInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.Set;
import java.util.TimeZone;
import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BufferedFSInputStream;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.fs.azure.AzureException;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.azure.storage.OperationContext;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.azure.storage.core.*;
/**
* A {@link FileSystem} for reading and writing files stored on <a
* href="http://store.azure.com/">Windows Azure</a>. This implementation is
* blob-based and stores files on Azure in their native form so they can be read
* by other Azure tools.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class NativeAzureFileSystem extends FileSystem {
private static final int USER_WX_PERMISION = 0300;
/**
* A description of a folder rename operation, including the source and
* destination keys, and descriptions of the files in the source folder.
*/
public static class FolderRenamePending {
private SelfRenewingLease folderLease;
private String srcKey;
private String dstKey;
private FileMetadata[] fileMetadata = null; // descriptions of source files
private ArrayList<String> fileStrings = null;
private NativeAzureFileSystem fs;
private static final int MAX_RENAME_PENDING_FILE_SIZE = 10000000;
private static final int FORMATTING_BUFFER = 10000;
private boolean committed;
public static final String SUFFIX = "-RenamePending.json";
// Prepare in-memory information needed to do or redo a folder rename.
public FolderRenamePending(String srcKey, String dstKey, SelfRenewingLease lease,
NativeAzureFileSystem fs) throws IOException {
this.srcKey = srcKey;
this.dstKey = dstKey;
this.folderLease = lease;
this.fs = fs;
ArrayList<FileMetadata> fileMetadataList = new ArrayList<FileMetadata>();
// List all the files in the folder.
String priorLastKey = null;
do {
PartialListing listing = fs.getStoreInterface().listAll(srcKey, AZURE_LIST_ALL,
AZURE_UNBOUNDED_DEPTH, priorLastKey);
for(FileMetadata file : listing.getFiles()) {
fileMetadataList.add(file);
}
priorLastKey = listing.getPriorLastKey();
} while (priorLastKey != null);
fileMetadata = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]);
this.committed = true;
}
// Prepare in-memory information needed to do or redo folder rename from
// a -RenamePending.json file read from storage. This constructor is to use during
// redo processing.
public FolderRenamePending(Path redoFile, NativeAzureFileSystem fs)
throws IllegalArgumentException, IOException {
this.fs = fs;
// open redo file
Path f = redoFile;
FSDataInputStream input = fs.open(f);
byte[] bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE];
int l = input.read(bytes);
if (l < 0) {
throw new IOException(
"Error reading pending rename file contents -- no data available");
}
if (l == MAX_RENAME_PENDING_FILE_SIZE) {
throw new IOException(
"Error reading pending rename file contents -- "
+ "maximum file size exceeded");
}
String contents = new String(bytes, 0, l, Charset.forName("UTF-8"));
// parse the JSON
ObjectMapper objMapper = new ObjectMapper();
objMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
JsonNode json = null;
try {
json = objMapper.readValue(contents, JsonNode.class);
this.committed = true;
} catch (JsonMappingException e) {
// The -RedoPending.json file is corrupted, so we assume it was
// not completely written
// and the redo operation did not commit.
this.committed = false;
} catch (JsonParseException e) {
this.committed = false;
} catch (IOException e) {
this.committed = false;
}
if (!this.committed) {
LOG.error("Deleting corruped rename pending file "
+ redoFile + "\n" + contents);
// delete the -RenamePending.json file
fs.delete(redoFile, false);
return;
}
// initialize this object's fields
ArrayList<String> fileStrList = new ArrayList<String>();
JsonNode oldFolderName = json.get("OldFolderName");
JsonNode newFolderName = json.get("NewFolderName");
if (oldFolderName == null || newFolderName == null) {
this.committed = false;
} else {
this.srcKey = oldFolderName.getTextValue();
this.dstKey = newFolderName.getTextValue();
if (this.srcKey == null || this.dstKey == null) {
this.committed = false;
} else {
JsonNode fileList = json.get("FileList");
if (fileList == null) {
this.committed = false;
} else {
for (int i = 0; i < fileList.size(); i++) {
fileStrList.add(fileList.get(i).getTextValue());
}
}
}
}
this.fileStrings = fileStrList;
}
public FileMetadata[] getFiles() {
return fileMetadata;
}
public SelfRenewingLease getFolderLease() {
return folderLease;
}
/**
* Write to disk the information needed to redo folder rename,
* in JSON format. The file name will be
* {@code wasb://<sourceFolderPrefix>/folderName-RenamePending.json}
* The file format will be:
* <pre>{@code
* {
* FormatVersion: "1.0",
* OperationTime: "<YYYY-MM-DD HH:MM:SS.MMM>",
* OldFolderName: "<key>",
* NewFolderName: "<key>",
* FileList: [ <string> , <string> , ... ]
* }
*
* Here's a sample:
* {
* FormatVersion: "1.0",
* OperationUTCTime: "2014-07-01 23:50:35.572",
* OldFolderName: "user/ehans/folderToRename",
* NewFolderName: "user/ehans/renamedFolder",
* FileList: [
* "innerFile",
* "innerFile2"
* ]
* } }</pre>
* @throws IOException
*/
public void writeFile(FileSystem fs) throws IOException {
Path path = getRenamePendingFilePath();
if (LOG.isDebugEnabled()){
LOG.debug("Preparing to write atomic rename state to " + path.toString());
}
OutputStream output = null;
String contents = makeRenamePendingFileContents();
// Write file.
try {
output = fs.create(path);
output.write(contents.getBytes(Charset.forName("UTF-8")));
} catch (IOException e) {
throw new IOException("Unable to write RenamePending file for folder rename from "
+ srcKey + " to " + dstKey, e);
} finally {
IOUtils.cleanup(LOG, output);
}
}
/**
* Return the contents of the JSON file to represent the operations
* to be performed for a folder rename.
*/
public String makeRenamePendingFileContents() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
String time = sdf.format(new Date());
// Make file list string
StringBuilder builder = new StringBuilder();
builder.append("[\n");
for (int i = 0; i != fileMetadata.length; i++) {
if (i > 0) {
builder.append(",\n");
}
builder.append(" ");
String noPrefix = StringUtils.removeStart(fileMetadata[i].getKey(), srcKey + "/");
// Quote string file names, escaping any possible " characters or other
// necessary characters in the name.
builder.append(quote(noPrefix));
if (builder.length() >=
MAX_RENAME_PENDING_FILE_SIZE - FORMATTING_BUFFER) {
// Give up now to avoid using too much memory.
LOG.error("Internal error: Exceeded maximum rename pending file size of "
+ MAX_RENAME_PENDING_FILE_SIZE + " bytes.");
// return some bad JSON with an error message to make it human readable
return "exceeded maximum rename pending file size";
}
}
builder.append("\n ]");
String fileList = builder.toString();
// Make file contents as a string. Again, quote file names, escaping
// characters as appropriate.
String contents = "{\n"
+ " FormatVersion: \"1.0\",\n"
+ " OperationUTCTime: \"" + time + "\",\n"
+ " OldFolderName: " + quote(srcKey) + ",\n"
+ " NewFolderName: " + quote(dstKey) + ",\n"
+ " FileList: " + fileList + "\n"
+ "}\n";
return contents;
}
/**
* This is an exact copy of org.codehaus.jettison.json.JSONObject.quote
* method.
*
* Produce a string in double quotes with backslash sequences in all the
* right places. A backslash will be inserted within </, allowing JSON
* text to be delivered in HTML. In JSON text, a string cannot contain a
* control character or an unescaped quote or backslash.
* @param string A String
* @return A String correctly formatted for insertion in a JSON text.
*/
private String quote(String string) {
if (string == null || string.length() == 0) {
return "\"\"";
}
char c = 0;
int i;
int len = string.length();
StringBuilder sb = new StringBuilder(len + 4);
String t;
sb.append('"');
for (i = 0; i < len; i += 1) {
c = string.charAt(i);
switch (c) {
case '\\':
case '"':
sb.append('\\');
sb.append(c);
break;
case '/':
sb.append('\\');
sb.append(c);
break;
case '\b':
sb.append("\\b");
break;
case '\t':
sb.append("\\t");
break;
case '\n':
sb.append("\\n");
break;
case '\f':
sb.append("\\f");
break;
case '\r':
sb.append("\\r");
break;
default:
if (c < ' ') {
t = "000" + Integer.toHexString(c);
sb.append("\\u" + t.substring(t.length() - 4));
} else {
sb.append(c);
}
}
}
sb.append('"');
return sb.toString();
}
public String getSrcKey() {
return srcKey;
}
public String getDstKey() {
return dstKey;
}
public FileMetadata getSourceMetadata() throws IOException {
return fs.getStoreInterface().retrieveMetadata(srcKey);
}
/**
* Execute a folder rename. This is the execution path followed
* when everything is working normally. See redo() for the alternate
* execution path for the case where we're recovering from a folder rename
* failure.
* @throws IOException
*/
public void execute() throws IOException {
for (FileMetadata file : this.getFiles()) {
// Rename all materialized entries under the folder to point to the
// final destination.
if (file.getBlobMaterialization() == BlobMaterialization.Explicit) {
String srcName = file.getKey();
String suffix = srcName.substring((this.getSrcKey()).length());
String dstName = this.getDstKey() + suffix;
// Rename gets exclusive access (via a lease) for files
// designated for atomic rename.
// The main use case is for HBase write-ahead log (WAL) and data
// folder processing correctness. See the rename code for details.
boolean acquireLease = fs.getStoreInterface().isAtomicRenameKey(srcName);
fs.getStoreInterface().rename(srcName, dstName, acquireLease, null);
}
}
// Rename the source folder 0-byte root file itself.
FileMetadata srcMetadata2 = this.getSourceMetadata();
if (srcMetadata2.getBlobMaterialization() ==
BlobMaterialization.Explicit) {
// It already has a lease on it from the "prepare" phase so there's no
// need to get one now. Pass in existing lease to allow file delete.
fs.getStoreInterface().rename(this.getSrcKey(), this.getDstKey(),
false, folderLease);
}
// Update the last-modified time of the parent folders of both source and
// destination.
fs.updateParentFolderLastModifiedTime(srcKey);
fs.updateParentFolderLastModifiedTime(dstKey);
}
/** Clean up after execution of rename.
* @throws IOException */
public void cleanup() throws IOException {
if (fs.getStoreInterface().isAtomicRenameKey(srcKey)) {
// Remove RenamePending file
fs.delete(getRenamePendingFilePath(), false);
// Freeing source folder lease is not necessary since the source
// folder file was deleted.
}
}
private Path getRenamePendingFilePath() {
String fileName = srcKey + SUFFIX;
Path fileNamePath = keyToPath(fileName);
Path path = fs.makeAbsolute(fileNamePath);
return path;
}
/**
* Recover from a folder rename failure by redoing the intended work,
* as recorded in the -RenamePending.json file.
*
* @throws IOException
*/
public void redo() throws IOException {
if (!committed) {
// Nothing to do. The -RedoPending.json file should have already been
// deleted.
return;
}
// Try to get a lease on source folder to block concurrent access to it.
// It may fail if the folder is already gone. We don't check if the
// source exists explicitly because that could recursively trigger redo
// and give an infinite recursion.
SelfRenewingLease lease = null;
boolean sourceFolderGone = false;
try {
lease = fs.leaseSourceFolder(srcKey);
} catch (AzureException e) {
// If the source folder was not found then somebody probably
// raced with us and finished the rename first, or the
// first rename failed right before deleting the rename pending
// file.
String errorCode = "";
try {
StorageException se = (StorageException) e.getCause();
errorCode = se.getErrorCode();
} catch (Exception e2) {
; // do nothing -- could not get errorCode
}
if (errorCode.equals("BlobNotFound")) {
sourceFolderGone = true;
} else {
throw new IOException(
"Unexpected error when trying to lease source folder name during "
+ "folder rename redo",
e);
}
}
if (!sourceFolderGone) {
// Make sure the target folder exists.
Path dst = fullPath(dstKey);
if (!fs.exists(dst)) {
fs.mkdirs(dst);
}
// For each file inside the folder to be renamed,
// make sure it has been renamed.
for(String fileName : fileStrings) {
finishSingleFileRename(fileName);
}
// Remove the source folder. Don't check explicitly if it exists,
// to avoid triggering redo recursively.
try {
fs.getStoreInterface().delete(srcKey, lease);
} catch (Exception e) {
LOG.info("Unable to delete source folder during folder rename redo. "
+ "If the source folder is already gone, this is not an error "
+ "condition. Continuing with redo.", e);
}
// Update the last-modified time of the parent folders of both source
// and destination.
fs.updateParentFolderLastModifiedTime(srcKey);
fs.updateParentFolderLastModifiedTime(dstKey);
}
// Remove the -RenamePending.json file.
fs.delete(getRenamePendingFilePath(), false);
}
// See if the source file is still there, and if it is, rename it.
private void finishSingleFileRename(String fileName)
throws IOException {
Path srcFile = fullPath(srcKey, fileName);
Path dstFile = fullPath(dstKey, fileName);
boolean srcExists = fs.exists(srcFile);
boolean dstExists = fs.exists(dstFile);
if (srcExists && !dstExists) {
// Rename gets exclusive access (via a lease) for HBase write-ahead log
// (WAL) file processing correctness. See the rename code for details.
String srcName = fs.pathToKey(srcFile);
String dstName = fs.pathToKey(dstFile);
fs.getStoreInterface().rename(srcName, dstName, true, null);
} else if (srcExists && dstExists) {
// Get a lease on source to block write access.
String srcName = fs.pathToKey(srcFile);
SelfRenewingLease lease = fs.acquireLease(srcFile);
// Delete the file. This will free the lease too.
fs.getStoreInterface().delete(srcName, lease);
} else if (!srcExists && dstExists) {
// The rename already finished, so do nothing.
;
} else {
throw new IOException(
"Attempting to complete rename of file " + srcKey + "/" + fileName
+ " during folder rename redo, and file was not found in source "
+ "or destination.");
}
}
// Return an absolute path for the specific fileName within the folder
// specified by folderKey.
private Path fullPath(String folderKey, String fileName) {
return new Path(new Path(fs.getUri()), "/" + folderKey + "/" + fileName);
}
private Path fullPath(String fileKey) {
return new Path(new Path(fs.getUri()), "/" + fileKey);
}
}
private static final String TRAILING_PERIOD_PLACEHOLDER = "[[.]]";
private static final Pattern TRAILING_PERIOD_PLACEHOLDER_PATTERN =
Pattern.compile("\\[\\[\\.\\]\\](?=$|/)");
private static final Pattern TRAILING_PERIOD_PATTERN = Pattern.compile("\\.(?=$|/)");
@Override
public String getScheme() {
return "wasb";
}
/**
* <p>
* A {@link FileSystem} for reading and writing files stored on <a
* href="http://store.azure.com/">Windows Azure</a>. This implementation is
* blob-based and stores files on Azure in their native form so they can be read
* by other Azure tools. This implementation uses HTTPS for secure network communication.
* </p>
*/
public static class Secure extends NativeAzureFileSystem {
@Override
public String getScheme() {
return "wasbs";
}
}
public static final Log LOG = LogFactory.getLog(NativeAzureFileSystem.class);
static final String AZURE_BLOCK_SIZE_PROPERTY_NAME = "fs.azure.block.size";
/**
* The time span in seconds before which we consider a temp blob to be
* dangling (not being actively uploaded to) and up for reclamation.
*
* So e.g. if this is 60, then any temporary blobs more than a minute old
* would be considered dangling.
*/
static final String AZURE_TEMP_EXPIRY_PROPERTY_NAME = "fs.azure.fsck.temp.expiry.seconds";
private static final int AZURE_TEMP_EXPIRY_DEFAULT = 3600;
static final String PATH_DELIMITER = Path.SEPARATOR;
static final String AZURE_TEMP_FOLDER = "_$azuretmpfolder$";
private static final int AZURE_LIST_ALL = -1;
private static final int AZURE_UNBOUNDED_DEPTH = -1;
private static final long MAX_AZURE_BLOCK_SIZE = 512 * 1024 * 1024L;
/**
* The configuration property that determines which group owns files created
* in WASB.
*/
private static final String AZURE_DEFAULT_GROUP_PROPERTY_NAME = "fs.azure.permissions.supergroup";
/**
* The default value for fs.azure.permissions.supergroup. Chosen as the same
* default as DFS.
*/
static final String AZURE_DEFAULT_GROUP_DEFAULT = "supergroup";
static final String AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME =
"fs.azure.block.location.impersonatedhost";
private static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT =
"localhost";
static final String AZURE_RINGBUFFER_CAPACITY_PROPERTY_NAME =
"fs.azure.ring.buffer.capacity";
static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME =
"fs.azure.output.stream.buffer.size";
public static final String SKIP_AZURE_METRICS_PROPERTY_NAME = "fs.azure.skip.metrics";
private class NativeAzureFsInputStream extends FSInputStream {
private InputStream in;
private final String key;
private long pos = 0;
private boolean closed = false;
private boolean isPageBlob;
// File length, valid only for streams over block blobs.
private long fileLength;
public NativeAzureFsInputStream(DataInputStream in, String key, long fileLength) {
this.in = in;
this.key = key;
this.isPageBlob = store.isPageBlobKey(key);
this.fileLength = fileLength;
}
/**
* Return the size of the remaining available bytes
* if the size is less than or equal to {@link Integer#MAX_VALUE},
* otherwise, return {@link Integer#MAX_VALUE}.
*
* This is to match the behavior of DFSInputStream.available(),
* which some clients may rely on (HBase write-ahead log reading in
* particular).
*/
@Override
public synchronized int available() throws IOException {
if (isPageBlob) {
return in.available();
} else {
if (closed) {
throw new IOException("Stream closed");
}
final long remaining = this.fileLength - pos;
return remaining <= Integer.MAX_VALUE ?
(int) remaining : Integer.MAX_VALUE;
}
}
/*
* Reads the next byte of data from the input stream. The value byte is
* returned as an integer in the range 0 to 255. If no byte is available
* because the end of the stream has been reached, the value -1 is returned.
* This method blocks until input data is available, the end of the stream
* is detected, or an exception is thrown.
*
* @returns int An integer corresponding to the byte read.
*/
@Override
public synchronized int read() throws IOException {
int result = 0;
result = in.read();
if (result != -1) {
pos++;
if (statistics != null) {
statistics.incrementBytesRead(1);
}
}
// Return to the caller with the result.
//
return result;
}
/*
* Reads up to len bytes of data from the input stream into an array of
* bytes. An attempt is made to read as many as len bytes, but a smaller
* number may be read. The number of bytes actually read is returned as an
* integer. This method blocks until input data is available, end of file is
* detected, or an exception is thrown. If len is zero, then no bytes are
* read and 0 is returned; otherwise, there is an attempt to read at least
* one byte. If no byte is available because the stream is at end of file,
* the value -1 is returned; otherwise, at least one byte is read and stored
* into b.
*
* @param b -- the buffer into which data is read
*
* @param off -- the start offset in the array b at which data is written
*
* @param len -- the maximum number of bytes read
*
* @ returns int The total number of byes read into the buffer, or -1 if
* there is no more data because the end of stream is reached.
*/
@Override
public synchronized int read(byte[] b, int off, int len) throws IOException {
int result = 0;
result = in.read(b, off, len);
if (result > 0) {
pos += result;
}
if (null != statistics) {
statistics.incrementBytesRead(result);
}
// Return to the caller with the result.
return result;
}
@Override
public void close() throws IOException {
in.close();
closed = true;
}
@Override
public synchronized void seek(long pos) throws IOException {
in.close();
in = store.retrieve(key);
this.pos = in.skip(pos);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Seek to position %d. Bytes skipped %d", pos,
this.pos));
}
}
@Override
public synchronized long getPos() throws IOException {
return pos;
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
}
private class NativeAzureFsOutputStream extends OutputStream {
// We should not override flush() to actually close current block and flush
// to DFS, this will break applications that assume flush() is a no-op.
// Applications are advised to use Syncable.hflush() for that purpose.
// NativeAzureFsOutputStream needs to implement Syncable if needed.
private String key;
private String keyEncoded;
private OutputStream out;
public NativeAzureFsOutputStream(OutputStream out, String aKey,
String anEncodedKey) throws IOException {
// Check input arguments. The output stream should be non-null and the
// keys
// should be valid strings.
if (null == out) {
throw new IllegalArgumentException(
"Illegal argument: the output stream is null.");
}
if (null == aKey || 0 == aKey.length()) {
throw new IllegalArgumentException(
"Illegal argument the key string is null or empty");
}
if (null == anEncodedKey || 0 == anEncodedKey.length()) {
throw new IllegalArgumentException(
"Illegal argument the encoded key string is null or empty");
}
// Initialize the member variables with the incoming parameters.
this.out = out;
setKey(aKey);
setEncodedKey(anEncodedKey);
}
@Override
public synchronized void close() throws IOException {
if (out != null) {
// Close the output stream and decode the key for the output stream
// before returning to the caller.
//
out.close();
restoreKey();
out = null;
}
}
/**
* Writes the specified byte to this output stream. The general contract for
* write is that one byte is written to the output stream. The byte to be
* written is the eight low-order bits of the argument b. The 24 high-order
* bits of b are ignored.
*
* @param b
* 32-bit integer of block of 4 bytes
*/
@Override
public void write(int b) throws IOException {
out.write(b);
}
/**
* Writes b.length bytes from the specified byte array to this output
* stream. The general contract for write(b) is that it should have exactly
* the same effect as the call write(b, 0, b.length).
*
* @param b
* Block of bytes to be written to the output stream.
*/
@Override
public void write(byte[] b) throws IOException {
out.write(b);
}
/**
* Writes <code>len</code> from the specified byte array starting at offset
* <code>off</code> to the output stream. The general contract for write(b,
* off, len) is that some of the bytes in the array <code>
* b</code b> are written to the output stream in order; element
* <code>b[off]</code> is the first byte written and
* <code>b[off+len-1]</code> is the last byte written by this operation.
*
* @param b
* Byte array to be written.
* @param off
* Write this offset in stream.
* @param len
* Number of bytes to be written.
*/
@Override
public void write(byte[] b, int off, int len) throws IOException {
out.write(b, off, len);
}
/**
* Get the blob name.
*
* @return String Blob name.
*/
public String getKey() {
return key;
}
/**
* Set the blob name.
*
* @param key
* Blob name.
*/
public void setKey(String key) {
this.key = key;
}
/**
* Get the blob name.
*
* @return String Blob name.
*/
public String getEncodedKey() {
return keyEncoded;
}
/**
* Set the blob name.
*
* @param anEncodedKey
* Blob name.
*/
public void setEncodedKey(String anEncodedKey) {
this.keyEncoded = anEncodedKey;
}
/**
* Restore the original key name from the m_key member variable. Note: The
* output file stream is created with an encoded blob store key to guarantee
* load balancing on the front end of the Azure storage partition servers.
* The create also includes the name of the original key value which is
* stored in the m_key member variable. This method should only be called
* when the stream is closed.
*/
private void restoreKey() throws IOException {
store.rename(getEncodedKey(), getKey());
}
}
private URI uri;
private NativeFileSystemStore store;
private AzureNativeFileSystemStore actualStore;
private Path workingDir;
private long blockSize = MAX_AZURE_BLOCK_SIZE;
private AzureFileSystemInstrumentation instrumentation;
private String metricsSourceName;
private boolean isClosed = false;
private static boolean suppressRetryPolicy = false;
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
public NativeAzureFileSystem() {
// set store in initialize()
}
public NativeAzureFileSystem(NativeFileSystemStore store) {
this.store = store;
}
/**
* Suppress the default retry policy for the Storage, useful in unit tests to
* test negative cases without waiting forever.
*/
@VisibleForTesting
static void suppressRetryPolicy() {
suppressRetryPolicy = true;
}
/**
* Undo the effect of suppressRetryPolicy.
*/
@VisibleForTesting
static void resumeRetryPolicy() {
suppressRetryPolicy = false;
}
/**
* Creates a new metrics source name that's unique within this process.
*/
@VisibleForTesting
public static String newMetricsSourceName() {
int number = metricsSourceNameCounter.incrementAndGet();
final String baseName = "AzureFileSystemMetrics";
if (number == 1) { // No need for a suffix for the first one
return baseName;
} else {
return baseName + number;
}
}
/**
* Checks if the given URI scheme is a scheme that's affiliated with the Azure
* File System.
*
* @param scheme
* The URI scheme.
* @return true iff it's an Azure File System URI scheme.
*/
private static boolean isWasbScheme(String scheme) {
// The valid schemes are: asv (old name), asvs (old name over HTTPS),
// wasb (new name), wasbs (new name over HTTPS).
return scheme != null
&& (scheme.equalsIgnoreCase("asv") || scheme.equalsIgnoreCase("asvs")
|| scheme.equalsIgnoreCase("wasb") || scheme
.equalsIgnoreCase("wasbs"));
}
/**
* Puts in the authority of the default file system if it is a WASB file
* system and the given URI's authority is null.
*
* @return The URI with reconstructed authority if necessary and possible.
*/
private static URI reconstructAuthorityIfNeeded(URI uri, Configuration conf) {
if (null == uri.getAuthority()) {
// If WASB is the default file system, get the authority from there
URI defaultUri = FileSystem.getDefaultUri(conf);
if (defaultUri != null && isWasbScheme(defaultUri.getScheme())) {
try {
// Reconstruct the URI with the authority from the default URI.
return new URI(uri.getScheme(), defaultUri.getAuthority(),
uri.getPath(), uri.getQuery(), uri.getFragment());
} catch (URISyntaxException e) {
// This should never happen.
throw new Error("Bad URI construction", e);
}
}
}
return uri;
}
@Override
protected void checkPath(Path path) {
// Make sure to reconstruct the path's authority if needed
super.checkPath(new Path(reconstructAuthorityIfNeeded(path.toUri(),
getConf())));
}
@Override
public void initialize(URI uri, Configuration conf)
throws IOException, IllegalArgumentException {
// Check authority for the URI to guarantee that it is non-null.
uri = reconstructAuthorityIfNeeded(uri, conf);
if (null == uri.getAuthority()) {
final String errMsg = String
.format("Cannot initialize WASB file system, URI authority not recognized.");
throw new IllegalArgumentException(errMsg);
}
super.initialize(uri, conf);
if (store == null) {
store = createDefaultStore(conf);
}
instrumentation = new AzureFileSystemInstrumentation(conf);
if(!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
// Make sure the metrics system is available before interacting with Azure
AzureFileSystemMetricsSystem.fileSystemStarted();
metricsSourceName = newMetricsSourceName();
String sourceDesc = "Azure Storage Volume File System metrics";
AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
instrumentation);
}
store.initialize(uri, conf, instrumentation);
setConf(conf);
this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
this.workingDir = new Path("/user", UserGroupInformation.getCurrentUser()
.getShortUserName()).makeQualified(getUri(), getWorkingDirectory());
this.blockSize = conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME,
MAX_AZURE_BLOCK_SIZE);
if (LOG.isDebugEnabled()) {
LOG.debug("NativeAzureFileSystem. Initializing.");
LOG.debug(" blockSize = "
+ conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE));
}
}
private NativeFileSystemStore createDefaultStore(Configuration conf) {
actualStore = new AzureNativeFileSystemStore();
if (suppressRetryPolicy) {
actualStore.suppressRetryPolicy();
}
return actualStore;
}
/**
* Azure Storage doesn't allow the blob names to end in a period,
* so encode this here to work around that limitation.
*/
private static String encodeTrailingPeriod(String toEncode) {
Matcher matcher = TRAILING_PERIOD_PATTERN.matcher(toEncode);
return matcher.replaceAll(TRAILING_PERIOD_PLACEHOLDER);
}
/**
* Reverse the encoding done by encodeTrailingPeriod().
*/
private static String decodeTrailingPeriod(String toDecode) {
Matcher matcher = TRAILING_PERIOD_PLACEHOLDER_PATTERN.matcher(toDecode);
return matcher.replaceAll(".");
}
/**
* Convert the path to a key. By convention, any leading or trailing slash is
* removed, except for the special case of a single slash.
*/
@VisibleForTesting
public String pathToKey(Path path) {
// Convert the path to a URI to parse the scheme, the authority, and the
// path from the path object.
URI tmpUri = path.toUri();
String pathUri = tmpUri.getPath();
// The scheme and authority is valid. If the path does not exist add a "/"
// separator to list the root of the container.
Path newPath = path;
if ("".equals(pathUri)) {
newPath = new Path(tmpUri.toString() + Path.SEPARATOR);
}
// Verify path is absolute if the path refers to a windows drive scheme.
if (!newPath.isAbsolute()) {
throw new IllegalArgumentException("Path must be absolute: " + path);
}
String key = null;
key = newPath.toUri().getPath();
key = removeTrailingSlash(key);
key = encodeTrailingPeriod(key);
if (key.length() == 1) {
return key;
} else {
return key.substring(1); // remove initial slash
}
}
// Remove any trailing slash except for the case of a single slash.
private static String removeTrailingSlash(String key) {
if (key.length() == 0 || key.length() == 1) {
return key;
}
if (key.charAt(key.length() - 1) == '/') {
return key.substring(0, key.length() - 1);
} else {
return key;
}
}
private static Path keyToPath(String key) {
if (key.equals("/")) {
return new Path("/"); // container
}
return new Path("/" + decodeTrailingPeriod(key));
}
/**
* Get the absolute version of the path (fully qualified).
* This is public for testing purposes.
*
* @param path
* @return fully qualified path
*/
@VisibleForTesting
public Path makeAbsolute(Path path) {
if (path.isAbsolute()) {
return path;
}
return new Path(workingDir, path);
}
/**
* For unit test purposes, retrieves the AzureNativeFileSystemStore store
* backing this file system.
*
* @return The store object.
*/
@VisibleForTesting
public AzureNativeFileSystemStore getStore() {
return actualStore;
}
NativeFileSystemStore getStoreInterface() {
return store;
}
/**
* Gets the metrics source for this file system.
* This is mainly here for unit testing purposes.
*
* @return the metrics source.
*/
public AzureFileSystemInstrumentation getInstrumentation() {
return instrumentation;
}
/** This optional operation is not yet supported. */
@Override
public FSDataOutputStream append(Path f, int bufferSize, Progressable progress)
throws IOException {
throw new IOException("Not supported");
}
@Override
public FSDataOutputStream create(Path f, FsPermission permission,
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
return create(f, permission, overwrite, true,
bufferSize, replication, blockSize, progress,
(SelfRenewingLease) null);
}
/**
* Get a self-renewing lease on the specified file.
*/
public SelfRenewingLease acquireLease(Path path) throws AzureException {
String fullKey = pathToKey(makeAbsolute(path));
return getStore().acquireLease(fullKey);
}
@Override
@SuppressWarnings("deprecation")
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
Path parent = f.getParent();
// Get exclusive access to folder if this is a directory designated
// for atomic rename. The primary use case of for HBase write-ahead
// log file management.
SelfRenewingLease lease = null;
if (store.isAtomicRenameKey(pathToKey(f))) {
try {
lease = acquireLease(parent);
} catch (AzureException e) {
String errorCode = "";
try {
StorageException e2 = (StorageException) e.getCause();
errorCode = e2.getErrorCode();
} catch (Exception e3) {
// do nothing if cast fails
}
if (errorCode.equals("BlobNotFound")) {
throw new FileNotFoundException("Cannot create file " +
f.getName() + " because parent folder does not exist.");
}
LOG.warn("Got unexpected exception trying to get lease on "
+ pathToKey(parent) + ". " + e.getMessage());
throw e;
}
}
// See if the parent folder exists. If not, throw error.
// The exists() check will push any pending rename operation forward,
// if there is one, and return false.
//
// At this point, we have exclusive access to the source folder
// via the lease, so we will not conflict with an active folder
// rename operation.
if (!exists(parent)) {
try {
// This'll let the keep-alive thread exit as soon as it wakes up.
lease.free();
} catch (Exception e) {
LOG.warn("Unable to free lease because: " + e.getMessage());
}
throw new FileNotFoundException("Cannot create file " +
f.getName() + " because parent folder does not exist.");
}
// Create file inside folder.
FSDataOutputStream out = null;
try {
out = create(f, permission, overwrite, false,
bufferSize, replication, blockSize, progress, lease);
} finally {
// Release exclusive access to folder.
try {
if (lease != null) {
lease.free();
}
} catch (Exception e) {
IOUtils.cleanup(LOG, out);
String msg = "Unable to free lease on " + parent.toUri();
LOG.error(msg);
throw new IOException(msg, e);
}
}
return out;
}
@Override
@SuppressWarnings("deprecation")
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
// Check if file should be appended or overwritten. Assume that the file
// is overwritten on if the CREATE and OVERWRITE create flags are set. Note
// that any other combinations of create flags will result in an open new or
// open with append.
final EnumSet<CreateFlag> createflags =
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
boolean overwrite = flags.containsAll(createflags);
// Delegate the create non-recursive call.
return this.createNonRecursive(f, permission, overwrite,
bufferSize, replication, blockSize, progress);
}
@Override
@SuppressWarnings("deprecation")
public FSDataOutputStream createNonRecursive(Path f,
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
return this.createNonRecursive(f, FsPermission.getFileDefault(),
overwrite, bufferSize, replication, blockSize, progress);
}
/**
* Create an Azure blob and return an output stream to use
* to write data to it.
*
* @param f
* @param permission
* @param overwrite
* @param createParent
* @param bufferSize
* @param replication
* @param blockSize
* @param progress
* @param parentFolderLease Lease on parent folder (or null if
* no lease).
* @return
* @throws IOException
*/
private FSDataOutputStream create(Path f, FsPermission permission,
boolean overwrite, boolean createParent, int bufferSize,
short replication, long blockSize, Progressable progress,
SelfRenewingLease parentFolderLease)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating file: " + f.toString());
}
if (containsColon(f)) {
throw new IOException("Cannot create file " + f
+ " through WASB that has colons in the name");
}
Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
FileMetadata existingMetadata = store.retrieveMetadata(key);
if (existingMetadata != null) {
if (existingMetadata.isDir()) {
throw new IOException("Cannot create file " + f
+ "; already exists as a directory.");
}
if (!overwrite) {
throw new IOException("File already exists:" + f);
}
}
Path parentFolder = absolutePath.getParent();
if (parentFolder != null && parentFolder.getParent() != null) { // skip root
// Update the parent folder last modified time if the parent folder
// already exists.
String parentKey = pathToKey(parentFolder);
FileMetadata parentMetadata = store.retrieveMetadata(parentKey);
if (parentMetadata != null && parentMetadata.isDir() &&
parentMetadata.getBlobMaterialization() == BlobMaterialization.Explicit) {
store.updateFolderLastModifiedTime(parentKey, parentFolderLease);
} else {
// Make sure that the parent folder exists.
// Create it using inherited permissions from the first existing directory going up the path
Path firstExisting = parentFolder.getParent();
FileMetadata metadata = store.retrieveMetadata(pathToKey(firstExisting));
while(metadata == null) {
// Guaranteed to terminate properly because we will eventually hit root, which will return non-null metadata
firstExisting = firstExisting.getParent();
metadata = store.retrieveMetadata(pathToKey(firstExisting));
}
mkdirs(parentFolder, metadata.getPermissionStatus().getPermission(), true);
}
}
// Mask the permission first (with the default permission mask as well).
FsPermission masked = applyUMask(permission, UMaskApplyMode.NewFile);
PermissionStatus permissionStatus = createPermissionStatus(masked);
OutputStream bufOutStream;
if (store.isPageBlobKey(key)) {
// Store page blobs directly in-place without renames.
bufOutStream = store.storefile(key, permissionStatus);
} else {
// This is a block blob, so open the output blob stream based on the
// encoded key.
//
String keyEncoded = encodeKey(key);
// First create a blob at the real key, pointing back to the temporary file
// This accomplishes a few things:
// 1. Makes sure we can create a file there.
// 2. Makes it visible to other concurrent threads/processes/nodes what
// we're
// doing.
// 3. Makes it easier to restore/cleanup data in the event of us crashing.
store.storeEmptyLinkFile(key, keyEncoded, permissionStatus);
// The key is encoded to point to a common container at the storage server.
// This reduces the number of splits on the server side when load balancing.
// Ingress to Azure storage can take advantage of earlier splits. We remove
// the root path to the key and prefix a random GUID to the tail (or leaf
// filename) of the key. Keys are thus broadly and randomly distributed over
// a single container to ease load balancing on the storage server. When the
// blob is committed it is renamed to its earlier key. Uncommitted blocks
// are not cleaned up and we leave it to Azure storage to garbage collect
// these
// blocks.
bufOutStream = new NativeAzureFsOutputStream(store.storefile(
keyEncoded, permissionStatus), key, keyEncoded);
}
// Construct the data output stream from the buffered output stream.
FSDataOutputStream fsOut = new FSDataOutputStream(bufOutStream, statistics);
// Increment the counter
instrumentation.fileCreated();
// Return data output stream to caller.
return fsOut;
}
@Override
@Deprecated
public boolean delete(Path path) throws IOException {
return delete(path, true);
}
@Override
public boolean delete(Path f, boolean recursive) throws IOException {
return delete(f, recursive, false);
}
/**
* Delete the specified file or folder. The parameter
* skipParentFolderLastModifidedTimeUpdate
* is used in the case of atomic folder rename redo. In that case, there is
* a lease on the parent folder, so (without reworking the code) modifying
* the parent folder update time will fail because of a conflict with the
* lease. Since we are going to delete the folder soon anyway so accurate
* modified time is not necessary, it's easier to just skip
* the modified time update.
*
* @param f
* @param recursive
* @param skipParentFolderLastModifidedTimeUpdate If true, don't update the folder last
* modified time.
* @return true if and only if the file is deleted
* @throws IOException
*/
public boolean delete(Path f, boolean recursive,
boolean skipParentFolderLastModifidedTimeUpdate) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting file: " + f.toString());
}
Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
// Capture the metadata for the path.
//
FileMetadata metaFile = store.retrieveMetadata(key);
if (null == metaFile) {
// The path to be deleted does not exist.
return false;
}
// The path exists, determine if it is a folder containing objects,
// an empty folder, or a simple file and take the appropriate actions.
if (!metaFile.isDir()) {
// The path specifies a file. We need to check the parent path
// to make sure it's a proper materialized directory before we
// delete the file. Otherwise we may get into a situation where
// the file we were deleting was the last one in an implicit directory
// (e.g. the blob store only contains the blob a/b and there's no
// corresponding directory blob a) and that would implicitly delete
// the directory as well, which is not correct.
Path parentPath = absolutePath.getParent();
if (parentPath.getParent() != null) {// Not root
String parentKey = pathToKey(parentPath);
FileMetadata parentMetadata = store.retrieveMetadata(parentKey);
if (!parentMetadata.isDir()) {
// Invalid state: the parent path is actually a file. Throw.
throw new AzureException("File " + f + " has a parent directory "
+ parentPath + " which is also a file. Can't resolve.");
}
if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
if (LOG.isDebugEnabled()) {
LOG.debug("Found an implicit parent directory while trying to"
+ " delete the file " + f + ". Creating the directory blob for"
+ " it in " + parentKey + ".");
}
store.storeEmptyFolder(parentKey,
createPermissionStatus(FsPermission.getDefault()));
} else {
if (!skipParentFolderLastModifidedTimeUpdate) {
store.updateFolderLastModifiedTime(parentKey, null);
}
}
}
store.delete(key);
instrumentation.fileDeleted();
} else {
// The path specifies a folder. Recursively delete all entries under the
// folder.
Path parentPath = absolutePath.getParent();
if (parentPath.getParent() != null) {
String parentKey = pathToKey(parentPath);
FileMetadata parentMetadata = store.retrieveMetadata(parentKey);
if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
if (LOG.isDebugEnabled()) {
LOG.debug("Found an implicit parent directory while trying to"
+ " delete the directory " + f
+ ". Creating the directory blob for" + " it in " + parentKey
+ ".");
}
store.storeEmptyFolder(parentKey,
createPermissionStatus(FsPermission.getDefault()));
}
}
// List all the blobs in the current folder.
String priorLastKey = null;
PartialListing listing = store.listAll(key, AZURE_LIST_ALL, 1,
priorLastKey);
FileMetadata[] contents = listing.getFiles();
if (!recursive && contents.length > 0) {
// The folder is non-empty and recursive delete was not specified.
// Throw an exception indicating that a non-recursive delete was
// specified for a non-empty folder.
throw new IOException("Non-recursive delete of non-empty directory "
+ f.toString());
}
// Delete all the files in the folder.
for (FileMetadata p : contents) {
// Tag on the directory name found as the suffix of the suffix of the
// parent directory to get the new absolute path.
String suffix = p.getKey().substring(
p.getKey().lastIndexOf(PATH_DELIMITER));
if (!p.isDir()) {
store.delete(key + suffix);
instrumentation.fileDeleted();
} else {
// Recursively delete contents of the sub-folders. Notice this also
// deletes the blob for the directory.
if (!delete(new Path(f.toString() + suffix), true)) {
return false;
}
}
}
store.delete(key);
// Update parent directory last modified time
Path parent = absolutePath.getParent();
if (parent != null && parent.getParent() != null) { // not root
String parentKey = pathToKey(parent);
if (!skipParentFolderLastModifidedTimeUpdate) {
store.updateFolderLastModifiedTime(parentKey, null);
}
}
instrumentation.directoryDeleted();
}
// File or directory was successfully deleted.
return true;
}
@Override
public FileStatus getFileStatus(Path f) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Getting the file status for " + f.toString());
}
// Capture the absolute path and the path to key.
Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
if (key.length() == 0) { // root always exists
return newDirectory(null, absolutePath);
}
// The path is either a folder or a file. Retrieve metadata to
// determine if it is a directory or file.
FileMetadata meta = store.retrieveMetadata(key);
if (meta != null) {
if (meta.isDir()) {
// The path is a folder with files in it.
//
if (LOG.isDebugEnabled()) {
LOG.debug("Path " + f.toString() + "is a folder.");
}
// If a rename operation for the folder was pending, redo it.
// Then the file does not exist, so signal that.
if (conditionalRedoFolderRename(f)) {
throw new FileNotFoundException(
absolutePath + ": No such file or directory.");
}
// Return reference to the directory object.
return newDirectory(meta, absolutePath);
}
// The path is a file.
if (LOG.isDebugEnabled()) {
LOG.debug("Found the path: " + f.toString() + " as a file.");
}
// Return with reference to a file object.
return newFile(meta, absolutePath);
}
// File not found. Throw exception no such file or directory.
//
throw new FileNotFoundException(
absolutePath + ": No such file or directory.");
}
// Return true if there is a rename pending and we redo it, otherwise false.
private boolean conditionalRedoFolderRename(Path f) throws IOException {
// Can't rename /, so return immediately in that case.
if (f.getName().equals("")) {
return false;
}
// Check if there is a -RenamePending.json file for this folder, and if so,
// redo the rename.
Path absoluteRenamePendingFile = renamePendingFilePath(f);
if (exists(absoluteRenamePendingFile)) {
FolderRenamePending pending =
new FolderRenamePending(absoluteRenamePendingFile, this);
pending.redo();
return true;
} else {
return false;
}
}
// Return the path name that would be used for rename of folder with path f.
private Path renamePendingFilePath(Path f) {
Path absPath = makeAbsolute(f);
String key = pathToKey(absPath);
key += "-RenamePending.json";
return keyToPath(key);
}
@Override
public URI getUri() {
return uri;
}
/**
* Retrieve the status of a given path if it is a file, or of all the
* contained files if it is a directory.
*/
@Override
public FileStatus[] listStatus(Path f) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Listing status for " + f.toString());
}
Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
Set<FileStatus> status = new TreeSet<FileStatus>();
FileMetadata meta = store.retrieveMetadata(key);
if (meta != null) {
if (!meta.isDir()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Found path as a file");
}
return new FileStatus[] { newFile(meta, absolutePath) };
}
String partialKey = null;
PartialListing listing = store.list(key, AZURE_LIST_ALL, 1, partialKey);
// For any -RenamePending.json files in the listing,
// push the rename forward.
boolean renamed = conditionalRedoFolderRenames(listing);
// If any renames were redone, get another listing,
// since the current one may have changed due to the redo.
if (renamed) {
listing = store.list(key, AZURE_LIST_ALL, 1, partialKey);
}
for (FileMetadata fileMetadata : listing.getFiles()) {
Path subpath = keyToPath(fileMetadata.getKey());
// Test whether the metadata represents a file or directory and
// add the appropriate metadata object.
//
// Note: There was a very old bug here where directories were added
// to the status set as files flattening out recursive listings
// using "-lsr" down the file system hierarchy.
if (fileMetadata.isDir()) {
// Make sure we hide the temp upload folder
if (fileMetadata.getKey().equals(AZURE_TEMP_FOLDER)) {
// Don't expose that.
continue;
}
status.add(newDirectory(fileMetadata, subpath));
} else {
status.add(newFile(fileMetadata, subpath));
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Found path as a directory with " + status.size()
+ " files in it.");
}
} else {
// There is no metadata found for the path.
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find any metadata for path: " + key);
}
throw new FileNotFoundException("File" + f + " does not exist.");
}
return status.toArray(new FileStatus[0]);
}
// Redo any folder renames needed if there are rename pending files in the
// directory listing. Return true if one or more redo operations were done.
private boolean conditionalRedoFolderRenames(PartialListing listing)
throws IllegalArgumentException, IOException {
boolean renamed = false;
for (FileMetadata fileMetadata : listing.getFiles()) {
Path subpath = keyToPath(fileMetadata.getKey());
if (isRenamePendingFile(subpath)) {
FolderRenamePending pending =
new FolderRenamePending(subpath, this);
pending.redo();
renamed = true;
}
}
return renamed;
}
// True if this is a folder rename pending file, else false.
private boolean isRenamePendingFile(Path path) {
return path.toString().endsWith(FolderRenamePending.SUFFIX);
}
private FileStatus newFile(FileMetadata meta, Path path) {
return new FileStatus (
meta.getLength(),
false,
1,
blockSize,
meta.getLastModified(),
0,
meta.getPermissionStatus().getPermission(),
meta.getPermissionStatus().getUserName(),
meta.getPermissionStatus().getGroupName(),
path.makeQualified(getUri(), getWorkingDirectory()));
}
private FileStatus newDirectory(FileMetadata meta, Path path) {
return new FileStatus (
0,
true,
1,
blockSize,
meta == null ? 0 : meta.getLastModified(),
0,
meta == null ? FsPermission.getDefault() : meta.getPermissionStatus().getPermission(),
meta == null ? "" : meta.getPermissionStatus().getUserName(),
meta == null ? "" : meta.getPermissionStatus().getGroupName(),
path.makeQualified(getUri(), getWorkingDirectory()));
}
private static enum UMaskApplyMode {
NewFile,
NewDirectory,
NewDirectoryNoUmask,
ChangeExistingFile,
ChangeExistingDirectory,
}
/**
* Applies the applicable UMASK's on the given permission.
*
* @param permission
* The permission to mask.
* @param applyMode
* Whether to also apply the default umask.
* @return The masked persmission.
*/
private FsPermission applyUMask(final FsPermission permission,
final UMaskApplyMode applyMode) {
FsPermission newPermission = new FsPermission(permission);
// Apply the default umask - this applies for new files or directories.
if (applyMode == UMaskApplyMode.NewFile
|| applyMode == UMaskApplyMode.NewDirectory) {
newPermission = newPermission
.applyUMask(FsPermission.getUMask(getConf()));
}
return newPermission;
}
/**
* Creates the PermissionStatus object to use for the given permission, based
* on the current user in context.
*
* @param permission
* The permission for the file.
* @return The permission status object to use.
* @throws IOException
* If login fails in getCurrentUser
*/
private PermissionStatus createPermissionStatus(FsPermission permission)
throws IOException {
// Create the permission status for this file based on current user
return new PermissionStatus(
UserGroupInformation.getCurrentUser().getShortUserName(),
getConf().get(AZURE_DEFAULT_GROUP_PROPERTY_NAME,
AZURE_DEFAULT_GROUP_DEFAULT),
permission);
}
@Override
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
return mkdirs(f, permission, false);
}
public boolean mkdirs(Path f, FsPermission permission, boolean noUmask) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating directory: " + f.toString());
}
if (containsColon(f)) {
throw new IOException("Cannot create directory " + f
+ " through WASB that has colons in the name");
}
Path absolutePath = makeAbsolute(f);
PermissionStatus permissionStatus = null;
if(noUmask) {
// ensure owner still has wx permissions at the minimum
permissionStatus = createPermissionStatus(
applyUMask(FsPermission.createImmutable((short) (permission.toShort() | USER_WX_PERMISION)),
UMaskApplyMode.NewDirectoryNoUmask));
} else {
permissionStatus = createPermissionStatus(
applyUMask(permission, UMaskApplyMode.NewDirectory));
}
ArrayList<String> keysToCreateAsFolder = new ArrayList<String>();
ArrayList<String> keysToUpdateAsFolder = new ArrayList<String>();
boolean childCreated = false;
// Check that there is no file in the parent chain of the given path.
for (Path current = absolutePath, parent = current.getParent();
parent != null; // Stop when you get to the root
current = parent, parent = current.getParent()) {
String currentKey = pathToKey(current);
FileMetadata currentMetadata = store.retrieveMetadata(currentKey);
if (currentMetadata != null && !currentMetadata.isDir()) {
throw new IOException("Cannot create directory " + f + " because " +
current + " is an existing file.");
} else if (currentMetadata == null) {
keysToCreateAsFolder.add(currentKey);
childCreated = true;
} else {
// The directory already exists. Its last modified time need to be
// updated if there is a child directory created under it.
if (childCreated) {
keysToUpdateAsFolder.add(currentKey);
}
childCreated = false;
}
}
for (String currentKey : keysToCreateAsFolder) {
store.storeEmptyFolder(currentKey, permissionStatus);
}
instrumentation.directoryCreated();
// otherwise throws exception
return true;
}
@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Opening file: " + f.toString());
}
Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
FileMetadata meta = store.retrieveMetadata(key);
if (meta == null) {
throw new FileNotFoundException(f.toString());
}
if (meta.isDir()) {
throw new FileNotFoundException(f.toString()
+ " is a directory not a file.");
}
return new FSDataInputStream(new BufferedFSInputStream(
new NativeAzureFsInputStream(store.retrieve(key), key, meta.getLength()), bufferSize));
}
@Override
public boolean rename(Path src, Path dst) throws IOException {
FolderRenamePending renamePending = null;
if (LOG.isDebugEnabled()) {
LOG.debug("Moving " + src + " to " + dst);
}
if (containsColon(dst)) {
throw new IOException("Cannot rename to file " + dst
+ " through WASB that has colons in the name");
}
String srcKey = pathToKey(makeAbsolute(src));
if (srcKey.length() == 0) {
// Cannot rename root of file system
return false;
}
// Figure out the final destination
Path absoluteDst = makeAbsolute(dst);
String dstKey = pathToKey(absoluteDst);
FileMetadata dstMetadata = store.retrieveMetadata(dstKey);
if (dstMetadata != null && dstMetadata.isDir()) {
// It's an existing directory.
dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
if (LOG.isDebugEnabled()) {
LOG.debug("Destination " + dst
+ " is a directory, adjusted the destination to be " + dstKey);
}
} else if (dstMetadata != null) {
// Attempting to overwrite a file using rename()
if (LOG.isDebugEnabled()) {
LOG.debug("Destination " + dst
+ " is an already existing file, failing the rename.");
}
return false;
} else {
// Check that the parent directory exists.
FileMetadata parentOfDestMetadata =
store.retrieveMetadata(pathToKey(absoluteDst.getParent()));
if (parentOfDestMetadata == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Parent of the destination " + dst
+ " doesn't exist, failing the rename.");
}
return false;
} else if (!parentOfDestMetadata.isDir()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Parent of the destination " + dst
+ " is a file, failing the rename.");
}
return false;
}
}
FileMetadata srcMetadata = store.retrieveMetadata(srcKey);
if (srcMetadata == null) {
// Source doesn't exist
if (LOG.isDebugEnabled()) {
LOG.debug("Source " + src + " doesn't exist, failing the rename.");
}
return false;
} else if (!srcMetadata.isDir()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Source " + src + " found as a file, renaming.");
}
store.rename(srcKey, dstKey);
} else {
// Prepare for, execute and clean up after of all files in folder, and
// the root file, and update the last modified time of the source and
// target parent folders. The operation can be redone if it fails part
// way through, by applying the "Rename Pending" file.
// The following code (internally) only does atomic rename preparation
// and lease management for page blob folders, limiting the scope of the
// operation to HBase log file folders, where atomic rename is required.
// In the future, we could generalize it easily to all folders.
renamePending = prepareAtomicFolderRename(srcKey, dstKey);
renamePending.execute();
if (LOG.isDebugEnabled()) {
LOG.debug("Renamed " + src + " to " + dst + " successfully.");
}
renamePending.cleanup();
return true;
}
// Update the last-modified time of the parent folders of both source
// and destination.
updateParentFolderLastModifiedTime(srcKey);
updateParentFolderLastModifiedTime(dstKey);
if (LOG.isDebugEnabled()) {
LOG.debug("Renamed " + src + " to " + dst + " successfully.");
}
return true;
}
/**
* Update the last-modified time of the parent folder of the file
* identified by key.
* @param key
* @throws IOException
*/
private void updateParentFolderLastModifiedTime(String key)
throws IOException {
Path parent = makeAbsolute(keyToPath(key)).getParent();
if (parent != null && parent.getParent() != null) { // not root
String parentKey = pathToKey(parent);
// ensure the parent is a materialized folder
FileMetadata parentMetadata = store.retrieveMetadata(parentKey);
// The metadata could be null if the implicit folder only contains a
// single file. In this case, the parent folder no longer exists if the
// file is renamed; so we can safely ignore the null pointer case.
if (parentMetadata != null) {
if (parentMetadata.isDir()
&& parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
store.storeEmptyFolder(parentKey,
createPermissionStatus(FsPermission.getDefault()));
}
if (store.isAtomicRenameKey(parentKey)) {
SelfRenewingLease lease = null;
try {
lease = leaseSourceFolder(parentKey);
store.updateFolderLastModifiedTime(parentKey, lease);
} catch (AzureException e) {
String errorCode = "";
try {
StorageException e2 = (StorageException) e.getCause();
errorCode = e2.getErrorCode();
} catch (Exception e3) {
// do nothing if cast fails
}
if (errorCode.equals("BlobNotFound")) {
throw new FileNotFoundException("Folder does not exist: " + parentKey);
}
LOG.warn("Got unexpected exception trying to get lease on "
+ parentKey + ". " + e.getMessage());
throw e;
} finally {
try {
if (lease != null) {
lease.free();
}
} catch (Exception e) {
LOG.error("Unable to free lease on " + parentKey, e);
}
}
} else {
store.updateFolderLastModifiedTime(parentKey, null);
}
}
}
}
/**
* If the source is a page blob folder,
* prepare to rename this folder atomically. This means to get exclusive
* access to the source folder, and record the actions to be performed for
* this rename in a "Rename Pending" file. This code was designed to
* meet the needs of HBase, which requires atomic rename of write-ahead log
* (WAL) folders for correctness.
*
* Before calling this method, the caller must ensure that the source is a
* folder.
*
* For non-page-blob directories, prepare the in-memory information needed,
* but don't take the lease or write the redo file. This is done to limit the
* scope of atomic folder rename to HBase, at least at the time of writing
* this code.
*
* @param srcKey Source folder name.
* @param dstKey Destination folder name.
* @throws IOException
*/
private FolderRenamePending prepareAtomicFolderRename(
String srcKey, String dstKey) throws IOException {
if (store.isAtomicRenameKey(srcKey)) {
// Block unwanted concurrent access to source folder.
SelfRenewingLease lease = leaseSourceFolder(srcKey);
// Prepare in-memory information needed to do or redo a folder rename.
FolderRenamePending renamePending =
new FolderRenamePending(srcKey, dstKey, lease, this);
// Save it to persistent storage to help recover if the operation fails.
renamePending.writeFile(this);
return renamePending;
} else {
FolderRenamePending renamePending =
new FolderRenamePending(srcKey, dstKey, null, this);
return renamePending;
}
}
/**
* Get a self-renewing Azure blob lease on the source folder zero-byte file.
*/
private SelfRenewingLease leaseSourceFolder(String srcKey)
throws AzureException {
return store.acquireLease(srcKey);
}
/**
* Return an array containing hostnames, offset and size of
* portions of the given file. For WASB we'll just lie and give
* fake hosts to make sure we get many splits in MR jobs.
*/
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file,
long start, long len) throws IOException {
if (file == null) {
return null;
}
if ((start < 0) || (len < 0)) {
throw new IllegalArgumentException("Invalid start or len parameter");
}
if (file.getLen() < start) {
return new BlockLocation[0];
}
final String blobLocationHost = getConf().get(
AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME,
AZURE_BLOCK_LOCATION_HOST_DEFAULT);
final String[] name = { blobLocationHost };
final String[] host = { blobLocationHost };
long blockSize = file.getBlockSize();
if (blockSize <= 0) {
throw new IllegalArgumentException(
"The block size for the given file is not a positive number: "
+ blockSize);
}
int numberOfLocations = (int) (len / blockSize)
+ ((len % blockSize == 0) ? 0 : 1);
BlockLocation[] locations = new BlockLocation[numberOfLocations];
for (int i = 0; i < locations.length; i++) {
long currentOffset = start + (i * blockSize);
long currentLength = Math.min(blockSize, start + len - currentOffset);
locations[i] = new BlockLocation(name, host, currentOffset, currentLength);
}
return locations;
}
/**
* Set the working directory to the given directory.
*/
@Override
public void setWorkingDirectory(Path newDir) {
workingDir = makeAbsolute(newDir);
}
@Override
public Path getWorkingDirectory() {
return workingDir;
}
@Override
public void setPermission(Path p, FsPermission permission) throws IOException {
Path absolutePath = makeAbsolute(p);
String key = pathToKey(absolutePath);
FileMetadata metadata = store.retrieveMetadata(key);
if (metadata == null) {
throw new FileNotFoundException("File doesn't exist: " + p);
}
permission = applyUMask(permission,
metadata.isDir() ? UMaskApplyMode.ChangeExistingDirectory
: UMaskApplyMode.ChangeExistingFile);
if (metadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
// It's an implicit folder, need to materialize it.
store.storeEmptyFolder(key, createPermissionStatus(permission));
} else if (!metadata.getPermissionStatus().getPermission().
equals(permission)) {
store.changePermissionStatus(key, new PermissionStatus(
metadata.getPermissionStatus().getUserName(),
metadata.getPermissionStatus().getGroupName(),
permission));
}
}
@Override
public void setOwner(Path p, String username, String groupname)
throws IOException {
Path absolutePath = makeAbsolute(p);
String key = pathToKey(absolutePath);
FileMetadata metadata = store.retrieveMetadata(key);
if (metadata == null) {
throw new FileNotFoundException("File doesn't exist: " + p);
}
PermissionStatus newPermissionStatus = new PermissionStatus(
username == null ?
metadata.getPermissionStatus().getUserName() : username,
groupname == null ?
metadata.getPermissionStatus().getGroupName() : groupname,
metadata.getPermissionStatus().getPermission());
if (metadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
// It's an implicit folder, need to materialize it.
store.storeEmptyFolder(key, newPermissionStatus);
} else {
store.changePermissionStatus(key, newPermissionStatus);
}
}
@Override
public synchronized void close() throws IOException {
if (isClosed) {
return;
}
// Call the base close() to close any resources there.
super.close();
// Close the store to close any resources there - e.g. the bandwidth
// updater thread would be stopped at this time.
store.close();
// Notify the metrics system that this file system is closed, which may
// trigger one final metrics push to get the accurate final file system
// metrics out.
long startTime = System.currentTimeMillis();
if(!getConf().getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName);
AzureFileSystemMetricsSystem.fileSystemClosed();
}
if (LOG.isDebugEnabled()) {
LOG.debug("Submitting metrics when file system closed took "
+ (System.currentTimeMillis() - startTime) + " ms.");
}
isClosed = true;
}
/**
* A handler that defines what to do with blobs whose upload was
* interrupted.
*/
private abstract class DanglingFileHandler {
abstract void handleFile(FileMetadata file, FileMetadata tempFile)
throws IOException;
}
/**
* Handler implementation for just deleting dangling files and cleaning
* them up.
*/
private class DanglingFileDeleter extends DanglingFileHandler {
@Override
void handleFile(FileMetadata file, FileMetadata tempFile)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting dangling file " + file.getKey());
}
store.delete(file.getKey());
store.delete(tempFile.getKey());
}
}
/**
* Handler implementation for just moving dangling files to recovery
* location (/lost+found).
*/
private class DanglingFileRecoverer extends DanglingFileHandler {
private final Path destination;
DanglingFileRecoverer(Path destination) {
this.destination = destination;
}
@Override
void handleFile(FileMetadata file, FileMetadata tempFile)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Recovering " + file.getKey());
}
// Move to the final destination
String finalDestinationKey =
pathToKey(new Path(destination, file.getKey()));
store.rename(tempFile.getKey(), finalDestinationKey);
if (!finalDestinationKey.equals(file.getKey())) {
// Delete the empty link file now that we've restored it.
store.delete(file.getKey());
}
}
}
/**
* Check if a path has colons in its name
*/
private boolean containsColon(Path p) {
return p.toUri().getPath().toString().contains(":");
}
/**
* Implements recover and delete (-move and -delete) behaviors for handling
* dangling files (blobs whose upload was interrupted).
*
* @param root
* The root path to check from.
* @param handler
* The handler that deals with dangling files.
*/
private void handleFilesWithDanglingTempData(Path root,
DanglingFileHandler handler) throws IOException {
// Calculate the cut-off for when to consider a blob to be dangling.
long cutoffForDangling = new Date().getTime()
- getConf().getInt(AZURE_TEMP_EXPIRY_PROPERTY_NAME,
AZURE_TEMP_EXPIRY_DEFAULT) * 1000;
// Go over all the blobs under the given root and look for blobs to
// recover.
String priorLastKey = null;
do {
PartialListing listing = store.listAll(pathToKey(root), AZURE_LIST_ALL,
AZURE_UNBOUNDED_DEPTH, priorLastKey);
for (FileMetadata file : listing.getFiles()) {
if (!file.isDir()) { // We don't recover directory blobs
// See if this blob has a link in it (meaning it's a place-holder
// blob for when the upload to the temp blob is complete).
String link = store.getLinkInFileMetadata(file.getKey());
if (link != null) {
// It has a link, see if the temp blob it is pointing to is
// existent and old enough to be considered dangling.
FileMetadata linkMetadata = store.retrieveMetadata(link);
if (linkMetadata != null
&& linkMetadata.getLastModified() >= cutoffForDangling) {
// Found one!
handler.handleFile(file, linkMetadata);
}
}
}
}
priorLastKey = listing.getPriorLastKey();
} while (priorLastKey != null);
}
/**
* Looks under the given root path for any blob that are left "dangling",
* meaning that they are place-holder blobs that we created while we upload
* the data to a temporary blob, but for some reason we crashed in the middle
* of the upload and left them there. If any are found, we move them to the
* destination given.
*
* @param root
* The root path to consider.
* @param destination
* The destination path to move any recovered files to.
* @throws IOException
*/
public void recoverFilesWithDanglingTempData(Path root, Path destination)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Recovering files with dangling temp data in " + root);
}
handleFilesWithDanglingTempData(root,
new DanglingFileRecoverer(destination));
}
/**
* Looks under the given root path for any blob that are left "dangling",
* meaning that they are place-holder blobs that we created while we upload
* the data to a temporary blob, but for some reason we crashed in the middle
* of the upload and left them there. If any are found, we delete them.
*
* @param root
* The root path to consider.
* @throws IOException
*/
public void deleteFilesWithDanglingTempData(Path root) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting files with dangling temp data in " + root);
}
handleFilesWithDanglingTempData(root, new DanglingFileDeleter());
}
@Override
protected void finalize() throws Throwable {
LOG.debug("finalize() called.");
close();
super.finalize();
}
/**
* Encode the key with a random prefix for load balancing in Azure storage.
* Upload data to a random temporary file then do storage side renaming to
* recover the original key.
*
* @param aKey
* @return Encoded version of the original key.
*/
private static String encodeKey(String aKey) {
// Get the tail end of the key name.
//
String fileName = aKey.substring(aKey.lastIndexOf(Path.SEPARATOR) + 1,
aKey.length());
// Construct the randomized prefix of the file name. The prefix ensures the
// file always drops into the same folder but with a varying tail key name.
String filePrefix = AZURE_TEMP_FOLDER + Path.SEPARATOR
+ UUID.randomUUID().toString();
// Concatenate the randomized prefix with the tail of the key name.
String randomizedKey = filePrefix + fileName;
// Return to the caller with the randomized key.
return randomizedKey;
}
}
