org.apache.hadoop.fs.swift.util.SwiftObjectPath Java Examples
The following examples show how to use
org.apache.hadoop.fs.swift.util.SwiftObjectPath.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestSwiftObjectPath.java From hadoop with Apache License 2.0 | 6 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testChildOfProbe() throws Throwable { SwiftObjectPath parent = new SwiftObjectPath("container", "/parent"); SwiftObjectPath parent2 = new SwiftObjectPath("container", "/parent2"); SwiftObjectPath child = new SwiftObjectPath("container", "/parent/child"); SwiftObjectPath sibling = new SwiftObjectPath("container", "/parent/sibling"); SwiftObjectPath grandchild = new SwiftObjectPath("container", "/parent/child/grandchild"); assertParentOf(parent, child); assertParentOf(parent, grandchild); assertParentOf(child, grandchild); assertParentOf(parent, parent); assertNotParentOf(child, parent); assertParentOf(child, child); assertNotParentOf(parent, parent2); assertNotParentOf(grandchild, parent); }
Example #2
Source File: SwiftNativeFileSystemStore.java From big-c with Apache License 2.0 | 6 votes |
/** * Upload part of a larger file. * * @param path destination path * @param partNumber item number in the path * @param inputStream input data * @param length length of the data * @throws IOException on a problem */ public void uploadFilePart(Path path, int partNumber, InputStream inputStream, long length) throws IOException { String stringPath = path.toUri().toString(); String partitionFilename = SwiftUtils.partitionFilenameFromNumber( partNumber); if (stringPath.endsWith("/")) { stringPath = stringPath.concat(partitionFilename); } else { stringPath = stringPath.concat("/").concat(partitionFilename); } swiftRestClient.upload( new SwiftObjectPath(toDirPath(path).getContainer(), stringPath), inputStream, length); }
Example #3
Source File: SwiftNativeFileSystemStore.java From sahara-extra with Apache License 2.0 | 6 votes |
/** * deletes object from Swift * * @param status FileStatus to delete * @return true if the path was deleted by this specific operation. * @throws IOException on a failure */ public boolean deleteObject(FileStatus status) throws IOException { SwiftObjectPath swiftObjectPath; if (status.isDir()) { swiftObjectPath = toDirPath(status.getPath()); } else { swiftObjectPath = toObjectPath(status.getPath()); } if (!SwiftUtils.isRootDir(swiftObjectPath)) { return swiftRestClient.delete(swiftObjectPath); } else { if (LOG.isDebugEnabled()) { LOG.debug("Not deleting root directory entry"); } return true; } }
Example #4
Source File: TestSwiftObjectPath.java From sahara-extra with Apache License 2.0 | 6 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testChildOfProbe() throws Throwable { SwiftObjectPath parent = new SwiftObjectPath("container", "/parent"); SwiftObjectPath parent2 = new SwiftObjectPath("container", "/parent2"); SwiftObjectPath child = new SwiftObjectPath("container", "/parent/child"); SwiftObjectPath sibling = new SwiftObjectPath("container", "/parent/sibling"); SwiftObjectPath grandchild = new SwiftObjectPath("container", "/parent/child/grandchild"); assertParentOf(parent, child); assertParentOf(parent, grandchild); assertParentOf(child, grandchild); assertParentOf(parent, parent); assertNotParentOf(child, parent); assertParentOf(child, child); assertNotParentOf(parent, parent2); assertNotParentOf(grandchild, parent); }
Example #5
Source File: SwiftNativeFileSystemStore.java From hadoop with Apache License 2.0 | 6 votes |
/** * Upload part of a larger file. * * @param path destination path * @param partNumber item number in the path * @param inputStream input data * @param length length of the data * @throws IOException on a problem */ public void uploadFilePart(Path path, int partNumber, InputStream inputStream, long length) throws IOException { String stringPath = path.toUri().toString(); String partitionFilename = SwiftUtils.partitionFilenameFromNumber( partNumber); if (stringPath.endsWith("/")) { stringPath = stringPath.concat(partitionFilename); } else { stringPath = stringPath.concat("/").concat(partitionFilename); } swiftRestClient.upload( new SwiftObjectPath(toDirPath(path).getContainer(), stringPath), inputStream, length); }
Example #6
Source File: TestSwiftFileSystemDirectories.java From sahara-extra with Apache License 2.0 | 6 votes |
private String[] getRawObjectNames() throws Exception { SwiftRestClient client; client = SwiftRestClient.getInstance(fs.getUri(), fs.getConf()); SwiftObjectPath path = SwiftObjectPath.fromPath(fs.getUri(), new Path("/")); byte[] bytes = client.listDeepObjectsInDirectory(path, true, true); final CollectionType collectionType = JSONUtil.getJsonMapper(). getTypeFactory().constructCollectionType(List.class, SwiftObjectFileStatus.class); final List<SwiftObjectFileStatus> fileStatusList = JSONUtil.toObject(new String(bytes), collectionType); final ArrayList<String> objects = new ArrayList(); for (SwiftObjectFileStatus status : fileStatusList) { if (status.getName() != null) { objects.add(status.getName()); } else if (status.getSubdir() != null) { objects.add(status.getSubdir()); } } return objects.toArray(new String[objects.size()]); }
Example #7
Source File: SwiftNativeFileSystemStore.java From sahara-extra with Apache License 2.0 | 6 votes |
/** * Upload part of a larger file. * * @param path destination path * @param partNumber item number in the path * @param inputStream input data * @param length length of the data * @throws IOException on a problem */ public void uploadFilePart(Path path, int partNumber, InputStream inputStream, long length) throws IOException { String stringPath = path.toUri().getPath(); String partitionFilename = SwiftUtils.partitionFilenameFromNumber( partNumber); if (stringPath.endsWith("/")) { stringPath = stringPath.concat(partitionFilename); } else { stringPath = stringPath.concat("/").concat(partitionFilename); } swiftRestClient.upload( new SwiftObjectPath(toDirPath(path).getContainer(), stringPath), inputStream, length); }
Example #8
Source File: SwiftNativeFileSystemStore.java From big-c with Apache License 2.0 | 5 votes |
/** * Copy an object then, if the copy worked, delete it. * If the copy failed, the source object is not deleted. * * @param srcObject source object path * @param destObject destination object path * @throws IOException IO problems */ private void copyThenDeleteObject(SwiftObjectPath srcObject, SwiftObjectPath destObject) throws IOException { //do the copy copyObject(srcObject, destObject); //getting here means the copy worked swiftRestClient.delete(srcObject); }
Example #9
Source File: SwiftRestClient.java From sahara-extra with Apache License 2.0 | 5 votes |
/** * Find objects in a directory * * @param path path prefix * @param addTrailingSlash should a trailing slash be added if there isn't one * @param requestHeaders optional request headers * @return byte[] file data or null if the object was not found * @throws IOException on IO Faults * @throws FileNotFoundException if nothing is at the end of the URI -that is, * the directory is empty */ public byte[] listDeepObjectsInDirectory(SwiftObjectPath path, boolean listDeep, boolean addTrailingSlash, final Header... requestHeaders) throws IOException { preRemoteCommand("listDeepObjectsInDirectory"); String endpoint = getEndpointURI().toString(); StringBuilder dataLocationURI = new StringBuilder(); dataLocationURI.append(endpoint); String object = path.getObject(); if (object.startsWith("/")) { object = object.substring(1); } if (addTrailingSlash && !object.endsWith("/")) { object = object.concat("/"); } if (object.equals("/")) { object = ""; } dataLocationURI = dataLocationURI.append("/") .append(path.getContainer()) .append("/?prefix=") .append(object) .append("&format=json"); //in listing deep set param to false if (listDeep == false) { dataLocationURI.append("&delimiter=/"); } return findObjects(dataLocationURI.toString(), requestHeaders); }
Example #10
Source File: TestSwiftObjectPath.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testParseAuthenticatedUrl() throws Exception { final String pathString = "swift://container.service1/v2/AUTH_00345h34l93459y4/home/tom/documents/finance.docx"; final URI uri = new URI(pathString); final Path path = new Path(pathString); final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path); final SwiftObjectPath actual = new SwiftObjectPath( RestClientBindings.extractContainerName(uri), "/home/tom/documents/finance.docx"); assertEquals(expected, actual); }
Example #11
Source File: SwiftRestClient.java From sahara-extra with Apache License 2.0 | 5 votes |
/** * Create a container -if it already exists, do nothing * * @param containerName the container name * @throws IOException IO problems * @throws SwiftBadRequestException invalid container name * @throws SwiftInvalidResponseException error from the server */ public void createContainer(String containerName) throws IOException { SwiftObjectPath objectPath = new SwiftObjectPath(containerName, ""); try { //see if the data is there headRequest("createContainer", objectPath, NEWEST); } catch (FileNotFoundException ex) { int status = 0; try { status = putRequest(objectPath); } catch (FileNotFoundException e) { //triggered by a very bad container name. //re-insert the 404 result into the status status = SC_NOT_FOUND; } if (status == SC_BAD_REQUEST) { throw new SwiftBadRequestException( "Bad request -authentication failure or bad container name?", status, "PUT", null); } if (!isStatusCodeExpected(status, SC_OK, SC_CREATED, SC_ACCEPTED, SC_NO_CONTENT)) { throw new SwiftInvalidResponseException("Couldn't create container " + containerName + " for storing data in Swift." + " Try to create container " + containerName + " manually ", status, "PUT", null); } else { throw ex; } } }
Example #12
Source File: SwiftRestClient.java From sahara-extra with Apache License 2.0 | 5 votes |
/** * Converts Swift path to URI to make request. * This is public for unit testing * * @param path path to object * @param endpointURI damain url e.g. http://domain.com * @return valid URI for object */ public static URI pathToURI(SwiftObjectPath path, URI endpointURI) throws SwiftException { checkNotNull(endpointURI, "Null Endpoint -client is not authenticated"); String dataLocationURI = endpointURI.toString(); try { dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath())); return new URI(dataLocationURI); } catch (URISyntaxException e) { throw new SwiftException("Failed to create URI from " + dataLocationURI, e); } }
Example #13
Source File: TestSwiftObjectPath.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testChildOfRoot() throws Throwable { SwiftObjectPath root = new SwiftObjectPath("container", "/"); SwiftObjectPath child = new SwiftObjectPath("container", "child"); SwiftObjectPath grandchild = new SwiftObjectPath("container", "/child/grandchild"); assertParentOf(root, child); assertParentOf(root, grandchild); assertParentOf(child, grandchild); assertParentOf(root, root); assertNotParentOf(child, root); assertParentOf(child, child); assertNotParentOf(grandchild, root); }
Example #14
Source File: TestSwiftObjectPath.java From sahara-extra with Apache License 2.0 | 5 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testConvertToPath() throws Throwable { String initialpath = "/dir/file1"; Path ipath = new Path(initialpath); SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath), ipath); URI endpoint = new URI(ENDPOINT); URI uri = SwiftRestClient.pathToURI(objectPath, endpoint); LOG.info("Inital Hadoop Path =" + initialpath); LOG.info("Merged URI=" + uri); }
Example #15
Source File: TestSwiftObjectPath.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testConvertToPath() throws Throwable { String initialpath = "/dir/file1"; Path ipath = new Path(initialpath); SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath), ipath); URI endpoint = new URI(ENDPOINT); URI uri = SwiftRestClient.pathToURI(objectPath, endpoint); LOG.info("Inital Hadoop Path =" + initialpath); LOG.info("Merged URI=" + uri); }
Example #16
Source File: TestSwiftObjectPath.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testParsePath() throws Exception { final String pathString = "/home/user/files/file1"; final Path path = new Path(pathString); final URI uri = new URI("http://container.localhost"); final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path); final SwiftObjectPath actual = new SwiftObjectPath( RestClientBindings.extractContainerName(uri), pathString); assertEquals(expected, actual); }
Example #17
Source File: SwiftNativeFileSystemStore.java From big-c with Apache License 2.0 | 5 votes |
/** * Builds a hadoop-Path from a swift path, inserting the URI authority * of this FS instance * @param path swift object path * @return Hadoop path * @throws SwiftException if the URI couldn't be created. */ private Path getCorrectSwiftPath(SwiftObjectPath path) throws SwiftException { try { final URI fullUri = new URI(uri.getScheme(), uri.getAuthority(), path.getObject(), null, null); return new Path(fullUri); } catch (URISyntaxException e) { throw new SwiftException("Specified path " + path + " is incorrect", e); } }
Example #18
Source File: TestSwiftRestClient.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testPutAndDelete() throws Throwable { assumeEnabled(); SwiftRestClient client = createClient(); client.authenticate(); Path path = new Path("restTestPutAndDelete"); SwiftObjectPath sobject = SwiftObjectPath.fromPath(serviceURI, path); byte[] stuff = new byte[1]; stuff[0] = 'a'; client.upload(sobject, new ByteArrayInputStream(stuff), stuff.length); //check file exists Duration head = new Duration(); Header[] responseHeaders = client.headRequest("expect success", sobject, SwiftRestClient.NEWEST); head.finished(); LOG.info("head request duration " + head); for (Header header: responseHeaders) { LOG.info(header.toString()); } //delete the file client.delete(sobject); //check file is gone try { Header[] headers = client.headRequest("expect fail", sobject, SwiftRestClient.NEWEST); Assert.fail("Expected deleted file, but object is still present: " + sobject); } catch (FileNotFoundException e) { //expected } for (DurationStats stats: client.getOperationStatistics()) { LOG.info(stats); } }
Example #19
Source File: SwiftNativeFileSystemStore.java From big-c with Apache License 2.0 | 5 votes |
/** * Debug action to dump directory statuses to the debug log * * @param message explanation * @param objectPath object path (can be null) * @param statuses listing output */ private void logDirectory(String message, SwiftObjectPath objectPath, Iterable<FileStatus> statuses) { if (LOG.isDebugEnabled()) { LOG.debug(message + ": listing of " + objectPath); for (FileStatus fileStatus : statuses) { LOG.debug(fileStatus.getPath()); } } }
Example #20
Source File: SwiftNativeFileSystemStore.java From big-c with Apache License 2.0 | 5 votes |
/** * Does the object exist * * @param path swift object path * @return true if the metadata of an object could be retrieved * @throws IOException IO problems other than FileNotFound, which * is downgraded to an object does not exist return code */ public boolean objectExists(SwiftObjectPath path) throws IOException { try { Header[] headers = swiftRestClient.headRequest("objectExists", path, SwiftRestClient.NEWEST); //no headers is treated as a missing file return headers.length != 0; } catch (FileNotFoundException e) { return false; } }
Example #21
Source File: SwiftNativeFileSystemStore.java From big-c with Apache License 2.0 | 5 votes |
/** * deletes object from Swift * * @param path path to delete * @return true if the path was deleted by this specific operation. * @throws IOException on a failure */ public boolean deleteObject(Path path) throws IOException { SwiftObjectPath swiftObjectPath = toObjectPath(path); if (!SwiftUtils.isRootDir(swiftObjectPath)) { return swiftRestClient.delete(swiftObjectPath); } else { if (LOG.isDebugEnabled()) { LOG.debug("Not deleting root directory entry"); } return true; } }
Example #22
Source File: SwiftNativeFileSystemStore.java From big-c with Apache License 2.0 | 5 votes |
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws IOException { Header[] headers; if (newest) { headers = swiftRestClient.headRequest("getObjectMetadata-newest", objectPath, SwiftRestClient.NEWEST); } else { headers = swiftRestClient.headRequest("getObjectMetadata", objectPath); } return headers; }
Example #23
Source File: SwiftFileStatus.java From sahara-extra with Apache License 2.0 | 5 votes |
public SwiftFileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time, Path path, SwiftObjectPath dloPrefix) { super(length, isdir, block_replication, blocksize, modification_time, path); this.dloPrefix = dloPrefix; }
Example #24
Source File: TestSwiftObjectPath.java From sahara-extra with Apache License 2.0 | 5 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testParseUrlPath() throws Exception { final String pathString = "swift://container.service1/home/user/files/file1"; final URI uri = new URI(pathString); final Path path = new Path(pathString); final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path); final SwiftObjectPath actual = new SwiftObjectPath( RestClientBindings.extractContainerName(uri), "/home/user/files/file1"); assertEquals(expected, actual); }
Example #25
Source File: SwiftRestClient.java From big-c with Apache License 2.0 | 5 votes |
/** * Converts Swift path to URI to make request. * This is public for unit testing * * @param path path to object * @param endpointURI damain url e.g. http://domain.com * @return valid URI for object * @throws SwiftException */ public static URI pathToURI(SwiftObjectPath path, URI endpointURI) throws SwiftException { checkNotNull(endpointURI, "Null Endpoint -client is not authenticated"); String dataLocationURI = endpointURI.toString(); try { dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath())); return new URI(dataLocationURI); } catch (URISyntaxException e) { throw new SwiftException("Failed to create URI from " + dataLocationURI, e); } }
Example #26
Source File: SwiftRestClient.java From big-c with Apache License 2.0 | 5 votes |
/** * Create a container -if it already exists, do nothing * * @param containerName the container name * @throws IOException IO problems * @throws SwiftBadRequestException invalid container name * @throws SwiftInvalidResponseException error from the server */ public void createContainer(String containerName) throws IOException { SwiftObjectPath objectPath = new SwiftObjectPath(containerName, ""); try { //see if the data is there headRequest("createContainer", objectPath, NEWEST); } catch (FileNotFoundException ex) { int status = 0; try { status = putRequest(objectPath); } catch (FileNotFoundException e) { //triggered by a very bad container name. //re-insert the 404 result into the status status = SC_NOT_FOUND; } if (status == SC_BAD_REQUEST) { throw new SwiftBadRequestException( "Bad request -authentication failure or bad container name?", status, "PUT", null); } if (!isStatusCodeExpected(status, SC_OK, SC_CREATED, SC_ACCEPTED, SC_NO_CONTENT)) { throw new SwiftInvalidResponseException("Couldn't create container " + containerName + " for storing data in Swift." + " Try to create container " + containerName + " manually ", status, "PUT", null); } else { throw ex; } } }
Example #27
Source File: SwiftRestClient.java From big-c with Apache License 2.0 | 5 votes |
/** * Find objects in a directory * * @param path path prefix * @param requestHeaders optional request headers * @return byte[] file data or null if the object was not found * @throws IOException on IO Faults * @throws FileNotFoundException if nothing is at the end of the URI -that is, * the directory is empty */ public byte[] listDeepObjectsInDirectory(SwiftObjectPath path, boolean listDeep, final Header... requestHeaders) throws IOException { preRemoteCommand("listDeepObjectsInDirectory"); String endpoint = getEndpointURI().toString(); StringBuilder dataLocationURI = new StringBuilder(); dataLocationURI.append(endpoint); String object = path.getObject(); if (object.startsWith("/")) { object = object.substring(1); } if (!object.endsWith("/")) { object = object.concat("/"); } if (object.equals("/")) { object = ""; } dataLocationURI = dataLocationURI.append("/") .append(path.getContainer()) .append("/?prefix=") .append(object) .append("&format=json"); //in listing deep set param to false if (listDeep == false) { dataLocationURI.append("&delimiter=/"); } return findObjects(dataLocationURI.toString(), requestHeaders); }
Example #28
Source File: SwiftRestClient.java From big-c with Apache License 2.0 | 5 votes |
/** * Make an HTTP GET request to Swift to get a range of data in the object. * * @param path path to object * @param offset offset from file beginning * @param length file length * @return The input stream -which must be closed afterwards. * @throws IOException Problems * @throws SwiftException swift specific error * @throws FileNotFoundException path is not there */ public HttpBodyContent getData(SwiftObjectPath path, long offset, long length) throws IOException { if (offset < 0) { throw new SwiftException("Invalid offset: " + offset + " in getDataAsInputStream( path=" + path + ", offset=" + offset + ", length =" + length + ")"); } if (length <= 0) { throw new SwiftException("Invalid length: " + length + " in getDataAsInputStream( path="+ path + ", offset=" + offset + ", length ="+ length + ")"); } final String range = String.format(SWIFT_RANGE_HEADER_FORMAT_PATTERN, offset, offset + length - 1); if (LOG.isDebugEnabled()) { LOG.debug("getData:" + range); } return getData(path, new Header(HEADER_RANGE, range), SwiftRestClient.NEWEST); }
Example #29
Source File: SwiftNativeFileSystemStore.java From sahara-extra with Apache License 2.0 | 5 votes |
/** * Debug action to dump directory statuses to the debug log * * @param message explanation * @param objectPath object path (can be null) * @param statuses listing output */ private void logDirectory(String message, SwiftObjectPath objectPath, Iterable<FileStatus> statuses) { if (LOG.isDebugEnabled()) { LOG.debug(message + ": listing of " + objectPath); for (FileStatus fileStatus : statuses) { LOG.debug(fileStatus.getPath()); } } }
Example #30
Source File: SwiftNativeFileSystemStore.java From big-c with Apache License 2.0 | 5 votes |
/** * Copy an object * @param srcObject source object path * @param destObject destination object path * @throws IOException IO problems */ private void copyObject(SwiftObjectPath srcObject, SwiftObjectPath destObject) throws IOException { if (srcObject.isEqualToOrParentOf(destObject)) { throw new SwiftException( "Can't copy " + srcObject + " onto " + destObject); } //do the copy boolean copySucceeded = swiftRestClient.copyObject(srcObject, destObject); if (!copySucceeded) { throw new SwiftException("Copy of " + srcObject + " to " + destObject + "failed"); } }