org.apache.hadoop.fs.swift.http.SwiftRestClient Java Examples

The following examples show how to use org.apache.hadoop.fs.swift.http.SwiftRestClient. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SwiftNativeFileSystemStore.java    From sahara-extra with Apache License 2.0 6 votes vote down vote up
/**
 * Get the object as an input stream
 *
 * @param path object path
 * @return the input stream -this must be closed to terminate the connection
 * @throws IOException           IO problems
 * @throws FileNotFoundException path doesn't resolve to an object
 */
public HttpBodyContent getObject(Path path) throws IOException {
  List<String> locations = getDataLocalEndpoints(path);

  for (String url : locations) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Reading " + path + " from location: " + url);
    }
    try {
      return swiftRestClient.getData(new URI(url),
          SwiftRestClient.NEWEST);
    } catch (Exception e) {
      // Ignore
      // It is possible that endpoint doesn't contains needed data.
    }
  }

  if (LOG.isDebugEnabled()) {
    LOG.debug("Reading " + path + " from proxy node");
  }
  return swiftRestClient.getData(toObjectPath(path),
                               SwiftRestClient.NEWEST);
}
 
Example #2
Source File: TestSwiftFileSystemDirectories.java    From sahara-extra with Apache License 2.0 6 votes vote down vote up
private String[] getRawObjectNames() throws Exception {
  SwiftRestClient client;
  client = SwiftRestClient.getInstance(fs.getUri(), fs.getConf());
  SwiftObjectPath path = SwiftObjectPath.fromPath(fs.getUri(), new Path("/"));
  byte[] bytes = client.listDeepObjectsInDirectory(path, true, true);
  final CollectionType collectionType = JSONUtil.getJsonMapper().
    getTypeFactory().constructCollectionType(List.class,
                                             SwiftObjectFileStatus.class);
  final List<SwiftObjectFileStatus> fileStatusList =
    JSONUtil.toObject(new String(bytes), collectionType);
  final ArrayList<String> objects = new ArrayList();
  for (SwiftObjectFileStatus status : fileStatusList) {
    if (status.getName() != null) {
      objects.add(status.getName());
    } else if (status.getSubdir() != null) {
      objects.add(status.getSubdir());
    }
  }
  return objects.toArray(new String[objects.size()]);
}
 
Example #3
Source File: TestSwiftConfig.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareFalsePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "false");
  SwiftRestClient restClient = mkInstance(configuration);
  assertFalse(restClient.isLocationAware());
}
 
Example #4
Source File: SwiftNativeFileSystemStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
                                                                  IOException {
  Header[] headers;
  if (newest) {
    headers = swiftRestClient.headRequest("getObjectMetadata-newest",
                                          objectPath, SwiftRestClient.NEWEST);
  } else {
    headers = swiftRestClient.headRequest("getObjectMetadata",
                                          objectPath);
  }
  return headers;
}
 
Example #5
Source File: TestSwiftConfig.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testPositivePartsize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getPartSizeKB());
}
 
Example #6
Source File: TestSwiftConfig.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testProxyData() throws Exception {
  final Configuration configuration = createCoreConfig();
  String proxy="web-proxy";
  int port = 8088;
  configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
  configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(proxy, restClient.getProxyHost());
  assertEquals(port, restClient.getProxyPort());
}
 
Example #7
Source File: TestSwiftObjectPath.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConvertToPath() throws Throwable {
  String initialpath = "/dir/file1";
  Path ipath = new Path(initialpath);
  SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
          ipath);
  URI endpoint = new URI(ENDPOINT);
  URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
  LOG.info("Inital Hadoop Path =" + initialpath);
  LOG.info("Merged URI=" + uri);
}
 
Example #8
Source File: SwiftNativeFileSystemStore.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
/**
 * Initialize the filesystem store -this creates the REST client binding.
 *
 * @param fsURI         URI of the filesystem, which is used to map to the filesystem-specific
 *                      options in the configuration file
 * @param configuration configuration
 * @throws IOException on any failure.
 */
public void initialize(URI fsURI, Configuration configuration) throws IOException {
  this.uri = fsURI;
  dnsToSwitchMapping = ReflectionUtils.newInstance(
      configuration.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
          DNSToSwitchMapping.class), configuration);

  this.swiftRestClient = SwiftRestClient.getInstance(fsURI, configuration);
}
 
Example #9
Source File: SwiftNativeFileSystemStore.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
                                                                  IOException {
  Header[] headers;
  if (newest) {
    headers = swiftRestClient.headRequest("getObjectMetadata-newest",
                                          objectPath, SwiftRestClient.NEWEST);
  } else {
    headers = swiftRestClient.headRequest("getObjectMetadata",
                                          objectPath);
  }
  return headers;
}
 
Example #10
Source File: SwiftNativeFileSystemStore.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
/**
 * Does the object exist
 *
 * @param path swift object path
 * @return true if the metadata of an object could be retrieved
 * @throws IOException IO problems other than FileNotFound, which
 *                     is downgraded to an object does not exist return code
 */
public boolean objectExists(SwiftObjectPath path) throws IOException {
  try {
    Header[] headers = swiftRestClient.headRequest("objectExists",
                                                   path,
                                                   SwiftRestClient.NEWEST);
    //no headers is treated as a missing file
    return headers.length != 0;
  } catch (FileNotFoundException e) {
    return false;
  }
}
 
Example #11
Source File: TestSwiftConfig.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
@Test
public void testPositiveBlocksize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getBlocksizeKB());
}
 
Example #12
Source File: TestSwiftConfig.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareTruePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "true");
  SwiftRestClient restClient = mkInstance(configuration);
  assertTrue(restClient.isLocationAware());
}
 
Example #13
Source File: TestSwiftConfig.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareFalsePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "false");
  SwiftRestClient restClient = mkInstance(configuration);
  assertFalse(restClient.isLocationAware());
}
 
Example #14
Source File: TestSwiftConfig.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
@Test
public void testPositivePartsize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getPartSizeKB());
}
 
Example #15
Source File: TestSwiftConfig.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
@Test
public void testProxyData() throws Exception {
  final Configuration configuration = createCoreConfig();
  String proxy="web-proxy";
  int port = 8088;
  configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
  configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(proxy, restClient.getProxyHost());
  assertEquals(port, restClient.getProxyPort());
}
 
Example #16
Source File: TestSwiftFileSystemRename.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenamePseudoDir() throws Throwable {
  assumeRenameSupported();

  // create file directory (don't create directory file)
  SwiftRestClient client;
  client = SwiftRestClient.getInstance(fs.getUri(), fs.getConf());
  SwiftObjectPath path = SwiftObjectPath.fromPath(fs.getUri(), new Path("/test/olddir/file"));
  client.upload(path, new ByteArrayInputStream(new byte[0]), 0);

  rename(path("/test/olddir"), path("/test/newdir"), true, false, true);
  SwiftTestUtils.assertIsDirectory(fs, path("/test/newdir"));
  assertIsFile(path("/test/newdir/file"));
}
 
Example #17
Source File: TestSwiftObjectPath.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConvertToPath() throws Throwable {
  String initialpath = "/dir/file1";
  Path ipath = new Path(initialpath);
  SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
          ipath);
  URI endpoint = new URI(ENDPOINT);
  URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
  LOG.info("Inital Hadoop Path =" + initialpath);
  LOG.info("Merged URI=" + uri);
}
 
Example #18
Source File: TestSwiftConfig.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testPositiveBlocksize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getBlocksizeKB());
}
 
Example #19
Source File: SwiftNativeFileSystemStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Does the object exist
 *
 * @param path swift object path
 * @return true if the metadata of an object could be retrieved
 * @throws IOException IO problems other than FileNotFound, which
 *                     is downgraded to an object does not exist return code
 */
public boolean objectExists(SwiftObjectPath path) throws IOException {
  try {
    Header[] headers = swiftRestClient.headRequest("objectExists",
                                                   path,
                                                   SwiftRestClient.NEWEST);
    //no headers is treated as a missing file
    return headers.length != 0;
  } catch (FileNotFoundException e) {
    return false;
  }
}
 
Example #20
Source File: TestSwiftConfig.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testPositiveBlocksize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getBlocksizeKB());
}
 
Example #21
Source File: TestSwiftConfig.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareTruePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "true");
  SwiftRestClient restClient = mkInstance(configuration);
  assertTrue(restClient.isLocationAware());
}
 
Example #22
Source File: TestSwiftConfig.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareFalsePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "false");
  SwiftRestClient restClient = mkInstance(configuration);
  assertFalse(restClient.isLocationAware());
}
 
Example #23
Source File: TestSwiftConfig.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testPositivePartsize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getPartSizeKB());
}
 
Example #24
Source File: TestSwiftConfig.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testProxyData() throws Exception {
  final Configuration configuration = createCoreConfig();
  String proxy="web-proxy";
  int port = 8088;
  configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
  configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(proxy, restClient.getProxyHost());
  assertEquals(port, restClient.getProxyPort());
}
 
Example #25
Source File: TestSwiftObjectPath.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConvertToPath() throws Throwable {
  String initialpath = "/dir/file1";
  Path ipath = new Path(initialpath);
  SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
          ipath);
  URI endpoint = new URI(ENDPOINT);
  URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
  LOG.info("Inital Hadoop Path =" + initialpath);
  LOG.info("Merged URI=" + uri);
}
 
Example #26
Source File: SwiftNativeFileSystemStore.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
                                                                  IOException {
  Header[] headers;
  if (newest) {
    headers = swiftRestClient.headRequest("getObjectMetadata-newest",
                                          objectPath, SwiftRestClient.NEWEST);
  } else {
    headers = swiftRestClient.headRequest("getObjectMetadata",
                                          objectPath);
  }
  return headers;
}
 
Example #27
Source File: SwiftNativeFileSystemStore.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Does the object exist
 *
 * @param path swift object path
 * @return true if the metadata of an object could be retrieved
 * @throws IOException IO problems other than FileNotFound, which
 *                     is downgraded to an object does not exist return code
 */
public boolean objectExists(SwiftObjectPath path) throws IOException {
  try {
    Header[] headers = swiftRestClient.headRequest("objectExists",
                                                   path,
                                                   SwiftRestClient.NEWEST);
    //no headers is treated as a missing file
    return headers.length != 0;
  } catch (FileNotFoundException e) {
    return false;
  }
}
 
Example #28
Source File: TestSwiftConfig.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareTruePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "true");
  SwiftRestClient restClient = mkInstance(configuration);
  assertTrue(restClient.isLocationAware());
}
 
Example #29
Source File: TestSwiftConfig.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private SwiftRestClient mkInstance(Configuration configuration) throws
        IOException,
        URISyntaxException {
  URI uri = new URI("swift://container.openstack/");
  return SwiftRestClient.getInstance(uri, configuration);
}
 
Example #30
Source File: TestSwiftConfig.java    From big-c with Apache License 2.0 4 votes vote down vote up
private SwiftRestClient mkInstance(Configuration configuration) throws
        IOException,
        URISyntaxException {
  URI uri = new URI("swift://container.openstack/");
  return SwiftRestClient.getInstance(uri, configuration);
}