Java Code Examples for org.apache.hadoop.fs.swift.http.SwiftRestClient

The following examples show how to use org.apache.hadoop.fs.swift.http.SwiftRestClient. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: sahara-extra   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get the object as an input stream
 *
 * @param path object path
 * @return the input stream -this must be closed to terminate the connection
 * @throws IOException           IO problems
 * @throws FileNotFoundException path doesn't resolve to an object
 */
public HttpBodyContent getObject(Path path) throws IOException {
  List<String> locations = getDataLocalEndpoints(path);

  for (String url : locations) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Reading " + path + " from location: " + url);
    }
    try {
      return swiftRestClient.getData(new URI(url),
          SwiftRestClient.NEWEST);
    } catch (Exception e) {
      // Ignore
      // It is possible that endpoint doesn't contains needed data.
    }
  }

  if (LOG.isDebugEnabled()) {
    LOG.debug("Reading " + path + " from proxy node");
  }
  return swiftRestClient.getData(toObjectPath(path),
                               SwiftRestClient.NEWEST);
}
 
Example 2
private String[] getRawObjectNames() throws Exception {
  SwiftRestClient client;
  client = SwiftRestClient.getInstance(fs.getUri(), fs.getConf());
  SwiftObjectPath path = SwiftObjectPath.fromPath(fs.getUri(), new Path("/"));
  byte[] bytes = client.listDeepObjectsInDirectory(path, true, true);
  final CollectionType collectionType = JSONUtil.getJsonMapper().
    getTypeFactory().constructCollectionType(List.class,
                                             SwiftObjectFileStatus.class);
  final List<SwiftObjectFileStatus> fileStatusList =
    JSONUtil.toObject(new String(bytes), collectionType);
  final ArrayList<String> objects = new ArrayList();
  for (SwiftObjectFileStatus status : fileStatusList) {
    if (status.getName() != null) {
      objects.add(status.getName());
    } else if (status.getSubdir() != null) {
      objects.add(status.getSubdir());
    }
  }
  return objects.toArray(new String[objects.size()]);
}
 
Example 3
Source Project: hadoop   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 5 votes vote down vote up
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
                                                                  IOException {
  Header[] headers;
  if (newest) {
    headers = swiftRestClient.headRequest("getObjectMetadata-newest",
                                          objectPath, SwiftRestClient.NEWEST);
  } else {
    headers = swiftRestClient.headRequest("getObjectMetadata",
                                          objectPath);
  }
  return headers;
}
 
Example 4
Source Project: hadoop   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Does the object exist
 *
 * @param path swift object path
 * @return true if the metadata of an object could be retrieved
 * @throws IOException IO problems other than FileNotFound, which
 *                     is downgraded to an object does not exist return code
 */
public boolean objectExists(SwiftObjectPath path) throws IOException {
  try {
    Header[] headers = swiftRestClient.headRequest("objectExists",
                                                   path,
                                                   SwiftRestClient.NEWEST);
    //no headers is treated as a missing file
    return headers.length != 0;
  } catch (FileNotFoundException e) {
    return false;
  }
}
 
Example 5
Source Project: hadoop   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPositiveBlocksize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getBlocksizeKB());
}
 
Example 6
Source Project: hadoop   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareTruePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "true");
  SwiftRestClient restClient = mkInstance(configuration);
  assertTrue(restClient.isLocationAware());
}
 
Example 7
Source Project: hadoop   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareFalsePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "false");
  SwiftRestClient restClient = mkInstance(configuration);
  assertFalse(restClient.isLocationAware());
}
 
Example 8
Source Project: hadoop   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPositivePartsize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getPartSizeKB());
}
 
Example 9
Source Project: hadoop   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProxyData() throws Exception {
  final Configuration configuration = createCoreConfig();
  String proxy="web-proxy";
  int port = 8088;
  configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
  configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(proxy, restClient.getProxyHost());
  assertEquals(port, restClient.getProxyPort());
}
 
Example 10
Source Project: hadoop   Source File: TestSwiftObjectPath.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConvertToPath() throws Throwable {
  String initialpath = "/dir/file1";
  Path ipath = new Path(initialpath);
  SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
          ipath);
  URI endpoint = new URI(ENDPOINT);
  URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
  LOG.info("Inital Hadoop Path =" + initialpath);
  LOG.info("Merged URI=" + uri);
}
 
Example 11
Source Project: big-c   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 5 votes vote down vote up
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
                                                                  IOException {
  Header[] headers;
  if (newest) {
    headers = swiftRestClient.headRequest("getObjectMetadata-newest",
                                          objectPath, SwiftRestClient.NEWEST);
  } else {
    headers = swiftRestClient.headRequest("getObjectMetadata",
                                          objectPath);
  }
  return headers;
}
 
Example 12
Source Project: big-c   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Does the object exist
 *
 * @param path swift object path
 * @return true if the metadata of an object could be retrieved
 * @throws IOException IO problems other than FileNotFound, which
 *                     is downgraded to an object does not exist return code
 */
public boolean objectExists(SwiftObjectPath path) throws IOException {
  try {
    Header[] headers = swiftRestClient.headRequest("objectExists",
                                                   path,
                                                   SwiftRestClient.NEWEST);
    //no headers is treated as a missing file
    return headers.length != 0;
  } catch (FileNotFoundException e) {
    return false;
  }
}
 
Example 13
Source Project: big-c   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPositiveBlocksize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getBlocksizeKB());
}
 
Example 14
Source Project: big-c   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareTruePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "true");
  SwiftRestClient restClient = mkInstance(configuration);
  assertTrue(restClient.isLocationAware());
}
 
Example 15
Source Project: big-c   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareFalsePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "false");
  SwiftRestClient restClient = mkInstance(configuration);
  assertFalse(restClient.isLocationAware());
}
 
Example 16
Source Project: big-c   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPositivePartsize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getPartSizeKB());
}
 
Example 17
Source Project: big-c   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProxyData() throws Exception {
  final Configuration configuration = createCoreConfig();
  String proxy="web-proxy";
  int port = 8088;
  configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
  configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(proxy, restClient.getProxyHost());
  assertEquals(port, restClient.getProxyPort());
}
 
Example 18
Source Project: big-c   Source File: TestSwiftObjectPath.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConvertToPath() throws Throwable {
  String initialpath = "/dir/file1";
  Path ipath = new Path(initialpath);
  SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
          ipath);
  URI endpoint = new URI(ENDPOINT);
  URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
  LOG.info("Inital Hadoop Path =" + initialpath);
  LOG.info("Merged URI=" + uri);
}
 
Example 19
Source Project: sahara-extra   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Initialize the filesystem store -this creates the REST client binding.
 *
 * @param fsURI         URI of the filesystem, which is used to map to the filesystem-specific
 *                      options in the configuration file
 * @param configuration configuration
 * @throws IOException on any failure.
 */
public void initialize(URI fsURI, Configuration configuration) throws IOException {
  this.uri = fsURI;
  dnsToSwitchMapping = ReflectionUtils.newInstance(
      configuration.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
          DNSToSwitchMapping.class), configuration);

  this.swiftRestClient = SwiftRestClient.getInstance(fsURI, configuration);
}
 
Example 20
Source Project: sahara-extra   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 5 votes vote down vote up
private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
                                                                  IOException {
  Header[] headers;
  if (newest) {
    headers = swiftRestClient.headRequest("getObjectMetadata-newest",
                                          objectPath, SwiftRestClient.NEWEST);
  } else {
    headers = swiftRestClient.headRequest("getObjectMetadata",
                                          objectPath);
  }
  return headers;
}
 
Example 21
Source Project: sahara-extra   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Does the object exist
 *
 * @param path swift object path
 * @return true if the metadata of an object could be retrieved
 * @throws IOException IO problems other than FileNotFound, which
 *                     is downgraded to an object does not exist return code
 */
public boolean objectExists(SwiftObjectPath path) throws IOException {
  try {
    Header[] headers = swiftRestClient.headRequest("objectExists",
                                                   path,
                                                   SwiftRestClient.NEWEST);
    //no headers is treated as a missing file
    return headers.length != 0;
  } catch (FileNotFoundException e) {
    return false;
  }
}
 
Example 22
Source Project: sahara-extra   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPositiveBlocksize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getBlocksizeKB());
}
 
Example 23
Source Project: sahara-extra   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareTruePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "true");
  SwiftRestClient restClient = mkInstance(configuration);
  assertTrue(restClient.isLocationAware());
}
 
Example 24
Source Project: sahara-extra   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLocationAwareFalsePropagates() throws Exception {
  final Configuration configuration = createCoreConfig();
  set(configuration, DOT_LOCATION_AWARE, "false");
  SwiftRestClient restClient = mkInstance(configuration);
  assertFalse(restClient.isLocationAware());
}
 
Example 25
Source Project: sahara-extra   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPositivePartsize() throws Exception {
  final Configuration configuration = createCoreConfig();
  int size = 127;
  configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(size, restClient.getPartSizeKB());
}
 
Example 26
Source Project: sahara-extra   Source File: TestSwiftConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testProxyData() throws Exception {
  final Configuration configuration = createCoreConfig();
  String proxy="web-proxy";
  int port = 8088;
  configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
  configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
  SwiftRestClient restClient = mkInstance(configuration);
  assertEquals(proxy, restClient.getProxyHost());
  assertEquals(port, restClient.getProxyPort());
}
 
Example 27
Source Project: sahara-extra   Source File: TestSwiftFileSystemRename.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenamePseudoDir() throws Throwable {
  assumeRenameSupported();

  // create file directory (don't create directory file)
  SwiftRestClient client;
  client = SwiftRestClient.getInstance(fs.getUri(), fs.getConf());
  SwiftObjectPath path = SwiftObjectPath.fromPath(fs.getUri(), new Path("/test/olddir/file"));
  client.upload(path, new ByteArrayInputStream(new byte[0]), 0);

  rename(path("/test/olddir"), path("/test/newdir"), true, false, true);
  SwiftTestUtils.assertIsDirectory(fs, path("/test/newdir"));
  assertIsFile(path("/test/newdir/file"));
}
 
Example 28
Source Project: sahara-extra   Source File: TestSwiftObjectPath.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testConvertToPath() throws Throwable {
  String initialpath = "/dir/file1";
  Path ipath = new Path(initialpath);
  SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
          ipath);
  URI endpoint = new URI(ENDPOINT);
  URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
  LOG.info("Inital Hadoop Path =" + initialpath);
  LOG.info("Merged URI=" + uri);
}
 
Example 29
Source Project: hadoop   Source File: TestSwiftConfig.java    License: Apache License 2.0 4 votes vote down vote up
@Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
public void testNegativePartsize() throws Exception {
  final Configuration configuration = createCoreConfig();
  configuration.set(SWIFT_PARTITION_SIZE, "-1");
  SwiftRestClient restClient = mkInstance(configuration);
}
 
Example 30
Source Project: hadoop   Source File: TestSwiftConfig.java    License: Apache License 2.0 4 votes vote down vote up
private SwiftRestClient mkInstance(Configuration configuration) throws
        IOException,
        URISyntaxException {
  URI uri = new URI("swift://container.openstack/");
  return SwiftRestClient.getInstance(uri, configuration);
}