Java Code Examples for org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem

The following examples show how to use org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestSwiftFileSystemBasicOps.java    License: Apache License 2.0 5 votes vote down vote up
private void delete(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, false)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
 
Example 2
Source Project: hadoop   Source File: TestSwiftFileSystemBasicOps.java    License: Apache License 2.0 5 votes vote down vote up
private void deleteR(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, true)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
 
Example 3
/**
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testManyPartitionedFile");

  int len = PART_SIZE_BYTES * 15;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = fs.listStatus(path);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
 
Example 4
Source Project: big-c   Source File: TestSwiftFileSystemBasicOps.java    License: Apache License 2.0 5 votes vote down vote up
private void delete(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, false)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
 
Example 5
Source Project: big-c   Source File: TestSwiftFileSystemBasicOps.java    License: Apache License 2.0 5 votes vote down vote up
private void deleteR(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, true)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
 
Example 6
/**
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testManyPartitionedFile");

  int len = PART_SIZE_BYTES * 15;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = fs.listStatus(path);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
 
Example 7
Source Project: sahara-extra   Source File: TestSwiftFileSystemBasicOps.java    License: Apache License 2.0 5 votes vote down vote up
private void delete(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, false)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
 
Example 8
Source Project: sahara-extra   Source File: TestSwiftFileSystemBasicOps.java    License: Apache License 2.0 5 votes vote down vote up
private void deleteR(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, true)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
 
Example 9
/**
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testManyPartitionedFile");

  int len = PART_SIZE_BYTES * 15;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = getStore().listSegments(fs.getFileStatus(path), true);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
 
Example 10
/**
 * Test writes partitioned file writing that path is qualified.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testQualifiedPath() throws Throwable {
  final Path path = path("/test/qualifiedPath");
  int len = PART_SIZE_BYTES * 4;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = getStore().listSegments(fs.getFileStatus(path), true);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
 
Example 11
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testGetSchemeImplemented() throws Throwable {
  String scheme = fs.getScheme();
  assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
}
 
Example 12
Source Project: hadoop   Source File: TestSwiftFileSystemContract.java    License: Apache License 2.0 4 votes vote down vote up
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
          new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
 
Example 13
Source Project: hadoop   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 4 votes vote down vote up
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
    new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
 
Example 14
Source Project: hadoop   Source File: SwiftContract.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public String getScheme() {
  return SwiftNativeFileSystem.SWIFT;
}
 
Example 15
Source Project: big-c   Source File: TestSwiftFileSystemExtendedContract.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testGetSchemeImplemented() throws Throwable {
  String scheme = fs.getScheme();
  assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
}
 
Example 16
Source Project: big-c   Source File: TestSwiftFileSystemContract.java    License: Apache License 2.0 4 votes vote down vote up
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
          new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
 
Example 17
Source Project: big-c   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 4 votes vote down vote up
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
    new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
 
Example 18
Source Project: big-c   Source File: SwiftContract.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public String getScheme() {
  return SwiftNativeFileSystem.SWIFT;
}
 
Example 19
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testGetSchemeImplemented() throws Throwable {
  String scheme = fs.getScheme();
  assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
}
 
Example 20
Source Project: sahara-extra   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 4 votes vote down vote up
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
    new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
 
Example 21
Source Project: hadoop   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get the filesystem
 * @return the current FS
 */
public SwiftNativeFileSystem getFs() {
  return fs;
}
 
Example 22
Source Project: hadoop   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get the number of partitions written from the Swift Native FS APIs
 * @param out output stream
 * @return the number of partitioned files written by the stream
 */
protected int getPartitionsWritten(FSDataOutputStream out) {
  return SwiftNativeFileSystem.getPartitionsWritten(out);
}
 
Example 23
Source Project: big-c   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get the filesystem
 * @return the current FS
 */
public SwiftNativeFileSystem getFs() {
  return fs;
}
 
Example 24
Source Project: big-c   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get the number of partitions written from the Swift Native FS APIs
 * @param out output stream
 * @return the number of partitioned files written by the stream
 */
protected int getPartitionsWritten(FSDataOutputStream out) {
  return SwiftNativeFileSystem.getPartitionsWritten(out);
}
 
Example 25
Source Project: sahara-extra   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get the filesystem
 * @return the current FS
 */
public SwiftNativeFileSystem getFs() {
  return fs;
}
 
Example 26
Source Project: sahara-extra   Source File: SwiftFileSystemBaseTest.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get the number of partitions written from the Swift Native FS APIs
 * @param out output stream
 * @return the number of partitioned files written by the stream
 */
protected int getPartitionsWritten(FSDataOutputStream out) {
  return SwiftNativeFileSystem.getPartitionsWritten(out);
}