Java Code Examples for org.apache.hadoop.fs.swift.util.SwiftUtils#partitionFilenameFromNumber()

The following examples show how to use org.apache.hadoop.fs.swift.util.SwiftUtils#partitionFilenameFromNumber() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SwiftNativeFileSystemStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Upload part of a larger file.
 *
 * @param path        destination path
 * @param partNumber  item number in the path
 * @param inputStream input data
 * @param length      length of the data
 * @throws IOException on a problem
 */
public void uploadFilePart(Path path, int partNumber,
                           InputStream inputStream, long length)
        throws IOException {

  String stringPath = path.toUri().toString();
  String partitionFilename = SwiftUtils.partitionFilenameFromNumber(
    partNumber);
  if (stringPath.endsWith("/")) {
    stringPath = stringPath.concat(partitionFilename);
  } else {
    stringPath = stringPath.concat("/").concat(partitionFilename);
  }

  swiftRestClient.upload(
    new SwiftObjectPath(toDirPath(path).getContainer(), stringPath),
          inputStream,
          length);
}
 
Example 2
Source File: TestSwiftFileSystemPartitionedUploads.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeleteSmallPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeleteSmallPartitionedFile");

  final int len1 = 1024;
  final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
  SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
}
 
Example 3
Source File: TestSwiftFileSystemPartitionedUploads.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeletePartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeletePartitionedFile");

  SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
}
 
Example 4
Source File: SwiftNativeFileSystemStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Upload part of a larger file.
 *
 * @param path        destination path
 * @param partNumber  item number in the path
 * @param inputStream input data
 * @param length      length of the data
 * @throws IOException on a problem
 */
public void uploadFilePart(Path path, int partNumber,
                           InputStream inputStream, long length)
        throws IOException {

  String stringPath = path.toUri().toString();
  String partitionFilename = SwiftUtils.partitionFilenameFromNumber(
    partNumber);
  if (stringPath.endsWith("/")) {
    stringPath = stringPath.concat(partitionFilename);
  } else {
    stringPath = stringPath.concat("/").concat(partitionFilename);
  }

  swiftRestClient.upload(
    new SwiftObjectPath(toDirPath(path).getContainer(), stringPath),
          inputStream,
          length);
}
 
Example 5
Source File: TestSwiftFileSystemPartitionedUploads.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeleteSmallPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeleteSmallPartitionedFile");

  final int len1 = 1024;
  final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
  SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
}
 
Example 6
Source File: TestSwiftFileSystemPartitionedUploads.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeletePartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeletePartitionedFile");

  SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
}
 
Example 7
Source File: SwiftNativeFileSystemStore.java    From sahara-extra with Apache License 2.0 6 votes vote down vote up
/**
 * Upload part of a larger file.
 *
 * @param path        destination path
 * @param partNumber  item number in the path
 * @param inputStream input data
 * @param length      length of the data
 * @throws IOException on a problem
 */
public void uploadFilePart(Path path, int partNumber,
                           InputStream inputStream, long length)
        throws IOException {

  String stringPath = path.toUri().getPath();
  String partitionFilename = SwiftUtils.partitionFilenameFromNumber(
    partNumber);
  if (stringPath.endsWith("/")) {
    stringPath = stringPath.concat(partitionFilename);
  } else {
    stringPath = stringPath.concat("/").concat(partitionFilename);
  }

  swiftRestClient.upload(
    new SwiftObjectPath(toDirPath(path).getContainer(), stringPath),
          inputStream,
          length);
}
 
Example 8
Source File: TestSwiftFileSystemPartitionedUploads.java    From sahara-extra with Apache License 2.0 6 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeleteSmallPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeleteSmallPartitionedFile");

  final int len1 = 1024;
  final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
  SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
}
 
Example 9
Source File: TestSwiftFileSystemPartitionedUploads.java    From sahara-extra with Apache License 2.0 6 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeletePartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeletePartitionedFile");

  SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
}
 
Example 10
Source File: TestSwiftFileSystemPartitionedUploads.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testRenamePartitionedFile() throws Throwable {
  Path src = new Path("/test/testRenamePartitionedFileSrc");

  int len = data.length;
  SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
  assertExists("Exists", src);

  String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
  Path srcPart = new Path(src, partOneName);
  Path dest = new Path("/test/testRenamePartitionedFileDest");
  Path destPart = new Path(src, partOneName);
  assertExists("Partition Exists", srcPart);
  fs.rename(src, dest);
  assertPathExists(fs, "dest file missing", dest);
  FileStatus status = fs.getFileStatus(dest);
  assertEquals("Length of renamed file is wrong", len, status.getLen());
  byte[] destData = readDataset(fs, dest, len);
  //compare data
  SwiftTestUtils.compareByteArrays(data, destData, len);
  String srcLs = SwiftTestUtils.ls(fs, src);
  String destLs = SwiftTestUtils.ls(fs, dest);

  assertPathDoesNotExist("deleted file still found in " + srcLs, src);

  assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
}
 
Example 11
Source File: TestSwiftFileSystemPartitionedUploads.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testRenamePartitionedFile() throws Throwable {
  Path src = new Path("/test/testRenamePartitionedFileSrc");

  int len = data.length;
  SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
  assertExists("Exists", src);

  String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
  Path srcPart = new Path(src, partOneName);
  Path dest = new Path("/test/testRenamePartitionedFileDest");
  Path destPart = new Path(src, partOneName);
  assertExists("Partition Exists", srcPart);
  fs.rename(src, dest);
  assertPathExists(fs, "dest file missing", dest);
  FileStatus status = fs.getFileStatus(dest);
  assertEquals("Length of renamed file is wrong", len, status.getLen());
  byte[] destData = readDataset(fs, dest, len);
  //compare data
  SwiftTestUtils.compareByteArrays(data, destData, len);
  String srcLs = SwiftTestUtils.ls(fs, src);
  String destLs = SwiftTestUtils.ls(fs, dest);

  assertPathDoesNotExist("deleted file still found in " + srcLs, src);

  assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
}
 
Example 12
Source File: TestSwiftFileSystemPartitionedUploads.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testRenamePartitionedFile() throws Throwable {
  Path src = new Path("/test/testRenamePartitionedFileSrc");

  int len = data.length;
  SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
  assertExists("Exists", src);

  String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
  Path srcPart = new Path(src, partOneName);
  Path dest = new Path("/test/testRenamePartitionedFileDest");
  Path destPart = new Path(src, partOneName);
  assertExists("Partition Exists", srcPart);
  fs.rename(src, dest);
  assertPathExists(fs, "dest file missing", dest);
  FileStatus status = fs.getFileStatus(dest);
  assertEquals("Length of renamed file is wrong", len, status.getLen());
  byte[] destData = readDataset(fs, dest, len);
  //compare data
  SwiftTestUtils.compareByteArrays(data, destData, len);
  String srcLs = SwiftTestUtils.ls(fs, src);
  String destLs = SwiftTestUtils.ls(fs, dest);

  assertPathDoesNotExist("deleted file still found in " + srcLs, src);

  assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
}