org.apache.hadoop.fs.contract.ContractTestUtils Java Examples

The following examples show how to use org.apache.hadoop.fs.contract.ContractTestUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
private void testDeleteCreatesFakeParentDir() throws Exception {
  Path grandparent = new Path("/testDeleteCreatesFakeParentDir");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);
  rootItemCount++; // grandparent

  // Verify that parent dir key does not exist
  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // Delete the child key
  fs.delete(child, false);

  // Deleting the only child should create the parent dir key if it does
  // not exist
  String parentKey = o3fs.pathToKey(parent) + "/";
  OzoneKeyDetails parentKeyInfo = getKey(parent, true);
  assertEquals(parentKey, parentKeyInfo.getName());
}
 
Example #2
Source File: TestOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
private void testListStatus() throws Exception {
  Path parent = new Path("/testListStatus");
  Path file1 = new Path(parent, "key1");
  Path file2 = new Path(parent, "key2");
  ContractTestUtils.touch(fs, file1);
  ContractTestUtils.touch(fs, file2);
  rootItemCount++; // parent

  // ListStatus on a directory should return all subdirs along with
  // files, even if there exists a file and sub-dir with the same name.
  FileStatus[] fileStatuses = o3fs.listStatus(parent);
  assertEquals("FileStatus did not return all children of the directory",
      2, fileStatuses.length);

  // ListStatus should return only the immediate children of a directory.
  Path file3 = new Path(parent, "dir1/key3");
  Path file4 = new Path(parent, "dir1/key4");
  ContractTestUtils.touch(fs, file3);
  ContractTestUtils.touch(fs, file4);
  fileStatuses = o3fs.listStatus(parent);
  assertEquals("FileStatus did not return all children of the directory",
      3, fileStatuses.length);
}
 
Example #3
Source File: TestOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
private void testCreateDoesNotAddParentDirKeys() throws Exception {
  Path grandparent = new Path("/testCreateDoesNotAddParentDirKeys");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);
  rootItemCount++; // grandparent

  OzoneKeyDetails key = getKey(child, false);
  assertEquals(key.getName(), o3fs.pathToKey(child));

  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // List status on the parent should show the child file
  assertEquals("List status of parent should include the 1 child file", 1L,
      fs.listStatus(parent).length);
  assertTrue("Parent directory does not appear to be a directory",
      fs.getFileStatus(parent).isDirectory());
}
 
Example #4
Source File: AbstractGoogleContractConcatTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void testConcatMultiple() throws Throwable {
  int numFiles = GoogleCloudStorage.MAX_COMPOSE_OBJECTS * 3 / 2;
  Path testPath = path("test");

  byte[][] blocks = new byte[numFiles][0];
  Path[] srcs = new Path[numFiles];
  for (int i = 0; i < numFiles; i++) {
    Path srcFile = new Path(testPath, "" + i);
    blocks[i] = dataset(TEST_FILE_LEN, i, 255);
    createFile(getFileSystem(), srcFile, true, blocks[i]);
    srcs[i] = srcFile;
  }
  Path target = new Path(testPath, "target");

  createFile(getFileSystem(), target, false, new byte[0]);
  getFileSystem().concat(target, srcs);
  assertFileHasLength(getFileSystem(), target, TEST_FILE_LEN * numFiles);
  ContractTestUtils.validateFileContent(
      ContractTestUtils.readDataset(getFileSystem(), target, TEST_FILE_LEN * numFiles), blocks);
}
 
Example #5
Source File: TestS3ADeleteManyFiles.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testOpenCreate() throws IOException {
  Path dir = new Path("/tests3a");
  ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);


  /*
  Enable to test the multipart upload
  try {
    ContractTestUtils.createAndVerifyFile(fs, dir,
        (long)6 * 1024 * 1024 * 1024);
  } catch (IOException e) {
    fail(e.getMessage());
  }
  */
}
 
Example #6
Source File: TestS3ADeleteManyFiles.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testOpenCreate() throws IOException {
  Path dir = new Path("/tests3a");
  ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);


  /*
  Enable to test the multipart upload
  try {
    ContractTestUtils.createAndVerifyFile(fs, dir,
        (long)6 * 1024 * 1024 * 1024);
  } catch (IOException e) {
    fail(e.getMessage());
  }
  */
}
 
Example #7
Source File: CephFSContract.java    From cephfs-hadoop with GNU Lesser General Public License v2.1 5 votes vote down vote up
/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 
Example #8
Source File: LocalFSContract.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 
Example #9
Source File: TestHDFSContractAppend.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void testRenameFileBeingAppended() throws Throwable {
  try {
    super.testRenameFileBeingAppended();
    fail("Expected a FileNotFoundException");
  } catch (FileNotFoundException e) {
    // downgrade
    ContractTestUtils.downgrade("Renaming an open file" +
                                "still creates the old path", e);

  }
}
 
Example #10
Source File: LocalFSContract.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 
Example #11
Source File: TestOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
/**
 * Tests listStatus on a path with subdirs.
 */
private void testListStatusOnSubDirs() throws Exception {
  // Create the following key structure
  //      /dir1/dir11/dir111
  //      /dir1/dir12
  //      /dir1/dir12/file121
  //      /dir2
  // ListStatus on /dir1 should return all its immediated subdirs only
  // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
  // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
  // listStatus.
  Path dir1 = new Path("/dir1");
  Path dir11 = new Path(dir1, "dir11");
  Path dir111 = new Path(dir11, "dir111");
  Path dir12 = new Path(dir1, "dir12");
  Path file121 = new Path(dir12, "file121");
  Path dir2 = new Path("/dir2");
  fs.mkdirs(dir111);
  fs.mkdirs(dir12);
  ContractTestUtils.touch(fs, file121);
  fs.mkdirs(dir2);

  FileStatus[] fileStatuses = o3fs.listStatus(dir1);
  assertEquals("FileStatus should return only the immediate children", 2,
      fileStatuses.length);

  // Verify that the two children of /dir1 returned by listStatus operation
  // are /dir1/dir11 and /dir1/dir12.
  String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
  String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
  assertTrue(fileStatus1.equals(dir11.toString()) ||
      fileStatus1.equals(dir12.toString()));
  assertTrue(fileStatus2.equals(dir11.toString()) ||
      fileStatus2.equals(dir12.toString()));
}
 
Example #12
Source File: TestOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
public void testSeekOnFileLength() throws IOException {
  Path file = new Path("/file");
  ContractTestUtils.createFile(fs, file, true, "a".getBytes());
  try (FSDataInputStream stream = fs.open(file)) {
    long fileLength = fs.getFileStatus(file).getLen();
    stream.seek(fileLength);
    assertEquals(-1, stream.read());
  }
}
 
Example #13
Source File: TestRootedOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateDoesNotAddParentDirKeys() throws Exception {
  Path grandparent = new Path(testBucketPath,
      "testCreateDoesNotAddParentDirKeys");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);

  OzoneKeyDetails key = getKey(child, false);
  OFSPath childOFSPath = new OFSPath(child);
  Assert.assertEquals(key.getName(), childOFSPath.getKeyName());

  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // List status on the parent should show the child file
  Assert.assertEquals(
      "List status of parent should include the 1 child file",
      1L, fs.listStatus(parent).length);
  Assert.assertTrue(
      "Parent directory does not appear to be a directory",
      fs.getFileStatus(parent).isDirectory());
}
 
Example #14
Source File: TestHDFSContractAppend.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void testRenameFileBeingAppended() throws Throwable {
  try {
    super.testRenameFileBeingAppended();
    fail("Expected a FileNotFoundException");
  } catch (FileNotFoundException e) {
    // downgrade
    ContractTestUtils.downgrade("Renaming an open file" +
                                "still creates the old path", e);

  }
}
 
Example #15
Source File: TestRootedOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
/**
 * Tests listStatus on a path with subdirs.
 */
@Test
public void testListStatusOnSubDirs() throws Exception {
  // Create the following key structure
  //      /dir1/dir11/dir111
  //      /dir1/dir12
  //      /dir1/dir12/file121
  //      /dir2
  // ListStatus on /dir1 should return all its immediated subdirs only
  // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
  // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
  // listStatus.
  Path dir1 = new Path(testBucketPath, "dir1");
  Path dir11 = new Path(dir1, "dir11");
  Path dir111 = new Path(dir11, "dir111");
  Path dir12 = new Path(dir1, "dir12");
  Path file121 = new Path(dir12, "file121");
  Path dir2 = new Path(testBucketPath, "dir2");
  fs.mkdirs(dir111);
  fs.mkdirs(dir12);
  ContractTestUtils.touch(fs, file121);
  fs.mkdirs(dir2);

  FileStatus[] fileStatuses = ofs.listStatus(dir1);
  Assert.assertEquals(
      "FileStatus should return only the immediate children",
      2, fileStatuses.length);

  // Verify that the two children of /dir1 returned by listStatus operation
  // are /dir1/dir11 and /dir1/dir12.
  String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
  String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
  Assert.assertTrue(fileStatus1.equals(dir11.toString()) ||
      fileStatus1.equals(dir12.toString()));
  Assert.assertTrue(fileStatus2.equals(dir11.toString()) ||
      fileStatus2.equals(dir12.toString()));
}
 
Example #16
Source File: TestRootedOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
@Test
public void testListStatus() throws Exception {
  Path parent = new Path(testBucketPath, "testListStatus");
  Path file1 = new Path(parent, "key1");
  Path file2 = new Path(parent, "key2");

  FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
  Assert.assertEquals("Should be empty", 0, fileStatuses.length);

  ContractTestUtils.touch(fs, file1);
  ContractTestUtils.touch(fs, file2);

  fileStatuses = ofs.listStatus(testBucketPath);
  Assert.assertEquals("Should have created parent",
      1, fileStatuses.length);
  Assert.assertEquals("Parent path doesn't match",
      fileStatuses[0].getPath().toUri().getPath(), parent.toString());

  // ListStatus on a directory should return all subdirs along with
  // files, even if there exists a file and sub-dir with the same name.
  fileStatuses = ofs.listStatus(parent);
  Assert.assertEquals(
      "FileStatus did not return all children of the directory",
      2, fileStatuses.length);

  // ListStatus should return only the immediate children of a directory.
  Path file3 = new Path(parent, "dir1/key3");
  Path file4 = new Path(parent, "dir1/key4");
  ContractTestUtils.touch(fs, file3);
  ContractTestUtils.touch(fs, file4);
  fileStatuses = ofs.listStatus(parent);
  Assert.assertEquals(
      "FileStatus did not return all children of the directory",
      3, fileStatuses.length);
}
 
Example #17
Source File: TestRootedOzoneFileSystem.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteCreatesFakeParentDir() throws Exception {
  Path grandparent = new Path(testBucketPath,
      "testDeleteCreatesFakeParentDir");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);

  // Verify that parent dir key does not exist
  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // Delete the child key
  Assert.assertTrue(fs.delete(child, false));

  // Deleting the only child should create the parent dir key if it does
  // not exist
  OFSPath parentOFSPath = new OFSPath(parent);
  String parentKey = parentOFSPath.getKeyName() + "/";
  OzoneKeyDetails parentKeyInfo = getKey(parent, true);
  Assert.assertEquals(parentKey, parentKeyInfo.getName());

  // Recursive delete with DeleteIterator
  Assert.assertTrue(fs.delete(grandparent, true));
}
 
Example #18
Source File: TestSwiftContractCreate.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip("blobstores can't distinguish empty directories from files");
}
 
Example #19
Source File: TestInMemoryGoogleContractAppend.java    From hadoop-connectors with Apache License 2.0 4 votes vote down vote up
@Override
public void testRenameFileBeingAppended() throws Throwable {
  ContractTestUtils.skip("blobstores can not rename file that being appended");
}
 
Example #20
Source File: TestInMemoryGoogleContractCreate.java    From hadoop-connectors with Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}
 
Example #21
Source File: TestGoogleContractCreate.java    From hadoop-connectors with Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip("blobstores can't distinguish empty directories from files");
}
 
Example #22
Source File: TestGoogleContractAppend.java    From hadoop-connectors with Apache License 2.0 4 votes vote down vote up
@Override
public void testRenameFileBeingAppended() throws Throwable {
  ContractTestUtils.skip("blobstores can not rename file that being appended");
}
 
Example #23
Source File: S3AScaleTestBase.java    From big-c with Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
  ContractTestUtils.rm(fs, getTestPath(), true, true);
}
 
Example #24
Source File: TestS3ADeleteManyFiles.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testBulkRenameAndDelete() throws Throwable {
  final Path scaleTestDir = getTestPath();
  final Path srcDir = new Path(scaleTestDir, "src");
  final Path finalDir = new Path(scaleTestDir, "final");
  final long count = getOperationCount();
  ContractTestUtils.rm(fs, scaleTestDir, true, false);

  fs.mkdirs(srcDir);
  fs.mkdirs(finalDir);

  int testBufferSize = fs.getConf()
      .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
          ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
  // use Executor to speed up file creation
  ExecutorService exec = Executors.newFixedThreadPool(16);
  final ExecutorCompletionService<Boolean> completionService =
      new ExecutorCompletionService<Boolean>(exec);
  try {
    final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');

    for (int i = 0; i < count; ++i) {
      final String fileName = "foo-" + i;
      completionService.submit(new Callable<Boolean>() {
        @Override
        public Boolean call() throws IOException {
          ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
              false, data);
          return fs.exists(new Path(srcDir, fileName));
        }
      });
    }
    for (int i = 0; i < count; ++i) {
      final Future<Boolean> future = completionService.take();
      try {
        if (!future.get()) {
          LOG.warn("cannot create file");
        }
      } catch (ExecutionException e) {
        LOG.warn("Error while uploading file", e.getCause());
        throw e;
      }
    }
  } finally {
    exec.shutdown();
  }

  int nSrcFiles = fs.listStatus(srcDir).length;
  fs.rename(srcDir, finalDir);
  assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + 0));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + count / 2));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + (count - 1)));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + 0));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + count/2));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + (count-1)));

  ContractTestUtils.assertDeleted(fs, finalDir, true, false);
}
 
Example #25
Source File: TestS3AFastOutputStream.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultiPartUpload() throws IOException {
  ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 *
      1024);
}
 
Example #26
Source File: TestS3AFastOutputStream.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testRegularUpload() throws IOException {
  ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
}
 
Example #27
Source File: TestS3AContractCreate.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}
 
Example #28
Source File: TestS3NContractCreate.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}
 
Example #29
Source File: TestSwiftContractOpen.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public void testOpenReadDir() throws Throwable {
  ContractTestUtils.skip("Skipping object-store quirk");
}
 
Example #30
Source File: TestSwiftContractOpen.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public void testOpenReadDirWithChild() throws Throwable {
  ContractTestUtils.skip("Skipping object-store quirk");
}