Java Code Examples for org.apache.hadoop.fs.contract.ContractTestUtils

The following examples show how to use org.apache.hadoop.fs.contract.ContractTestUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop-ozone   Source File: TestOzoneFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
private void testCreateDoesNotAddParentDirKeys() throws Exception {
  Path grandparent = new Path("/testCreateDoesNotAddParentDirKeys");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);
  rootItemCount++; // grandparent

  OzoneKeyDetails key = getKey(child, false);
  assertEquals(key.getName(), o3fs.pathToKey(child));

  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // List status on the parent should show the child file
  assertEquals("List status of parent should include the 1 child file", 1L,
      fs.listStatus(parent).length);
  assertTrue("Parent directory does not appear to be a directory",
      fs.getFileStatus(parent).isDirectory());
}
 
Example 2
Source Project: hadoop-ozone   Source File: TestOzoneFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
private void testDeleteCreatesFakeParentDir() throws Exception {
  Path grandparent = new Path("/testDeleteCreatesFakeParentDir");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);
  rootItemCount++; // grandparent

  // Verify that parent dir key does not exist
  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // Delete the child key
  fs.delete(child, false);

  // Deleting the only child should create the parent dir key if it does
  // not exist
  String parentKey = o3fs.pathToKey(parent) + "/";
  OzoneKeyDetails parentKeyInfo = getKey(parent, true);
  assertEquals(parentKey, parentKeyInfo.getName());
}
 
Example 3
Source Project: hadoop-ozone   Source File: TestOzoneFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
private void testListStatus() throws Exception {
  Path parent = new Path("/testListStatus");
  Path file1 = new Path(parent, "key1");
  Path file2 = new Path(parent, "key2");
  ContractTestUtils.touch(fs, file1);
  ContractTestUtils.touch(fs, file2);
  rootItemCount++; // parent

  // ListStatus on a directory should return all subdirs along with
  // files, even if there exists a file and sub-dir with the same name.
  FileStatus[] fileStatuses = o3fs.listStatus(parent);
  assertEquals("FileStatus did not return all children of the directory",
      2, fileStatuses.length);

  // ListStatus should return only the immediate children of a directory.
  Path file3 = new Path(parent, "dir1/key3");
  Path file4 = new Path(parent, "dir1/key4");
  ContractTestUtils.touch(fs, file3);
  ContractTestUtils.touch(fs, file4);
  fileStatuses = o3fs.listStatus(parent);
  assertEquals("FileStatus did not return all children of the directory",
      3, fileStatuses.length);
}
 
Example 4
Source Project: hadoop   Source File: TestS3ADeleteManyFiles.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOpenCreate() throws IOException {
  Path dir = new Path("/tests3a");
  ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);


  /*
  Enable to test the multipart upload
  try {
    ContractTestUtils.createAndVerifyFile(fs, dir,
        (long)6 * 1024 * 1024 * 1024);
  } catch (IOException e) {
    fail(e.getMessage());
  }
  */
}
 
Example 5
Source Project: big-c   Source File: TestS3ADeleteManyFiles.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOpenCreate() throws IOException {
  Path dir = new Path("/tests3a");
  ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);


  /*
  Enable to test the multipart upload
  try {
    ContractTestUtils.createAndVerifyFile(fs, dir,
        (long)6 * 1024 * 1024 * 1024);
  } catch (IOException e) {
    fail(e.getMessage());
  }
  */
}
 
Example 6
@Test
public void testConcatMultiple() throws Throwable {
  int numFiles = GoogleCloudStorage.MAX_COMPOSE_OBJECTS * 3 / 2;
  Path testPath = path("test");

  byte[][] blocks = new byte[numFiles][0];
  Path[] srcs = new Path[numFiles];
  for (int i = 0; i < numFiles; i++) {
    Path srcFile = new Path(testPath, "" + i);
    blocks[i] = dataset(TEST_FILE_LEN, i, 255);
    createFile(getFileSystem(), srcFile, true, blocks[i]);
    srcs[i] = srcFile;
  }
  Path target = new Path(testPath, "target");

  createFile(getFileSystem(), target, false, new byte[0]);
  getFileSystem().concat(target, srcs);
  assertFileHasLength(getFileSystem(), target, TEST_FILE_LEN * numFiles);
  ContractTestUtils.validateFileContent(
      ContractTestUtils.readDataset(getFileSystem(), target, TEST_FILE_LEN * numFiles), blocks);
}
 
Example 7
Source Project: hadoop-ozone   Source File: TestOzoneFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests listStatus on a path with subdirs.
 */
private void testListStatusOnSubDirs() throws Exception {
  // Create the following key structure
  //      /dir1/dir11/dir111
  //      /dir1/dir12
  //      /dir1/dir12/file121
  //      /dir2
  // ListStatus on /dir1 should return all its immediated subdirs only
  // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
  // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
  // listStatus.
  Path dir1 = new Path("/dir1");
  Path dir11 = new Path(dir1, "dir11");
  Path dir111 = new Path(dir11, "dir111");
  Path dir12 = new Path(dir1, "dir12");
  Path file121 = new Path(dir12, "file121");
  Path dir2 = new Path("/dir2");
  fs.mkdirs(dir111);
  fs.mkdirs(dir12);
  ContractTestUtils.touch(fs, file121);
  fs.mkdirs(dir2);

  FileStatus[] fileStatuses = o3fs.listStatus(dir1);
  assertEquals("FileStatus should return only the immediate children", 2,
      fileStatuses.length);

  // Verify that the two children of /dir1 returned by listStatus operation
  // are /dir1/dir11 and /dir1/dir12.
  String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
  String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
  assertTrue(fileStatus1.equals(dir11.toString()) ||
      fileStatus1.equals(dir12.toString()));
  assertTrue(fileStatus2.equals(dir11.toString()) ||
      fileStatus2.equals(dir12.toString()));
}
 
Example 8
Source Project: hadoop-ozone   Source File: TestOzoneFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
public void testSeekOnFileLength() throws IOException {
  Path file = new Path("/file");
  ContractTestUtils.createFile(fs, file, true, "a".getBytes());
  try (FSDataInputStream stream = fs.open(file)) {
    long fileLength = fs.getFileStatus(file).getLen();
    stream.seek(fileLength);
    assertEquals(-1, stream.read());
  }
}
 
Example 9
Source Project: hadoop-ozone   Source File: TestRootedOzoneFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateDoesNotAddParentDirKeys() throws Exception {
  Path grandparent = new Path(testBucketPath,
      "testCreateDoesNotAddParentDirKeys");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);

  OzoneKeyDetails key = getKey(child, false);
  OFSPath childOFSPath = new OFSPath(child);
  Assert.assertEquals(key.getName(), childOFSPath.getKeyName());

  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // List status on the parent should show the child file
  Assert.assertEquals(
      "List status of parent should include the 1 child file",
      1L, fs.listStatus(parent).length);
  Assert.assertTrue(
      "Parent directory does not appear to be a directory",
      fs.getFileStatus(parent).isDirectory());
}
 
Example 10
Source Project: hadoop-ozone   Source File: TestRootedOzoneFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteCreatesFakeParentDir() throws Exception {
  Path grandparent = new Path(testBucketPath,
      "testDeleteCreatesFakeParentDir");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);

  // Verify that parent dir key does not exist
  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // Delete the child key
  Assert.assertTrue(fs.delete(child, false));

  // Deleting the only child should create the parent dir key if it does
  // not exist
  OFSPath parentOFSPath = new OFSPath(parent);
  String parentKey = parentOFSPath.getKeyName() + "/";
  OzoneKeyDetails parentKeyInfo = getKey(parent, true);
  Assert.assertEquals(parentKey, parentKeyInfo.getName());

  // Recursive delete with DeleteIterator
  Assert.assertTrue(fs.delete(grandparent, true));
}
 
Example 11
Source Project: hadoop-ozone   Source File: TestRootedOzoneFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testListStatus() throws Exception {
  Path parent = new Path(testBucketPath, "testListStatus");
  Path file1 = new Path(parent, "key1");
  Path file2 = new Path(parent, "key2");

  FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
  Assert.assertEquals("Should be empty", 0, fileStatuses.length);

  ContractTestUtils.touch(fs, file1);
  ContractTestUtils.touch(fs, file2);

  fileStatuses = ofs.listStatus(testBucketPath);
  Assert.assertEquals("Should have created parent",
      1, fileStatuses.length);
  Assert.assertEquals("Parent path doesn't match",
      fileStatuses[0].getPath().toUri().getPath(), parent.toString());

  // ListStatus on a directory should return all subdirs along with
  // files, even if there exists a file and sub-dir with the same name.
  fileStatuses = ofs.listStatus(parent);
  Assert.assertEquals(
      "FileStatus did not return all children of the directory",
      2, fileStatuses.length);

  // ListStatus should return only the immediate children of a directory.
  Path file3 = new Path(parent, "dir1/key3");
  Path file4 = new Path(parent, "dir1/key4");
  ContractTestUtils.touch(fs, file3);
  ContractTestUtils.touch(fs, file4);
  fileStatuses = ofs.listStatus(parent);
  Assert.assertEquals(
      "FileStatus did not return all children of the directory",
      3, fileStatuses.length);
}
 
Example 12
Source Project: hadoop-ozone   Source File: TestRootedOzoneFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests listStatus on a path with subdirs.
 */
@Test
public void testListStatusOnSubDirs() throws Exception {
  // Create the following key structure
  //      /dir1/dir11/dir111
  //      /dir1/dir12
  //      /dir1/dir12/file121
  //      /dir2
  // ListStatus on /dir1 should return all its immediated subdirs only
  // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
  // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
  // listStatus.
  Path dir1 = new Path(testBucketPath, "dir1");
  Path dir11 = new Path(dir1, "dir11");
  Path dir111 = new Path(dir11, "dir111");
  Path dir12 = new Path(dir1, "dir12");
  Path file121 = new Path(dir12, "file121");
  Path dir2 = new Path(testBucketPath, "dir2");
  fs.mkdirs(dir111);
  fs.mkdirs(dir12);
  ContractTestUtils.touch(fs, file121);
  fs.mkdirs(dir2);

  FileStatus[] fileStatuses = ofs.listStatus(dir1);
  Assert.assertEquals(
      "FileStatus should return only the immediate children",
      2, fileStatuses.length);

  // Verify that the two children of /dir1 returned by listStatus operation
  // are /dir1/dir11 and /dir1/dir12.
  String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
  String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
  Assert.assertTrue(fileStatus1.equals(dir11.toString()) ||
      fileStatus1.equals(dir12.toString()));
  Assert.assertTrue(fileStatus2.equals(dir11.toString()) ||
      fileStatus2.equals(dir12.toString()));
}
 
Example 13
Source Project: hadoop   Source File: TestHDFSContractAppend.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void testRenameFileBeingAppended() throws Throwable {
  try {
    super.testRenameFileBeingAppended();
    fail("Expected a FileNotFoundException");
  } catch (FileNotFoundException e) {
    // downgrade
    ContractTestUtils.downgrade("Renaming an open file" +
                                "still creates the old path", e);

  }
}
 
Example 14
Source Project: hadoop   Source File: LocalFSContract.java    License: Apache License 2.0 5 votes vote down vote up
/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 
Example 15
Source Project: big-c   Source File: TestHDFSContractAppend.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void testRenameFileBeingAppended() throws Throwable {
  try {
    super.testRenameFileBeingAppended();
    fail("Expected a FileNotFoundException");
  } catch (FileNotFoundException e) {
    // downgrade
    ContractTestUtils.downgrade("Renaming an open file" +
                                "still creates the old path", e);

  }
}
 
Example 16
Source Project: big-c   Source File: LocalFSContract.java    License: Apache License 2.0 5 votes vote down vote up
/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 
Example 17
/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 
Example 18
Source Project: hadoop   Source File: TestSwiftContractOpen.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOpenReadDir() throws Throwable {
  ContractTestUtils.skip("Skipping object-store quirk");
}
 
Example 19
Source Project: hadoop   Source File: TestSwiftContractOpen.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOpenReadDirWithChild() throws Throwable {
  ContractTestUtils.skip("Skipping object-store quirk");
}
 
Example 20
Source Project: hadoop   Source File: TestSwiftContractCreate.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip("blobstores can't distinguish empty directories from files");
}
 
Example 21
Source Project: hadoop   Source File: TestS3NContractCreate.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}
 
Example 22
Source Project: hadoop   Source File: TestS3AContractCreate.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}
 
Example 23
Source Project: hadoop   Source File: TestS3AFastOutputStream.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRegularUpload() throws IOException {
  ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
}
 
Example 24
Source Project: hadoop   Source File: TestS3AFastOutputStream.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMultiPartUpload() throws IOException {
  ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 *
      1024);
}
 
Example 25
Source Project: hadoop   Source File: TestS3ADeleteManyFiles.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBulkRenameAndDelete() throws Throwable {
  final Path scaleTestDir = getTestPath();
  final Path srcDir = new Path(scaleTestDir, "src");
  final Path finalDir = new Path(scaleTestDir, "final");
  final long count = getOperationCount();
  ContractTestUtils.rm(fs, scaleTestDir, true, false);

  fs.mkdirs(srcDir);
  fs.mkdirs(finalDir);

  int testBufferSize = fs.getConf()
      .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
          ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
  // use Executor to speed up file creation
  ExecutorService exec = Executors.newFixedThreadPool(16);
  final ExecutorCompletionService<Boolean> completionService =
      new ExecutorCompletionService<Boolean>(exec);
  try {
    final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');

    for (int i = 0; i < count; ++i) {
      final String fileName = "foo-" + i;
      completionService.submit(new Callable<Boolean>() {
        @Override
        public Boolean call() throws IOException {
          ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
              false, data);
          return fs.exists(new Path(srcDir, fileName));
        }
      });
    }
    for (int i = 0; i < count; ++i) {
      final Future<Boolean> future = completionService.take();
      try {
        if (!future.get()) {
          LOG.warn("cannot create file");
        }
      } catch (ExecutionException e) {
        LOG.warn("Error while uploading file", e.getCause());
        throw e;
      }
    }
  } finally {
    exec.shutdown();
  }

  int nSrcFiles = fs.listStatus(srcDir).length;
  fs.rename(srcDir, finalDir);
  assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + 0));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + count / 2));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + (count - 1)));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + 0));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + count/2));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + (count-1)));

  ContractTestUtils.assertDeleted(fs, finalDir, true, false);
}
 
Example 26
Source Project: hadoop   Source File: S3AScaleTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
  ContractTestUtils.rm(fs, getTestPath(), true, true);
}
 
Example 27
Source Project: big-c   Source File: TestSwiftContractOpen.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOpenReadDir() throws Throwable {
  ContractTestUtils.skip("Skipping object-store quirk");
}
 
Example 28
Source Project: big-c   Source File: TestSwiftContractOpen.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOpenReadDirWithChild() throws Throwable {
  ContractTestUtils.skip("Skipping object-store quirk");
}
 
Example 29
Source Project: big-c   Source File: TestSwiftContractCreate.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip("blobstores can't distinguish empty directories from files");
}
 
Example 30
Source Project: big-c   Source File: TestS3NContractCreate.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}