Java Code Examples for org.apache.hadoop.fs.contract.ContractTestUtils#assertDeleted()

The following examples show how to use org.apache.hadoop.fs.contract.ContractTestUtils#assertDeleted() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestS3ADeleteManyFiles.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBulkRenameAndDelete() throws Throwable {
  final Path scaleTestDir = getTestPath();
  final Path srcDir = new Path(scaleTestDir, "src");
  final Path finalDir = new Path(scaleTestDir, "final");
  final long count = getOperationCount();
  ContractTestUtils.rm(fs, scaleTestDir, true, false);

  fs.mkdirs(srcDir);
  fs.mkdirs(finalDir);

  int testBufferSize = fs.getConf()
      .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
          ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
  // use Executor to speed up file creation
  ExecutorService exec = Executors.newFixedThreadPool(16);
  final ExecutorCompletionService<Boolean> completionService =
      new ExecutorCompletionService<Boolean>(exec);
  try {
    final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');

    for (int i = 0; i < count; ++i) {
      final String fileName = "foo-" + i;
      completionService.submit(new Callable<Boolean>() {
        @Override
        public Boolean call() throws IOException {
          ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
              false, data);
          return fs.exists(new Path(srcDir, fileName));
        }
      });
    }
    for (int i = 0; i < count; ++i) {
      final Future<Boolean> future = completionService.take();
      try {
        if (!future.get()) {
          LOG.warn("cannot create file");
        }
      } catch (ExecutionException e) {
        LOG.warn("Error while uploading file", e.getCause());
        throw e;
      }
    }
  } finally {
    exec.shutdown();
  }

  int nSrcFiles = fs.listStatus(srcDir).length;
  fs.rename(srcDir, finalDir);
  assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + 0));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + count / 2));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + (count - 1)));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + 0));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + count/2));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + (count-1)));

  ContractTestUtils.assertDeleted(fs, finalDir, true, false);
}
Example 2
Source File: TestS3ADeleteManyFiles.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBulkRenameAndDelete() throws Throwable {
  final Path scaleTestDir = getTestPath();
  final Path srcDir = new Path(scaleTestDir, "src");
  final Path finalDir = new Path(scaleTestDir, "final");
  final long count = getOperationCount();
  ContractTestUtils.rm(fs, scaleTestDir, true, false);

  fs.mkdirs(srcDir);
  fs.mkdirs(finalDir);

  int testBufferSize = fs.getConf()
      .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
          ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
  // use Executor to speed up file creation
  ExecutorService exec = Executors.newFixedThreadPool(16);
  final ExecutorCompletionService<Boolean> completionService =
      new ExecutorCompletionService<Boolean>(exec);
  try {
    final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');

    for (int i = 0; i < count; ++i) {
      final String fileName = "foo-" + i;
      completionService.submit(new Callable<Boolean>() {
        @Override
        public Boolean call() throws IOException {
          ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
              false, data);
          return fs.exists(new Path(srcDir, fileName));
        }
      });
    }
    for (int i = 0; i < count; ++i) {
      final Future<Boolean> future = completionService.take();
      try {
        if (!future.get()) {
          LOG.warn("cannot create file");
        }
      } catch (ExecutionException e) {
        LOG.warn("Error while uploading file", e.getCause());
        throw e;
      }
    }
  } finally {
    exec.shutdown();
  }

  int nSrcFiles = fs.listStatus(srcDir).length;
  fs.rename(srcDir, finalDir);
  assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + 0));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + count / 2));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + (count - 1)));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + 0));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + count/2));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + (count-1)));

  ContractTestUtils.assertDeleted(fs, finalDir, true, false);
}