Java Code Examples for org.apache.hadoop.fs.contract.ContractTestUtils#assertDeleted()
The following examples show how to use
org.apache.hadoop.fs.contract.ContractTestUtils#assertDeleted() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestS3ADeleteManyFiles.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testBulkRenameAndDelete() throws Throwable { final Path scaleTestDir = getTestPath(); final Path srcDir = new Path(scaleTestDir, "src"); final Path finalDir = new Path(scaleTestDir, "final"); final long count = getOperationCount(); ContractTestUtils.rm(fs, scaleTestDir, true, false); fs.mkdirs(srcDir); fs.mkdirs(finalDir); int testBufferSize = fs.getConf() .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE, ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE); // use Executor to speed up file creation ExecutorService exec = Executors.newFixedThreadPool(16); final ExecutorCompletionService<Boolean> completionService = new ExecutorCompletionService<Boolean>(exec); try { final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z'); for (int i = 0; i < count; ++i) { final String fileName = "foo-" + i; completionService.submit(new Callable<Boolean>() { @Override public Boolean call() throws IOException { ContractTestUtils.createFile(fs, new Path(srcDir, fileName), false, data); return fs.exists(new Path(srcDir, fileName)); } }); } for (int i = 0; i < count; ++i) { final Future<Boolean> future = completionService.take(); try { if (!future.get()) { LOG.warn("cannot create file"); } } catch (ExecutionException e) { LOG.warn("Error while uploading file", e.getCause()); throw e; } } } finally { exec.shutdown(); } int nSrcFiles = fs.listStatus(srcDir).length; fs.rename(srcDir, finalDir); assertEquals(nSrcFiles, fs.listStatus(finalDir).length); ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + 0)); ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + count / 2)); ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + (count - 1))); ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + 0)); ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + count/2)); ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + (count-1))); ContractTestUtils.assertDeleted(fs, finalDir, true, false); }
Example 2
Source File: TestS3ADeleteManyFiles.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testBulkRenameAndDelete() throws Throwable { final Path scaleTestDir = getTestPath(); final Path srcDir = new Path(scaleTestDir, "src"); final Path finalDir = new Path(scaleTestDir, "final"); final long count = getOperationCount(); ContractTestUtils.rm(fs, scaleTestDir, true, false); fs.mkdirs(srcDir); fs.mkdirs(finalDir); int testBufferSize = fs.getConf() .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE, ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE); // use Executor to speed up file creation ExecutorService exec = Executors.newFixedThreadPool(16); final ExecutorCompletionService<Boolean> completionService = new ExecutorCompletionService<Boolean>(exec); try { final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z'); for (int i = 0; i < count; ++i) { final String fileName = "foo-" + i; completionService.submit(new Callable<Boolean>() { @Override public Boolean call() throws IOException { ContractTestUtils.createFile(fs, new Path(srcDir, fileName), false, data); return fs.exists(new Path(srcDir, fileName)); } }); } for (int i = 0; i < count; ++i) { final Future<Boolean> future = completionService.take(); try { if (!future.get()) { LOG.warn("cannot create file"); } } catch (ExecutionException e) { LOG.warn("Error while uploading file", e.getCause()); throw e; } } } finally { exec.shutdown(); } int nSrcFiles = fs.listStatus(srcDir).length; fs.rename(srcDir, finalDir); assertEquals(nSrcFiles, fs.listStatus(finalDir).length); ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + 0)); ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + count / 2)); ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + (count - 1))); ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + 0)); ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + count/2)); ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + (count-1))); ContractTestUtils.assertDeleted(fs, finalDir, true, false); }