org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem Java Examples
The following examples show how to use
org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestSwiftFileSystemPartitionedUploads.java From sahara-extra with Apache License 2.0 | 5 votes |
/** * Test writes partitioned file writing that path is qualified. * @throws Throwable */ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT) public void testQualifiedPath() throws Throwable { final Path path = path("/test/qualifiedPath"); int len = PART_SIZE_BYTES * 4; final byte[] src = SwiftTestUtils.dataset(len, 32, 144); FSDataOutputStream out = fs.create(path, false, getBufferSize(), (short) 1, BLOCK_SIZE); out.write(src, 0, src.length); int expected = getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true); out.close(); assertPartitionsWritten("write completed", out, expected); assertEquals("too few bytes written", len, SwiftNativeFileSystem.getBytesWritten(out)); assertEquals("too few bytes uploaded", len, SwiftNativeFileSystem.getBytesUploaded(out)); //now we verify that the data comes back. If it //doesn't, it means that the ordering of the partitions //isn't right byte[] dest = readDataset(fs, path, len); //compare data SwiftTestUtils.compareByteArrays(src, dest, len); //finally, check the data FileStatus[] stats = getStore().listSegments(fs.getFileStatus(path), true); assertEquals("wrong entry count in " + SwiftTestUtils.dumpStats(path.toString(), stats), expected, stats.length); }
Example #2
Source File: TestSwiftFileSystemPartitionedUploads.java From sahara-extra with Apache License 2.0 | 5 votes |
/** * Test sticks up a very large partitioned file and verifies that * it comes back unchanged. * @throws Throwable */ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT) public void testManyPartitionedFile() throws Throwable { final Path path = new Path("/test/testManyPartitionedFile"); int len = PART_SIZE_BYTES * 15; final byte[] src = SwiftTestUtils.dataset(len, 32, 144); FSDataOutputStream out = fs.create(path, false, getBufferSize(), (short) 1, BLOCK_SIZE); out.write(src, 0, src.length); int expected = getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true); out.close(); assertPartitionsWritten("write completed", out, expected); assertEquals("too few bytes written", len, SwiftNativeFileSystem.getBytesWritten(out)); assertEquals("too few bytes uploaded", len, SwiftNativeFileSystem.getBytesUploaded(out)); //now we verify that the data comes back. If it //doesn't, it means that the ordering of the partitions //isn't right byte[] dest = readDataset(fs, path, len); //compare data SwiftTestUtils.compareByteArrays(src, dest, len); //finally, check the data FileStatus[] stats = getStore().listSegments(fs.getFileStatus(path), true); assertEquals("wrong entry count in " + SwiftTestUtils.dumpStats(path.toString(), stats), expected, stats.length); }
Example #3
Source File: TestSwiftFileSystemBasicOps.java From sahara-extra with Apache License 2.0 | 5 votes |
private void deleteR(SwiftNativeFileSystem fs, Path path) { try { if (!fs.delete(path, true)) { LOG.warn("Failed to delete " + path); } } catch (IOException e) { LOG.warn("deleting " + path, e); } }
Example #4
Source File: TestSwiftFileSystemBasicOps.java From sahara-extra with Apache License 2.0 | 5 votes |
private void delete(SwiftNativeFileSystem fs, Path path) { try { if (!fs.delete(path, false)) { LOG.warn("Failed to delete " + path); } } catch (IOException e) { LOG.warn("deleting " + path, e); } }
Example #5
Source File: TestSwiftFileSystemBasicOps.java From hadoop with Apache License 2.0 | 5 votes |
private void delete(SwiftNativeFileSystem fs, Path path) { try { if (!fs.delete(path, false)) { LOG.warn("Failed to delete " + path); } } catch (IOException e) { LOG.warn("deleting " + path, e); } }
Example #6
Source File: TestSwiftFileSystemBasicOps.java From hadoop with Apache License 2.0 | 5 votes |
private void deleteR(SwiftNativeFileSystem fs, Path path) { try { if (!fs.delete(path, true)) { LOG.warn("Failed to delete " + path); } } catch (IOException e) { LOG.warn("deleting " + path, e); } }
Example #7
Source File: TestSwiftFileSystemPartitionedUploads.java From hadoop with Apache License 2.0 | 5 votes |
/** * Test sticks up a very large partitioned file and verifies that * it comes back unchanged. * @throws Throwable */ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT) public void testManyPartitionedFile() throws Throwable { final Path path = new Path("/test/testManyPartitionedFile"); int len = PART_SIZE_BYTES * 15; final byte[] src = SwiftTestUtils.dataset(len, 32, 144); FSDataOutputStream out = fs.create(path, false, getBufferSize(), (short) 1, BLOCK_SIZE); out.write(src, 0, src.length); int expected = getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true); out.close(); assertPartitionsWritten("write completed", out, expected); assertEquals("too few bytes written", len, SwiftNativeFileSystem.getBytesWritten(out)); assertEquals("too few bytes uploaded", len, SwiftNativeFileSystem.getBytesUploaded(out)); //now we verify that the data comes back. If it //doesn't, it means that the ordering of the partitions //isn't right byte[] dest = readDataset(fs, path, len); //compare data SwiftTestUtils.compareByteArrays(src, dest, len); //finally, check the data FileStatus[] stats = fs.listStatus(path); assertEquals("wrong entry count in " + SwiftTestUtils.dumpStats(path.toString(), stats), expected, stats.length); }
Example #8
Source File: TestSwiftFileSystemPartitionedUploads.java From big-c with Apache License 2.0 | 5 votes |
/** * Test sticks up a very large partitioned file and verifies that * it comes back unchanged. * @throws Throwable */ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT) public void testManyPartitionedFile() throws Throwable { final Path path = new Path("/test/testManyPartitionedFile"); int len = PART_SIZE_BYTES * 15; final byte[] src = SwiftTestUtils.dataset(len, 32, 144); FSDataOutputStream out = fs.create(path, false, getBufferSize(), (short) 1, BLOCK_SIZE); out.write(src, 0, src.length); int expected = getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true); out.close(); assertPartitionsWritten("write completed", out, expected); assertEquals("too few bytes written", len, SwiftNativeFileSystem.getBytesWritten(out)); assertEquals("too few bytes uploaded", len, SwiftNativeFileSystem.getBytesUploaded(out)); //now we verify that the data comes back. If it //doesn't, it means that the ordering of the partitions //isn't right byte[] dest = readDataset(fs, path, len); //compare data SwiftTestUtils.compareByteArrays(src, dest, len); //finally, check the data FileStatus[] stats = fs.listStatus(path); assertEquals("wrong entry count in " + SwiftTestUtils.dumpStats(path.toString(), stats), expected, stats.length); }
Example #9
Source File: TestSwiftFileSystemBasicOps.java From big-c with Apache License 2.0 | 5 votes |
private void deleteR(SwiftNativeFileSystem fs, Path path) { try { if (!fs.delete(path, true)) { LOG.warn("Failed to delete " + path); } } catch (IOException e) { LOG.warn("deleting " + path, e); } }
Example #10
Source File: TestSwiftFileSystemBasicOps.java From big-c with Apache License 2.0 | 5 votes |
private void delete(SwiftNativeFileSystem fs, Path path) { try { if (!fs.delete(path, false)) { LOG.warn("Failed to delete " + path); } } catch (IOException e) { LOG.warn("deleting " + path, e); } }
Example #11
Source File: TestSwiftFileSystemContract.java From hadoop with Apache License 2.0 | 4 votes |
protected SwiftNativeFileSystem createSwiftFS() throws IOException { SwiftNativeFileSystem swiftNativeFileSystem = new SwiftNativeFileSystem(); return swiftNativeFileSystem; }
Example #12
Source File: SwiftFileSystemBaseTest.java From sahara-extra with Apache License 2.0 | 4 votes |
protected SwiftNativeFileSystem createSwiftFS() throws IOException { SwiftNativeFileSystem swiftNativeFileSystem = new SwiftNativeFileSystem(); return swiftNativeFileSystem; }
Example #13
Source File: TestSwiftFileSystemExtendedContract.java From sahara-extra with Apache License 2.0 | 4 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testGetSchemeImplemented() throws Throwable { String scheme = fs.getScheme(); assertEquals(SwiftNativeFileSystem.SWIFT,scheme); }
Example #14
Source File: SwiftContract.java From big-c with Apache License 2.0 | 4 votes |
@Override public String getScheme() { return SwiftNativeFileSystem.SWIFT; }
Example #15
Source File: SwiftFileSystemBaseTest.java From big-c with Apache License 2.0 | 4 votes |
protected SwiftNativeFileSystem createSwiftFS() throws IOException { SwiftNativeFileSystem swiftNativeFileSystem = new SwiftNativeFileSystem(); return swiftNativeFileSystem; }
Example #16
Source File: TestSwiftFileSystemContract.java From big-c with Apache License 2.0 | 4 votes |
protected SwiftNativeFileSystem createSwiftFS() throws IOException { SwiftNativeFileSystem swiftNativeFileSystem = new SwiftNativeFileSystem(); return swiftNativeFileSystem; }
Example #17
Source File: TestSwiftFileSystemExtendedContract.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testGetSchemeImplemented() throws Throwable { String scheme = fs.getScheme(); assertEquals(SwiftNativeFileSystem.SWIFT,scheme); }
Example #18
Source File: SwiftContract.java From hadoop with Apache License 2.0 | 4 votes |
@Override public String getScheme() { return SwiftNativeFileSystem.SWIFT; }
Example #19
Source File: SwiftFileSystemBaseTest.java From hadoop with Apache License 2.0 | 4 votes |
protected SwiftNativeFileSystem createSwiftFS() throws IOException { SwiftNativeFileSystem swiftNativeFileSystem = new SwiftNativeFileSystem(); return swiftNativeFileSystem; }
Example #20
Source File: TestSwiftFileSystemExtendedContract.java From hadoop with Apache License 2.0 | 4 votes |
@Test(timeout = SWIFT_TEST_TIMEOUT) public void testGetSchemeImplemented() throws Throwable { String scheme = fs.getScheme(); assertEquals(SwiftNativeFileSystem.SWIFT,scheme); }
Example #21
Source File: SwiftFileSystemBaseTest.java From big-c with Apache License 2.0 | 2 votes |
/** * Get the number of partitions written from the Swift Native FS APIs * @param out output stream * @return the number of partitioned files written by the stream */ protected int getPartitionsWritten(FSDataOutputStream out) { return SwiftNativeFileSystem.getPartitionsWritten(out); }
Example #22
Source File: SwiftFileSystemBaseTest.java From big-c with Apache License 2.0 | 2 votes |
/** * Get the filesystem * @return the current FS */ public SwiftNativeFileSystem getFs() { return fs; }
Example #23
Source File: SwiftFileSystemBaseTest.java From sahara-extra with Apache License 2.0 | 2 votes |
/** * Get the filesystem * @return the current FS */ public SwiftNativeFileSystem getFs() { return fs; }
Example #24
Source File: SwiftFileSystemBaseTest.java From sahara-extra with Apache License 2.0 | 2 votes |
/** * Get the number of partitions written from the Swift Native FS APIs * @param out output stream * @return the number of partitioned files written by the stream */ protected int getPartitionsWritten(FSDataOutputStream out) { return SwiftNativeFileSystem.getPartitionsWritten(out); }
Example #25
Source File: SwiftFileSystemBaseTest.java From hadoop with Apache License 2.0 | 2 votes |
/** * Get the number of partitions written from the Swift Native FS APIs * @param out output stream * @return the number of partitioned files written by the stream */ protected int getPartitionsWritten(FSDataOutputStream out) { return SwiftNativeFileSystem.getPartitionsWritten(out); }
Example #26
Source File: SwiftFileSystemBaseTest.java From hadoop with Apache License 2.0 | 2 votes |
/** * Get the filesystem * @return the current FS */ public SwiftNativeFileSystem getFs() { return fs; }