org.apache.hadoop.fs.ByteBufferReadable Java Examples
The following examples show how to use
org.apache.hadoop.fs.ByteBufferReadable.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CryptoInputStream.java From hadoop with Apache License 2.0 | 6 votes |
public CryptoInputStream(InputStream in, CryptoCodec codec, int bufferSize, byte[] key, byte[] iv, long streamOffset) throws IOException { super(in); CryptoStreamUtils.checkCodec(codec); this.bufferSize = CryptoStreamUtils.checkBufferSize(codec, bufferSize); this.codec = codec; this.key = key.clone(); this.initIV = iv.clone(); this.iv = iv.clone(); this.streamOffset = streamOffset; isByteBufferReadable = in instanceof ByteBufferReadable; isReadableByteChannel = in instanceof ReadableByteChannel; inBuffer = ByteBuffer.allocateDirect(this.bufferSize); outBuffer = ByteBuffer.allocateDirect(this.bufferSize); decryptor = getDecryptor(); resetStreamOffset(streamOffset); }
Example #2
Source File: CryptoInputStream.java From big-c with Apache License 2.0 | 6 votes |
public CryptoInputStream(InputStream in, CryptoCodec codec, int bufferSize, byte[] key, byte[] iv, long streamOffset) throws IOException { super(in); CryptoStreamUtils.checkCodec(codec); this.bufferSize = CryptoStreamUtils.checkBufferSize(codec, bufferSize); this.codec = codec; this.key = key.clone(); this.initIV = iv.clone(); this.iv = iv.clone(); this.streamOffset = streamOffset; isByteBufferReadable = in instanceof ByteBufferReadable; isReadableByteChannel = in instanceof ReadableByteChannel; inBuffer = ByteBuffer.allocateDirect(this.bufferSize); outBuffer = ByteBuffer.allocateDirect(this.bufferSize); decryptor = getDecryptor(); resetStreamOffset(streamOffset); }
Example #3
Source File: IOUtil.java From indexr with Apache License 2.0 | 6 votes |
public static void readFully(FSDataInputStream reader, long offset, ByteBuffer buffer) throws IOException { if (offset >= 0) { reader.seek(offset); } InputStream is = reader.getWrappedStream(); if (!(is instanceof ByteBufferReadable)) { logger.trace("Using read bytes method"); byte[] bytes = new byte[buffer.remaining()]; reader.readFully(bytes); buffer.put(bytes); } else { while (buffer.hasRemaining()) { int pos = buffer.position(); int rt = reader.read(buffer); if (rt < 0) { throw new IOException("End of stream"); } buffer.position(pos + rt); } } Preconditions.checkState(!buffer.hasRemaining()); }
Example #4
Source File: TextInput.java From Bats with Apache License 2.0 | 5 votes |
/** * Creates a new instance with the mandatory characters for handling newlines * transparently. lineSeparator the sequence of characters that represent a * newline, as defined in {@link Format#getLineSeparator()} * normalizedLineSeparator the normalized newline character (as defined in * {@link Format#getNormalizedNewline()}) that is used to replace any * lineSeparator sequence found in the input. */ public TextInput(TextParsingSettings settings, InputStream input, DrillBuf readBuffer, long startPos, long endPos) { this.lineSeparator = settings.getNewLineDelimiter(); byte normalizedLineSeparator = settings.getNormalizedNewLine(); Preconditions.checkArgument(input instanceof Seekable, "Text input only supports an InputStream that supports Seekable."); boolean isCompressed = input instanceof CompressionInputStream; Preconditions.checkArgument(!isCompressed || startPos == 0, "Cannot use split on compressed stream."); // splits aren't allowed with compressed data. The split length will be the compressed size which means we'll normally end prematurely. if (isCompressed && endPos > 0) { endPos = Long.MAX_VALUE; } this.input = input; this.seekable = (Seekable) input; this.settings = settings; if (input instanceof FSDataInputStream) { this.inputFS = (FSDataInputStream) input; this.bufferReadable = inputFS.getWrappedStream() instanceof ByteBufferReadable; } else { this.inputFS = null; this.bufferReadable = false; } this.startPos = startPos; this.endPos = endPos; this.normalizedLineSeparator = normalizedLineSeparator; this.buffer = readBuffer; this.bStart = buffer.memoryAddress(); this.bStartMinus1 = bStart -1; this.underlyingBuffer = buffer.nioBuffer(0, buffer.capacity()); }
Example #5
Source File: CryptoStreamsTestBase.java From hadoop with Apache License 2.0 | 5 votes |
private void byteBufferReadCheck(InputStream in, ByteBuffer buf, int bufPos) throws Exception { buf.position(bufPos); int n = ((ByteBufferReadable) in).read(buf); Assert.assertEquals(bufPos + n, buf.position()); byte[] readData = new byte[n]; buf.rewind(); buf.position(bufPos); buf.get(readData); byte[] expectedData = new byte[n]; System.arraycopy(data, 0, expectedData, 0, n); Assert.assertArrayEquals(readData, expectedData); }
Example #6
Source File: FSDataInputStreamWrapper.java From dremio-oss with Apache License 2.0 | 5 votes |
public static FSInputStream of(FSDataInputStream in) throws IOException { if (in.getWrappedStream() instanceof ByteBufferReadable) { return new FSDataInputStreamWrapper(in); } return new ByteArrayFSInputStream(in); }
Example #7
Source File: TestFSDataInputStreamWrapper.java From dremio-oss with Apache License 2.0 | 5 votes |
@Test public void test() throws Exception { Class<?> byteBufferPositionedReadableClass = getClass("org.apache.hadoop.fs.ByteBufferPositionedReadable"); assumeNonMaprProfile(); final IOException ioException = new IOException("test io exception"); final FSError fsError = newFSError(ioException); FSDataInputStream fdis = new FSDataInputStream(mock(InputStream.class, withSettings().extraInterfaces(Seekable.class, byteBufferPositionedReadableClass == null ? AutoCloseable.class : byteBufferPositionedReadableClass, PositionedReadable.class, ByteBufferReadable.class).defaultAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { throw fsError; } }))); FSInputStream fdisw = FSDataInputStreamWrapper.of(fdis); Object[] params = getDummyArguments(method); try { method.invoke(fdisw, params); } catch(InvocationTargetException e) { if (byteBufferPositionedReadableClass == null) { assertThat(e.getTargetException(), anyOf(is(instanceOf(IOException.class)), is(instanceOf(UnsupportedOperationException.class)))); } else { assertThat(e.getTargetException(), is(instanceOf(IOException.class))); } if (e.getTargetException() instanceof IOException) { assertThat((IOException) e.getTargetException(), is(sameInstance(ioException))); } } }
Example #8
Source File: FSDataInputStreamWrapper.java From dremio-oss with Apache License 2.0 | 5 votes |
public static FSInputStream of(FSDataInputStream in) throws IOException { if (in.getWrappedStream() instanceof ByteBufferReadable) { return new FSDataInputStreamWrapper(in); } return new ByteArrayFSInputStream(in); }
Example #9
Source File: FSDataInputStreamWrapper.java From dremio-oss with Apache License 2.0 | 5 votes |
public static FSInputStream of(FSDataInputStream in) throws IOException { if (in.getWrappedStream() instanceof ByteBufferReadable) { return new FSDataInputStreamWrapper(in); } return new ByteArrayFSInputStream(in); }
Example #10
Source File: CryptoStreamsTestBase.java From big-c with Apache License 2.0 | 5 votes |
private void byteBufferReadCheck(InputStream in, ByteBuffer buf, int bufPos) throws Exception { buf.position(bufPos); int n = ((ByteBufferReadable) in).read(buf); Assert.assertEquals(bufPos + n, buf.position()); byte[] readData = new byte[n]; buf.rewind(); buf.position(bufPos); buf.get(readData); byte[] expectedData = new byte[n]; System.arraycopy(data, 0, expectedData, 0, n); Assert.assertArrayEquals(readData, expectedData); }
Example #11
Source File: FSDataInputChannel.java From tajo with Apache License 2.0 | 5 votes |
public FSDataInputChannel(FSDataInputStream inputStream) throws IOException { if (inputStream.getWrappedStream() instanceof ByteBufferReadable) { this.isDirectRead = true; } else { /* LocalFileSystem, S3 does not support ByteBufferReadable */ this.channel = Channels.newChannel(inputStream); } this.inputStream = inputStream; this.size = inputStream.getPos() + inputStream.available(); }
Example #12
Source File: ByteBufInputChannel.java From tajo with Apache License 2.0 | 5 votes |
public ByteBufInputChannel(InputStream inputStream) { if (inputStream instanceof ByteBufferReadable) { this.byteBufferReadable = (ByteBufferReadable) inputStream; } else { this.channel = Channels.newChannel(inputStream); } this.inputStream = inputStream; }
Example #13
Source File: BlockIOUtils.java From hbase with Apache License 2.0 | 5 votes |
public static boolean isByteBufferReadable(FSDataInputStream is) { InputStream cur = is.getWrappedStream(); for (;;) { if ((cur instanceof FSDataInputStream)) { cur = ((FSDataInputStream) cur).getWrappedStream(); } else { break; } } return cur instanceof ByteBufferReadable; }
Example #14
Source File: DirectBufInputStream.java From Bats with Apache License 2.0 | 4 votes |
protected void checkStreamSupportsByteBuffer() throws UnsupportedOperationException { // Check input stream supports ByteBuffer if (!(in instanceof ByteBufferReadable)) { throw new UnsupportedOperationException("The input stream is not ByteBuffer readable."); } }
Example #15
Source File: CryptoStreamsTestBase.java From hadoop with Apache License 2.0 | 4 votes |
@Test(timeout=120000) public void testCombinedOp() throws Exception { OutputStream out = getOutputStream(defaultBufferSize); writeData(out); final int len1 = dataLen / 8; final int len2 = dataLen / 10; InputStream in = getInputStream(defaultBufferSize); // Read len1 data. byte[] readData = new byte[len1]; readAll(in, readData, 0, len1); byte[] expectedData = new byte[len1]; System.arraycopy(data, 0, expectedData, 0, len1); Assert.assertArrayEquals(readData, expectedData); long pos = ((Seekable) in).getPos(); Assert.assertEquals(len1, pos); // Seek forward len2 ((Seekable) in).seek(pos + len2); // Skip forward len2 long n = in.skip(len2); Assert.assertEquals(len2, n); // Pos: 1/4 dataLen positionedReadCheck(in , dataLen / 4); // Pos should be len1 + len2 + len2 pos = ((Seekable) in).getPos(); Assert.assertEquals(len1 + len2 + len2, pos); // Read forward len1 ByteBuffer buf = ByteBuffer.allocate(len1); int nRead = ((ByteBufferReadable) in).read(buf); Assert.assertEquals(nRead, buf.position()); readData = new byte[nRead]; buf.rewind(); buf.get(readData); expectedData = new byte[nRead]; System.arraycopy(data, (int)pos, expectedData, 0, nRead); Assert.assertArrayEquals(readData, expectedData); long lastPos = pos; // Pos should be lastPos + nRead pos = ((Seekable) in).getPos(); Assert.assertEquals(lastPos + nRead, pos); // Pos: 1/3 dataLen positionedReadCheck(in , dataLen / 3); // Read forward len1 readData = new byte[len1]; readAll(in, readData, 0, len1); expectedData = new byte[len1]; System.arraycopy(data, (int)pos, expectedData, 0, len1); Assert.assertArrayEquals(readData, expectedData); lastPos = pos; // Pos should be lastPos + len1 pos = ((Seekable) in).getPos(); Assert.assertEquals(lastPos + len1, pos); // Read forward len1 buf = ByteBuffer.allocate(len1); nRead = ((ByteBufferReadable) in).read(buf); Assert.assertEquals(nRead, buf.position()); readData = new byte[nRead]; buf.rewind(); buf.get(readData); expectedData = new byte[nRead]; System.arraycopy(data, (int)pos, expectedData, 0, nRead); Assert.assertArrayEquals(readData, expectedData); lastPos = pos; // Pos should be lastPos + nRead pos = ((Seekable) in).getPos(); Assert.assertEquals(lastPos + nRead, pos); // ByteBuffer read after EOF ((Seekable) in).seek(dataLen); buf.clear(); n = ((ByteBufferReadable) in).read(buf); Assert.assertEquals(n, -1); in.close(); }
Example #16
Source File: CryptoStreamsTestBase.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout=120000) public void testCombinedOp() throws Exception { OutputStream out = getOutputStream(defaultBufferSize); writeData(out); final int len1 = dataLen / 8; final int len2 = dataLen / 10; InputStream in = getInputStream(defaultBufferSize); // Read len1 data. byte[] readData = new byte[len1]; readAll(in, readData, 0, len1); byte[] expectedData = new byte[len1]; System.arraycopy(data, 0, expectedData, 0, len1); Assert.assertArrayEquals(readData, expectedData); long pos = ((Seekable) in).getPos(); Assert.assertEquals(len1, pos); // Seek forward len2 ((Seekable) in).seek(pos + len2); // Skip forward len2 long n = in.skip(len2); Assert.assertEquals(len2, n); // Pos: 1/4 dataLen positionedReadCheck(in , dataLen / 4); // Pos should be len1 + len2 + len2 pos = ((Seekable) in).getPos(); Assert.assertEquals(len1 + len2 + len2, pos); // Read forward len1 ByteBuffer buf = ByteBuffer.allocate(len1); int nRead = ((ByteBufferReadable) in).read(buf); Assert.assertEquals(nRead, buf.position()); readData = new byte[nRead]; buf.rewind(); buf.get(readData); expectedData = new byte[nRead]; System.arraycopy(data, (int)pos, expectedData, 0, nRead); Assert.assertArrayEquals(readData, expectedData); long lastPos = pos; // Pos should be lastPos + nRead pos = ((Seekable) in).getPos(); Assert.assertEquals(lastPos + nRead, pos); // Pos: 1/3 dataLen positionedReadCheck(in , dataLen / 3); // Read forward len1 readData = new byte[len1]; readAll(in, readData, 0, len1); expectedData = new byte[len1]; System.arraycopy(data, (int)pos, expectedData, 0, len1); Assert.assertArrayEquals(readData, expectedData); lastPos = pos; // Pos should be lastPos + len1 pos = ((Seekable) in).getPos(); Assert.assertEquals(lastPos + len1, pos); // Read forward len1 buf = ByteBuffer.allocate(len1); nRead = ((ByteBufferReadable) in).read(buf); Assert.assertEquals(nRead, buf.position()); readData = new byte[nRead]; buf.rewind(); buf.get(readData); expectedData = new byte[nRead]; System.arraycopy(data, (int)pos, expectedData, 0, nRead); Assert.assertArrayEquals(readData, expectedData); lastPos = pos; // Pos should be lastPos + nRead pos = ((Seekable) in).getPos(); Assert.assertEquals(lastPos + nRead, pos); // ByteBuffer read after EOF ((Seekable) in).seek(dataLen); buf.clear(); n = ((ByteBufferReadable) in).read(buf); Assert.assertEquals(n, -1); in.close(); }