org.apache.hadoop.hdfs.client.HdfsDataInputStream Java Examples

The following examples show how to use org.apache.hadoop.hdfs.client.HdfsDataInputStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ProxiedDFSClient.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
        throws IOException {
    final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
    if (feInfo != null) {
        // File is encrypted, wrap the stream in a crypto stream.
        // Currently only one version, so no special logic based on the version #
        getCryptoProtocolVersion(feInfo);
        final CryptoCodec codec = getCryptoCodec(getConfiguration(), feInfo);
        final KeyProvider.KeyVersion decrypted = decryptEncryptedDataEncryptionKey(dfsis, feInfo);
        final CryptoInputStream cryptoIn =
                new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
                        feInfo.getIV());
        return new HdfsDataInputStream(cryptoIn);
    } else {
        // No FileEncryptionInfo so no encryption.
        return new HdfsDataInputStream(dfsis);
    }
}
 
Example #2
Source File: DFSClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Wraps the stream in a CryptoInputStream if the underlying file is
 * encrypted.
 */
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
    throws IOException {
  final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
  if (feInfo != null) {
    // File is encrypted, wrap the stream in a crypto stream.
    // Currently only one version, so no special logic based on the version #
    getCryptoProtocolVersion(feInfo);
    final CryptoCodec codec = getCryptoCodec(conf, feInfo);
    final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
    final CryptoInputStream cryptoIn =
        new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
            feInfo.getIV());
    return new HdfsDataInputStream(cryptoIn);
  } else {
    // No FileEncryptionInfo so no encryption.
    return new HdfsDataInputStream(dfsis);
  }
}
 
Example #3
Source File: FSDataInputStreamWrapper.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void updateInputStreamStatistics(FSDataInputStream stream) {
  // If the underlying file system is HDFS, update read statistics upon close.
  if (stream instanceof HdfsDataInputStream) {
    /**
     * Because HDFS ReadStatistics is calculated per input stream, it is not
     * feasible to update the aggregated number in real time. Instead, the
     * metrics are updated when an input stream is closed.
     */
    HdfsDataInputStream hdfsDataInputStream = (HdfsDataInputStream)stream;
    synchronized (readStatistics) {
      readStatistics.totalBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalBytesRead();
      readStatistics.totalLocalBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalLocalBytesRead();
      readStatistics.totalShortCircuitBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalShortCircuitBytesRead();
      readStatistics.totalZeroCopyBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalZeroCopyBytesRead();
    }
  }
}
 
Example #4
Source File: TestReadWhileWriting.java    From big-c with Apache License 2.0 6 votes vote down vote up
static void checkFile(Path p, int expectedsize, final Configuration conf
    ) throws IOException, InterruptedException {
  //open the file with another user account
  final String username = UserGroupInformation.getCurrentUser().getShortUserName()
      + "_" + ++userCount;

  UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username, 
                               new String[] {"supergroup"});
  
  final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
  
  final HdfsDataInputStream in = (HdfsDataInputStream)fs.open(p);

  //Check visible length
  Assert.assertTrue(in.getVisibleLength() >= expectedsize);

  //Able to read?
  for(int i = 0; i < expectedsize; i++) {
    Assert.assertEquals((byte)i, (byte)in.read());  
  }

  in.close();
}
 
Example #5
Source File: TestReadWhileWriting.java    From hadoop with Apache License 2.0 6 votes vote down vote up
static void checkFile(Path p, int expectedsize, final Configuration conf
    ) throws IOException, InterruptedException {
  //open the file with another user account
  final String username = UserGroupInformation.getCurrentUser().getShortUserName()
      + "_" + ++userCount;

  UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username, 
                               new String[] {"supergroup"});
  
  final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
  
  final HdfsDataInputStream in = (HdfsDataInputStream)fs.open(p);

  //Check visible length
  Assert.assertTrue(in.getVisibleLength() >= expectedsize);

  //Able to read?
  for(int i = 0; i < expectedsize; i++) {
    Assert.assertEquals((byte)i, (byte)in.read());  
  }

  in.close();
}
 
Example #6
Source File: DFSClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Wraps the stream in a CryptoInputStream if the underlying file is
 * encrypted.
 */
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
    throws IOException {
  final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
  if (feInfo != null) {
    // File is encrypted, wrap the stream in a crypto stream.
    // Currently only one version, so no special logic based on the version #
    getCryptoProtocolVersion(feInfo);
    final CryptoCodec codec = getCryptoCodec(conf, feInfo);
    final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
    final CryptoInputStream cryptoIn =
        new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
            feInfo.getIV());
    return new HdfsDataInputStream(cryptoIn);
  } else {
    // No FileEncryptionInfo so no encryption.
    return new HdfsDataInputStream(dfsis);
  }
}
 
Example #7
Source File: Hdfs.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Override
public HdfsDataInputStream open(Path f, int bufferSize) 
    throws IOException, UnresolvedLinkException {
  final DFSInputStream dfsis = dfs.open(getUriPath(f),
    bufferSize, verifyChecksum);
  return dfs.createWrappedInputStream(dfsis);
}
 
Example #8
Source File: TestScrLazyPersistFiles.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR,
    new StorageType[]{RAM_DISK, DEFAULT},
    2 * BLOCK_SIZE - 1, true);  // 1 replica + delta, SCR read
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
  FSDataInputStream fis = fs.open(path);

  // Verify SCR read counters
  try {
    fis = fs.open(path);
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 
Example #9
Source File: TestShortCircuitLocalRead.java    From big-c with Apache License 2.0 5 votes vote down vote up
private boolean checkUnsupportedMethod(FileSystem fs, Path file,
                                         byte[] expected, int readOffset) throws IOException {
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
  IOUtils.skipFully(stm, readOffset);
  try {
    stm.read(actual);
  } catch(UnsupportedOperationException unex) {
    return true;
  }
  return false;
}
 
Example #10
Source File: TestWriteRead.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Open the file to read from begin to end. Then close the file. 
 * Return number of bytes read. 
 * Support both sequential read and position read.
 */
private long readData(String fname, byte[] buffer, long byteExpected, long beginPosition)
    throws IOException {
  long totalByteRead = 0;
  Path path = getFullyQualifiedPath(fname);

  FSDataInputStream in = null;
  try {
    in = openInputStream(path);

    long visibleLenFromReadStream = ((HdfsDataInputStream)in).getVisibleLength();

    if (visibleLenFromReadStream < byteExpected)
    {
      throw new IOException(visibleLenFromReadStream
          + " = visibleLenFromReadStream < bytesExpected= "
          + byteExpected);
    }

    totalByteRead = readUntilEnd(in, buffer, buffer.length, fname,
        beginPosition, visibleLenFromReadStream, positionReadOption);
    in.close();

    // reading more data than visibleLeng is OK, but not less
    if (totalByteRead + beginPosition < byteExpected ){
      throw new IOException("readData mismatch in byte read: expected=" 
          + byteExpected + " ; got " +  (totalByteRead + beginPosition));
    }
    return totalByteRead + beginPosition;

  } catch (IOException e) {
    throw new IOException("##### Caught Exception in readData. "
        + "Total Byte Read so far = " + totalByteRead + " beginPosition = "
        + beginPosition, e);
  } finally {
    if (in != null)
      in.close();
  }
}
 
Example #11
Source File: DFSTestUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  try {
    in.readByte();
    return in.getCurrentBlock();
  } finally {
    in.close();
  }
}
 
Example #12
Source File: WebHdfsHandler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void onOpen(ChannelHandlerContext ctx) throws IOException {
  final String nnId = params.namenodeId();
  final int bufferSize = params.bufferSize();
  final long offset = params.offset();
  final long length = params.length();

  DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  HttpHeaders headers = response.headers();
  // Allow the UI to access the file
  headers.set(ACCESS_CONTROL_ALLOW_METHODS, GET);
  headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
  headers.set(CONTENT_TYPE, APPLICATION_OCTET_STREAM);
  headers.set(CONNECTION, CLOSE);

  final DFSClient dfsclient = newDfsClient(nnId, conf);
  HdfsDataInputStream in = dfsclient.createWrappedInputStream(
    dfsclient.open(path, bufferSize, true));
  in.seek(offset);

  long contentLength = in.getVisibleLength() - offset;
  if (length >= 0) {
    contentLength = Math.min(contentLength, length);
  }
  final InputStream data;
  if (contentLength >= 0) {
    headers.set(CONTENT_LENGTH, contentLength);
    data = new LimitInputStream(in, contentLength);
  } else {
    data = in;
  }

  ctx.write(response);
  ctx.writeAndFlush(new ChunkedStream(data) {
    @Override
    public void close() throws Exception {
      super.close();
      dfsclient.close();
    }
  }).addListener(ChannelFutureListener.CLOSE);
}
 
Example #13
Source File: TestWriteRead.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Open the file to read from begin to end. Then close the file. 
 * Return number of bytes read. 
 * Support both sequential read and position read.
 */
private long readData(String fname, byte[] buffer, long byteExpected, long beginPosition)
    throws IOException {
  long totalByteRead = 0;
  Path path = getFullyQualifiedPath(fname);

  FSDataInputStream in = null;
  try {
    in = openInputStream(path);

    long visibleLenFromReadStream = ((HdfsDataInputStream)in).getVisibleLength();

    if (visibleLenFromReadStream < byteExpected)
    {
      throw new IOException(visibleLenFromReadStream
          + " = visibleLenFromReadStream < bytesExpected= "
          + byteExpected);
    }

    totalByteRead = readUntilEnd(in, buffer, buffer.length, fname,
        beginPosition, visibleLenFromReadStream, positionReadOption);
    in.close();

    // reading more data than visibleLeng is OK, but not less
    if (totalByteRead + beginPosition < byteExpected ){
      throw new IOException("readData mismatch in byte read: expected=" 
          + byteExpected + " ; got " +  (totalByteRead + beginPosition));
    }
    return totalByteRead + beginPosition;

  } catch (IOException e) {
    throw new IOException("##### Caught Exception in readData. "
        + "Total Byte Read so far = " + totalByteRead + " beginPosition = "
        + beginPosition, e);
  } finally {
    if (in != null)
      in.close();
  }
}
 
Example #14
Source File: Hdfs.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Override
public HdfsDataInputStream open(Path f, int bufferSize) 
    throws IOException, UnresolvedLinkException {
  final DFSInputStream dfsis = dfs.open(getUriPath(f),
    bufferSize, verifyChecksum);
  return dfs.createWrappedInputStream(dfsis);
}
 
Example #15
Source File: WebHdfsHandler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void onOpen(ChannelHandlerContext ctx) throws IOException {
  final String nnId = params.namenodeId();
  final int bufferSize = params.bufferSize();
  final long offset = params.offset();
  final long length = params.length();

  DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  HttpHeaders headers = response.headers();
  // Allow the UI to access the file
  headers.set(ACCESS_CONTROL_ALLOW_METHODS, GET);
  headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
  headers.set(CONTENT_TYPE, APPLICATION_OCTET_STREAM);
  headers.set(CONNECTION, CLOSE);

  final DFSClient dfsclient = newDfsClient(nnId, conf);
  HdfsDataInputStream in = dfsclient.createWrappedInputStream(
    dfsclient.open(path, bufferSize, true));
  in.seek(offset);

  long contentLength = in.getVisibleLength() - offset;
  if (length >= 0) {
    contentLength = Math.min(contentLength, length);
  }
  final InputStream data;
  if (contentLength >= 0) {
    headers.set(CONTENT_LENGTH, contentLength);
    data = new LimitInputStream(in, contentLength);
  } else {
    data = in;
  }

  ctx.write(response);
  ctx.writeAndFlush(new ChunkedStream(data) {
    @Override
    public void close() throws Exception {
      super.close();
      dfsclient.close();
    }
  }).addListener(ChannelFutureListener.CLOSE);
}
 
Example #16
Source File: DFSTestUtil.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  try {
    in.readByte();
    return in.getCurrentBlock();
  } finally {
    in.close();
  }
}
 
Example #17
Source File: TestShortCircuitLocalRead.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private boolean checkUnsupportedMethod(FileSystem fs, Path file,
                                         byte[] expected, int readOffset) throws IOException {
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
  IOUtils.skipFully(stm, readOffset);
  try {
    stm.read(actual);
  } catch(UnsupportedOperationException unex) {
    return true;
  }
  return false;
}
 
Example #18
Source File: TestScrLazyPersistFiles.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR,
    new StorageType[]{RAM_DISK, DEFAULT},
    2 * BLOCK_SIZE - 1, true);  // 1 replica + delta, SCR read
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
  FSDataInputStream fis = fs.open(path);

  // Verify SCR read counters
  try {
    fis = fs.open(path);
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 
Example #19
Source File: TestShortCircuitLocalRead.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 
Example #20
Source File: DFSTestUtil.java    From big-c with Apache License 2.0 4 votes vote down vote up
public static List<LocatedBlock> getAllBlocks(FSDataInputStream in)
    throws IOException {
  return ((HdfsDataInputStream) in).getAllBlocks();
}
 
Example #21
Source File: DFSTestUtil.java    From big-c with Apache License 2.0 4 votes vote down vote up
public static List<LocatedBlock> getAllBlocks(FileSystem fs, Path path)
    throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  return in.getAllBlocks();
}
 
Example #22
Source File: TestScrLazyPersistFiles.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Eviction of lazy persisted blocks with Short Circuit Read handle open
 * Note: the test uses faked RAM_DISK from physical disk.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRamDiskEvictionWithShortCircuitReadHandle()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
    (6 * BLOCK_SIZE -1), true);  // 5 replica + delta, SCR.
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
  Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
  final int SEED = 0xFADED;

  makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path1, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job.
  // However the block replica should not be evicted from RAM_DISK yet.
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  // No eviction should happen as the free ratio is below the threshold
  FSDataInputStream fis = fs.open(path1);
  try {
    // Keep and open read handle to path1 while creating path2
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);

    // Create the 2nd file that will trigger RAM_DISK eviction.
    makeTestFile(path2, BLOCK_SIZE * 2, true);
    ensureFileReplicasOnStorageType(path2, RAM_DISK);

    // Ensure path1 is still readable from the open SCR handle.
    fis.read(fis.getPos(), buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    IOUtils.closeQuietly(fis);
  }

  // After the open handle is closed, path1 should be evicted to DISK.
  triggerBlockReport();
  ensureFileReplicasOnStorageType(path1, DEFAULT);
}
 
Example #23
Source File: TestShortCircuitLocalRead.java    From big-c with Apache License 2.0 4 votes vote down vote up
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 
Example #24
Source File: DFSTestUtil.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public static List<LocatedBlock> getAllBlocks(FileSystem fs, Path path)
    throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  return in.getAllBlocks();
}
 
Example #25
Source File: DFSTestUtil.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public static List<LocatedBlock> getAllBlocks(FSDataInputStream in)
    throws IOException {
  return ((HdfsDataInputStream) in).getAllBlocks();
}
 
Example #26
Source File: TestScrLazyPersistFiles.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Eviction of lazy persisted blocks with Short Circuit Read handle open
 * Note: the test uses faked RAM_DISK from physical disk.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRamDiskEvictionWithShortCircuitReadHandle()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
    (6 * BLOCK_SIZE -1), true);  // 5 replica + delta, SCR.
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
  Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
  final int SEED = 0xFADED;

  makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path1, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job.
  // However the block replica should not be evicted from RAM_DISK yet.
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  // No eviction should happen as the free ratio is below the threshold
  FSDataInputStream fis = fs.open(path1);
  try {
    // Keep and open read handle to path1 while creating path2
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);

    // Create the 2nd file that will trigger RAM_DISK eviction.
    makeTestFile(path2, BLOCK_SIZE * 2, true);
    ensureFileReplicasOnStorageType(path2, RAM_DISK);

    // Ensure path1 is still readable from the open SCR handle.
    fis.read(fis.getPos(), buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    IOUtils.closeQuietly(fis);
  }

  // After the open handle is closed, path1 should be evicted to DISK.
  triggerBlockReport();
  ensureFileReplicasOnStorageType(path1, DEFAULT);
}
 
Example #27
Source File: TestDecommission.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Verify that the number of replicas are as expected for each block in
 * the given file.
 * For blocks with a decommissioned node, verify that their replication
 * is 1 more than what is specified.
 * For blocks without decommissioned nodes, verify their replication is
 * equal to what is specified.
 * 
 * @param downnode - if null, there is no decommissioned node for this file.
 * @return - null if no failure found, else an error message string.
 */
private static String checkFile(FileSystem fileSys, Path name, int repl,
  String downnode, int numDatanodes) throws IOException {
  boolean isNodeDown = (downnode != null);
  // need a raw stream
  assertTrue("Not HDFS:"+fileSys.getUri(),
      fileSys instanceof DistributedFileSystem);
  HdfsDataInputStream dis = (HdfsDataInputStream)
      fileSys.open(name);
  Collection<LocatedBlock> dinfo = dis.getAllBlocks();
  for (LocatedBlock blk : dinfo) { // for each block
    int hasdown = 0;
    DatanodeInfo[] nodes = blk.getLocations();
    for (int j = 0; j < nodes.length; j++) { // for each replica
      if (isNodeDown && nodes[j].getXferAddr().equals(downnode)) {
        hasdown++;
        //Downnode must actually be decommissioned
        if (!nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is given as downnode, " +
            "but is not decommissioned";
        }
        //Decommissioned node (if any) should only be last node in list.
        if (j != nodes.length - 1) {
          return "For block " + blk.getBlock() + " decommissioned node "
            + nodes[j] + " was not last node in list: "
            + (j + 1) + " of " + nodes.length;
        }
        LOG.info("Block " + blk.getBlock() + " replica on " +
          nodes[j] + " is decommissioned.");
      } else {
        //Non-downnodes must not be decommissioned
        if (nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is unexpectedly decommissioned";
        }
      }
    }

    LOG.info("Block " + blk.getBlock() + " has " + hasdown
      + " decommissioned replica.");
    if(Math.min(numDatanodes, repl+hasdown) != nodes.length) {
      return "Wrong number of replicas for block " + blk.getBlock() +
        ": " + nodes.length + ", expected " +
        Math.min(numDatanodes, repl+hasdown);
    }
  }
  return null;
}
 
Example #28
Source File: FileSystemTracerTest.java    From garmadon with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, URISyntaxException, InstantiationException {
    classLoader = new ByteArrayClassLoader.ChildFirst(FileSystemTracerTest.class.getClassLoader(),
        ClassFileExtraction.of(
            Tracer.class,
            MethodTracer.class,
            FileSystemTracer.class,
            FileSystemTracer.WriteTracer.class,
            FileSystemTracer.ReadTracer.class,
            FileSystemTracer.AddBlockTracer.class,
            FileSystemTracer.ListStatusTracer.class,
            FileSystemTracer.GetContentSummaryTracer.class,
            FileSystemTracer.RenameTracer.class,
            FileSystemTracer.DeleteTracer.class,
            DFSClient.class,
            DFSClient.Conf.class,
            ClientContext.class,
            DistributedFileSystem.class,
            DomainSocketFactory.class,
            DFSInputStream.class,
            DFSOutputStream.class,
            HdfsDataInputStream.class,
            HdfsDataOutputStream.class,
            ClientNamenodeProtocolTranslatorPB.class,
            Class.forName("org.apache.hadoop.hdfs.DFSOpsCountStatistics"),
            Class.forName("org.apache.hadoop.hdfs.BlockReaderLocal"),
            Class.forName(DFSOutputStream.class.getName() + "$Packet"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$1"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$2"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$ResponseProcessor"),
            Class.forName(DistributedFileSystem.class.getName() + "$1"),
            Class.forName(DistributedFileSystem.class.getName() + "$4"),
            Class.forName(DistributedFileSystem.class.getName() + "$7"),
            Class.forName(DistributedFileSystem.class.getName() + "$13"),
            Class.forName(DistributedFileSystem.class.getName() + "$14"),
            Class.forName(DistributedFileSystem.class.getName() + "$16"),
            Class.forName(DistributedFileSystem.class.getName() + "$19"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$1"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$Factory"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$Factory$Key"),
            Class.forName("org.apache.hadoop.fs.Hdfs"),
            HdfsDataOutputStream.class
        ),
        ByteArrayClassLoader.PersistenceHandler.MANIFEST);

    argument = ArgumentCaptor.forClass(DataAccessEventProtos.FsEvent.class);

    eventHandler = mock(BiConsumer.class);

    ReflectionHelper.setField(null, classLoader.loadClass(FileSystemTracer.class.getName()), "eventHandler", eventHandler);
    ReflectionHelper.setField(conf, Configuration.class, "classLoader", classLoader);
    assertThat(ByteBuddyAgent.install(), instanceOf(Instrumentation.class));

    new FileSystemTracer.WriteTracer().installOnByteBuddyAgent();
    new FileSystemTracer.ReadTracer().installOnByteBuddyAgent();
    new FileSystemTracer.ListStatusTracer().installOnByteBuddyAgent();
    new FileSystemTracer.GetContentSummaryTracer().installOnByteBuddyAgent();
    new FileSystemTracer.DeleteTracer().installOnByteBuddyAgent();
    new FileSystemTracer.RenameTracer().installOnByteBuddyAgent();

    // MiniDfsCluster
    File baseDir = new File("./target/hdfs/test").getAbsoluteFile();
    FileUtil.fullyDelete(baseDir);
    conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf)
        .simulatedCapacities(new long[] {10240000L});
    hdfsCluster = builder.build();
    hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort();

    initDFS();
}
 
Example #29
Source File: TestDecommission.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Verify that the number of replicas are as expected for each block in
 * the given file.
 * For blocks with a decommissioned node, verify that their replication
 * is 1 more than what is specified.
 * For blocks without decommissioned nodes, verify their replication is
 * equal to what is specified.
 * 
 * @param downnode - if null, there is no decommissioned node for this file.
 * @return - null if no failure found, else an error message string.
 */
private static String checkFile(FileSystem fileSys, Path name, int repl,
  String downnode, int numDatanodes) throws IOException {
  boolean isNodeDown = (downnode != null);
  // need a raw stream
  assertTrue("Not HDFS:"+fileSys.getUri(),
      fileSys instanceof DistributedFileSystem);
  HdfsDataInputStream dis = (HdfsDataInputStream)
      fileSys.open(name);
  Collection<LocatedBlock> dinfo = dis.getAllBlocks();
  for (LocatedBlock blk : dinfo) { // for each block
    int hasdown = 0;
    DatanodeInfo[] nodes = blk.getLocations();
    for (int j = 0; j < nodes.length; j++) { // for each replica
      if (isNodeDown && nodes[j].getXferAddr().equals(downnode)) {
        hasdown++;
        //Downnode must actually be decommissioned
        if (!nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is given as downnode, " +
            "but is not decommissioned";
        }
        //Decommissioned node (if any) should only be last node in list.
        if (j != nodes.length - 1) {
          return "For block " + blk.getBlock() + " decommissioned node "
            + nodes[j] + " was not last node in list: "
            + (j + 1) + " of " + nodes.length;
        }
        LOG.info("Block " + blk.getBlock() + " replica on " +
          nodes[j] + " is decommissioned.");
      } else {
        //Non-downnodes must not be decommissioned
        if (nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is unexpectedly decommissioned";
        }
      }
    }

    LOG.info("Block " + blk.getBlock() + " has " + hasdown
      + " decommissioned replica.");
    if(Math.min(numDatanodes, repl+hasdown) != nodes.length) {
      return "Wrong number of replicas for block " + blk.getBlock() +
        ": " + nodes.length + ", expected " +
        Math.min(numDatanodes, repl+hasdown);
    }
  }
  return null;
}