Java Code Examples for org.apache.hadoop.hdfs.client.HdfsDataInputStream

The following examples show how to use org.apache.hadoop.hdfs.client.HdfsDataInputStream. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: DFSClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Wraps the stream in a CryptoInputStream if the underlying file is
 * encrypted.
 */
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
    throws IOException {
  final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
  if (feInfo != null) {
    // File is encrypted, wrap the stream in a crypto stream.
    // Currently only one version, so no special logic based on the version #
    getCryptoProtocolVersion(feInfo);
    final CryptoCodec codec = getCryptoCodec(conf, feInfo);
    final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
    final CryptoInputStream cryptoIn =
        new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
            feInfo.getIV());
    return new HdfsDataInputStream(cryptoIn);
  } else {
    // No FileEncryptionInfo so no encryption.
    return new HdfsDataInputStream(dfsis);
  }
}
 
Example 2
Source Project: hadoop   Source File: TestReadWhileWriting.java    License: Apache License 2.0 6 votes vote down vote up
static void checkFile(Path p, int expectedsize, final Configuration conf
    ) throws IOException, InterruptedException {
  //open the file with another user account
  final String username = UserGroupInformation.getCurrentUser().getShortUserName()
      + "_" + ++userCount;

  UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username, 
                               new String[] {"supergroup"});
  
  final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
  
  final HdfsDataInputStream in = (HdfsDataInputStream)fs.open(p);

  //Check visible length
  Assert.assertTrue(in.getVisibleLength() >= expectedsize);

  //Able to read?
  for(int i = 0; i < expectedsize; i++) {
    Assert.assertEquals((byte)i, (byte)in.read());  
  }

  in.close();
}
 
Example 3
Source Project: big-c   Source File: DFSClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Wraps the stream in a CryptoInputStream if the underlying file is
 * encrypted.
 */
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
    throws IOException {
  final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
  if (feInfo != null) {
    // File is encrypted, wrap the stream in a crypto stream.
    // Currently only one version, so no special logic based on the version #
    getCryptoProtocolVersion(feInfo);
    final CryptoCodec codec = getCryptoCodec(conf, feInfo);
    final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
    final CryptoInputStream cryptoIn =
        new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
            feInfo.getIV());
    return new HdfsDataInputStream(cryptoIn);
  } else {
    // No FileEncryptionInfo so no encryption.
    return new HdfsDataInputStream(dfsis);
  }
}
 
Example 4
Source Project: big-c   Source File: TestReadWhileWriting.java    License: Apache License 2.0 6 votes vote down vote up
static void checkFile(Path p, int expectedsize, final Configuration conf
    ) throws IOException, InterruptedException {
  //open the file with another user account
  final String username = UserGroupInformation.getCurrentUser().getShortUserName()
      + "_" + ++userCount;

  UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username, 
                               new String[] {"supergroup"});
  
  final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
  
  final HdfsDataInputStream in = (HdfsDataInputStream)fs.open(p);

  //Check visible length
  Assert.assertTrue(in.getVisibleLength() >= expectedsize);

  //Able to read?
  for(int i = 0; i < expectedsize; i++) {
    Assert.assertEquals((byte)i, (byte)in.read());  
  }

  in.close();
}
 
Example 5
Source Project: hbase   Source File: FSDataInputStreamWrapper.java    License: Apache License 2.0 6 votes vote down vote up
private void updateInputStreamStatistics(FSDataInputStream stream) {
  // If the underlying file system is HDFS, update read statistics upon close.
  if (stream instanceof HdfsDataInputStream) {
    /**
     * Because HDFS ReadStatistics is calculated per input stream, it is not
     * feasible to update the aggregated number in real time. Instead, the
     * metrics are updated when an input stream is closed.
     */
    HdfsDataInputStream hdfsDataInputStream = (HdfsDataInputStream)stream;
    synchronized (readStatistics) {
      readStatistics.totalBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalBytesRead();
      readStatistics.totalLocalBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalLocalBytesRead();
      readStatistics.totalShortCircuitBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalShortCircuitBytesRead();
      readStatistics.totalZeroCopyBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalZeroCopyBytesRead();
    }
  }
}
 
Example 6
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
        throws IOException {
    final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
    if (feInfo != null) {
        // File is encrypted, wrap the stream in a crypto stream.
        // Currently only one version, so no special logic based on the version #
        getCryptoProtocolVersion(feInfo);
        final CryptoCodec codec = getCryptoCodec(getConfiguration(), feInfo);
        final KeyProvider.KeyVersion decrypted = decryptEncryptedDataEncryptionKey(dfsis, feInfo);
        final CryptoInputStream cryptoIn =
                new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
                        feInfo.getIV());
        return new HdfsDataInputStream(cryptoIn);
    } else {
        // No FileEncryptionInfo so no encryption.
        return new HdfsDataInputStream(dfsis);
    }
}
 
Example 7
Source Project: hadoop   Source File: Hdfs.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Override
public HdfsDataInputStream open(Path f, int bufferSize) 
    throws IOException, UnresolvedLinkException {
  final DFSInputStream dfsis = dfs.open(getUriPath(f),
    bufferSize, verifyChecksum);
  return dfs.createWrappedInputStream(dfsis);
}
 
Example 8
Source Project: hadoop   Source File: WebHdfsHandler.java    License: Apache License 2.0 5 votes vote down vote up
private void onOpen(ChannelHandlerContext ctx) throws IOException {
  final String nnId = params.namenodeId();
  final int bufferSize = params.bufferSize();
  final long offset = params.offset();
  final long length = params.length();

  DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  HttpHeaders headers = response.headers();
  // Allow the UI to access the file
  headers.set(ACCESS_CONTROL_ALLOW_METHODS, GET);
  headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
  headers.set(CONTENT_TYPE, APPLICATION_OCTET_STREAM);
  headers.set(CONNECTION, CLOSE);

  final DFSClient dfsclient = newDfsClient(nnId, conf);
  HdfsDataInputStream in = dfsclient.createWrappedInputStream(
    dfsclient.open(path, bufferSize, true));
  in.seek(offset);

  long contentLength = in.getVisibleLength() - offset;
  if (length >= 0) {
    contentLength = Math.min(contentLength, length);
  }
  final InputStream data;
  if (contentLength >= 0) {
    headers.set(CONTENT_LENGTH, contentLength);
    data = new LimitInputStream(in, contentLength);
  } else {
    data = in;
  }

  ctx.write(response);
  ctx.writeAndFlush(new ChunkedStream(data) {
    @Override
    public void close() throws Exception {
      super.close();
      dfsclient.close();
    }
  }).addListener(ChannelFutureListener.CLOSE);
}
 
Example 9
Source Project: hadoop   Source File: DFSTestUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  try {
    in.readByte();
    return in.getCurrentBlock();
  } finally {
    in.close();
  }
}
 
Example 10
Source Project: hadoop   Source File: TestWriteRead.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Open the file to read from begin to end. Then close the file. 
 * Return number of bytes read. 
 * Support both sequential read and position read.
 */
private long readData(String fname, byte[] buffer, long byteExpected, long beginPosition)
    throws IOException {
  long totalByteRead = 0;
  Path path = getFullyQualifiedPath(fname);

  FSDataInputStream in = null;
  try {
    in = openInputStream(path);

    long visibleLenFromReadStream = ((HdfsDataInputStream)in).getVisibleLength();

    if (visibleLenFromReadStream < byteExpected)
    {
      throw new IOException(visibleLenFromReadStream
          + " = visibleLenFromReadStream < bytesExpected= "
          + byteExpected);
    }

    totalByteRead = readUntilEnd(in, buffer, buffer.length, fname,
        beginPosition, visibleLenFromReadStream, positionReadOption);
    in.close();

    // reading more data than visibleLeng is OK, but not less
    if (totalByteRead + beginPosition < byteExpected ){
      throw new IOException("readData mismatch in byte read: expected=" 
          + byteExpected + " ; got " +  (totalByteRead + beginPosition));
    }
    return totalByteRead + beginPosition;

  } catch (IOException e) {
    throw new IOException("##### Caught Exception in readData. "
        + "Total Byte Read so far = " + totalByteRead + " beginPosition = "
        + beginPosition, e);
  } finally {
    if (in != null)
      in.close();
  }
}
 
Example 11
Source Project: hadoop   Source File: TestShortCircuitLocalRead.java    License: Apache License 2.0 5 votes vote down vote up
private boolean checkUnsupportedMethod(FileSystem fs, Path file,
                                         byte[] expected, int readOffset) throws IOException {
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
  IOUtils.skipFully(stm, readOffset);
  try {
    stm.read(actual);
  } catch(UnsupportedOperationException unex) {
    return true;
  }
  return false;
}
 
Example 12
Source Project: hadoop   Source File: TestScrLazyPersistFiles.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR,
    new StorageType[]{RAM_DISK, DEFAULT},
    2 * BLOCK_SIZE - 1, true);  // 1 replica + delta, SCR read
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
  FSDataInputStream fis = fs.open(path);

  // Verify SCR read counters
  try {
    fis = fs.open(path);
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 
Example 13
Source Project: big-c   Source File: Hdfs.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Override
public HdfsDataInputStream open(Path f, int bufferSize) 
    throws IOException, UnresolvedLinkException {
  final DFSInputStream dfsis = dfs.open(getUriPath(f),
    bufferSize, verifyChecksum);
  return dfs.createWrappedInputStream(dfsis);
}
 
Example 14
Source Project: big-c   Source File: WebHdfsHandler.java    License: Apache License 2.0 5 votes vote down vote up
private void onOpen(ChannelHandlerContext ctx) throws IOException {
  final String nnId = params.namenodeId();
  final int bufferSize = params.bufferSize();
  final long offset = params.offset();
  final long length = params.length();

  DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  HttpHeaders headers = response.headers();
  // Allow the UI to access the file
  headers.set(ACCESS_CONTROL_ALLOW_METHODS, GET);
  headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
  headers.set(CONTENT_TYPE, APPLICATION_OCTET_STREAM);
  headers.set(CONNECTION, CLOSE);

  final DFSClient dfsclient = newDfsClient(nnId, conf);
  HdfsDataInputStream in = dfsclient.createWrappedInputStream(
    dfsclient.open(path, bufferSize, true));
  in.seek(offset);

  long contentLength = in.getVisibleLength() - offset;
  if (length >= 0) {
    contentLength = Math.min(contentLength, length);
  }
  final InputStream data;
  if (contentLength >= 0) {
    headers.set(CONTENT_LENGTH, contentLength);
    data = new LimitInputStream(in, contentLength);
  } else {
    data = in;
  }

  ctx.write(response);
  ctx.writeAndFlush(new ChunkedStream(data) {
    @Override
    public void close() throws Exception {
      super.close();
      dfsclient.close();
    }
  }).addListener(ChannelFutureListener.CLOSE);
}
 
Example 15
Source Project: big-c   Source File: DFSTestUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  try {
    in.readByte();
    return in.getCurrentBlock();
  } finally {
    in.close();
  }
}
 
Example 16
Source Project: big-c   Source File: TestWriteRead.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Open the file to read from begin to end. Then close the file. 
 * Return number of bytes read. 
 * Support both sequential read and position read.
 */
private long readData(String fname, byte[] buffer, long byteExpected, long beginPosition)
    throws IOException {
  long totalByteRead = 0;
  Path path = getFullyQualifiedPath(fname);

  FSDataInputStream in = null;
  try {
    in = openInputStream(path);

    long visibleLenFromReadStream = ((HdfsDataInputStream)in).getVisibleLength();

    if (visibleLenFromReadStream < byteExpected)
    {
      throw new IOException(visibleLenFromReadStream
          + " = visibleLenFromReadStream < bytesExpected= "
          + byteExpected);
    }

    totalByteRead = readUntilEnd(in, buffer, buffer.length, fname,
        beginPosition, visibleLenFromReadStream, positionReadOption);
    in.close();

    // reading more data than visibleLeng is OK, but not less
    if (totalByteRead + beginPosition < byteExpected ){
      throw new IOException("readData mismatch in byte read: expected=" 
          + byteExpected + " ; got " +  (totalByteRead + beginPosition));
    }
    return totalByteRead + beginPosition;

  } catch (IOException e) {
    throw new IOException("##### Caught Exception in readData. "
        + "Total Byte Read so far = " + totalByteRead + " beginPosition = "
        + beginPosition, e);
  } finally {
    if (in != null)
      in.close();
  }
}
 
Example 17
Source Project: big-c   Source File: TestShortCircuitLocalRead.java    License: Apache License 2.0 5 votes vote down vote up
private boolean checkUnsupportedMethod(FileSystem fs, Path file,
                                         byte[] expected, int readOffset) throws IOException {
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
  IOUtils.skipFully(stm, readOffset);
  try {
    stm.read(actual);
  } catch(UnsupportedOperationException unex) {
    return true;
  }
  return false;
}
 
Example 18
Source Project: big-c   Source File: TestScrLazyPersistFiles.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR,
    new StorageType[]{RAM_DISK, DEFAULT},
    2 * BLOCK_SIZE - 1, true);  // 1 replica + delta, SCR read
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
  FSDataInputStream fis = fs.open(path);

  // Verify SCR read counters
  try {
    fis = fs.open(path);
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 
Example 19
Source Project: garmadon   Source File: FileSystemTracerTest.java    License: Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, URISyntaxException, InstantiationException {
    classLoader = new ByteArrayClassLoader.ChildFirst(FileSystemTracerTest.class.getClassLoader(),
        ClassFileExtraction.of(
            Tracer.class,
            MethodTracer.class,
            FileSystemTracer.class,
            FileSystemTracer.WriteTracer.class,
            FileSystemTracer.ReadTracer.class,
            FileSystemTracer.AddBlockTracer.class,
            FileSystemTracer.ListStatusTracer.class,
            FileSystemTracer.GetContentSummaryTracer.class,
            FileSystemTracer.RenameTracer.class,
            FileSystemTracer.DeleteTracer.class,
            DFSClient.class,
            DFSClient.Conf.class,
            ClientContext.class,
            DistributedFileSystem.class,
            DomainSocketFactory.class,
            DFSInputStream.class,
            DFSOutputStream.class,
            HdfsDataInputStream.class,
            HdfsDataOutputStream.class,
            ClientNamenodeProtocolTranslatorPB.class,
            Class.forName("org.apache.hadoop.hdfs.DFSOpsCountStatistics"),
            Class.forName("org.apache.hadoop.hdfs.BlockReaderLocal"),
            Class.forName(DFSOutputStream.class.getName() + "$Packet"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$1"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$2"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$ResponseProcessor"),
            Class.forName(DistributedFileSystem.class.getName() + "$1"),
            Class.forName(DistributedFileSystem.class.getName() + "$4"),
            Class.forName(DistributedFileSystem.class.getName() + "$7"),
            Class.forName(DistributedFileSystem.class.getName() + "$13"),
            Class.forName(DistributedFileSystem.class.getName() + "$14"),
            Class.forName(DistributedFileSystem.class.getName() + "$16"),
            Class.forName(DistributedFileSystem.class.getName() + "$19"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$1"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$Factory"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$Factory$Key"),
            Class.forName("org.apache.hadoop.fs.Hdfs"),
            HdfsDataOutputStream.class
        ),
        ByteArrayClassLoader.PersistenceHandler.MANIFEST);

    argument = ArgumentCaptor.forClass(DataAccessEventProtos.FsEvent.class);

    eventHandler = mock(BiConsumer.class);

    ReflectionHelper.setField(null, classLoader.loadClass(FileSystemTracer.class.getName()), "eventHandler", eventHandler);
    ReflectionHelper.setField(conf, Configuration.class, "classLoader", classLoader);
    assertThat(ByteBuddyAgent.install(), instanceOf(Instrumentation.class));

    new FileSystemTracer.WriteTracer().installOnByteBuddyAgent();
    new FileSystemTracer.ReadTracer().installOnByteBuddyAgent();
    new FileSystemTracer.ListStatusTracer().installOnByteBuddyAgent();
    new FileSystemTracer.GetContentSummaryTracer().installOnByteBuddyAgent();
    new FileSystemTracer.DeleteTracer().installOnByteBuddyAgent();
    new FileSystemTracer.RenameTracer().installOnByteBuddyAgent();

    // MiniDfsCluster
    File baseDir = new File("./target/hdfs/test").getAbsoluteFile();
    FileUtil.fullyDelete(baseDir);
    conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf)
        .simulatedCapacities(new long[] {10240000L});
    hdfsCluster = builder.build();
    hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort();

    initDFS();
}
 
Example 20
Source Project: hadoop   Source File: DFSTestUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static List<LocatedBlock> getAllBlocks(FSDataInputStream in)
    throws IOException {
  return ((HdfsDataInputStream) in).getAllBlocks();
}
 
Example 21
Source Project: hadoop   Source File: DFSTestUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static List<LocatedBlock> getAllBlocks(FileSystem fs, Path path)
    throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  return in.getAllBlocks();
}
 
Example 22
Source Project: hadoop   Source File: TestShortCircuitLocalRead.java    License: Apache License 2.0 4 votes vote down vote up
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 
Example 23
Source Project: hadoop   Source File: TestScrLazyPersistFiles.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Eviction of lazy persisted blocks with Short Circuit Read handle open
 * Note: the test uses faked RAM_DISK from physical disk.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRamDiskEvictionWithShortCircuitReadHandle()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
    (6 * BLOCK_SIZE -1), true);  // 5 replica + delta, SCR.
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
  Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
  final int SEED = 0xFADED;

  makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path1, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job.
  // However the block replica should not be evicted from RAM_DISK yet.
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  // No eviction should happen as the free ratio is below the threshold
  FSDataInputStream fis = fs.open(path1);
  try {
    // Keep and open read handle to path1 while creating path2
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);

    // Create the 2nd file that will trigger RAM_DISK eviction.
    makeTestFile(path2, BLOCK_SIZE * 2, true);
    ensureFileReplicasOnStorageType(path2, RAM_DISK);

    // Ensure path1 is still readable from the open SCR handle.
    fis.read(fis.getPos(), buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    IOUtils.closeQuietly(fis);
  }

  // After the open handle is closed, path1 should be evicted to DISK.
  triggerBlockReport();
  ensureFileReplicasOnStorageType(path1, DEFAULT);
}
 
Example 24
Source Project: hadoop   Source File: TestDecommission.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Verify that the number of replicas are as expected for each block in
 * the given file.
 * For blocks with a decommissioned node, verify that their replication
 * is 1 more than what is specified.
 * For blocks without decommissioned nodes, verify their replication is
 * equal to what is specified.
 * 
 * @param downnode - if null, there is no decommissioned node for this file.
 * @return - null if no failure found, else an error message string.
 */
private static String checkFile(FileSystem fileSys, Path name, int repl,
  String downnode, int numDatanodes) throws IOException {
  boolean isNodeDown = (downnode != null);
  // need a raw stream
  assertTrue("Not HDFS:"+fileSys.getUri(),
      fileSys instanceof DistributedFileSystem);
  HdfsDataInputStream dis = (HdfsDataInputStream)
      fileSys.open(name);
  Collection<LocatedBlock> dinfo = dis.getAllBlocks();
  for (LocatedBlock blk : dinfo) { // for each block
    int hasdown = 0;
    DatanodeInfo[] nodes = blk.getLocations();
    for (int j = 0; j < nodes.length; j++) { // for each replica
      if (isNodeDown && nodes[j].getXferAddr().equals(downnode)) {
        hasdown++;
        //Downnode must actually be decommissioned
        if (!nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is given as downnode, " +
            "but is not decommissioned";
        }
        //Decommissioned node (if any) should only be last node in list.
        if (j != nodes.length - 1) {
          return "For block " + blk.getBlock() + " decommissioned node "
            + nodes[j] + " was not last node in list: "
            + (j + 1) + " of " + nodes.length;
        }
        LOG.info("Block " + blk.getBlock() + " replica on " +
          nodes[j] + " is decommissioned.");
      } else {
        //Non-downnodes must not be decommissioned
        if (nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is unexpectedly decommissioned";
        }
      }
    }

    LOG.info("Block " + blk.getBlock() + " has " + hasdown
      + " decommissioned replica.");
    if(Math.min(numDatanodes, repl+hasdown) != nodes.length) {
      return "Wrong number of replicas for block " + blk.getBlock() +
        ": " + nodes.length + ", expected " +
        Math.min(numDatanodes, repl+hasdown);
    }
  }
  return null;
}
 
Example 25
Source Project: big-c   Source File: DFSTestUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static List<LocatedBlock> getAllBlocks(FSDataInputStream in)
    throws IOException {
  return ((HdfsDataInputStream) in).getAllBlocks();
}
 
Example 26
Source Project: big-c   Source File: DFSTestUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static List<LocatedBlock> getAllBlocks(FileSystem fs, Path path)
    throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  return in.getAllBlocks();
}
 
Example 27
Source Project: big-c   Source File: TestShortCircuitLocalRead.java    License: Apache License 2.0 4 votes vote down vote up
/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 
Example 28
Source Project: big-c   Source File: TestScrLazyPersistFiles.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Eviction of lazy persisted blocks with Short Circuit Read handle open
 * Note: the test uses faked RAM_DISK from physical disk.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRamDiskEvictionWithShortCircuitReadHandle()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
    (6 * BLOCK_SIZE -1), true);  // 5 replica + delta, SCR.
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
  Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
  final int SEED = 0xFADED;

  makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path1, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job.
  // However the block replica should not be evicted from RAM_DISK yet.
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  // No eviction should happen as the free ratio is below the threshold
  FSDataInputStream fis = fs.open(path1);
  try {
    // Keep and open read handle to path1 while creating path2
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);

    // Create the 2nd file that will trigger RAM_DISK eviction.
    makeTestFile(path2, BLOCK_SIZE * 2, true);
    ensureFileReplicasOnStorageType(path2, RAM_DISK);

    // Ensure path1 is still readable from the open SCR handle.
    fis.read(fis.getPos(), buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    IOUtils.closeQuietly(fis);
  }

  // After the open handle is closed, path1 should be evicted to DISK.
  triggerBlockReport();
  ensureFileReplicasOnStorageType(path1, DEFAULT);
}
 
Example 29
Source Project: big-c   Source File: TestDecommission.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Verify that the number of replicas are as expected for each block in
 * the given file.
 * For blocks with a decommissioned node, verify that their replication
 * is 1 more than what is specified.
 * For blocks without decommissioned nodes, verify their replication is
 * equal to what is specified.
 * 
 * @param downnode - if null, there is no decommissioned node for this file.
 * @return - null if no failure found, else an error message string.
 */
private static String checkFile(FileSystem fileSys, Path name, int repl,
  String downnode, int numDatanodes) throws IOException {
  boolean isNodeDown = (downnode != null);
  // need a raw stream
  assertTrue("Not HDFS:"+fileSys.getUri(),
      fileSys instanceof DistributedFileSystem);
  HdfsDataInputStream dis = (HdfsDataInputStream)
      fileSys.open(name);
  Collection<LocatedBlock> dinfo = dis.getAllBlocks();
  for (LocatedBlock blk : dinfo) { // for each block
    int hasdown = 0;
    DatanodeInfo[] nodes = blk.getLocations();
    for (int j = 0; j < nodes.length; j++) { // for each replica
      if (isNodeDown && nodes[j].getXferAddr().equals(downnode)) {
        hasdown++;
        //Downnode must actually be decommissioned
        if (!nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is given as downnode, " +
            "but is not decommissioned";
        }
        //Decommissioned node (if any) should only be last node in list.
        if (j != nodes.length - 1) {
          return "For block " + blk.getBlock() + " decommissioned node "
            + nodes[j] + " was not last node in list: "
            + (j + 1) + " of " + nodes.length;
        }
        LOG.info("Block " + blk.getBlock() + " replica on " +
          nodes[j] + " is decommissioned.");
      } else {
        //Non-downnodes must not be decommissioned
        if (nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is unexpectedly decommissioned";
        }
      }
    }

    LOG.info("Block " + blk.getBlock() + " has " + hasdown
      + " decommissioned replica.");
    if(Math.min(numDatanodes, repl+hasdown) != nodes.length) {
      return "Wrong number of replicas for block " + blk.getBlock() +
        ": " + nodes.length + ", expected " +
        Math.min(numDatanodes, repl+hasdown);
    }
  }
  return null;
}